From: Jeremy Sowden Date: Mon, 3 Jun 2024 21:05:55 +0000 (+0100) Subject: Import maildir-utils_1.12.5.orig.tar.gz X-Git-Tag: archive/raspbian/1.12.9-1+rpi1~6 X-Git-Url: https://dgit.raspbian.org/?a=commitdiff_plain;h=9b4164a748baf0015963b4e8901ecc54982cd262;p=maildir-utils.git Import maildir-utils_1.12.5.orig.tar.gz [dgit import orig maildir-utils_1.12.5.orig.tar.gz] --- 9b4164a748baf0015963b4e8901ecc54982cd262 diff --git a/.dir-locals.el b/.dir-locals.el new file mode 100644 index 0000000..997a80e --- /dev/null +++ b/.dir-locals.el @@ -0,0 +1,26 @@ +;;; -*- no-byte-compile: t; -*- +((nil . ((tab-width . 8) + (fill-column . 80) + ;; (commment-fill-column . 80) + (emacs-lisp-docstring-fill-column . 65) + (bug-reference-url-format . "https://github.com/djcb/mu/issues/%s"))) + (c-mode . ((c-file-style . "linux") + (indent-tabs-mode . t) + (mode . bug-reference-prog))) + (c-ts-mode . ((indent-tabs-mode . t) + (c-ts-mode-indent-style . linux) + (c-ts-mode-indent-offset . 8) + (mode . bug-reference-prog))) + (c++-mode . ((c-file-style . "linux") + (fill-column . 100) + ;; (comment-fill-column . 80) + (mode . bug-reference-prog))) + (c++-ts-mode . ((indent-tabs-mode . t) + (c-ts-mode-indent-style . linux) + (c-ts-mode-indent-offset . 8) + (mode . bug-reference-prog))) + (emacs-lisp-mode . ((indent-tabs-mode . nil) + (mode . bug-reference-prog))) + (lisp-data-mode . ((indent-tabs-mode . nil))) + (texinfo-mode . ((mode . bug-reference-prog))) + (org-mode . ((mode . bug-reference)))) diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..824f406 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,34 @@ +#-*-mode:conf-*- +# editorconfig file (see EditorConfig.org), with some +# lowest-denominator settings that should work for many editors. + + +root = true # this is the top-level + +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true + +# The "best" answer is "tabs-for-indentation; spaces for alignment". + +[*.{cc,cpp,hh,hpp}] +indent_style = tab +indent_size = 8 +max_line_length = 90 + +[*.{c,h}] +indent_style = tab +indent_size = 8 +max_line_length = 80 + +[configure.ac] +indent_style = tab +indent_size = 4 +max_line_length = 100 + +[Makefile.am] +indent_style = tab +indent_size = 8 +max_line_length = 100 diff --git a/.github/ISSUE_TEMPLATE/feature-request.md b/.github/ISSUE_TEMPLATE/feature-request.md new file mode 100644 index 0000000..77f0195 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.md @@ -0,0 +1,22 @@ +--- +name: Mu4e Feature request +about: Suggest an idea for this project +title: "[mu4e rfe]" +labels: rfe, mu4e, new +assignees: '' + +--- +Note, please see the IDEAS.org file in repository root for existing ideas; +maybe it's already there. + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/guile.md b/.github/ISSUE_TEMPLATE/guile.md new file mode 100644 index 0000000..020b849 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/guile.md @@ -0,0 +1,20 @@ +--- +name: Guile +about: mu-guile related item +title: "[guile]" +labels: new, guile +assignees: '' + +--- + +**Describe the item** +A clear and concise description of what you expected or wished to happen and what actually happened while using mu-guile. + +**To Reproduce** +Steps to reproduce the behavior. + +**Environment** +Please describe the versions of OS, Emacs, mu/mu4e etc. you are using. + +**Checklist** +- [ ] you are running either the latest 1.4.x release, or a 1.5.11+ development release (otherwise, please upgrade). diff --git a/.github/ISSUE_TEMPLATE/misc.md b/.github/ISSUE_TEMPLATE/misc.md new file mode 100644 index 0000000..7f942cd --- /dev/null +++ b/.github/ISSUE_TEMPLATE/misc.md @@ -0,0 +1,16 @@ +--- +name: Misc +about: Miscellaneous items you want to share +title: "[misc]" +labels: new +assignees: '' + +--- + +**Note**: for questions, please use the mailing-list: https://groups.google.com/g/mu-discuss + +**Describe the issue** +A clear and concise description, i.e. what you expected/desired to happen and what actually happened. + +**Environment** +If applicable, please describe the versions of OS, Emacs, mu etc. you are using. diff --git a/.github/ISSUE_TEMPLATE/mu-bug-report.md b/.github/ISSUE_TEMPLATE/mu-bug-report.md new file mode 100644 index 0000000..ef2b3f4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/mu-bug-report.md @@ -0,0 +1,20 @@ +--- +name: Mu Bug Report +about: Create a report to help us improve +title: "[mu bug]" +labels: bug, mu, new +assignees: '' + +--- + +**Describe the bug** +A clear and concise description of what the bug is, what you expected to happen and what actually happened. + +**To Reproduce** +Detailed steps to reproduce the behavior. If this is about a specific (kind of) message, **always** attach an (anonymized as need) example message. + +**Environment** +Please describe the versions of OS, Emacs, mu etc. you are using. + +**Checklist** +- [ ] you are running either the latest 1.8.x/1.10.x release or `master` (otherwise, please upgrade). diff --git a/.github/ISSUE_TEMPLATE/mu4e-bug-report.md b/.github/ISSUE_TEMPLATE/mu4e-bug-report.md new file mode 100644 index 0000000..63d857f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/mu4e-bug-report.md @@ -0,0 +1,41 @@ +--- +name: Mu4e Bug Report +about: Create a report to help us improve +title: "[mu4e bug]" +labels: bug, mu4e, new +assignees: '' +--- + +**Describe the bug** + +Give the bug a good title. + +Please provide a clear and concise description of what you expected to happen +and what actually happened, and follow the steps below. + +**How to Reproduce** + +Include the exact steps of what you were doing (commands executed etc.). Include +any relevant logs and outputs: + +- Best start from `emacs -Q`, and load a minimal `mu4e` setup; describe the steps + that lead up to the bug. +- Does the problem happen each time? Sometimes? +- If this is about a specific (kind of) message, attach an example message. + (Open the message, press `.` (`mu4e-view-raw-message`), then `C-x C-w` and + attach. Anonymize as needed, all that matters is that the issue still + reproduces. + +**Environment** + +Please describe the versions of OS, Emacs, mu/mu4e etc. you are using. + +**Checklist** + +- [ ] you are running either an 1.10.x/1.12.x release or `master` (otherwise please upgrade) +- [ ] you can reproduce the problem without 3rd party extensions (including Doom/Evil, various extensions etc.) +- [ ] you have read all of the above + +Please make sure you all items in the checklist are set/met before filing the ticket. + +Thank you! diff --git a/.github/issue_template.md b/.github/issue_template.md new file mode 100644 index 0000000..a684c39 --- /dev/null +++ b/.github/issue_template.md @@ -0,0 +1,39 @@ +# Important! Before filing an issue, please consider the following: + + * Ensure your mu/mu4e setup is no older than the latest stable release (1.6.x). + + * Disable any third-party mu4e extensions; this includes customizations like the ones in "Doom" / + "Evil" etc. + + * If a problem occurs with a certain (type of) message, attach an (anonymized) example of + such a message + + * Please provide some minimal steps to reproduce + + * Please follow the below template + + Thanks! + +## Expected or desired behavior + +Please describe the behavior you expect or want + +## Actual behavior + +Please describe the behavior you are actually seeing. + +For bug-reports, if applicable, include error messages, emacs stack traces, example messages +etc. Try to be as specific as possible - when do you see this happening? Does it happen always? +Sometimes? How often? + +## Steps to reproduce + +For bug-reports, please describe in as much detail as possible how one can reproduce the problem. + +If there's a problem with a specific (type of) message, please attach such a message to the report. + +## Versions of mu, mu4e/emacs, operating system etc. + +## Any other detail + +E.g. are you using the gnus-based message view? diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml new file mode 100644 index 0000000..508901c --- /dev/null +++ b/.github/workflows/build-and-test.yml @@ -0,0 +1,39 @@ +name: Build & run tests + +on: + - push + - pull_request + +jobs: + build: + runs-on: ${{ matrix.os }} + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-latest + + steps: + - uses: actions/checkout@v2 + + - if: contains(matrix.os, 'ubuntu') + name: ubuntu-deps + run: | + sudo apt update + sudo apt-get install meson ninja-build libglib2.0-dev libxapian-dev libgmime-3.0-dev libcld2-dev pkg-config guile-3.0-dev emacs texinfo + + - if: contains(matrix.os, 'macos') + name: macos-deps + run: | + brew install meson ninja libgpg-error libtool pkg-config glib gmime xapian guile emacs texinfo + + - name: configure + run: ./autogen.sh # '-Db_sanitize=address' + + - name: build + run: make + + - name: test + run: make test-verbose-if-fail diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7d785f6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,141 @@ +www/mu +mug +/mu/mu +/mu/mu-help-strings.h +mug2 +.desktop +*html +.deps +.libs +autom4te* +Makefile +Makefile.in +INSTALL +aclocal.m4 +config.* +configure +install-sh +depcomp +libtool +ltmain.sh + +# Added automatically by `autoreconf` +m4/libtool.m4 +m4/ltoptions.m4 +m4/ltsugar.m4 +m4/ltversion.m4 +m4/lt~obsolete.m4 + +missing +nohup.out +vgdump +stamp-h1 +GPATH +GRTAGS +GSYMS +GTAGS +*.lo +*.o +*.la +*.x +*.go +*.gz +*.bz2 +\#* +*.aux +*.cp +*.fn +*.info +*.ky +*.log +*.pg +*.toc +*.tp +*.vr +*.elc +*.gcda +*.gcno +*.trs +*.exe +*.lib +aminclude_static.am +elisp-comp +elc-stamp +dummy.cc +msg2pdf +gmime-test +test-mu-cmd +test-mu-cmd-cfind +test-mu-contacts +test-mu-container +test-mu-date +test-mu-flags +test-mu-maildir +test-mu-msg +test-mu-msg-fields +test-mu-query +test-mu-runtime +test-mu-store +test-mu-str +test-mu-threads +test-mu-util +test-parser +test-tokenizer +test-utils +tokenize +test-command-parser +test-mu-utils +test-sexp-parser +test-scanner +/guile/tests/test-mu-guile + +mu4e-config.el +mu4e.pdf +texinfo.tex +texi.texi +*.tex +*.pdf +/www/auto/* +configure.lineno +/test.xml +/mu4e/mdate-sh +/mu4e/mu4e-about.el +/mu4e/stamp-vti +/mu4e/version.texi +/lib/doxyfile +/version.texi +/compile +/TAGS +parse + +*_flymake.* +*_flymake_* +/perf.data +perf.data +perf.data.old +*vgdump +/lib/asan.log* +/man/mu-mfind.1 +/mu/mu-memcheck +mu-*-coverage +mu*tar.xz +compile_commands.json +/lib/utils/test-sexp +/lib/utils/test-option +/lib/test-mu-threader +/lib/test-mu-tokenizer +/lib/test-mu-parser +/lib/test-mu-query-threader +/lib/test-contacts +/lib/test-flags +/lib/test-maildir +/lib/test-msg +/lib/test-msg-fields +/lib/test-query +/lib/test-store +/lib/test-threader +/mu/test-cmd +/mu/test-cmd-cfind +/mu/test-query +/mu/test-threads +/lib/test-threads diff --git a/.mailmap b/.mailmap new file mode 100644 index 0000000..3a54641 --- /dev/null +++ b/.mailmap @@ -0,0 +1 @@ +Dirk-Jan C. Binnema diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..3a54641 --- /dev/null +++ b/AUTHORS @@ -0,0 +1 @@ +Dirk-Jan C. Binnema diff --git a/COPYING b/COPYING new file mode 100644 index 0000000..94a9ed0 --- /dev/null +++ b/COPYING @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/IDEAS.org b/IDEAS.org new file mode 100644 index 0000000..e3e954e --- /dev/null +++ b/IDEAS.org @@ -0,0 +1,49 @@ +#+STARTUP:showall +* IDEAS + +Ideas for future enhancements. We collect those here so they don't clutter up +the Github issue list, i.e. without any clear plan for adding in the near +future. + +- Ability to _mute_ message threads. This is useful but also requires quite bit of extra infra; we could add some blacklist for "muted" messages, perhaps on the 'mu server' side, but then we'd need some way to manage that (ie., unmute). + https://github.com/djcb/mu/issues/636 + +- Support automatic handling for List-Unsubscribe headers + https://github.com/djcb/mu/issues/2623 This seems useful, but probably + requires a lot of testing to get right. + +- Allow for *muting* messages https://github.com/djcb/mu/issues/636 Useful; + probably need to do this by *remembering* the thread-id of muted messages; and + management (unmute etc.). Perhaps at the mu side, a list of thread-id to add + to each query for what *not* to match. + +- Support *creating* calendar invitations. + https://github.com/djcb/mu/issues/2308 + Shouldn't be _too_ hard, for someone that uses the functionality. + +- Make sorting stable if there are multiple messages with the same date. We + _could_ do this by adding some random millisecs to each messasge's timestamp; _or_ + complicating the search (i.e., the message hash?). Maybe leave as is? + https://github.com/djcb/mu/issues/2527 + +- Include "message summary" in message information, for display in the headers + buffer: https://github.com/djcb/mu/issues/1821 It's not so easy to get a + useful one line description... perhaps the first line after the "Dear x,"? + Moreover, this requires new functionality on the headers-view side as well. + +- Support indexing PDF (and other) attachments. This can be done extending + process_message_part in mu-message.cc; instead of using something + PDF-specific, we could pipe a PDF through some tool to extract text; and we'd + need some way for users to specify a MIME-type => tool mapping (in Config). + https://github.com/djcb/mu/issues/2117 + +- Support "aggregate actions" apply to a set of messages, e.g. apply patch-set + in a set of messages. That'll require some advanced scripting, maybe using + Guile. + https://github.com/djcb/mu/issues/301 + +* Done + +- Support mu4e-mark-handle-when also for when leaving emacs + (kill-emacs-query-functions). + https://github.com/djcb/mu/issues/2649 diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..662eda7 --- /dev/null +++ b/Makefile @@ -0,0 +1,158 @@ +## Copyright (C) 2008-2023 Dirk-Jan C. Binnema +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# Makefile with some useful targets for meson/ninja +V ?= 0 + +BUILDDIR ?= $(CURDIR)/build +BUILDDIR_COVERAGE ?= $(CURDIR)/build-coverage +BUILDDIR_VALGRIND ?= $(CURDIR)/build-valgrind +BUILDDIR_BENCHMARK ?= $(CURDIR)/build-benchmark + +GENHTML ?= genhtml +LCOV ?= lcov +MAKEINFO ?= makeinfo +MESON ?= meson +NINJA ?= ninja +VALGRIND ?= valgrind + +ifneq ($(V),0) + VERBOSE=--verbose +endif + +# when MU_HACKER is set, do a debug build +# MU_HACKER is for djcb & compatible developers +# note that mu uses C++17, we only pass C++23 here +# for the better error messages (esp. for fmt). +ifneq (${MU_HACKER},) +MESON_FLAGS:=$(MESON_FLAGS) '-Dbuildtype=debug' \ + '-Db_sanitize=address' \ + '-Dreadline=enabled' \ + '-Dcpp_std=c++23' +endif + +.PHONY: all build-valgrind +.PHONY: check test test-verbose-if-fail test-valgrind test-helgrind +.PHONY: benchmark coverage +.PHONY: dist install uninstall clean distclean +.PHONY: mu4e-doc-html + +# MESON_FLAGS, e.g. "-Dreadline=enabled" + +# examples: +# 1. build with clang, and the thread-sanitizer +# make clean all MESON_FLAGS="-Db_sanitize=thread" CXX=clang++ CC=clang +all: $(BUILDDIR) + @$(MESON) compile -C $(BUILDDIR) $(VERBOSE) + @ln -sf $(BUILDDIR)/compile_commands.json $(CURDIR) || /bin/true + +$(BUILDDIR): + @$(MESON) setup $(MESON_FLAGS) $(BUILDDIR) + +check: test + +test: all + @$(MESON) test $(VERBOSE) -C $(BUILDDIR) + +install: $(BUILDDIR) + @$(MESON) install -C $(BUILDDIR) $(VERBOSE) + +uninstall: $(BUILDDIR) + @$(NINJA) -C $(BUILDDIR) uninstall + +clean: + @rm -rf $(BUILDDIR) $(BUILDDIR_COVERAGE) $(BUILDDIR_VALGRIND) $(BUILDDIR_BENCHMARK) + @rm -rf compile_commands.json + +# +# below targets are just for development/testing/debugging. They may or +# may not work on your system. +# +test-verbose-if-fail: all + $(MESON) test -C $(BUILDDIR) || $(MESON) test -C $(BUILDDIR) --verbose + +build-valgrind: $(BUILDDIR_VALGRIND) + @$(MESON) compile -C $(BUILDDIR_VALGRIND) $(VERBOSE) + +$(BUILDDIR_VALGRIND): + @$(MESON) setup --buildtype=debug $(BUILDDIR_VALGRIND) + +vg_opts:=--enable-debuginfod=no --leak-check=full --error-exitcode=1 +test-valgrind: export G_SLICE=always-malloc +test-valgrind: export G_DEBUG=gc-friendly +test-valgrind: build-valgrind + @$(MESON) test -C $(BUILDDIR_VALGRIND) \ + --wrap="$(VALGRIND) $(vg_opts)" \ + --timeout-multiplier 100 + +check-valgrind: test-valgrind + +# we do _not_ pass helgrind; but this seems to be a false-alarm +# https://gitlab.gnome.org/GNOME/glib/-/issues/2662 +test-helgrind: $(BUILDDIR_VALGRIND) + $(MESON) -C $(BUILDDIR_VALGRIND) test \ + --wrap="$(VALGRIND) --tool=helgrind --error-exitcode=1" \ + --timeout-multiplier 100 + +check-helgrind: test-helgrind + +# +# benchmarking +# + +$(BUILDDIR_BENCHMARK): + @$(MESON) setup --buildtype=debugoptimized $(BUILDDIR_BENCHMARK) + +build-benchmark-target: $(BUILDDIR_BENCHMARK) + @$(MESON) compile -C $(BUILDDIR_BENCHMARK) $(VERBOSE) + +benchmark: build-benchmark-target + $(NINJA) -C $(BUILDDIR_BENCHMARK) benchmark + +# +# coverage +# + +$(BUILDDIR_COVERAGE): + $(MESON) setup -Db_coverage=true --buildtype=debug $(BUILDDIR_COVERAGE) + +covfile:=$(BUILDDIR_COVERAGE)/meson-logs/coverage.info + +# generate by hand, meson's built-ins are rather inflexible +coverage: $(BUILDDIR_COVERAGE) + @$(MESON) compile -C $(BUILDDIR_COVERAGE) + @$(MESON) test -C $(BUILDDIR_COVERAGE) $(VERBOSE) + $(LCOV) --capture --directory . --output-file $(covfile) + @$(LCOV) --remove $(covfile) '/usr/*' '*guile*' '*thirdparty*' '*/tests/*' '*mime-object*' --output $(covfile) + @$(LCOV) --remove $(covfile) '*mu/mu/*' --output $(covfile) + @mkdir -p $(BUILDDIR_COVERAGE)/meson-logs/coverage + @$(GENHTML) $(covfile) --output-directory $(BUILDDIR_COVERAGE)/meson-logs/coverage/ + @echo "coverage report at: file://$(BUILDDIR_COVERAGE)/meson-logs/coverage/index.html" + +# +# misc +# + +dist: $(BUILDDIR) + $(MESON) compile -C $(BUILDDIR) $(VERBOSE) + $(MESON) dist -C $(BUILDDIR) $(VERBOSE) + +distclean: clean + +HTMLPATH=${BUILDDIR}/mu4e/mu4e +mu4e-doc-html: + @mkdir -p ${HTMLPATH} && cp mu4e/texinfo-klare.css ${HTMLPATH} + @cd mu4e; makeinfo -v -I ${BUILDDIR} -I ${BUILDDIR}/mu4e --html --css-ref=texinfo-klare.css -o ${HTMLPATH} mu4e.texi diff --git a/NEWS.org b/NEWS.org new file mode 100644 index 0000000..442eb4e --- /dev/null +++ b/NEWS.org @@ -0,0 +1,1582 @@ +#+STARTUP:showall +* NEWS (user visible changes & bigger non-visible ones) + +* 1.12 (released on February 24, 2024) + +** Some highlights + + - Significant speedups in both ~mu~ and ~mu4e~ + - Reworked message composition, closer to its Gnus origins which adds many of its features + - Overhauled the query parser; squashing a number of bugs/limitations, incl. dealing + with CJK messages + - Experimental folding of message threads + - Better and faster indexing of HTML messages + - Experimental search by (human) language wit CLD2 + + For details & more, see below. Note a few minor new features were added + _after_ the initial 1.12.0. + +*** mu + + - new command ~mu move~ to move messages across maildirs and/or change their + flags. See the manpage for all the details. + + - ~mu~ commands ~extract~ ~verify~ and ~view~ can now read the message from + standard input; see their man-pages for details + + - ~mu init~ gained the ~--ignored-address~ option for email-addresses / regexps + that should _not_ be included in the contacts-cache (i.e., for ~mu cfind~ and + Mu4e address completion). See the ~mu-init~ manpage for details. + + It's not unusual for ~noreply~-type e-mail addresses to be the majority in + an e-mail corpus; to get rid of those, with something like + ~--ignored-address=/.*no.*reply.*/~ + + - what used to be the ~mu fields~ command has been merged into ~mu info~; i.e., + ~mu fields~ is now ~mu info fields~. + + - ~mu view~ gained ~--format=html~ which compels it to output the HTML body of + the message rather than the (default) plain-text body. See its updated + manpage for details. + + - when encountering an HTML message part during indexing, previously (i.e., + ~mu 1.10~) we would attempt to process that as-is, with HTML-tags etc.; this + is now improved by employing a custom html->text scraper which extracts + the human-readable text from the html. + + - mu querying and (esp.) showing results has been made significantly faster; + e.g., in one big ~mu find~ query we went from ~47s to only ~7s + + - /experimental/: if you build ~mu~ with [[https://github.com/CLD2Owners/cld2][CLD2]] support (available in many Linux + distros), ~mu~ will try to detect the language of the body of e-mail + messages; you can then search by their ISO-639-1 code, e.g.: + ~$ mu find lang:en~ + + the matching is not perfect, and seems to favor non-English if there's a + mostly English message with some other language mixed in. + + this does require re-indexing the database. + + - set the default database batch-size (using the ~mu init~ command) to 50000 + rather than 250000; the latter was too high for systems with limited + memory. You can of course change that with ~--batch-size=...~ + + - restore expansion for path options such as ~--maildir=~/Maildir~ (to e.g. + ~/home/user/Maildir~) for shells that do not do that, such as Bash. + + - overhauled the query-parser; this is (should be) compatible with the older + one, apart from a number of fixes. There is a new option ~--analyze~ for the + ~mu find~ command, which shows the parsed query in a (hopefully) + human-readable s-expression form; this can be used to debug your queries + (this replaces the older ~--format=mquery|xquery~) + + Furthermore, there now support for "ngram"-based indexing and querying, + which is useful for languages/scripts without explicit word-breaks, such + as Chinese/Japanese/Korean. See the *mu-init* manpages, in particular the + ~--support-ngrams~ option, and why you may (or may not) want to enable that. + + - the build has been made reproducible + +*** mu4e + +**** message composer + + - Overhaul of the message composer; it is now closer to the Gnus/Message + composer functions (e.g. the whole mu4e-specific draft setup is gone); + this reduces code size and offers some new capabilities. + + More of the ~message-~ functionality can be used now in ~mu4e~. + + - Variables ~mu4e-compose-signature~, ~mu4e-compose-cite-function~ are gone + (with aliases in place), use ~message-signature~, ~message-cite-function~ + instead. There's a special ~mu4e-message-cite-nothing~ for the case where + you do not want to cite anything. + + - There's a new function ~mu4e-compose-wide-reply~ (bound to =W=) which does a + wide-reply, a.k.a., 'reply to all'. So ~mu4e-compose-reply-recipients~ is + not needed anymore and has been obsoleted (and doesn't do anything). + ~mu4e-compose-reply-ignore-address~ is no longer supported, use + ~message-prune-recipient-rules~ instead. + + Same for ~mu4e-compose-dont-reply-to-self~; roughly the same effect can be + achieved by setting ~message-dont-reply-to-names~ to + ~#'mu4e-personal-or-alternative-address-p~. This only works for + [[info:(message) Wide Reply][wide-replies]]. + + - Another new function is ~mu4e-compose-supersede~ (not bound to any key by + default), with which you can /supersede/ your own messages; that is, send + the message as a kind-of reply to the same recipients. This only works if + you were the sender. + + - The special mailing list handling is gone; ~mu4e-compose-reply~ and + ~mu4e-compose-wide-reply~ should take care of that. There's also + ~message-reply-to-function~ for ultimate control; see [[info:(message) + Reply][info (message) Reply]] for details. + + - ~mu4e-compose-in-new-frame~ has been generalized (in a backward-compatible + way) to ~mu4e-compose-switch~, which lets you decide whether a message + should be composed in the current window (default), a new window or a new + frame. + + - ~mu4e-compose-context-switch~ is gone; it was a little too fragile. Best + change when creating the message (=mu4e= asks you by default, see + ~mu4e-compose-context-policy~). + + - there's a new hook ~mu4e-compose-post-hook~ which fires when message + composition is complete - either a message has been sent, it is postponed, + canceled etc. (1.12.5). + + - iCalendar support is a work-in-progress with the new editor. One change is + that support is now _automatically_ available. + +**** other + + - New command ~mu4e-search-query~ (bound to =c=) which lets you pick a query + (from bookmark / maildir shortcuts) with completion in main / headers / + view buffers. + + - improved support for dealing with attachments and other MIME-parts in the + message view; they gained completions support with annotations in the + minibuffer + + It is possible to save all attachments at once with =C-c C-a=, except with + Helm, which uses its own mechanism for this. This same has been extended + to the MIME-part actions. + + - experimental: support folding message threads (with =TAB= / =S-TAB=). See the + [[info:mu4e:Folding threads][entry in the Mu4e manual]] for further details. + + - mailing list support was modernized a bit; the format changed (see the + ~mu4e-mailing-lists~ and ~mu4e-user-mailing-lists~ docstrings. There is + ~M-x mu4e-mailing-list-info-refresh~ to update to the new values after + changing them. + + - also, there are now actions ('a' in view/header) to get to online archives + for some (selected) mailing-list archives. + + - ~mu4e-quit~ now takes a prefix argument which, if provided, causes it to + bury the mu main buffer, rather than quitting mu. ~mu4e~ will now just + switch the mu4e buffer if it exists (otherwise it starts ~mu4e~). + + - ~mu4e~ queries are much snappier now, due to the mentioned speed-ups in + querying; ~mu4e~ also adds a new optimization =mu4e-mu-allow-temp-file= + (turned off by default), which speed up things further; e.g., for showing + 500 messages (debug build), we went from 642ms to 247ms, given an + in-memory temp file. + + If and how much this helps, depends on your setup, see the + =mu4e-mu-allow-temp-file= docstring for details on how to determine this. + + - Maildir lists are now generated server-side; so e.g. jumping to the 'jo' + /other/ Maildirs used to be quite slow the first time, but is now very fast. + + ~mu4e-cache-maildir-list~ is obsolete / non-functional now. + + - after retrieving mail (~mu4e-update-mail-and-index~), save the output of the + retrieval command in a buffer =*mu4e-last-update*=, = which can be useful + for diagnosis. + + - links (in text-mode emails) are now clickable through , to be + consistent with eww. + + - support new-mail notifications on MacOS out-of-the-box + + - allow sorting by tag + + - ~mu4e~ now follows Emacs' ~package~ guidelines + +*** Contributors + + Thanks to our contributors - code committers belows, but also to everyone + who filed tickets, asked questions, answered them etc. + + Babak Farrokhi, Christophe Troestler, Christoph Reichenbach, Daniel Fleischer, + David Edmondson, Davide Masserut, Dirk-Jan C. Binnema, Jeremy Sowden, + Lin Jian, Martin R. Albrecht, Nacho Barrientos, Nicholas Vollmer, + Nicolas P. Rougier, ramon diaz-uriarte (at Phelsuma), reindert, Ruijie Yu, + Sean Farley, stardiviner, Tassilo Horn and Thierry Volpiatto + + +* Old news + :PROPERTIES: + :VISIBILITY: folded + :END: + +** 1.10 (released on March 26, 2023) + +*** mu + + - a new command-line parser, which allows (hopefully!) for a better user + interaction; better error checking and more + + - Invalid e-mail addresses are no longer added to the contacts-cache. + + - The ~cfind~ command gained ~--format=json~, which makes it easy to further + process contact information, e.g. using ~jq~. See the manpage for more + details. + + - The ~init~ command learned ~--reinit~ to reinitialize the database with the + settings of an existing one + + - The ~script~ command is gone, and integrated with ~mu~ directly, i.e. the + scripts (when enabled) are directly visible in the ~mu~ output. Also see the + Guile section. + + - The ~extract~ command gained the ~--uncooked~ option to tell it to _not_ replace + spaces with dashes in extracted filenames (and a few other things). + + - Revamped manpages which are now generated from ~org~ descriptions + + - Standardize on PCRE-flavored regular expressions throughout *mu*. + + - ~mu~ no longer attempts to 'expand' the =~= (and some other characters) in + command line options that take filenames, since it was a bit unpredictable. + So write e.g. ~--option=/home/user/hello~ instead of ~--option=~/hello~ + + - Experimental: as bit of a hack, html message bodies are processed as if + they were plain text, similar how "old mu" would do it (1.6.x and earlier). + A nicer solution would be to convert to text, but this something for the + future. + + - the MSYS2 (Windows) builds is _experimental_ now; some things may not work; + see e.g. https://github.com/djcb/mu/issues?q=is%3Aissue+label%3Amsys, but + we welcome efforts to fix those things. + +*** mu4e + + - ~emacs~ 26.3 or higher is now required for ~mu4e~ + + - ~mu4e-view-mode-hook~ now fires before the message is rendered. If you have + hook-functions that depend on the message contents, you should use + the new ~mu4e-view-rendered-hook~. + + - mu4e window management has been completely reworked and cleaned up, + affecting the message loading as well as the window-layout. As a + user-visible feature, there's now the =z= binding (~mu4e-view-detach~), to + 'detach' view and alllow for keV Detaching and reattaching][manual entry]] for further + details. + + - As a result of that, ~mu4e-split-view~ can no longer be a function; the new + way is to use ~display-buffer-alist~ as explained in the [[info:mu4e:Buffer Display][manual]] + + - ~mu4e~ now keeps track of 'baseline' query results and shows the difference + from that in the main view and modeline (you'll might see something like + =1(+1)/2= for your bookmarks or in the modeline; that means that there is + one more unread message since baseline; see the [[info:mu4e#Bookmarks and Maildirs][manual entry]] for details. + + The idea is that you get a quick overview of where changes happened while + you were doing something else. This is a somewhat experimental feature + which is under active development + + - Related to that, you can now crown one of your bookmarks in =mu4e-bookmarks= + with ~:favorite t~, causing it to be highlighted in the main view and used + in the mode-line. See the new [[info:mu4e#Modeline][modeline entry]] in the manual; this uses the + new =mu4e-modeline-mode= minor-mode. + + - Expanding on that further, you can also get desktop notifications for new + mail (on systems with DBus for now; see [[info:mu4e:#Desktop notifications][Desktop notifications]] in the + manual. + + - If your search query matches some bookmark, the modeline now shows the + bookmark's name rather than the query; this can be controlled through + =mu4e-modeline-prefer-bookmark-name= (default: =t=). + + - You can now tell mu4e to use emacs' completion system rather than the mu4e + built-in one; see the variables ~mu4e-read-option-use-builtin~ and + ~mu4e-completing-read-function~; e.g. to always emacs completion (which + may have been enhanced by various completion frameworks), use: + #+begin_src elisp + (setq mu4e-read-option-use-builtin nil + mu4e-completing-read-function 'completing-read) + #+end_src + + - when moving messages (which includes changing flags), file-flags changes + are propagated to duplicates of the messages; that is, e.g. the /Seen/ or + /Replied/ status is propagated to all duplicates (earlier, this was only + done when marking a message as read). Note, /Draft/, /Flagged/ and /Trashed/ + flags are deliberately *not* propagated. + + - Teach ~mu4e-copy-thing-at-point~ about ~shr~ links + + - The ~mu4e-headers-toggle-setting~ has been renamed + ~mu4e-headers-toggle-property~ and has the new default binding ~P~, which + works in both the headers-view and message-view. The older functions + ~mu4e-headers-toggle-threading~, ~mu4e-headers-toggle-threading~, + ~mu4e-headers-toggle-full-search~ ~mu4e-headers-toggle-include-related~, + ~full-search~skip-duplicates~ have been removed (with their keybindings) in + favor of ~mu4e-headers-toggle-property~. + + - There's also a new property ~mu4e-headers-hide-enabled~, which controls + wheter ~mu4e-headers-hide-predicate~ is applied (when non-~nil~). This can be + used to temporarily turn the predicate off/on. + + - You can now jump to previous / next threads in headers-view, message view. + Default binding is ~{~ and ~}~, respectively. + + - When searching, the number of hidden messages is now shown in the + message footer along with the number of Found messages + + - The ~eldoc~ support in header-mode is now optional and disabled by default; + set ~mu4e-eldoc-support~ to non-nil to enable it. + + - In the main view, the keybindings shown are a representation of the actual + keybindings, rather than just the defaults. This is for the benefit for + people who want to use different keybindings. + + - As a side-effect of that, ~mu4e-main-mode~ and ~mu4e-main-mode-hook~ functions + are now invoked _before_ the rendering takes place; if you're customizations + depend on happening after rendering is completed, use the new + ~mu4e-main-rendered-hook~ instead. + + - ~mu4e-cache-maildir-list~ has been promoted to be a =defcustom=, enabled by + default. This caches the list of "other" maildirs (i.e., without a + shortcut). + + - For testing, a new command ~mu4e-server-repl~ to start a ~mu~ server just as + ~mu4e~ does it. Note that this cannot run at the same time when ~mu4e~ runs. + + - all the obsolete function and variable aliases have been moved to + ~mu4e-obsolete.el~ so we can unclutter the non-obsolete code a bit. + +*** guile + + - in the 1.8 release, the /current/ Guile API was deprecated; that does not + mean that Guile support goes way, just that it will look different. + + - Guile script commands are now integrated with the main ~mu~, so without + further parameters ~mu~ shows both subcommands and scripts. This is a + work-in-progress! + + - The per-(week|day|year|year-month) scripts have been combined into a + ~histogram~ script. If you have Guile-support enabled, and have ~gnuplot~ + installed, you can do e.g., + +#+begin_example + mu histogram -- --time-unit=day --query="hello" +#+end_example + + to get a histogram of such messages. Note, this area is under active + development and will likely change. + +*** building and installation + + - the autotools build (which was deprecated since 1.8) has now been removed. + we thank it for its services since 2008. We continue with ~meson~. + + However, we still have ~autogen.sh~ and a ~Makefile~ which can be helpful for + driving ~meson~-based builds. Think of the ~Makefile~ as a convenient place to + put common action for which I always forget the ~meson~ incantation.** + + - ~meson~ 56.0 or higher is required for building + + - ~emacs~ 26.3 or higher is needed for ~mu4e~ + +*** internals + + As usual, there have been a number of internal updates in the ~mu~ codebase: + + - reworked the internal s-expression parser + + - new command-line argument parser (based on CLI11) + + - message-move flag propagation moved from the mu4e-server to mu-store + + - more =mu4e~= internals have been renamed/reworked in to ~mu4e--~. + +*** contributor to this release + + Aimé Bertrand, Aleksei Atavin, Al Haji-Ali, Andreas Hindborg, Anton Tetov, + Arsen Arsenović, Babak Farrokhi, Ben Cohen, Damon Kwok, Daniel Colascione, + Derek Zhou, Dirk-Jan C. Binnema, John Hamelink, Leo Gaskin, Manuel + Wiesinger, Marcel van der Boom, Mark Knoop, Mickey Petersen, Nicholas + Vollmer, Protesilaos Stavrou, Remco van 't Veer, Sean Allred, Sean Farley, + Stephen Eglen, Tassilo Horn + + And of course all the people how filed tickets, asked question, provided + suggestions. + + +** 1.8 (released on June 25, 2022) + + (there are some changes in the installation procedure compared to 1.6.x; see + Installation below) + +**** mu + + - The server protocol (as used my mu4e) has seen a number of updates, to + allow for faster rendering. As before, there's no compatibility between + minor release numbers (1.4 vs 1.6 vs 1.8) nor within development series + (such as 1.7). However, within a stable release (such as all 1.6.x) the + protocol won't change (except if required to fix some severe bug; this + never happened in practice) + + - The ~processed~ number in the indexing statistics has been renamed into + ~checked~ and describes the number of message files considered for updating, + which is a bit more useful that the old value, which was more-or-less + synonymous with the ~updated~ number (which are the messages that got + (re)parsed / (re)added to the database. + + Basically, it counts all the messages for which we checked their timestamp. + + - The internals of the message handling in ~mu~ have been heavily reworked; + much of this is not immediately visible but is an enabler for some new + features. + + - instead of passing ~--muhome~, you can now also set an environment variable + ~MUHOME~. + + - the ~info~ command now includes information about the last indexing + operation and the last database change that took place; note that the + information may be slightly delayed due to database caching. + + - the ~verify~ command for checking signatures has been updated, and is more + informative + + - a new command ~fields~ provides information about the message fields and + flags for use in queries. The information is the same information that ~mu~ + uses and so stays up to date. + + - a new message field ~changed~, which refers to the time/date of the last + time a message was changed (the file ~ctime~) + + - new message flags ~personal~ to search for "personal" messages, which are + defined as a message with at least one personal contact, and ~calendar~ for + messages with calendar-invitations. + + - message sexps are now cached in the store, which makes delivering + sexp-based search results (as used by ~mu4e~) much faster. + + - Windows/MSYS support is deprecated; it doesn't work well (if at all) and + there's currently not sufficient developer interest/expertise to change + this. + +**** mu4e + + - the old mu4e-view is *gone*; only the gnus-based one remains. This allowed + for removing quite a bit of old code. + + - the mu4e headers rendering is much faster (a factor of 3+), which makes + displaying big results snappier. This required some updates in the headers + handling and in the server protocol. Separate from that, the cached + message sexps (see the ~mu~ section) make getting the results much faster. + This becomes esp. clear when there are a lot of query results. + + - "related" messages are now recognizable as such in the headers-view, with + their own face, ~mu4e-related-face~; by default with an italic slant. + + - For performance testing, you can set the variable + ~mu4e-headers-report-render-time~ to ~t~ and ~mu4e~ will report the + search/rendering speed of each query operation. + + - Removed header-fields ~:attachments~, ~:signature~, ~:encryption~ and + ~:user-agent~. They're obsolete with the Gnus-based message viewer. + + - The various "toggles" for the headers-view (full-search, include-related, + skip-duplicates, threading) were a bit hard to find and with non-obvious + key-bindings. For that, there is now ~mu4e-headers-toggle-setting~ (bound + to ~M~) to handle all of that. The toggles are also reflected in the + mode-line; so e.g. 'RTU' means we're including [R]elated messages, and show + [T]hreads, skip duplicates ([U]nique). + + - A new ~defcustom~, ~mu4e-view-open-program~ for starting the appropriate + program for a give file (e.g., ~xdg-open~). There are some reasonable + defaults for various systems. This can also be set to a function. + + - indexing happens in the background now and mu4e can interact with the + server while it is ongoing; this allows for using mu4e during lengthy + indexing operations. + + - ~mu4e-index-updated-hook~ now fires after indexing completed, regardless of + whether anything changed (before, it fired only if something changed). In + your hook-functions (or elsewhere) you can check if anything changed using + the new variable ~mu4e-index-update-status~. And note that ~processed~ has + been renamed into ~checked~, with a slightly different meaning, see the mu + section. + + - ~message-user-organization~ can now be used to set the ~Organization:~ + header. See its docstring for details. + + - ~mu4e-compose-context-switch~ no longer attempts to update the draft folder + (which turned out to be a little fragile). However, it has been updated to + automatically change the ~Organization:~ header, and attempts to update the + message signature. Also, there's a key-binding now: ~C-c ;~ + + - Changed the default for ~mu4e-compose-complete-only-after~ to 2018-01-01, + to filter out contacts not seen after that date. + + - As an additional measure to limit the number of contacts that mu4e loads + for auto-completions, there's ~mu4e-compose-complete-max~, to set a precise + numerical match (*before* any possible filtering). Set to ~nil~ (no maximum + by default). + + - Updated the "fancy" characters for some header fields. Added new ones for + personal and list messages. + + - Removed ~make-mu4e-bookmark~ which was obsoleted in version 1.3.9. + + - Add command ~mu4e-sexp-at-point~ for showing/hiding the s-expression for + the message-at-point. Useful for development / debugging. Bound to ~,~ in + headers and view mode. + + - undo is now supported across message-saves + + - a lot of the internals have been changed: + + - =mu4e= is slowly moving from using the '=~'= to the more common '=--'= + separator for private functions; i.e., =mu4e-foo= becomes =mu4e--foo=. + + - =mu4e-utils.el= had become a bit of a dumping ground for bits of code; + it's gone now, with the functionality move to topic-specific files -- + =mu4e-folders.el=, =mu4e-bookmarks.el=, =mu4e-update.el=, and included in + existing files. + + - the remaining common functionality has ended up in =mu4e-helpers.el= + + - =mu4e-search.el= takes the search-specific code from =mu4e-headers.el=, + and adds a minor-mode for the keybindings. + + - =mu4e-context.el= and =mu4e-update.el= also define minor modes with + keybindings, which saves a lot of code in the various views, since they + don't need explicitly bind all those function. + + - also =mu4e-vars.el= had become very big, we're refactoring the =defvar= / + =defcustom= declarations to the topic-specific files. + + - =mu4e-proc.el= has been renamed =mu4e-server.el=. + + - Between =mu= and =mu4e=, contact cells are now represented as a plist ~(:name + "Foo Bar" :email "foobar@example.com")~ rather than a cons-cell ~("Foo + Bar" . "foobar@example.com").~ + + If you have scripts depending on the old format, there's the + ~mu4e-contact-cons~ function which takes a news-style contact and yields + the old form. + + - Because of all these changes, it is recommended you remove older version + of ~mu4e~ before reinstalling. + +**** guile + + - the current guile support has been deprecated. It may be revamped at some + point, but will be different from the current one, which is to be removed + after 1.8 + +**** toys + + - the ~toys~ (~mug~) has been removed, as they no longer worked with the rest of + the code. + +*** Installation + + - =mu= switched to the [[https://mesonbuild.com][meson]] build system by default. The existing =autotools= + is still available, but is to be removed after the 1.8 release. + + Using =meson= (which you may need to install), you can use something like + the following in the mu top source directory: + +#+BEGIN_SRC sh + $ meson build && ninja -C build +#+END_SRC + + - However, note that =autogen.sh= has been updated, and there's a + convenience =Makefile= with some useful targets, so you can also do: +#+BEGIN_SRC sh + $ ./autogen.sh && make # and optionally, 'sudo make install' +#+END_SRC + + - After that, either =ninja -C build= or =make= should be enough to rebuild + + - NOTE: development versions 1.7.18 - 17.7.25 had a bug where the mail file + names sometimes got misnamed (with some extra ':2,'). This can be restored + with something like: +#+begin_example + $ find ~/Maildir -name '*:2,*:*' | \ + sed "s/\(\([^:]*\)\(:2,\)\{1,\}\(:2,.*$\)\)/mv '\0' '\2\4'/" > rename.sh +#+end_example + (replace 'Maildir' with the path to your maildir) + + once this is done, do check the generated 'rename.sh' and after convincing + yourself it does the right thing, do +#+begin_example + $ sh rename.sh +#+end_example + after that, re-index. + + - Before installing, it is recommended that you *remove* any older versions + of ~mu~ and especially ~mu4e~, since they may conflict with the newer ones. + + - =mu= now requires C++17 support for building + + +*** Contributor for this release + + - As per ~git~: c0dev0id, Christophe Troestler, Daniel Fleischer, Daniel Nagy, + Dirk-Jan C. Binnema, Dr. Rich Cordero, Kai von Fintel, Marcelo Henrique + Cerri, Nicholas Vollmer, PRESFIL, Tassilo Horn, Thierry Volpiatto, Yaman + Qalieh, Yuri D'Elia, Zero King + - And of course all the people filing issues, suggesting features and helping + out on the maling list. + + + + +** 1.6 (released, as of July 27 2021) + + NOTE: After upgrading, you need to call ~mu init~, with your prefered parameters + before you can use ~mu~ / ~mu4e~. This is because the underlying database-schema + has changed. + +*** mu + + - Where available (and with suitably equiped ~libglib~), log to the ~systemd~ + journal instead of =~/.cache/mu.log=. Passing the ~--debug~ option to ~mu~ + increases the amount that is logged. + + - Follow symlinks in maildirs, and support moving messsages across + filesystems. Obviously, that is typically quite a bit slower than the + single-filesystem case, but can be still be useful. + + - Optionally provide readline support for the ~mu~ server (when in tty-mode) + + - Reworked the way mu generates s-expressions for mu4e; they are created + programmatically now instead of through string building. + + - The indexer (the part of mu that scans maildirs and updates the message + store) has been rewritten so it can work asynchronously and take advantage + of multiple cores. Note that for now, indexing in ~mu4e~ is still a blocking + operation. + + - Portability updates for dealing with non-POSIX systems, and in particular + VFAT filesystem, and building using Clang/libc++. + + - The personal addresses (as per ~--my-address=~ for ~mu init~) can now also + include regular expressions (basic POSIX); wrap the expression in ~/~, e.g., + ~--my-address='/.*@example.*/~'. + + - Modernized the querying/threading machinery; this makes some old code a + lot easier to understand and maintain, and even while not an explicit + goal, is also faster. + + - Experimental support for the Meson build system. + +*** mu4e + + - Use the gnus-based message viewer as the default; the new viewer has quite + a few extra features compared to the old, mu4e-specific one, such as + faster crypto, support for S/MIME, syntax-highlighting, calendar + invitations and more. + + The new view is superior in most ways, but if you still depend on + something from the old one, you can use: + #+begin_example + ;; set *before* loading mu4e; and restart emacs if you want to change it + ;; users of use-packag~ should can use the :init section for this. + (setq mu4e-view-use-old t) + #+end_example + + (The older variable ~mu4e-view-use-gnus~ with the opposite meaning is + obsolete now, and no longer in use). + + - Include maildir-shortcuts in the main-view with overall/unread counts, + similar to bookmarks, and with the same ~:hide~ and ~:hide-unread~ properties. + Note that for the latter, you need to update your maildir-shortcuts to the + new format, as explained in the ~mu4e-maildir-shortcuts~ docstring. + + You can set ~mu4e-main-hide-fully-read~ to hide any bookmarks/maildirs that + have no unread messages. + + - Add some more properties for use in capturing org-mode links to messages / + queries. See [[info:mu4e#Org-mode links][the mu4e manual]] for details. + + - Honor ~truncate-string-ellipsis~ so you can now use 'fancy' ellipses for + truncated strings with ~(setq truncate-string-ellipsis "…")~ + + - Add a variable ~mu4e-mu-debug~ which, when set to non-~nil,~ makes the ~mu~ + server log more verbosely (to ~mu.log~ or the journal) + + - Better alignment in headers-buffers; this looks nicer, but is also a bit + slower, hence you need to enable ~mu4e-headers-precise-alignment~ for this. + + - Support ~mu~'s new regexp-based personal addresses, and add + ~mu4e-personal-address-p~ to check whether a given string matches a personal + address. + + - TAB-Completion for writing ~mu~ queries + + - Switch the context for existing draft messages using + ~mu4e-compose-context-switch~ or ~C-c C-;~ in ~mu4e-compose-mode~. + + +** 1.4 (released, as of April 18 2020) + +*** mu + + - mu now defaults to the [[https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html][XDG Base Directory Specification]] for the default + locations for various files. E.g. on Unix the mu database now lives under + ~~/.cache/mu/~ rather than ~~/.mu~. You can still use the old location by + passing ~--muhome=~/.mu~ to various ~mu~ commands, or setting ~(setq + mu4e-mu-home "~/.mu")~ for ~mu4e~. + + If your ~~/.cache~ is volatile (e.g., is cleared on reboot), you may want + use ~--muhome~. Some mailing-list dicussion suggest that's fairly rare + though. + + After upgrading, you may wish to delete the files in the old location to + recover some diskspace. + + - There's a new subcommand ~mu init~ to initialize the mu database, which + takes the ~--maildir~ and ~--my-address~ parameters that ~index~ used to take. + These parameters are persistent so ~index~ does not need (or accept) them + anymore. ~mu4e~ now depends on those parameters. + + ~init~ only needs to be run once or when changing these parameters. That + implies that you need to re-index after changing these parameters. The + ~.noupdate~ files are ignored when indexing the first time after ~mu init~ (or + in general, when the database is empty). + + - There is another new subcommand ~mu info~ to get information about the mu + database, the personal addresses etc. + + - The contacts cache (which is used by ~mu cfind~ and ~mu4e~'s + contact-completion) is now stored as part of the Xapian database rather + than as a separate file. + + - The ~--xbatchsize~ and ~--autoupgrade~ options for indexing are gone; both are + determined implicitly now. + +*** mu4e + + - ~mu4e~ no longer uses the ~mu4e-maildir~ and ~mu4e-user-mail-address-list~ + variables; instead it uses the information it gets from ~mu~ (see the ~mu~ + section above). If you have a non-default ~mu4e-mu-home~, make sure to set + it before ~mu4e~ starts. + + It is strongly recommended that you run ~mu init~ with the appropriate + parameters to (re)initialize the Xapian database, as mentioned in the + mu-section above. + + The main screen shows your address(es), and issues a warning if + ~user-email-address~ is not part of that (and refer you to ~mu init~). You can + avoid the addresses in the main screen and the warning by setting + ~mu4e-main-view-hide-addresses~ to non-nil. + + - In many cases, ~mu4e~ used to receive /all/ contacts after each indexing + operation; this was slow for some users, so we have updated this to /only/ + get the contacts that have changed since the last round. + + We also moved sorting the contacts to the mu-side, which speeds things up + further. However, as a side-effect of this, ~mu4e-contact-rewrite-function~ + and ~mu4e-compose-complete-ignore-address-regexp~ have been obsoleted; users + of those should migrate to ~mu4e-contact-process-function~; see its + docstring for details. + + - Christophe Troestler contributed support for Gnus' calender-invitation + handling in mu4e (i.e., you should be able to accept/reject invitations + etc.). It's very fresh code, and likely it'll be tweaked in the future. + But it's available now for testing. Note that this requires the gnus-based + viewer, as per ~(setq mu4e-view-use-gnus t)~ + + - In addition, he added support for custom headers, so the ones for for the + non-gnus-view should work just as well. + + - ~org-mode~ support is enabled by default now. ~speedbar~ support is disabled + by default. The support org functionality has been moved to ~mu4e-org.el~, + with ~org-mu4e.el~ remaining for older things. + + - ~mu4e~ now adds message-ids to messages when saving drafts, so we can find + them even with ~mu4e-headers-skip-duplicates~. + + - Bookmarks (as in ~mu4e-bookmarks~) are now simple plists (instead of cl + structs). ~make-mu4e-bookmark~ has been updated to produce such plists (for + backward compatibility). A bookmark now looks like a list of e.g. ~(:name + "My bookmark" :query "banana OR pear" :key ?f)~ this format is a bit easier + extensible. + + - ~mu4e~ recognizes an attribute ~:hide t~, which will hide the bookmark item + from the main-screen (and speedbar), but keep it available through the + completion UI. + + - ~mu4e-maildir-shortcuts~ have also become plists. The older format is still + recognized for backward compatibility, but you are encouraged to upgrade. + + - Replying to mailing-lists has been improved, allowing for choosing for + replying to all, sender, list-only. + + - A very visible change, ~mu4e~ now shows unread/all counts for bookmarks in + the main screen that are strings. This is on by default, but can be + disabled by setting ~:hide-unread~ in the bookmark ~plist~ to ~t~. For + speed-reasons, these counts do _not_ filter out duplicates nor messages that + have been removed from the filesystem. + + - ~mu4e-attachment-dir~ now also applies to composing messages; it determines + the default directory for inclusion. + + - The mu4e <-> mu interaction has been rewritten to communicate using + s-expressions, with a repl for testing. + +*** guile + + - guile 3.0 is now supported; guile 2.2 still works. + +*** toys + + - Updated the ~mug~ toy UI to use Webkit2/GTK+. Note that this is just a toy + which is not meant for distribution. ~msg2pdf~ is disabled for now. + + +*** How to upgrade mu4e + + - upgrade ~mu~ to the latest stable version (1.4.x) + + - shut down emacs + + - Run ~mu init~ in a terminal + + - Make sure ~mu init~ points to the right Maildir folder and add your email + address(es) the following way: + + ~mu init --maildir=~/Maildir --my-address=jim@example.com --my-address=bob@example.com~ + + - once this is done, run ~mu index~ + + - Don't forget to delete your old mail cache location if necessary (see + release notes for more detail). + +** 1.2 + + After a bit over a year since version 1.0, here is version 1.2. This is + mostly a bugfix release, but there are also a number of new features. + +*** mu + + - Substantial (algorithmic) speed-up of message-threading; this also (or + especially) affects mu4e, since threading is the default. See commit + eb9bfbb1ca3c for all the details, and thanks to Nicolas Avrutin. + + - The query-parser now generates better queries for wildcard searches, by + using the Xapian machinery for that (when available) rather than + transforming into regexp queries. + + - The perl backend is hardly used and will be removed; for now we just + disable it in the build. + + - Allow outputting messages in json format, closely following the sexp + output. This adds an (optional) dependency on the Json-Glib library. + +*** mu4e + + - Bump the minimal required emacs version to 24.4. This was already de-facto + true, now it is enforced. + + - In mu4e-bookmarks, allow the `:query` element to take a function (or + lambda) to dynamically generate the query string. + + - There is a new message-view for mu4e, based on the Gnus' article-view. + This bring a lot of (but not all) of the very rich Gnus article-mode + feature-set to mu4e, such as S/MIME-support, syntax-highlighting, + + For now this is experimental ("tech preview"), but might replace the + current message-view in a future release. Enable it with: + (setq mu4e-view-use-gnus t) + + Thanks to Christophe Troestler for his work on fixing various encoding + issues. + + - Many bug fixes + +*** guile + + - Now requires guile 2.2. + +*** Contributors for this release: + + Ævar Arnfjörð Bjarmason, Albert Krewinkel, Alberto Luaces, Alex Bennée, Alex + Branham, Alex Murray, Cheong Yiu Fung, Chris Nixon, Christian Egli, + Christophe Troestler, Dirk-Jan C. Binnema, Eric Danan, Evan Klitzke, Ian + Kelling, ibizaman, James P. Ascher, John Whitbeck, Junyeong Jeong, Kevin + Foley, Marcelo Henrique Cerri, Nicolas Avrutin, Oleh Krehel, Peter W. V. + Tran-Jørgensen, Piotr Oleskiewicz, Sebastian Miele, Ulrich Ölmann, + +** 1.0 + + After a decade of development, *mu 1.0*! + + Note: the new release requires a C++14 capable compiler. + +*** mu + + - New, custom query parser which replaces Xapian's 'QueryParser' + both in mu and mu4e. Existing queries should still work, but the new + engine handles non-alphanumeric queries much better. + - Support regular expressions in queries (with the new query engine), + e.g. "subject:/foo.*bar/". See the new `mu-query` and updated `mu-easy` + manpages for examples. + - cfind: ensure nicks are unique + - auxiliary programs invoked from mu/mu4e survive terminating the + shell / emacs + +*** mu4e + + - Allow for rewriting message bodies + - Toggle-menus for header settings + - electric-quote-(local-)mode work when composing emails + - Respect format=flowed and delsp=yes for viewing plain-text + messages + - Added new mu4e-split-view mode: single-window + - Add menu item for `untrash'. + - Unbreak abbrevs in mu4e-compose-mode + - Allow forwarding messages as attachments + (`mu4e-compose-forward-as-attachment') + - New defaults: default to 'skip duplicates' and 'include related' + in headers-view, which should be good defaults for most users. Can be + customized using `mu4e-headers-skip-duplicates' and + `mu4e-headers-include-related', respectively. + - Many bug fixed (see github for all the details). + - Updated documentation + +*** Contributors for this release: + + Ævar Arnfjörð Bjarmason, Alex Bennée, Arne Köhn, Christophe Troestler, + Damien Garaud, Dirk-Jan C. Binnema, galaunay, Hong Xu, Ian Kelling, John + Whitbeck, Josiah Schwab, Jun Hao, Krzysztof Jurewicz, maxime, Mekeor Melire, + Nathaniel Nicandro, Ronald Evers, Sean 'Shaleh' Perry, Sébastien Le + Callonnec, Stig Brautaset, Thierry Volpiatto, Titus von der Malsburg, + Vladimir Sedach, Wataru Ashihara, Yuri D'Elia. + + And all the people on the mailing-list and in github, with bug reports, + questions and suggestions. + + +** 0.9.18 + + New development series which will lead to 0.9.18. + +*** mu + + - Increase the default maximum size for messages to index to 500 + Mb; you can customize this using the --max-msg-size parameter to mu index. + - implement "lazy-checking", which makes mu not descend into + subdirectories when the directory-timestamp is up to date; greatly speeds + up indexing (see --lazy-check) + - prefer gpg2 for crypto + - fix a crash when running on OpenBSD + - fix --clear-links (broken filenames) + - You can now set the MU_HOME environment variable as an + alternative way of setting the mu homedir via the --muhome command-line + parameter. + +*** mu4e + +**** reading messages + + - Add `mu4e-action-view-with-xwidget`, and action for viewing + e-mails inside a Webkit-widget inside emacs (requires emacs 25.x with + xwidget/webkit/gtk3 support) + - Explicitly specify utf8 for external html viewing, so browsers + can handle it correctly. + - Make `shr' the default renderer for rich-text emails (when + available) + - Add a :user-agent field to the message-sexp (in mu4e-view), which + is either the User-Agent or X-Mailer field, when present. + +**** composing messages + + - Cleanly handle early exits from message composition as well as while + composing. + - Allow for resending existing messages, possibly editing them. M-x + mu4e-compose-resend, or use the menu; no shortcut. + - Better handle the closing of separate compose frames + - Improved font-locking for the compose buffers, and more extensive + checks for cited parts. + - automatically sign/encrypt replies to signed/encrypted messages + (subject to `mu4e-compose-crypto-reply-policy') + +**** searching & marking + + - Add a hook `mu4e-mark-execute-pre-hook`, which is run just before + executing marks. + - Just before executing any search, a hook-function + `mu4e-headers-search-hook` is invoked, which receives the search + expression as its parameter. + - In addition, there's a `mu4e-headers-search-bookmark-hook` which + gets called when searches get invoked as a bookmark (note that + `mu4e-headers-search-hook` will also be called just afterwards). This + hook also receives the search expression as its parameter. + - Remove the 'z' keybinding for leaving the headers + view. Keybindings are precious! + - Fix parentheses/precedence in narrowing search terms + +**** indexing + + - Allow for indexing in the background; see + `mu4e-index-update-in-background`. + - Better handle mbsync output in the update buffer + - Add variables mu4e-index-cleanup and mu4e-index-lazy to enable + lazy checking from mu4e; you can sit from mu4e using something like: +#+begin_src elisp +(setq mu4e-index-cleanup nil ;; don't do a full cleanup check + mu4e-index-lazy-check t) ;; don't consider up-to-date dirs #+END_SRC +#+end_src +**** misc + + - don't overwrite global-mode-string, append to it. + - Make org-links (and more general, all users of + mu4e-view-message-with-message-id) use a headers buffer, then view the + message. This way, those linked message are just like any other, and can + be deleted, moved etc. + - Support org-mode 9.x + - Improve file-name escaping, and make it support non-ascii filenames + - Attempt to jump to the same messages after a re-search update operation + - Add action for spam-filter options + - Let `mu4e~read-char-choice' become case-insensitive if there is + no exact match; small convenience that affects most the single-char + option-reading in mu4e. + +*** Perl + + - an experimental Perl binding ("mup") is available now. See + perl/README.md for details. + +*** Contributors: + + Aaron LI, Abdo Roig-Maranges, Ævar Arnfjörð Bjarmason, Alex Bennée, Allen, + Anders Johansson, Antoine Levitt, Arthur Lee, attila, Charles-H. Schulz, + Christophe Troestler, Chunyang Xu, Dirk-Jan C. Binnema, Jakub Sitnicki, + Josiah Schwab, jsrjenkins, Jun Hao, Klaus Holst, Lukas Fürmetz, Magnus + Therning, Maximilian Matthe, Nicolas Richard, Piotr Trojanek, Prashant + Sachdeva, Remco van 't Veer, Stephen Eglen, Stig Brautaset, Thierry + Volpiatto, Thomas Moulia, Titus von der Malsburg, Yuri D'Elia, Vladimir + Sedach + +** 0.9.16 + +*** Release + + 2016-01-20: Release from the 0.9.15 series + +*** Contributors: + + Adam Sampson, Ævar Arnfjörð Bjarmason, Bar Shirtcliff, Charles-H. Schulz, + Clément Pit--Claudel, Damien Cassou, Declan Qian, Dima Kogan, Dirk-Jan C. + Binnema, Foivos S. Zakkak, Hinrik Örn Sigurðsson, Jeroen Tiebout, JJ Asghar, + Jonas Bernoulli, Jun Hao, Martin Yrjölä, Maximilian Matthé, Piotr Trojanek, + prsarv, Thierry Volpiatto, Titus von der Malsburg + + (and of course all people who reported issues, provided suggestions etc.) + +** 0.9.15 + + - bump version to 0.9.15. From now on, odd minor version numbers + are for development versions; thus, 0.9.16 is to be the next stable + release. + - special case text/calendar attachments to get .vcs + extensions. This makes it easier to process those with external tools. + - change the message file names to better conform to the maildir + spec; this was confusing some tools. + - fix navigation when not running in split-view mode + - add `mu4e-view-body-face', so the body-face for message in the + view can be customized; e.g. (set-face-attribute 'mu4e-view-body-face nil + :font "Liberation Serif-10") + - add `mu4e-action-show-thread`, an action for the headers and view + buffers to search for messages in the same thread as the current one. + - allow for transforming mailing-list names for display, using + `mu4e-mailing-list-patterns'. + - some optimizations in indexing (~30% faster in some cases) + - new variable mu4e-user-agent-string, to customize the User-Agent: + header. + - when removing the "In-reply-to" header from replies, mu4e will + also remove the (hidden) References header, effectively creating a new + message-thread. + - implement 'mu4e-context', for defining and switching between + various contexts, which are groups of settings. This can be used for + instance for switch between e-mail accounts. See the section in the manual + for details. + - correctly decode mailing-list headers + - allow for "fancy" mark-characters; and improve the default set + - by default, the maildirs are no longer cached; please see the + variable ~mu4e-cache-maildir-list~ if you have a lot of maildirs and it + gets slow. + - change the default value for + ~org-mu4e-link-query-in-headers-mode~ to ~nil~, ie. by default link to the + message, not the query, as this is usually more useful behavior. + - overwrite target message files that already exist, rather than + erroring out. + - set mu4e-view-html-plaintext-ratio-heuristic to 5, as 10 was too + high to detect some effectively html-only messages + - add mu4e-view-toggle-html (keybinding: 'h') to toggle between + text and html display. The existing 'mu4e-view-toggle-hide-cited' gets the + new binding '#'. + - add a customization variable `mu4e-view-auto-mark-as-read' + (defaults to t); if set to nil, mu4e won't mark messages as read when you + open them. This can be useful on read-only file-systems, since + marking-as-read implies a file-move operation. + - use smaller chunks for mu server on Cygwin, allowing for better + mu4e support there. + +** 0.9.13 + +*** contributors + + Attila, Daniele Pizzolli, Charles-H.Schulz, David C Sterrat, Dirk-Jan C. + Binnema, Eike Kettner, Florian Lindner, Foivos S. Zakkak, Gour, KOMURA + Takaaki, Pan Jie, Phil Hagelberg, thdox, Tiago Saboga, Titus von der + Malsburg + + (and of course all people who reported issues, provided suggestions etc.) + +*** mu/mu4e/guile + + - NEWS (this file) is now visible from within mu4e – "N" in the main-menu. + + - make `mu4e-headers-sort-field', `mu4e-headers-sort-direction' + public (that, is change the prefix from mu4e~ to mu4e-), so users can + manipulate them + + - make it possible the 'fancy' (unicode) characters separately for + headers and marks (see the variable `mu4e-use-fancy-chars'.) + + - allow for composing in a separate frame (see + `mu4e-compose-in-new-frame') + + - add the `:thread-subject' header field, for showing the subject + for a thread only once. So, instead of (from the manual): + +#+begin_example +06:32 Nu To Edmund Dantès GstDev + Re: Gstreamer-V4L... +15:08 Nu Abbé Busoni GstDev + Re: Gstreamer-V... +18:20 Nu Pierre Morrel GstDev \ Re: Gstreamer... +2013-03-18 S Jacopo EmacsUsr + emacs server on win... +2013-03-18 S Mercédès EmacsUsr \ RE: emacs server ... +2013-03-18 S Beachamp EmacsUsr + Re: Copying a whole... +22:07 Nu Albert de Moncerf EmacsUsr \ Re: Copying a who... +2013-03-18 S Gaspard Caderousse GstDev | Issue with GESSimpl... +2013-03-18 Ss Baron Danglars GuileUsr | Guile-SDL 0.4.2 ava... +End of search results +#+end_example + +the headers list would now look something like: +#+begin_example +06:32 Nu To Edmund Dantès GstDev + Re: Gstreamer-V4L... +15:08 Nu Abbé Busoni GstDev + +18:20 Nu Pierre Morrel GstDev \ Re: Gstreamer... +2013-03-18 S Jacopo EmacsUsr + emacs server on win... +2013-03-18 S Mercédès EmacsUsr \ +2013-03-18 S Beachamp EmacsUsr + Re: Copying a whole... +22:07 Nu Albert de Moncerf EmacsUsr \ +2013-03-18 S Gaspard Caderousse GstDev | Issue with GESSimpl... +2013-03-18 Ss Baron Danglars GuileUsr | Guile-SDL 0.4.2 ava... +End of search results +#+end_example + + This is a feature known from e.g. `mutt' and `gnus` and many other + clients, and can be enabled by customizing `mu4e-headers-fields' + (replacing `:subject' with `:thread-subject') + + It's not the default yet, but may become so in the future. + + - add some spam-handling actions to mu4e-contrib.el + + - mu4e now targets org 8.x, which support for previous versions + relegated to `org-old-mu4e.el`. Some of the new org-features are improved + capture templates. + + - updates to the documentation, in particular about using BBDB. + + - improved URL-handling (use emacs built-in functionality) + + - many bug fixes, including some crash fixes on BSD + +*** guile + + – add --delete option to the find-dups scripts, to automatically delete + them. Use with care! + +** Release 0.9.12 + +*** mu + + - truncate /all/ terms the go beyond xapian's max term length + - lowercase the domain-part of email addresses in mu cfind (and mu4e), if + the domain is in ascii + - give messages without msgids fake-message-ids; this fixes the problem + where such messages were not found in --include-related queries + - cleanup of the query parser + - provide fake message-ids for messages without it; fixes #183 + - allow showing tags in 'mu find' output + - fix CSV quoting + +*** mu4e + + - update the emacs <-> backend protocol; documented in the mu-server man page + - show 'None' as date for messages without it (Headers View) + - add `mu4e-headers-found-hook', `mu4e-update-pre-hook'. + - split org support in org-old-mu4e.el (org <= 7.x) and org-mu4e.el + - org: improve template keywords + - rework URL handling + +** Release 0.9.10 + +*** mu + + - allow 'contact:' as a shortcut in queries for 'from:foo OR to:foo OR + cc:foo OR bcc:foo', and 'recip:' as a shortcut for 'to:foo OR cc:foo OR + bcc:foo' + - support getting related messages (--include-related), which includes + messages that may not match the query, but that are in the same threads as + messages that were + - support "list:"/"v:" for matching mailing list names, and the "v" + format-field to show them. E.g 'mu find list:emacs-orgmode.gnu.org' + +*** mu4e + + - scroll down in message view takes you to next message (but see + `mu4e-view-scroll-to-next') + - support 'human dates', that is, show the time for today's messages, and + the date for older messages in the headers view + - replace `mu4e-user-mail-address-regexp' and `mu4e-my-mail-addresses' with + `mu4e-user-mail-address-list' + - support tags (i.e.., X-Keywords and friends) in the headers-view, and the + message view. Thanks to Abdó Roig-Maranges. New field ":tags". + - automatically update the headers buffer when new messages are found during + indexing; set `mu4e-headers-auto-update' to nil to disable this. + - update mail/index with M-x mu4e-update-mail-and-index; which everywhere in + mu4e is available with key C-S-u. Use prefix argument to run in + background. + - add function `mu4e-update-index' to only update the index + - add 'friendly-names' for mailing lists, so they should up nicely in the + headers view + +*** guile + + - add 'mu script' command to run mu script, for example to do statistics on + your message corpus. See the mu-script man-page. + +*** mug + + - ported to gtk+ 3; remove gtk+ 2.x code + + + +** Release 0.9.9 <2012-10-14> + +*** mu4e + - view: address can be toggled long/short, compose message + - sanitize opening urls (mouse-1, and not too eager) + - tooltips for header labels, flags + - add sort buttons to header-labels + - support signing / decryption of messages + - improve address-autocompletion (e.g., ensure it's case-insensitive) + - much faster when there are many maildirs + - improved line wrapping + - better handle attached messages + - improved URL-matching + - improved messages to user (mu4e-(warn|error|message)) + - add refiling functionality + - support fancy non-ascii in the UI + - dynamic folders (i.e.., allow mu4e-(sent|draft|trash|refile)-folder) to + be a function + - dynamic attachment download folder (can be a function now) + - much improved manual + +*** mu + - remove --summary (use --summary-len instead) + - add --after for mu find, to limit to messages after T + - add new command `mu verify', to verify signatures + - fix iso-2022-jp decoding (and other 7-bit clean non-ascii) + - add support for X-keywords + - performance improvements for threaded display (~ 25% for 23K msgs) + - mu improved user-help (and the 'mu help' command) + - toys/mug2 replaces toys/mug + +*** mu-guile + - automated tests + - add mu:timestamp, mu:count + - handle db reopenings in the background + + +** Release 0.9.8.5 <2012-07-01> + +*** mu4e + + - auto-completion of e-mail addresses + - inline display of images (see `mu4e-view-show-images'), uses imagemagick + if available + - interactively change number of headers / columns for showing headers with + C-+ and C-- in headers, view mode + - support flagging message + - navigate to previous/next queries like a web browser (with , + ) + - narrow search results with '/' + - next/previous take a prefix arg now, to move to the nth previous/next message + - allow for writing rich-text messages with org-mode + - enable marking messages as Flagged + - custom marker functions (see manual) + - better "dwim" handling of buffer switching / killing + - deferred marking of message (i.e.., mark now, decide what to mark for + later) + - enable changing of sort order, display of threads + - clearer marks for marked messages + - fix sorting by subject (disregarding Re:, Fwd: etc.) + - much faster handling when there are many maildirs (speedbar) + - handle mailto: links + - improved, extended documentation + +*** mu + + - support .noupdate files (parallel to .noindex, dir is ignored unless we're + doing a --rebuild). + - append all inline text parts, when getting the text body + - respect custom maildir flags + - correctly handle the case where g_utf8_strdown (str) > len (str) + - make gtk, guile, webkit dependency optional, even if they are installed + + +** Release 0.9.8.4 <2012-05-08> + +*** mu4e + + - much faster header buffers + - split view mode (headers, view); see `mu4e-split-view'. + - add search history for queries + - ability to open attachments with arbitrary programs, pipe through shell + commands or open in the current emacs + - quote names in recipient addresses + - mu4e-get-maildirs works now for recursive maildirs as well + - define arbitrary operations for headers/messages/attachments using the + actions system -- see the chapter 'Actions' in the manual + - allow mu4e to be uses as the default emacs mailer (`mu4e-user-agent') + - mark headers based on a regexp, `mu4e-mark-matches', or '%' + - mark threads, sub-threads (mu4e-hdrs-mark-thread, + mu4e-hdrs-mark-subthread, or 'T', 't') + - add msg2pdf toy + - easy logging (using `mu4e-toggle-logging') + - improve mu4e-speedbar for use in headers/view + - use the message-mode FCC system for saving messages to the sent-messages + folder + - fix: off-by-one in number of matches shown + +*** general + + - fix for opening files with non-ascii names + - much improved support for searching non-Latin (Cyrillic etc.) languages + we can now match 'Тесла' or 'Аркона' without problems + - smarter escaping (fixes issues with finding message ids) + - fixes for queries with brackets + - allow --summary-len for the length of message summaries + - numerous other small fixes + + +** Release 0.9.8.3 <2012-04-06> + + *NOTE*: existing mu/mu4e are recommended to run `mu index --rebuild' after + installation. + +*** mu4e + + - allow for searching by editing bookmarks + (`mu4e-search-bookmark-edit-first') (keybinding 'B') + - make it configurable what to do with sent messages (see + `mu4e-sent-messages-behavior') + - speedbar support (initial patch by Antono V) + - better handling of drafts: + - don't save too early + - more descriptive buffer names (based on Subject, if any) + - don't put "--text-follows-this-line--" markers in files + - automatically include signatures, if set + - add user-settable variables mu4e-view-wrap-lines and mu4e-view-hide-cited, + which determine the initial way a message is displayed + - improved documentation + +*** general + + - much improved searching for GMail folders (i.e. maildir:/ matching); + this requires a 'mu index --rebuild' + - correctly handle utf-8 messages, even if they don't specify this explicitly + - fix compiler warnings for newer/older gcc and clang/clang++ + - fix unit tests (and some code) for Ubuntu 10.04 and FreeBSD9 + - fix warnings for compilation with GTK+ 3.2 and recent glib (g_set_error) + - fix mu_msg_move_to_maildir for top-level messages + - fix in maildir scanning + - plug some memleaks + +** Release 0.9.8.2 <2012-03-11> + +*** mu4e: + + - make mail updating non-blocking + - allow for automatic periodic update ('mu4e-update-interval') + - allow for external triggering of update + - make behavior when leaving the headers buffer customizable, ie. + ask/apply/ignore ('mu4e-headers-leave-behaviour') + +*** general + + - fix output for some non-UTF8 locales + - open ('play') file names with spaces + - don't show unnecessary errors for --format=links + - make build warning-free for clang/clang++ + - allow for slightly older autotools + - fix unit tests for some hidden assumptions (locale, dir structure etc.) + - some documentation updates / clarifications + +** Release 0.9.8.1 <2012-02-18 Sat> + +*** mu + - show only leaf/rfc822 MIME-parts + +*** mu4e + + - allow for shell commands with arguments in `mu4e-get-mail-command'. + - support marking messages as 'read' and 'unread' + - show the current query in the the mode-line (`global-mode-string'). + - don't repeat 'Re:' / 'Fwd:' + - colorize cited message parts + - better handling of text-based, embedded message attachments + - for text-bodies, concatenate all text/plain parts + - make filladapt dep optional + - documentation improvements + +** Release 0.9.8 <2012-01-31> + + - '--descending' has been renamed into '--reverse' + - search for attachment MIME-type using 'mime:' or 'y:' + - search for text in text-attachments using 'embed:' or 'e:' + - searching for attachment file names now uses 'file:' (was: 'attach:') + - experimental emacs-based mail client -- "mu4e" + - added more unit tests + - improved guile binding - no special binary is needed anymore, it's + installable are works with the normal guile system; code has been + substantially improved. still 'experimental' + +** Release 0.9.7 <2011-09-03 Sat> + + - don't enforce UTF-8 output, use locale (fixes issue #11) + - add mail threading to mu-find (using -t/--threads) (sorta fixes issue #13) + - add header line to --format=mutt-ab (mu cfind), (fixes issue #42) + - terminate mu view results with a form-feed marker (use --terminate) (fixes + issue #41) + - search X-Label: tags (fixes issue #40) + - added toys/muile, the mu guile shells, which allows for message stats etc. + - fix date handling (timezones) + +** Release 0.9.6 <2011-05-28 Sat> + + - FreeBSD build fix + - fix matching for mu cfind to be as expected + - fix mu-contacts for broken names/emails + - clear the contacts-cache too when doing a --rebuild + - wildcard searches ('*') for fields (except for path/maildir) + - search for attachment file names (with 'a:'/'attach:') -- also works with + wildcards + - remove --xquery completely; use --output=xquery instead + - fix progress info in 'mu index' + - display the references for a message using the 'r' character (xmu find) + - remove --summary-len/-k, instead use --summary for mu view and mu find, and + - support colorized output for some sub-commands (view, cfind and + extract). Disabled by default, use --color to enable, or set env MU_COLORS + to non-empty + - update documentation, added more examples + +** Release 0.9.5 <2011-04-25 Mon> + + - bug fix for infinite loop in Maildir detection + - minor fixes in tests, small optimizations + +** Release 0.9.4 <2011-04-12 Tue> + + - add the 'cfind' command, to search/export contact information + - add 'flag:unread' as a synonym for 'flag:new OR NOT flag:unseen' + - updated documentation + +** Release 0.9.3 <2011-02-13 Sun> + + - don't warn about missing files with --quiet + +** Release 0.9.2 <2011-02-02 Wed> + + - stricter checking of options; and options must now *follow* the sub-command + (if any); so, something like: 'mu index --maildir=/foo/bar' + - output searches as plain text (default), XML, JSON or s-expressions using + --format=plain|xml|json|sexp. For example: 'mu find foobar --output=json'. + These format options are experimental (except for 'plain') + - the --xquery option should now be used as --format=xquery, for output + symlinks, use --format=links. This is a change in the options. + - search output can include the message size using the 'z' shortcut + - match message size ranges (i.e.. size:500k..2M) + - fix: honor the --overwrite (or lack thereof) parameter + - support folder names with special characters (@, ' ', '.' and so on) + - better check for already-running mu index + - when --maildir= is not provided for mu index, default to the last one + - add --max-msg-size, to specify a new maximum message size + - move the 'mug' UI to toys/mug; no longer installable + - better support for Solaris builds, Gentoo. + +** Release 0.9.1 <2010-12-05 Sun> + + - Add missing icon for mug + - Fix unit tests (Issue #30) + - Fix Fedora 14 build (broken GTK+ 3) (Issue #31) + +** Release 0.9 <2010-12-04 Sat> + + - you can now search for the message priority ('prio:high', 'prio:low', + 'prio:normal') + - you can now search for message flags, e.g. 'flag:attach' for messages with + attachment, or 'flag:encrypted' for encrypted messages + - you can search for time-intervals, e.g. 'date:2010-11-26..2010-11-29' for + messages in that range. See the mu-find(1) and mu-easy(1) man-pages for + details and examples. + - you can store bookmarked queries in ~/.mu/bookmarks + - the 'flags' parameter has been renamed in 'flag' + - add a simple graphical UI for searching, called 'mug' + - fix --clearlinks for file systems without entry->d_type (fixes issue #28) + - make matching case-insensitive and accent-insensitive (accent-insensitive + for characters in Unicode Blocks 'Latin-1 Supplement' and 'Latin + Extended-A') + - more extensive pre-processing is done to make searching for email-addresses + and message-ids less likely to not work (issue #21) + - updated the man-pages + - experimental support for Fedora 14, which uses GMime 2.5.x (fixes issue #29) + +** Release 0.8 <2010-10-30 Sat> + + - There's now 'mu extract' for getting information about MIME-parts + (attachments) and extracting them + - Queries are now internally converted to lowercase; this solves some of the + false-negative issues + - All mu sub-commands now have their own man-page + - 'mu find' now takes a --summary-len= argument to print a summary of + up-to-n lines of the message + - Same for 'mu view'; the summary replaces the full body + - Setting the mu home dir now goes with -m, --muhome + - --log-stderr, --reindex, --rebuild, --autoupgrade, --nocleanup, --mode, + --linksdir, --clearlinks lost their single char version + +** Release 0.7 <2010-02-27 Sat> + + - Database format changed + - Automatic database scheme version check, notifies users when an upgrade + is needed + - 'mu view', to view mail message files + - Support for >10K matches + - Support for unattended upgrades - that is, the database can automatically + by upgraded (--autoupgrade). Also, the log file is automatically cleaned + when it gets too big (unless you use --nocleanup) + - Search for a certain Maildir using the maildir:,m: search prefixes. For + example, you can find all messages located in ~/Maildir/foo/bar/cur/msg + ~/Maildir/foo/bar/new/msg and with m:/foo/bar this replace the search for + path/p in 0.6 + - Fixes for reported issues () + - A test suite with a growing number of unit tests + + +** Release 0.6 <2010-01-23 Sat> + + - First new release of mu since 2008 + - No longer depends on sqlite + + +# Local Variables: +# mode: org; org-startup-folded: nil +# fill-column:80 +# End: diff --git a/README.org b/README.org new file mode 100644 index 0000000..ec5c648 --- /dev/null +++ b/README.org @@ -0,0 +1,104 @@ +#+TITLE:mu +[[https://github.com/djcb/mu/blob/master/COPYING][https://img.shields.io/github/license/djcb/mu?logo=gnu&.svg]] +[[https://en.cppreference.com][https://img.shields.io/badge/Made%20with-C/CPP-1f425f?logo=c&.svg]] +[[https://img.shields.io/github/v/release/djcb/mu][https://img.shields.io/github/v/release/djcb/mu.svg]] +[[https://github.com/djcb/mu/graphs/contributors][https://img.shields.io/github/contributors/djcb/mu.svg]] +[[https://github.com/djcb/mu/issues][https://img.shields.io/github/issues/djcb/mu.svg]] +[[https://github.com/djcb/mu/issues?q=is%3Aissue+is%3Aopen+label%3Arfe][https://img.shields.io/github/issues/djcb/mu/rfe?color=008b8b.svg]] +[[https://github.com/djcb/mu/pull/new][https://img.shields.io/badge/PRs-welcome-brightgreen.svg]]\\ +[[https://www.gnu.org/software/emacs/][https://img.shields.io/badge/Emacs-26.3-922793?logo=gnu-emacs&logoColor=b39ddb&.svg]] +[[https://www.djcbsoftware.nl/code/mu/mu4e/Installation.html#Dependencies-for-Debian_002fUbuntu][https://img.shields.io/badge/Platform-Linux-2e8b57?logo=linux&.svg]] +[[https://www.djcbsoftware.nl/code/mu/mu4e/Installation.html#Building-from-a-release-tarball-1][https://img.shields.io/badge/Platform-FreeBSD-8b3a3a?logo=freebsd&logoColor=c32136&.svg]] +[[https://formulae.brew.sh/formula/mu#default][https://img.shields.io/badge/Platform-macOS-101010?logo=apple&logoColor=ffffff&.svg]] + + [ *Note*: you are looking at the *development* branch, which is where new code is + being developed and tested, and which may occasionally break. Distributions and + non-adventurous users are instead recommended to use the [[https://github.com/djcb/mu/tree/release/1.10][1.10 Release Branch]] or + to pick up one of the [[https://github.com/djcb/mu/releases][1.10 Releases]]. ] + +Welcome to ~mu~! + +Latest development news: [[NEWS.org]]. + +With the enormous amounts of e-mail many people gather and the importance of +e-mail message in our work-flows, it's essential to quickly deal with all that +mail - in particular, to instantly find that one important e-mail you need right +now, and quickly file away message for later use. + +~mu~ is a tool for dealing with e-mail messages stored in the Maildir-format. ~mu~'s +purpose in life is to help you to quickly find the messages you need; in +addition, it allows you to view messages, extract attachments, create new +maildirs, and so on. + +After indexing your messages into a [[http://www.xapian.org][Xapian]]-database, you can search them using a +custom query language. You can use various message fields or words in the body +text to find the right messages. + +Built on top of ~mu~ are some extensions (included in this package): + +- ~mu4e~: a full-featured e-mail client that runs inside emacs + +- ~mu-guile~: bindings for the Guile/Scheme programming language (version 3.0 and + later) + +~mu~ is written in C++; ~mu4e~ is written in ~elisp~ and ~mu-guile~ in a mix of C++ and +Scheme. + +~mu~ is available in Linux distributions (e.g. Debian/Ubuntu and Fedora) under the +name ~maildir-utils~; apparently because they don't like short names. All of the +code is distributed under the terms of the [[https://www.gnu.org/licenses/gpl-3.0.en.html][GNU General Public License version 3]] +(or higher). + +* Installation + +Note: building from source is an /advanced/ subject, especially if something goes +wrong. The below simple examples are a start, but all tools involved have many +options; there are differences between systems, versions etc. So if this is all +a bit daunting we recommend to wait for someone else to build it for you, such +as a Linux distribution. Many have packages available. + +** Requirements + +To be able to build ~mu~, ensure you have: + +- a C++17 compiler (~gcc~ or ~clang~ are known to work) +- development packages for /Xapian/ and /GMime/ and /GLib/ (see ~meson.build~ for thex + versions) +- basic tools such as ~make~, ~sed~, ~grep~ +- ~meson~ + +For ~mu4e~, you also need ~emacs~. + +Note, support for Windows is very much _experimental_, that is, it works for some +people, but we can't really support it due to lack of the specific expertise. +Help is welcome! + +** Building + +#+begin_example +$ git clone git://github.com/djcb/mu.git +$ cd mu +#+end_example + +~mu~ uses ~meson~ for building, so you can use that directly, and all the usual +commands apply. You can also use it _indirectly_ through the provided ~Makefile~, +which provides a number of useful targets. + +For instance, using the ~Makefile~, you could install ~mu~ using: + +#+begin_example +$ ./autogen.sh && make +$ sudo make install +#+end_example + +Alternatively, you can run ~meson~ directly (see the ~meson~ documentation for +more details): +#+begin_example +$ meson setup -C build +$ meson compile -C build +$ meson install -C build +#+end_example + +** Contributing + +Contributions are welcome! See the Github issue list and [[IDEAS.org]]. diff --git a/autogen.sh b/autogen.sh new file mode 100755 index 0000000..a0eabf8 --- /dev/null +++ b/autogen.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# Run this to generate all the initial makefiles, etc. + +echo "*** meson build setup" + +test -f mu/mu.cc || { + echo "*** Run this script from the top-level mu source directory" + exit 1 +} +BUILDDIR=build + +command -v meson 2> /dev/null +if [ $? != 0 ]; then + echo "*** 'meson' not found, please install it ***" + exit 1 +fi + +# we could remove build/ but let's avoid rm -rf risks... +if test -d ${BUILDDIR}; then + meson setup --reconfigure ${BUILDDIR} $@ || exit 1 +else + meson setup ${BUILDDIR} $@ || exit 1 +fi + +echo "*** Now run either 'ninja -C ${BUILDDIR}' or 'make' to build mu" +echo "*** Check the Makefile for other useful targets" diff --git a/build-aux/date.py b/build-aux/date.py new file mode 100755 index 0000000..d93b13d --- /dev/null +++ b/build-aux/date.py @@ -0,0 +1,15 @@ +#!/usr/bin/env python3 + +""" +Script to get date strings, since the MacOS 'date' is not quite up to GNU +standards + +E.g.. + date.py 2023-10-14 "The year-month is %y %m" +""" + +import sys +from datetime import datetime + +date=datetime.strptime(sys.argv[1],'%Y-%m-%d') +print(date.strftime(sys.argv[2])) diff --git a/build-aux/meson-install-info.sh b/build-aux/meson-install-info.sh new file mode 100644 index 0000000..0019249 --- /dev/null +++ b/build-aux/meson-install-info.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +infodir=$1 +infofile=$2 + +# Meson post-install script to update info metadata + +# If DESTDIR is set, do _not_ install-info, since it's only a temporary +# install +if test -z "${DESTDIR}"; then + install-info --info-dir "${infodir}" "${infodir}/${infofile}" + gzip --best --force "${infodir}/${infofile}" +fi diff --git a/build-aux/version.texi.in b/build-aux/version.texi.in new file mode 100644 index 0000000..aa13bab --- /dev/null +++ b/build-aux/version.texi.in @@ -0,0 +1,5 @@ +@set UPDATED @UPDATED@ +@set UPDATED-MONTH @UPDATEDMONTH@ +@set UPDATED-YEAR @UPDATEDYEAR@ +@set EDITION @VERSION@ +@set VERSION @VERSION@ diff --git a/contrib/mu-completion.zsh b/contrib/mu-completion.zsh new file mode 100644 index 0000000..ea2bdbd --- /dev/null +++ b/contrib/mu-completion.zsh @@ -0,0 +1,124 @@ +#compdef mu + +## Copyright (C) 2011-2012 Dirk-Jan C. Binnema +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# zsh completion for mu. Install this by copying/linking to this file somewhere in +# your $fpath; the link/copy must have a name starting with an underscore "_" + +# main dispatcher function +_mu() { + if (( CURRENT > 2 )) ; then + local cmd=${words[2]} + curcontext="${curcontext%:*:*}:mu-$cmd" + (( CURRENT-- )) + shift words + _call_function ret _mu_$cmd + return ret + else + _mu_commands + fi +} + + + +_mu_commands() { + local -a mu_commands + mu_commands=( + 'index:scan your maildirs and import their metadata in the database' + 'find:search for messages in the database' + 'view:display specific messages' + 'cfind:search for contacts (name + email) in the database' + 'extract:extract message-parts (attachments) and save or open them' + 'mkdir:create maildirs' +# below are not generally very useful, so let's not auto-complete them +# 'add: add a message to the database.' +# 'remove:remove a message from the database.' +# 'server:sart the mu server' +) + + _describe -t command 'command' mu_commands +} + +_mu_common_options=( + '--debug[output information useful for debugging mu]' + '--quiet[do not give any non-critical information]' + '--nocolor[do not use colors in some of the output]' + '--version[display mu version and copyright information]' + '--log-stderr[log to standard error]' +) + +_mu_db_options=( + '--muhome[use some non-default location for the mu database]:directory:_files' +) + +_mu_find_options=( + '--fields[fields to display in the output]' + '--sortfield[field to sort the output by]' + '--descending[sort in descending order]' + '--summary[include a summary of the message]' + '--summary-len[number of lines to use for the summary]' + '--bookmark[use a named bookmark]' + '--output[set the kind of output for the query]' +) + +_mu_view_options=( + '--summary[only show a summary of the message]' + '--summary-len[number of lines to use for the summary]' +) + + +_mu_view() { + _arguments -s : \ + $_mu_common_options \ + $_mu_view_options +} + +_mu_extract() { + _files +} + +_mu_find() { + _arguments -s : \ + $_mu_common_options \ + $_mu_db_options \ + $_mu_find_options +} + +_mu_index() { + _arguments -s : \ + $_mu_db_options \ + $_mu_common_options +}mu + +_mu_cleanup() { + _arguments -s : \ + $_mu_db_options \ + $_mu_common_options +} + + +_mu_mkdir() { + _arguments -s : \ + '--mode=[file mode for the new Maildir]:file mode: ' \ + $_mu_common_options +} + +_mu "$@" + +# Local variables: +# mode: sh +# End: diff --git a/contrib/mu-sexp-convert b/contrib/mu-sexp-convert new file mode 100755 index 0000000..b2835ac --- /dev/null +++ b/contrib/mu-sexp-convert @@ -0,0 +1,204 @@ +#!/bin/sh +exec guile -e main -s $0 $@ +!# + +;; Copyright (C) 2012 Dirk-Jan C. Binnema +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +;; +;; a little hack to convert the output of +;; mu find --format=sexp +;; and +;; mu view --format=sexp +;; into XML or JSON + +(use-modules (ice-9 getopt-long) (ice-9 format) (ice-9 regex)) +(use-modules (sxml simple)) + +(define (mapconcat func lst sepa) + "Apply FUNC to elements of LST, concat the result as strings +separated by SEPA." + (if (null? lst) + "" + (string-append + (func (car lst)) + (if (null? (cdr lst)) + "" + (string-append sepa (mapconcat func (cdr lst) sepa)))))) + +(define (property-list? obj) + "Is OBJ a elisp-style property list (ie. a list of the +form (:symbol1 something :symbol2 somethingelse), as in an elisp +proplilst." + (and (list? obj) + (not (null? obj)) + (symbol? (car obj)) + (string= ":" (substring (symbol->string (car obj)) 0 1)))) + +(define (plist->pairs plist) + "Convert an elisp-style property list; e.g: + (:prop1 foo :prop2: bar ...) +into a list of pairs + ((prop1 . foo) (prop2 . bar) ...)." + (if (null? plist) + '() + (cons + (cons + (substring (symbol->string (car plist)) 1) + (cadr plist)) + (plist->pairs (cddr plist))))) + +(define (string->xml str) + "XML-encode STR." + ;; sneakily re-using sxml->xml + (call-with-output-string (lambda (port) (sxml->xml str port)))) + +(define (string->json str) + "Convert string into a JSON-encoded string." + (letrec ((convert + (lambda (lst) + (if (null? lst) + "" + (string-append + (cond + ((equal? (car lst) #\") "\\\"") + ((equal? (car lst) #\\) "\\\\") + ((equal? (car lst) #\/) "\\/") + ((equal? (car lst) #\bs) "\\b") + ((equal? (car lst) #\ff) "\\f") + ((equal? (car lst) #\lf) "\\n") + ((equal? (car lst) #\cr) "\\r") + ((equal? (car lst) #\ht) "\\t") + (#t (string (car lst)))) + (convert (cdr lst))))))) + (convert (string->list str)))) + +(define (etime->time_t t) + "Convert elisp time object T into a time_t value." + (logior (ash (car t) 16) (car (cdr t)))) + +(define (sexp->xml) + "Convert string INPUT to XML, return the XML (string)." + (letrec ((convert-xml + (lambda* (expr #:optional parent) + (cond + ((property-list? expr) + (mapconcat + (lambda (pair) + (format #f "\t<~a>~a\n" + (car pair) (convert-xml (cdr pair) (car pair)) (car pair))) + (plist->pairs expr) " ")) + ((list? expr) + (cond + ((member parent '("from" "to" "cc" "bcc")) + (mapconcat (lambda (addr) + (format #f "
~a~a
" + (if (string? (car addr)) + (format #f "~a" + (string->xml (car addr))) "") + (if (string? (cdr addr)) + (format #f "~a" + (string->xml (cdr addr))) ""))) + expr " ")) + ((string= parent "parts") "") ;; for now, ignore + ;; convert the crazy emacs time thingy to time_t... + ((string= parent "date") (format #f "~a" (etime->time_t expr))) + (#t + (mapconcat + (lambda (elm) (format #f "~a" (convert-xml elm))) expr "")))) + ((string? expr) (string->xml expr)) + ((symbol? expr) (format #f "~a" expr)) + ((number? expr) (number->string expr)) + (#t ".")))) + (msg->xml + (lambda () + (let ((expr (read))) + (if (not (eof-object? expr)) + (string-append (format #f "\n~a\n" (convert-xml expr)) (msg->xml)) + ""))))) + (format #f "\n\n~a" (msg->xml)))) + + +(define (sexp->json) + "Convert string INPUT to JSON, return the JSON (string)." + (letrec ((convert-json + (lambda* (expr #:optional parent) + (cond + ((property-list? expr) + (mapconcat + (lambda (pair) + (format #f "\n\t\"~a\": ~a" + (car pair) (convert-json (cdr pair) (car pair)))) + (plist->pairs expr) ", ")) + ((list? expr) + (cond + ((member parent '("from" "to" "cc" "bcc")) + (string-append "[" + (mapconcat (lambda (addr) + (format #f "{~a~a}" + (if (string? (car addr)) + (format #f "\"name\": \"~a\"," + (string->json (car addr))) "") + (if (string? (cdr addr)) + (format #f "\"email\": \"~a\"" + (string->json (cdr addr))) ""))) + expr ", ") + "]")) + ((string= parent "parts") "[]") ;; todo + ;; convert the crazy emacs time thingy to time_t... + ((string= parent "date") + (format #f "~a" (format #f "~a" (etime->time_t expr)))) + (#t + (string-append "[" + (mapconcat (lambda (elm) (format #f "~a" (convert-json elm))) expr ",") "]")))) + ((string? expr) + (format #f "\"~a\"" (string->json expr))) + ((symbol? expr) + (format #f "\"~a\"" expr)) + ((number? expr) (number->string expr)) + (#t ".")))) + (msg->json + (lambda (first) + (let ((expr (read))) + (if (not (eof-object? expr)) + (string-append (format #f "~a{~a\n}" + (if first "" ",\n") + (convert-json expr)) (msg->json #f)) + ""))))) + (format #f "[\n~a\n]" (msg->json #t)))) + +(define (main args) + (let* ((optionspec '((format (value #t)))) + (options (getopt-long args optionspec)) + (msg (string-append + "usage: mu-sexp-convert " + "--format=\n" + "reads from standard-input and prints to standard output\n")) + (outformat (or (option-ref options 'format #f) + (begin (display msg) (exit 1))))) + (cond + ((string= outformat "xml") + (format #t "~a\n" (sexp->xml))) + ((string= outformat "json") + (format #t "~a\n" (sexp->json))) + (#t (begin + (display msg) + (exit 1)))))) + +;; Local Variables: +;; mode: scheme +;; End: diff --git a/contrib/mu.spec b/contrib/mu.spec new file mode 100644 index 0000000..1ed66d4 --- /dev/null +++ b/contrib/mu.spec @@ -0,0 +1,129 @@ + +# These refer to the release version +# When 0.9.9.6 gets out, remove the global pre line +%global pre pre2 +%global rel 1 + +Summary: A lightweight email search engine for Maildirs +Name: mu +Version: 0.9.9.6 +URL: https://github.com/djcb/mu +# From Packaging:NamingGuidelines for pre-relase versions: +# Release: 0.%{X}.%{alphatag} where %{X} is the release number +%if %{pre} +Release: 0.%{rel}.%{prerelease}%{?dist} +%else +Release: %{rel}%{?dist} +%endif + +License: GPLv3 +Group: Applications/Internet +BuildRoot: %{_tmppath}/%{name}-%{version}-build + +# Source is at ssaavedra repo because djcb has not yet this version tag created +Source0: http://github.com/ssaavedra/%{name}/archive/v%{version}%{?pre}.tar.gz +BuildRequires: emacs-el +BuildRequires: emacs +BuildRequires: gmime-devel +BuildRequires: guile-devel +BuildRequires: xapian-core-devel +BuildRequires: libuuid-devel +BuildRequires: texinfo +Requires: gmime +Requires: guile +Requires: xapian-core-libs +Requires: emacs-filesystem >= %{_emacs_version} + + +%description +E-mail is the 'flow' in the work flow of many people. Consequently, one spends a lot of time searching for old e-mails, to dig up some important piece of information. With people having tens of thousands of e-mails (or more), this is becoming harder and harder. How to find that one e-mail in an ever-growing haystack? +Enter mu. +'mu' is a set of command-line tools for Linux/Unix that enable you to quickly find the e-mails you are looking for, assuming that you store your e-mails in Maildirs (if you don't know what 'Maildirs' are, you are probably not using them). + +%package gtk +Group: Applications/Internet +Summary: GUI for using mu (called mug) +BuildRequires: gtk3-devel +BuildRequires: webkitgtk3-devel +Requires: gtk3 +Requires: gmime +Requires: webkitgtk3 +Requires: mu = %{version}-%{release} + +%description gtk +Mug is a simple GUI for mu from version 0.9. + +%package guile +Group: Applications/Internet +Summary: Guile scripting capabilities for mu +Requires: guile +Requires: mu = %{version}-%{release} +Requires(post): info +Requires(preun): info + +%description guile +Bindings for Guile to interact with mu. + + +%prep +%setup -n %{name}-%{version}%{?pre} -q + +%build +autoreconf -i +%configure +make %{?_smp_mflags} + +%install +rm -rf %{buildroot} +make install DESTDIR=%{buildroot} +install -p -c -m 755 %{_builddir}/%{buildsubdir}/toys/mug/mug %{buildroot}%{_bindir}/mug +cp -p %{_builddir}/%{buildsubdir}/mu4e/*.el %{buildroot}%{_emacs_sitelispdir}/mu4e/ +rm -f %{buildroot}%{_infodir}/dir + +%clean +rm -rf %{buildroot} + +%post +/sbin/install-info \ + --info-dir=%{_infodir} %{_infodir}/mu4e.info.gz || : +%preun +if [ $1 = 0 -a -f %{_infodir}/mu4e.info.gz ]; then + /sbin/install-info --delete \ + --info-dir=%{_infodir} %{_infodir}/mu4e.info.gz || : +fi + +%post guile +/sbin/install-info \ + --info-dir=%{_infodir} %{_infodir}/mu-guile.info.gz || : + +%preun guile +if [ $1 = 0 -a -f %{_infodir}/mu-guile.info.gz ]; then + /sbin/install-info --delete \ + --info-dir=%{_infodir} %{_infodir}/mu-guile.info.gz || : +fi + + +%files +%defattr(-,root,root) +%{_bindir}/mu +%{_mandir}/man1/* +%{_mandir}/man5/* +%{_datadir}/mu/* + +%{_emacs_sitelispdir}/mu4e +%{_emacs_sitelispdir}/mu4e/*.elc +%{_emacs_sitelispdir}/mu4e/*.el +%{_infodir}/mu4e.info.gz + +%files gtk +%{_bindir}/mug + +%files guile +%{_libdir}/libguile-mu.* +%{_datadir}/guile/site/2.0/mu/* +%{_datadir}/guile/site/2.0/mu.scm +%{_infodir}/mu-guile.info.gz + +%changelog +* Wed Feb 12 2014 Santiago Saavedra - 0.9.9.5-1 +- Create first SPEC. diff --git a/guile/compile-scm.in b/guile/compile-scm.in new file mode 100644 index 0000000..04cc0f9 --- /dev/null +++ b/guile/compile-scm.in @@ -0,0 +1,22 @@ +#!/bin/sh +## Copyright (C) 2021 Dirk-Jan C. Binnema +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +@abs_builddir@/build-env @guild@ compile "$@" + +# Local-Variables: +# mode: sh +# End: diff --git a/guile/examples/contacts-export b/guile/examples/contacts-export new file mode 100755 index 0000000..7e33c54 --- /dev/null +++ b/guile/examples/contacts-export @@ -0,0 +1,85 @@ +#!/bin/sh +exec guile -e main -s $0 $@ +!# + +;; +;; Copyright (C) 2012 Dirk-Jan C. Binnema +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + +(use-modules (ice-9 getopt-long) (ice-9 format)) +(use-modules (srfi srfi-1)) +(use-modules (mu)) + +(define (sort-by-freq c1 c2) + (< (mu:frequency c1) (mu:frequency c2))) + +(define (sort-by-newness c1 c2) + (< (mu:last-seen c1) (mu:last-seen c2))) + +(define (main args) + (let* ((optionspec '( (muhome (value #t)) + (sort-by (value #t)) + (revert (value #f)) + (format (value #t)) + (limit (value #t)) + (help (single-char #\h) (value #f)))) + (options (getopt-long args optionspec)) + (msg (string-append + "usage: contacts-export [--help] [--muhome=] " + "--format= " + "--sort-by= [--revert] [--limit=]\n")) + (help (option-ref options 'help #f)) + (muhome (option-ref options 'muhome #f)) + (sort-by (or (option-ref options 'sort-by #f) "frequency")) + (revert (option-ref options 'revert #f)) + (form (or (option-ref options 'format #f) "plain")) + (limit (string->number (option-ref options 'limit "1000000")))) + (if help + (begin + (display msg) + (exit 0)) + (begin + (setlocale LC_ALL "") + (mu:initialize muhome) + (let* ((sort-func + (cond + ((string= sort-by "frequency") sort-by-freq) + ((string= sort-by "newness") sort-by-newness) + (else (begin (display msg) (exit 1))))) + (contacts '())) + ;; make a list of all contacts + (mu:for-each-contact + (lambda (c) (set! contacts (cons c contacts)))) + + ;; should we sort it? + (if sort-by + (set! contacts (sort! contacts + (if revert (negate sort-func) sort-func)))) + + ;; should we limit the number? + (if (and limit (< limit (length contacts))) + (set! contacts (take! contacts limit))) + ;; export! + (for-each + (lambda (c) + (format #t "~a\n" (mu:contact->string c form))) + contacts)))))) + +;; Local Variables: +;; mode: scheme +;; End: diff --git a/guile/examples/msg-graphs b/guile/examples/msg-graphs new file mode 100755 index 0000000..654dd28 --- /dev/null +++ b/guile/examples/msg-graphs @@ -0,0 +1,133 @@ +#!/bin/sh +exec guile -e main -s $0 $@ +!# +;; +;; Copyright (C) 2011-2012 Dirk-Jan C. Binnema +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +(setlocale LC_ALL "") + +(use-modules (ice-9 getopt-long) (ice-9 optargs) (ice-9 popen) (ice-9 format)) +(use-modules (mu) (mu stats) (mu plot)) +;;(use-modules (mu) (mu message) (mu stats) (mu plot)) + +(define (per-hour expr output) + "Count the total number of messages for each weekday (0-6 for Sun..Sat) that +match EXPR. OUTPUT corresponds to the output format, as per gnuplot's 'set +terminal'." + (mu:plot + (sort + (mu:tabulate + (lambda (msg) + (tm:hour (localtime (mu:date msg)))) expr) + (lambda (x y) (< (car x) (car y)))) + (format #f "Messages per hour matching ~a" expr) "Hour" "Messages" output)) + +(define (per-day expr output) + "Count the total number of messages for each weekday (0-6 for Sun..Sat) that +match EXPR. OUTPUT corresponds to the output format, as per gnuplot's 'set +terminal'." + (mu:plot + (mu:weekday-numbers->names + (sort (mu:tabulate + (lambda (msg) + (tm:wday (localtime (mu:date msg)))) expr) + (lambda (x y) (< (car x) (car y))))) + (format #f "Messages per weekday matching ~a" expr) "Day" "Messages" output)) + +(define (per-month expr output) + "Count the total number of messages for each weekday (0-6 for Sun..Sat) that +match EXPR. OUTPUT corresponds to the output format, as per gnuplot's 'set +terminal'." + (mu:plot + (mu:month-numbers->names + (sort + (mu:tabulate + (lambda (msg) + (tm:mon (localtime (mu:date msg)))) expr) + (lambda (x y) (< (car x) (car y))))) + (format #f "Messages per month matching ~a" expr) "Month" "Messages" output)) + + +(define (per-year-month expr output) + "Count the total number of messages for each weekday (0-6 for Sun..Sat) that +match EXPR. OUTPUT corresponds to the output format, as per gnuplot's 'set +terminal'." + (mu:plot + (sort (mu:tabulate + (lambda (msg) + (string->number + (format #f "~d~2'0d" + (+ 1900 (tm:year (localtime (mu:date msg)))) + (tm:mon (localtime (mu:date msg)))))) + expr) + (lambda (x y) (< (car x) (car y)))) + (format #f "Messages per year/month matching ~a" expr) + "Year/Month" "Messages" output)) + + + +(define (per-year expr output) + "Count the total number of messages for each weekday (0-6 for Sun..Sat) that +match EXPR. OUTPUT corresponds to the output format, as per gnuplot's 'set +terminal'." + (mu:plot + (sort (mu:tabulate + (lambda (msg) + (+ 1900 (tm:year (localtime (mu:date msg))))) expr) + (lambda (x y) (< (car x) (car y)))) + (format #f "Messages per year matching ~a" expr) "Year" "Messages" output)) + + + +(define (main args) + (let* ((optionspec '( (muhome (value #t)) + (what (value #t)) + (text (value #f)) + (help (single-char #\h) (value #f)))) + (options (getopt-long args optionspec)) + (msg (string-append + "usage: mu-msg-stats [--help] [--text] " + "[--muhome=] " + "--what= [searchexpr]\n")) + (help (option-ref options 'help #f)) + (what (option-ref options 'what #f)) + (text (option-ref options 'text #f)) + ;; if `text' is `#f', use a graphical window by setting output to "wxt", + ;; else use text-mode plotting ("dumb") + (output (if text "dumb" "wxt")) + (muhome (option-ref options 'muhome #f)) + (restargs (option-ref options '() #f)) + (expr (if restargs (string-join restargs) ""))) + (if (or help (not what)) + (begin + (display msg) + (exit (if help 0 1)))) + (mu:initialize muhome) + (cond + ((string= what "per-hour") (per-hour expr output)) + ((string= what "per-day") (per-day expr output)) + ((string= what "per-month") (per-month expr output)) + ((string= what "per-year-month") (per-year-month expr output)) + ((string= what "per-year") (per-year expr output)) + (else (begin + (display msg) + (exit 1)))))) + +;; Local Variables: +;; mode: scheme +;; End: diff --git a/guile/examples/mu-biff b/guile/examples/mu-biff new file mode 100755 index 0000000..bc6d507 --- /dev/null +++ b/guile/examples/mu-biff @@ -0,0 +1,59 @@ +#!/bin/sh +exec guile -e main -s $0 $@ +!# + +;; +;; Copyright (C) 2012 Dirk-Jan C. Binnema +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +;; script to list the message matching which are newer than +;; minutes + +;; use it, eg. like: +;; $ mu-biff --newer-than=`date +%s --date='5 minutes ago'` "maildir:/inbox" + + +(use-modules (ice-9 getopt-long) (ice-9 format)) +(use-modules (mu)) + +(define (main args) + (let* ((optionspec '((muhome (value #t)) + (newer-than (value #t)) + (help (single-char #\h) (value #f)))) + (options (getopt-long args optionspec)) + (msg (string-append + "usage: mu-biff [--help] [--muhome=]" + " [--newer-than=] ")) + (help (option-ref options 'help #f)) + (newer-than (string->number (option-ref options 'newer-than "0"))) + (muhome (option-ref options 'muhome #f)) + (query (string-concatenate (option-ref options '() '())))) + (if help + (begin (display msg) (newline) (exit 0)) + (begin + (mu:initialize muhome) + (mu:for-each-message + (lambda (msg) + (if (> (mu:timestamp msg) newer-than) + (format #t "~a ~a\n" + (mu:from msg) + (mu:subject msg)))) + query))))) + +;; Local Variables: +;; mode: scheme +;; End: diff --git a/guile/examples/org2mu4e b/guile/examples/org2mu4e new file mode 100755 index 0000000..3556b9a --- /dev/null +++ b/guile/examples/org2mu4e @@ -0,0 +1,78 @@ +#!/bin/sh +exec guile -e main -s $0 $@ +!# + +;; +;; Copyright (C) 2011-2012 Dirk-Jan C. Binnema +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +(use-modules (ice-9 getopt-long) (ice-9 format)) +(use-modules (mu)) + +(define (display-org-header query) + "Print the header for the org-file for QUERY." + (format #t "* Messages matching '~a'\n\n" query)) + +(define (org-mu4e-link msg) + "Create a link for this message understandable by org-mu4e." + (let* ((subject ;; cleanup subject + (string-map + (lambda (kar) + (if (member kar '(#\] #\[)) #\space kar)) + (or (mu:subject msg) "No subject")))) + (format #f "[[mu4e:msgid:~a][~s]]" + (mu:message-id msg) subject))) + +(define (display-org-entry msg tag) + "Write an org entry for MSG." + (format #t "** ~a ~a\n\t~s\n\t~s\n" + (org-mu4e-link msg) + (if tag (string-concatenate `(":" ,tag "::")) "") + (or (mu:from msg) "?") + (let ((body (mu:body-txt msg))) + (if (not body) ;; get a 'summary' of the body text + "" + (string-map + (lambda (c) + (if (or (char=? c #\newline) (char=? c #\return)) + #\space + c)) + (substring body 0 (min (string-length body) 100))))))) + +(define (main args) + (let* ((optionspec '( (muhome (value #t)) + (tag (value #t)) + (help (single-char #\h) (value #f)))) + (options (getopt-long args optionspec)) + (msg (string-append + "usage: mu4e-org [--help] [--muhome=] [--tag=] ")) + (help (option-ref options 'help #f)) + (tag (option-ref options 'tag #f)) + (muhome (option-ref options 'muhome #f)) + (query (string-concatenate (option-ref options '() '())))) + (if help + (begin (display msg) (exit 0)) + (begin + (mu:initialize muhome) + (display-org-header query) + (mu:for-each-message + (lambda (msg) (display-org-entry msg tag)) + query))))) + +;; Local Variables: +;; mode: scheme +;; End: diff --git a/guile/fdl.texi b/guile/fdl.texi new file mode 100644 index 0000000..96ce74e --- /dev/null +++ b/guile/fdl.texi @@ -0,0 +1,451 @@ +@c The GNU Free Documentation License. +@center Version 1.2, November 2002 + +@c This file is intended to be included within another document, +@c hence no sectioning command or @node. + +@display +Copyright @copyright{} 2000,2001,2002 Free Software Foundation, Inc. +51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + +Everyone is permitted to copy and distribute verbatim copies +of this license document, but changing it is not allowed. +@end display + +@enumerate 0 +@item +PREAMBLE + +The purpose of this License is to make a manual, textbook, or other +functional and useful document @dfn{free} in the sense of freedom: to +assure everyone the effective freedom to copy and redistribute it, +with or without modifying it, either commercially or noncommercially. +Secondarily, this License preserves for the author and publisher a way +to get credit for their work, while not being considered responsible +for modifications made by others. + +This License is a kind of ``copyleft'', which means that derivative +works of the document must themselves be free in the same sense. It +complements the GNU General Public License, which is a copyleft +license designed for free software. + +We have designed this License in order to use it for manuals for free +software, because free software needs free documentation: a free +program should come with manuals providing the same freedoms that the +software does. But this License is not limited to software manuals; +it can be used for any textual work, regardless of subject matter or +whether it is published as a printed book. We recommend this License +principally for works whose purpose is instruction or reference. + +@item +APPLICABILITY AND DEFINITIONS + +This License applies to any manual or other work, in any medium, that +contains a notice placed by the copyright holder saying it can be +distributed under the terms of this License. Such a notice grants a +world-wide, royalty-free license, unlimited in duration, to use that +work under the conditions stated herein. The ``Document'', below, +refers to any such manual or work. Any member of the public is a +licensee, and is addressed as ``you''. You accept the license if you +copy, modify or distribute the work in a way requiring permission +under copyright law. + +A ``Modified Version'' of the Document means any work containing the +Document or a portion of it, either copied verbatim, or with +modifications and/or translated into another language. + +A ``Secondary Section'' is a named appendix or a front-matter section +of the Document that deals exclusively with the relationship of the +publishers or authors of the Document to the Document's overall +subject (or to related matters) and contains nothing that could fall +directly within that overall subject. (Thus, if the Document is in +part a textbook of mathematics, a Secondary Section may not explain +any mathematics.) The relationship could be a matter of historical +connection with the subject or with related matters, or of legal, +commercial, philosophical, ethical or political position regarding +them. + +The ``Invariant Sections'' are certain Secondary Sections whose titles +are designated, as being those of Invariant Sections, in the notice +that says that the Document is released under this License. If a +section does not fit the above definition of Secondary then it is not +allowed to be designated as Invariant. The Document may contain zero +Invariant Sections. If the Document does not identify any Invariant +Sections then there are none. + +The ``Cover Texts'' are certain short passages of text that are listed, +as Front-Cover Texts or Back-Cover Texts, in the notice that says that +the Document is released under this License. A Front-Cover Text may +be at most 5 words, and a Back-Cover Text may be at most 25 words. + +A ``Transparent'' copy of the Document means a machine-readable copy, +represented in a format whose specification is available to the +general public, that is suitable for revising the document +straightforwardly with generic text editors or (for images composed of +pixels) generic paint programs or (for drawings) some widely available +drawing editor, and that is suitable for input to text formatters or +for automatic translation to a variety of formats suitable for input +to text formatters. A copy made in an otherwise Transparent file +format whose markup, or absence of markup, has been arranged to thwart +or discourage subsequent modification by readers is not Transparent. +An image format is not Transparent if used for any substantial amount +of text. A copy that is not ``Transparent'' is called ``Opaque''. + +Examples of suitable formats for Transparent copies include plain +@sc{ascii} without markup, Texinfo input format, La@TeX{} input +format, @acronym{SGML} or @acronym{XML} using a publicly available +@acronym{DTD}, and standard-conforming simple @acronym{HTML}, +PostScript or @acronym{PDF} designed for human modification. Examples +of transparent image formats include @acronym{PNG}, @acronym{XCF} and +@acronym{JPG}. Opaque formats include proprietary formats that can be +read and edited only by proprietary word processors, @acronym{SGML} or +@acronym{XML} for which the @acronym{DTD} and/or processing tools are +not generally available, and the machine-generated @acronym{HTML}, +PostScript or @acronym{PDF} produced by some word processors for +output purposes only. + +The ``Title Page'' means, for a printed book, the title page itself, +plus such following pages as are needed to hold, legibly, the material +this License requires to appear in the title page. For works in +formats which do not have any title page as such, ``Title Page'' means +the text near the most prominent appearance of the work's title, +preceding the beginning of the body of the text. + +A section ``Entitled XYZ'' means a named subunit of the Document whose +title either is precisely XYZ or contains XYZ in parentheses following +text that translates XYZ in another language. (Here XYZ stands for a +specific section name mentioned below, such as ``Acknowledgements'', +``Dedications'', ``Endorsements'', or ``History''.) To ``Preserve the Title'' +of such a section when you modify the Document means that it remains a +section ``Entitled XYZ'' according to this definition. + +The Document may include Warranty Disclaimers next to the notice which +states that this License applies to the Document. These Warranty +Disclaimers are considered to be included by reference in this +License, but only as regards disclaiming warranties: any other +implication that these Warranty Disclaimers may have is void and has +no effect on the meaning of this License. + +@item +VERBATIM COPYING + +You may copy and distribute the Document in any medium, either +commercially or noncommercially, provided that this License, the +copyright notices, and the license notice saying this License applies +to the Document are reproduced in all copies, and that you add no other +conditions whatsoever to those of this License. You may not use +technical measures to obstruct or control the reading or further +copying of the copies you make or distribute. However, you may accept +compensation in exchange for copies. If you distribute a large enough +number of copies you must also follow the conditions in section 3. + +You may also lend copies, under the same conditions stated above, and +you may publicly display copies. + +@item +COPYING IN QUANTITY + +If you publish printed copies (or copies in media that commonly have +printed covers) of the Document, numbering more than 100, and the +Document's license notice requires Cover Texts, you must enclose the +copies in covers that carry, clearly and legibly, all these Cover +Texts: Front-Cover Texts on the front cover, and Back-Cover Texts on +the back cover. Both covers must also clearly and legibly identify +you as the publisher of these copies. The front cover must present +the full title with all words of the title equally prominent and +visible. You may add other material on the covers in addition. +Copying with changes limited to the covers, as long as they preserve +the title of the Document and satisfy these conditions, can be treated +as verbatim copying in other respects. + +If the required texts for either cover are too voluminous to fit +legibly, you should put the first ones listed (as many as fit +reasonably) on the actual cover, and continue the rest onto adjacent +pages. + +If you publish or distribute Opaque copies of the Document numbering +more than 100, you must either include a machine-readable Transparent +copy along with each Opaque copy, or state in or with each Opaque copy +a computer-network location from which the general network-using +public has access to download using public-standard network protocols +a complete Transparent copy of the Document, free of added material. +If you use the latter option, you must take reasonably prudent steps, +when you begin distribution of Opaque copies in quantity, to ensure +that this Transparent copy will remain thus accessible at the stated +location until at least one year after the last time you distribute an +Opaque copy (directly or through your agents or retailers) of that +edition to the public. + +It is requested, but not required, that you contact the authors of the +Document well before redistributing any large number of copies, to give +them a chance to provide you with an updated version of the Document. + +@item +MODIFICATIONS + +You may copy and distribute a Modified Version of the Document under +the conditions of sections 2 and 3 above, provided that you release +the Modified Version under precisely this License, with the Modified +Version filling the role of the Document, thus licensing distribution +and modification of the Modified Version to whoever possesses a copy +of it. In addition, you must do these things in the Modified Version: + +@enumerate A +@item +Use in the Title Page (and on the covers, if any) a title distinct +from that of the Document, and from those of previous versions +(which should, if there were any, be listed in the History section +of the Document). You may use the same title as a previous version +if the original publisher of that version gives permission. + +@item +List on the Title Page, as authors, one or more persons or entities +responsible for authorship of the modifications in the Modified +Version, together with at least five of the principal authors of the +Document (all of its principal authors, if it has fewer than five), +unless they release you from this requirement. + +@item +State on the Title page the name of the publisher of the +Modified Version, as the publisher. + +@item +Preserve all the copyright notices of the Document. + +@item +Add an appropriate copyright notice for your modifications +adjacent to the other copyright notices. + +@item +Include, immediately after the copyright notices, a license notice +giving the public permission to use the Modified Version under the +terms of this License, in the form shown in the Addendum below. + +@item +Preserve in that license notice the full lists of Invariant Sections +and required Cover Texts given in the Document's license notice. + +@item +Include an unaltered copy of this License. + +@item +Preserve the section Entitled ``History'', Preserve its Title, and add +to it an item stating at least the title, year, new authors, and +publisher of the Modified Version as given on the Title Page. If +there is no section Entitled ``History'' in the Document, create one +stating the title, year, authors, and publisher of the Document as +given on its Title Page, then add an item describing the Modified +Version as stated in the previous sentence. + +@item +Preserve the network location, if any, given in the Document for +public access to a Transparent copy of the Document, and likewise +the network locations given in the Document for previous versions +it was based on. These may be placed in the ``History'' section. +You may omit a network location for a work that was published at +least four years before the Document itself, or if the original +publisher of the version it refers to gives permission. + +@item +For any section Entitled ``Acknowledgements'' or ``Dedications'', Preserve +the Title of the section, and preserve in the section all the +substance and tone of each of the contributor acknowledgements and/or +dedications given therein. + +@item +Preserve all the Invariant Sections of the Document, +unaltered in their text and in their titles. Section numbers +or the equivalent are not considered part of the section titles. + +@item +Delete any section Entitled ``Endorsements''. Such a section +may not be included in the Modified Version. + +@item +Do not retitle any existing section to be Entitled ``Endorsements'' or +to conflict in title with any Invariant Section. + +@item +Preserve any Warranty Disclaimers. +@end enumerate + +If the Modified Version includes new front-matter sections or +appendices that qualify as Secondary Sections and contain no material +copied from the Document, you may at your option designate some or all +of these sections as invariant. To do this, add their titles to the +list of Invariant Sections in the Modified Version's license notice. +These titles must be distinct from any other section titles. + +You may add a section Entitled ``Endorsements'', provided it contains +nothing but endorsements of your Modified Version by various +parties---for example, statements of peer review or that the text has +been approved by an organization as the authoritative definition of a +standard. + +You may add a passage of up to five words as a Front-Cover Text, and a +passage of up to 25 words as a Back-Cover Text, to the end of the list +of Cover Texts in the Modified Version. Only one passage of +Front-Cover Text and one of Back-Cover Text may be added by (or +through arrangements made by) any one entity. If the Document already +includes a cover text for the same cover, previously added by you or +by arrangement made by the same entity you are acting on behalf of, +you may not add another; but you may replace the old one, on explicit +permission from the previous publisher that added the old one. + +The author(s) and publisher(s) of the Document do not by this License +give permission to use their names for publicity for or to assert or +imply endorsement of any Modified Version. + +@item +COMBINING DOCUMENTS + +You may combine the Document with other documents released under this +License, under the terms defined in section 4 above for modified +versions, provided that you include in the combination all of the +Invariant Sections of all of the original documents, unmodified, and +list them all as Invariant Sections of your combined work in its +license notice, and that you preserve all their Warranty Disclaimers. + +The combined work need only contain one copy of this License, and +multiple identical Invariant Sections may be replaced with a single +copy. If there are multiple Invariant Sections with the same name but +different contents, make the title of each such section unique by +adding at the end of it, in parentheses, the name of the original +author or publisher of that section if known, or else a unique number. +Make the same adjustment to the section titles in the list of +Invariant Sections in the license notice of the combined work. + +In the combination, you must combine any sections Entitled ``History'' +in the various original documents, forming one section Entitled +``History''; likewise combine any sections Entitled ``Acknowledgements'', +and any sections Entitled ``Dedications''. You must delete all +sections Entitled ``Endorsements.'' + +@item +COLLECTIONS OF DOCUMENTS + +You may make a collection consisting of the Document and other documents +released under this License, and replace the individual copies of this +License in the various documents with a single copy that is included in +the collection, provided that you follow the rules of this License for +verbatim copying of each of the documents in all other respects. + +You may extract a single document from such a collection, and distribute +it individually under this License, provided you insert a copy of this +License into the extracted document, and follow this License in all +other respects regarding verbatim copying of that document. + +@item +AGGREGATION WITH INDEPENDENT WORKS + +A compilation of the Document or its derivatives with other separate +and independent documents or works, in or on a volume of a storage or +distribution medium, is called an ``aggregate'' if the copyright +resulting from the compilation is not used to limit the legal rights +of the compilation's users beyond what the individual works permit. +When the Document is included in an aggregate, this License does not +apply to the other works in the aggregate which are not themselves +derivative works of the Document. + +If the Cover Text requirement of section 3 is applicable to these +copies of the Document, then if the Document is less than one half of +the entire aggregate, the Document's Cover Texts may be placed on +covers that bracket the Document within the aggregate, or the +electronic equivalent of covers if the Document is in electronic form. +Otherwise they must appear on printed covers that bracket the whole +aggregate. + +@item +TRANSLATION + +Translation is considered a kind of modification, so you may +distribute translations of the Document under the terms of section 4. +Replacing Invariant Sections with translations requires special +permission from their copyright holders, but you may include +translations of some or all Invariant Sections in addition to the +original versions of these Invariant Sections. You may include a +translation of this License, and all the license notices in the +Document, and any Warranty Disclaimers, provided that you also include +the original English version of this License and the original versions +of those notices and disclaimers. In case of a disagreement between +the translation and the original version of this License or a notice +or disclaimer, the original version will prevail. + +If a section in the Document is Entitled ``Acknowledgements'', +``Dedications'', or ``History'', the requirement (section 4) to Preserve +its Title (section 1) will typically require changing the actual +title. + +@item +TERMINATION + +You may not copy, modify, sublicense, or distribute the Document except +as expressly provided for under this License. Any other attempt to +copy, modify, sublicense or distribute the Document is void, and will +automatically terminate your rights under this License. However, +parties who have received copies, or rights, from you under this +License will not have their licenses terminated so long as such +parties remain in full compliance. + +@item +FUTURE REVISIONS OF THIS LICENSE + +The Free Software Foundation may publish new, revised versions +of the GNU Free Documentation License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. See +@uref{http://www.gnu.org/copyleft/}. + +Each version of the License is given a distinguishing version number. +If the Document specifies that a particular numbered version of this +License ``or any later version'' applies to it, you have the option of +following the terms and conditions either of that specified version or +of any later version that has been published (not as a draft) by the +Free Software Foundation. If the Document does not specify a version +number of this License, you may choose any version ever published (not +as a draft) by the Free Software Foundation. +@end enumerate + +@page +@heading ADDENDUM: How to use this License for your documents + +To use this License in a document you have written, include a copy of +the License in the document and put the following copyright and +license notices just after the title page: + +@smallexample +@group + Copyright (C) @var{year} @var{your name}. + Permission is granted to copy, distribute and/or modify this document + under the terms of the GNU Free Documentation License, Version 1.2 + or any later version published by the Free Software Foundation; + with no Invariant Sections, no Front-Cover Texts, and no Back-Cover + Texts. A copy of the license is included in the section entitled ``GNU + Free Documentation License''. +@end group +@end smallexample + +If you have Invariant Sections, Front-Cover Texts and Back-Cover Texts, +replace the ``with@dots{}Texts.'' line with this: + +@smallexample +@group + with the Invariant Sections being @var{list their titles}, with + the Front-Cover Texts being @var{list}, and with the Back-Cover Texts + being @var{list}. +@end group +@end smallexample + +If you have Invariant Sections without Cover Texts, or some other +combination of the three, merge those two alternatives to suit the +situation. + +If your document contains nontrivial examples of program code, we +recommend releasing these examples in parallel under your choice of +free software license, such as the GNU General Public License, +to permit their use in free software. + +@c Local Variables: +@c ispell-local-pdict: "ispell-dict" +@c End: + diff --git a/guile/meson.build b/guile/meson.build new file mode 100644 index 0000000..aceada3 --- /dev/null +++ b/guile/meson.build @@ -0,0 +1,114 @@ +## Copyright (C) 2022-2024 Dirk-Jan C. Binnema +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# +# create a shell script for compiling from the source dirs +compile_scm_conf = configuration_data() +compile_scm_conf.set('abs_builddir', meson.current_build_dir()) +compile_scm_conf.set('guild', 'guild') +compile_scm=configure_file( + input: 'compile-scm.in', + output: 'compile-scm', + configuration: compile_scm_conf, + install: false +) +run_command('chmod', '+x', compile_scm, check: true) +scm_compiler=join_paths(meson.current_build_dir(), 'compile-scm') + +# +# NOTE: snarfing works but you get: +# ,---- +# | cc1plus: warning: command-line option ‘-std=gnu11’ is valid for C/ObjC +# | but not for C++ +# `---- +# this is because the snarf-script hardcodes the '-std=gnu11' but we're +# building for c++; even worse, e.g. on some MacOS, the warning is a +# hard error. +# +# We can override flag through a env variable CPP; but then we _also_ need to +# override the compiler, so e.g. CPP="g++ -std=c++17'; but it's a bit +# hairy/ugly/fragile to derive the raw compiler name in meson; also the +# generator expression doesn't take an 'env:' parameter, so we'd need +# to rewrite using custom_target... +# +# for now, we avoid all that by simply including the generated files. +do_snarf=false + +if do_snarf + snarf = find_program('guile-snarf3.0','guile-snarf') + # there must be a better way of feeding the include paths to snarf... + snarf_args=['-o', '@OUTPUT@', '@INPUT@', '-I' + meson.current_source_dir() + '/..', + '-I' + meson.current_source_dir() + '/../lib', + '-I' + meson.current_build_dir() + '/..'] + snarf_args += '-I' + join_paths(glib_dep.get_pkgconfig_variable('includedir'), + 'glib-2.0') + snarf_args += '-I' + join_paths(glib_dep.get_pkgconfig_variable('libdir'), + 'glib-2.0', 'include') + snarf_args += '-I' + join_paths(guile_dep.get_pkgconfig_variable('includedir'), + 'guile', '3.0') + snarf_gen=generator(snarf, + output: '@BASENAME@.x', + arguments: snarf_args) + snarf_srcs=['mu-guile.cc', 'mu-guile-message.cc'] + snarf_x=snarf_gen.process(snarf_srcs) +else + snarf_x = [ 'mu-guile-message.x', 'mu-guile.x' ] +endif + +lib_guile_mu = shared_module( + 'guile-mu', + [ 'mu-guile.cc', + 'mu-guile-message.cc' ], + dependencies: [guile_dep, glib_dep, lib_mu_dep, config_h_dep, thread_dep ], + install: true, + install_dir: guile_extension_dir +) + +if makeinfo.found() + custom_target('mu_guile_info', + input: 'mu-guile.texi', + output: 'mu-guile.info', + install: true, + install_dir: infodir, + command: [makeinfo, + '-o', join_paths(meson.current_build_dir(), 'mu-guile.info'), + join_paths(meson.current_source_dir(), 'mu-guile.texi'), + '-I', join_paths(meson.current_build_dir(), '..')]) + if install_info.found() + infodir = join_paths(get_option('prefix') / get_option('infodir')) + meson.add_install_script(install_info_script, infodir, 'mu-guile.info') + endif +endif + +guile_scm_dir=join_paths(datadir, 'guile', 'site', '3.0') +install_data(['mu.scm'], install_dir: guile_scm_dir) +guile_scm_mu_dir=join_paths(guile_scm_dir, 'mu') +foreach mod : ['script.scm', 'message.scm', 'stats.scm', 'plot.scm'] + install_data(join_paths('mu', mod), install_dir: guile_scm_mu_dir) +endforeach + +mu_guile_scripts=[ + join_paths('scripts', 'find-dups.scm'), + join_paths('scripts', 'msgs-count.scm'), + join_paths('scripts', 'histogram.scm')] +mu_guile_script_dir=join_paths(datadir, 'mu', 'scripts') +install_data(mu_guile_scripts, install_dir: mu_guile_script_dir) + +guile_builddir=meson.current_build_dir() + +if not get_option('tests').disabled() + subdir('tests') +endif diff --git a/guile/mu-guile-message.cc b/guile/mu-guile-message.cc new file mode 100644 index 0000000..281ed7c --- /dev/null +++ b/guile/mu-guile-message.cc @@ -0,0 +1,485 @@ +/* +** Copyright (C) 2011-2023 Dirk-Jan C. Binnema +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include +#include "mu-guile-message.hh" + +#include "message/mu-message.hh" +#include "utils/mu-utils.hh" + +#include +#include +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wredundant-decls" +#include +#pragma GCC diagnostic pop + +#include "mu-guile.hh" + +#include +#include + +using namespace Mu; + +/* pseudo field, not in Xapian */ +constexpr auto MU_GUILE_MSG_FIELD_ID_TIMESTAMP = Field::id_size() + 1; + +/* some symbols */ +static SCM SYMB_PRIO_LOW, SYMB_PRIO_NORMAL, SYMB_PRIO_HIGH; +static std::array SYMB_FLAGS; +static SCM SYMB_CONTACT_TO, SYMB_CONTACT_CC, SYMB_CONTACT_BCC, SYMB_CONTACT_FROM; +static long MSG_TAG; + + +using MessageSPtr = std::unique_ptr; + +static gboolean +mu_guile_scm_is_msg(SCM scm) +{ + return SCM_NIMP(scm) && (long)SCM_CAR(scm) == MSG_TAG; +} + +static SCM +message_scm_create(Xapian::Document&& doc) +{ + /* placement-new */ + + void *scm_mem{scm_gc_malloc(sizeof(Message), "msg")}; + Message* msgp = new(scm_mem)Message(std::move(doc)); + + SCM_RETURN_NEWSMOB(MSG_TAG, msgp); +} + +static const Message* +message_from_scm(SCM msg_smob) +{ + return reinterpret_cast(SCM_CDR(msg_smob)); +} + +static size_t +message_scm_free(SCM msg_smob) +{ + if (auto msg = message_from_scm(msg_smob); msg) + msg->~Message(); + + return sizeof(Message); +} + +static int +message_scm_print(SCM msg_smob, SCM port, scm_print_state* pstate) +{ + scm_puts("#path().c_str(), port); + + scm_puts(">", port); + return 1; +} + +struct FlagData { + Flags flags; + SCM lst; +}; + +#define MU_GUILE_INITIALIZED_OR_ERROR \ + do { \ + if (!(mu_guile_initialized())) { \ + mu_guile_error(FUNC_NAME, \ + 0, \ + "mu not initialized; call mu:initialize", \ + SCM_UNDEFINED); \ + return SCM_UNSPECIFIED; \ + } \ + } while (0) + + +static SCM +get_flags_scm(const Message& msg) +{ + SCM lst{SCM_EOL}; + const auto flags{msg.flags()}; + + for (auto i = 0; i != AllMessageFlagInfos.size(); ++i) { + const auto& info{AllMessageFlagInfos.at(i)}; + if (any_of(info.flag & flags)) + scm_append_x(scm_list_2(lst, scm_list_1(SYMB_FLAGS.at(i)))); + } + + return lst; +} + +static SCM +get_prio_scm(const Message& msg) +{ + switch (msg.priority()) { + case Priority::Low: return SYMB_PRIO_LOW; + case Priority::Normal: return SYMB_PRIO_NORMAL; + case Priority::High: return SYMB_PRIO_HIGH; + + default: g_return_val_if_reached(SCM_UNDEFINED); + } +} + +static SCM +msg_string_list_field(const Message& msg, Field::Id field_id) +{ + SCM scmlst{SCM_EOL}; + for (auto&& val: msg.document().string_vec_value(field_id)) { + SCM item; + item = scm_list_1(mu_guile_scm_from_string(val)); + scmlst = scm_append_x(scm_list_2(scmlst, item)); + } + + return scmlst; +} + +static SCM +msg_contact_list_field(const Message& msg, Field::Id field_id) +{ + return scm_from_utf8_string( + to_string(msg.document().contacts_value(field_id)).c_str()); +} + +static SCM +get_body(const Message& msg, bool html) +{ + if (const auto body = html ? msg.body_html() : msg.body_text(); body) + return mu_guile_scm_from_string(*body); + else + return SCM_BOOL_F; +} + +SCM_DEFINE(get_field, + "mu:c:get-field", + 2, + 0, + 0, + (SCM MSG, SCM FIELD), + "Get the field FIELD from message MSG.\n") +#define FUNC_NAME s_get_field +{ + SCM_ASSERT(mu_guile_scm_is_msg(MSG), MSG, SCM_ARG1, FUNC_NAME); + auto msg{message_from_scm(MSG)}; + SCM_ASSERT(msg, MSG, SCM_ARG1, FUNC_NAME); + + SCM_ASSERT(scm_integer_p(FIELD), FIELD, SCM_ARG2, FUNC_NAME); + const auto field_opt{field_from_number(static_cast(scm_to_int(FIELD)))}; + SCM_ASSERT(!!field_opt, FIELD, SCM_ARG2, FUNC_NAME); +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" + switch (field_opt->id) { + case Field::Id::Priority: + return get_prio_scm(*msg); + case Field::Id::Flags: + return get_flags_scm(*msg); + case Field::Id::BodyText: + return get_body(*msg, false); + default: + break; + } +#pragma GCC diagnostic pop + + switch (field_opt->type) { + case Field::Type::String: + return mu_guile_scm_from_string(msg->document().string_value(field_opt->id)); + case Field::Type::ByteSize: + case Field::Type::TimeT: + case Field::Type::Integer: + return scm_from_uint(msg->document().integer_value(field_opt->id)); + case Field::Type::StringList: + return msg_string_list_field(*msg, field_opt->id); + case Field::Type::ContactList: + return msg_contact_list_field(*msg, field_opt->id); + default: + SCM_ASSERT(0, FIELD, SCM_ARG2, FUNC_NAME); + } + + return SCM_UNSPECIFIED; +} +#undef FUNC_NAME + +static SCM +contacts_to_list(const Message& msg, Option field_id) +{ + SCM list{SCM_EOL}; + + const auto contacts{field_id ? + msg.document().contacts_value(*field_id) : + msg.all_contacts()}; + + for (auto&& contact: contacts) { + SCM item{scm_list_1( + scm_cons(mu_guile_scm_from_string(contact.name), + mu_guile_scm_from_string(contact.email)))}; + list = scm_append_x(scm_list_2(list, item)); + } + + return list; +} + +SCM_DEFINE(get_contacts, + "mu:c:get-contacts", + 2, + 0, + 0, + (SCM MSG, SCM CONTACT_TYPE), + "Get a list of contact information pairs.\n") +#define FUNC_NAME s_get_contacts +{ + SCM list; + + MU_GUILE_INITIALIZED_OR_ERROR; + + SCM_ASSERT(mu_guile_scm_is_msg(MSG), MSG, SCM_ARG1, FUNC_NAME); + auto msg{message_from_scm(MSG)}; + SCM_ASSERT(msg, MSG, SCM_ARG1, FUNC_NAME); + + SCM_ASSERT(scm_symbol_p(CONTACT_TYPE) || scm_is_bool(CONTACT_TYPE), + CONTACT_TYPE, + SCM_ARG2, + FUNC_NAME); + + if (CONTACT_TYPE == SCM_BOOL_F) + return SCM_UNSPECIFIED; /* nothing to do */ + + Option field_id; + if (CONTACT_TYPE == SCM_BOOL_T) + field_id = {}; /* get all */ + else { + if (scm_is_eq(CONTACT_TYPE, SYMB_CONTACT_TO)) + field_id = Field::Id::To; + else if (scm_is_eq(CONTACT_TYPE, SYMB_CONTACT_CC)) + field_id = Field::Id::Cc; + else if (scm_is_eq(CONTACT_TYPE, SYMB_CONTACT_BCC)) + field_id = Field::Id::Bcc; + else if (scm_is_eq(CONTACT_TYPE, SYMB_CONTACT_FROM)) + field_id = Field::Id::From; + else { + mu_guile_error(FUNC_NAME, 0, "invalid contact type", SCM_UNDEFINED); + return SCM_UNSPECIFIED; + } + } + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wcast-function-type" + list = contacts_to_list(*msg, field_id); +#pragma GCC diagnostic pop + + /* explicitly close the file backend, so we won't run out of fds */ + + + return list; +} +#undef FUNC_NAME + +SCM_DEFINE(get_parts, + "mu:c:get-parts", + 1, + 1, + 0, + (SCM MSG, SCM ATTS_ONLY), + "Get the list of mime-parts for MSG. If ATTS_ONLY is #t, only" + "get parts that are (look like) attachments. The resulting list has " + "elements which are list of the form (index name mime-type size).\n") +#define FUNC_NAME s_get_parts +{ + MU_GUILE_INITIALIZED_OR_ERROR; + + SCM_ASSERT(mu_guile_scm_is_msg(MSG), MSG, SCM_ARG1, FUNC_NAME); + auto msg{message_from_scm(MSG)}; + SCM_ASSERT(msg, MSG, SCM_ARG1, FUNC_NAME); + SCM_ASSERT(scm_is_bool(ATTS_ONLY), ATTS_ONLY, SCM_ARG2, FUNC_NAME); + + SCM attlist = SCM_EOL; /* empty list */ + bool attachments_only = ATTS_ONLY == SCM_BOOL_T ? TRUE : FALSE; + + size_t n{}; + for (auto&& part: msg->parts()) { + + if (attachments_only && !part.is_attachment()) + continue; + + const auto mime_type{part.mime_type()}; + const auto filename{part.cooked_filename()}; + + SCM elm = scm_list_5( + /* msg */ + mu_guile_scm_from_string(msg->path().c_str()), + /* index */ + scm_from_uint(n++), + /* filename or #f */ + filename ? mu_guile_scm_from_string(*filename) : SCM_BOOL_F, + /* mime-type */ + mime_type ? mu_guile_scm_from_string(*mime_type) : SCM_BOOL_F, + /* size */ + part.size() > 0 ? scm_from_uint(part.size()) : SCM_BOOL_F); + + attlist = scm_cons(elm, attlist); + } + + /* explicitly close the file backend, so we won't run of fds */ + msg->unload_mime_message(); + + return attlist; +} +#undef FUNC_NAME + +SCM_DEFINE(get_header, + "mu:c:get-header", + 2, + 0, + 0, + (SCM MSG, SCM HEADER), + "Get an arbitrary HEADER from MSG.\n") +#define FUNC_NAME s_get_header +{ + MU_GUILE_INITIALIZED_OR_ERROR; + + SCM_ASSERT(mu_guile_scm_is_msg(MSG), MSG, SCM_ARG1, FUNC_NAME); + auto msg{message_from_scm(MSG)}; + SCM_ASSERT(msg, MSG, SCM_ARG1, FUNC_NAME); + + SCM_ASSERT(scm_is_string(HEADER) || HEADER == SCM_UNDEFINED, HEADER, SCM_ARG2, FUNC_NAME); + + char *header = scm_to_utf8_string(HEADER); + SCM val = mu_guile_scm_from_string(msg->header(header).value_or("")); + free(header); + + /* explicitly close the file backend, so we won't run of fds */ + msg->unload_mime_message(); + + return val; +} +#undef FUNC_NAME +SCM_DEFINE(for_each_message, + "mu:c:for-each-message", + 3, + 0, + 0, + (SCM FUNC, SCM EXPR, SCM MAXNUM), + "Call FUNC for each msg in the message store matching EXPR. EXPR is" + "either a string containing a mu search expression or a boolean; in the former " + "case, limit the messages to only those matching the expression, in the " + "latter case, match /all/ messages if the EXPR equals #t, and match " + "none if EXPR equals #f.") +#define FUNC_NAME s_for_each_message +{ + char* expr{}; + + MU_GUILE_INITIALIZED_OR_ERROR; + + SCM_ASSERT(scm_procedure_p(FUNC), FUNC, SCM_ARG1, FUNC_NAME); + SCM_ASSERT(scm_is_bool(EXPR) || scm_is_string(EXPR), EXPR, SCM_ARG2, FUNC_NAME); + SCM_ASSERT(scm_is_integer(MAXNUM), MAXNUM, SCM_ARG3, FUNC_NAME); + + if (EXPR == SCM_BOOL_F) + return SCM_UNSPECIFIED; /* nothing to do */ + + if (EXPR == SCM_BOOL_T) + expr = strdup("\"\""); /* note, "" matches *all* messages */ + else + expr = scm_to_utf8_string(EXPR); + + const auto res = mu_guile_store().run_query(expr,{}, {}, scm_to_int(MAXNUM)); + free(expr); + if (!res) + return SCM_UNSPECIFIED; + + for (auto&& mi : *res) { + if (auto xdoc{mi.document()}; xdoc) { + scm_call_1(FUNC, message_scm_create(std::move(xdoc.value()))); + } + } + + return SCM_UNSPECIFIED; +} +#undef FUNC_NAME + +static SCM +register_symbol(const char* name) +{ + SCM scm; + + scm = scm_from_utf8_symbol(name); + scm_c_define(name, scm); + scm_c_export(name, NULL); + + return scm; +} + +static void +define_symbols(void) +{ + SYMB_CONTACT_TO = register_symbol("mu:contact:to"); + SYMB_CONTACT_CC = register_symbol("mu:contact:cc"); + SYMB_CONTACT_FROM = register_symbol("mu:contact:from"); + SYMB_CONTACT_BCC = register_symbol("mu:contact:bcc"); + + SYMB_PRIO_LOW = register_symbol("mu:prio:low"); + SYMB_PRIO_NORMAL = register_symbol("mu:prio:normal"); + SYMB_PRIO_HIGH = register_symbol("mu:prio:high"); + + for (auto i = 0U; i != AllMessageFlagInfos.size(); ++i) { + const auto& info{AllMessageFlagInfos.at(i)}; + const auto name = "mu:flag:" + std::string{info.name}; + SYMB_FLAGS[i] = register_symbol(name.c_str()); + } +} +static void +define_vars(void) +{ + field_for_each([](auto&& field){ + + auto defvar = [&](auto&& fname, auto&& ffield) { + const auto name{"mu:field:" + std::string{fname}}; + scm_c_define(name.c_str(), scm_from_uint(field.value_no())); + scm_c_export(name.c_str(), NULL); + }; + + // define for both name and (if exists) alias. + if (!field.name.empty()) + defvar(field.name, field); + if (!field.alias.empty()) + defvar(field.alias, field); + }); + + /* non-Xapian field: timestamp */ + scm_c_define("mu:field:timestamp", + scm_from_uint(MU_GUILE_MSG_FIELD_ID_TIMESTAMP)); + scm_c_export("mu:field:timestamp", NULL); + +} + +void* +mu_guile_message_init(void* data) +{ + MSG_TAG = scm_make_smob_type("message", sizeof(Message)); + + scm_set_smob_free(MSG_TAG, message_scm_free); + scm_set_smob_print(MSG_TAG, message_scm_print); + + define_vars(); + define_symbols(); + +#ifndef SCM_MAGIC_SNARFER +#include "mu-guile-message.x" +#endif /*SCM_MAGIC_SNARFER*/ + + return NULL; +} diff --git a/guile/mu-guile-message.hh b/guile/mu-guile-message.hh new file mode 100644 index 0000000..0e7201d --- /dev/null +++ b/guile/mu-guile-message.hh @@ -0,0 +1,34 @@ +/* +** Copyright (C) 2011-2020 Dirk-Jan C. Binnema +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_GUILE_MESSAGE_H__ +#define MU_GUILE_MESSAGE_H__ + +/** + * Initialize this mu guile module. + * + * @param data +q * + * @return + */ +extern "C" { +void* mu_guile_message_init(void* data); +} + +#endif /*MU_GUILE_MESSAGE_HH__*/ diff --git a/guile/mu-guile-message.x b/guile/mu-guile-message.x new file mode 100644 index 0000000..6127b39 --- /dev/null +++ b/guile/mu-guile-message.x @@ -0,0 +1,6 @@ +/* cpp arguments: mu-guile-message.cc -DHAVE_CONFIG_H -I. -I.. -I../lib -I/usr/local/include/guile/3.0 -pthread -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include -I/usr/include/sysprof-4 -I/usr/include/libmount -I/usr/include/blkid -pthread -fno-strict-aliasing -Wall -Wextra -Wundef -Wwrite-strings -Wpointer-arith -Wmissing-declarations -Wredundant-decls -Wno-unused-parameter -Wno-missing-field-initializers -Wformat=2 -Wcast-align -Wformat-nonliteral -Wformat-security -Wsign-compare -Wstrict-aliasing -Wshadow -Winline -Wpacked -Wmissing-format-attribute -Wmissing-noreturn -Winit-self -Wmissing-include-dirs -Wunused-but-set-variable -Warray-bounds -Wreturn-type -Wno-overloaded-virtual -Wswitch-enum -Wswitch-default -Wno-error=unused-parameter -Wno-error=missing-field-initializers -Wno-error=overloaded-virtual -Wno-redundant-decls -Wno-missing-declarations -Wno-suggest-attribute=noreturn -O2 -Wno-inline */ +scm_c_define_gsubr (s_get_field, 2, 0, 0, (scm_t_subr) get_field);; +scm_c_define_gsubr (s_get_contacts, 2, 0, 0, (scm_t_subr) get_contacts);; +scm_c_define_gsubr (s_get_parts, 1, 1, 0, (scm_t_subr) get_parts);; +scm_c_define_gsubr (s_get_header, 2, 0, 0, (scm_t_subr) get_header);; +scm_c_define_gsubr (s_for_each_message, 3, 0, 0, (scm_t_subr) for_each_message);; diff --git a/guile/mu-guile.cc b/guile/mu-guile.cc new file mode 100644 index 0000000..44659aa --- /dev/null +++ b/guile/mu-guile.cc @@ -0,0 +1,250 @@ +/* +** Copyright (C) 2011-2023 Dirk-Jan C. Binnema +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include + +#include "mu-guile.hh" + +#include +#include + +#include +#include + +#include + +using namespace Mu; + +SCM +mu_guile_scm_from_string(const std::string& str) +{ + if (str.empty()) + return SCM_BOOL_F; + else + return scm_from_stringn(str.c_str(), str.size(), + "UTF-8", + SCM_FAILED_CONVERSION_QUESTION_MARK); +} + +SCM +mu_guile_error(const char* func_name, int status, const char* fmt, SCM args) +{ + scm_error_scm(scm_from_locale_symbol("MuError"), + scm_from_utf8_string(func_name ? func_name : ""), + scm_from_utf8_string(fmt), + args, + scm_list_1(scm_from_int(status))); + + return SCM_UNSPECIFIED; +} + +SCM +mu_guile_g_error(const char* func_name, GError* err) +{ + scm_error_scm(scm_from_locale_symbol("MuError"), + scm_from_utf8_string(func_name), + scm_from_utf8_string(err ? err->message : "error"), + SCM_UNDEFINED, + SCM_UNDEFINED); + + return SCM_UNSPECIFIED; +} + +/* there can be only one */ + +static Option StoreSingleton = Nothing; + +static bool +mu_guile_init_instance(const std::string& muhome) try { + setlocale(LC_ALL, ""); + + const auto path{runtime_path(RuntimePath::XapianDb, muhome)}; + auto store = Store::make(path); + if (!store) { + mu_critical("error creating store @ %s: %s", path, + store.error().what()); + throw store.error(); + } else + StoreSingleton.emplace(std::move(store.value())); + + mu_debug("mu-guile: opened store @ {} (n={}); maildir: {}", + StoreSingleton->path(), + StoreSingleton->size(), + StoreSingleton->root_maildir()); + + return true; + +} catch (const Xapian::Error& xerr) { + mu_critical("{}: xapian error '{}'", __func__, xerr.get_msg()); + return false; +} catch (const std::runtime_error& re) { + mu_critical("{}: error: {}", __func__, re.what()); + return false; +} catch (const std::exception& e) { + mu_critical("{}: caught exception: {}", __func__, e.what()); + return false; +} catch (...) { + mu_critical("{}: caught exception", __func__); + return false; +} + +static void +mu_guile_uninit_instance() +{ + StoreSingleton.reset(); +} + +Mu::Store& +mu_guile_store() +{ + if (!StoreSingleton) + mu_error("mu guile not initialized"); + + return StoreSingleton.value(); +} + +gboolean +mu_guile_initialized() +{ + g_debug("initialized ? %u", !!StoreSingleton); + + return !!StoreSingleton; +} + +SCM_DEFINE_PUBLIC(mu_initialize, + "mu:initialize", + 0, + 1, + 0, + (SCM MUHOME), + "Initialize mu - needed before you call any of the other " + "functions. Optionally, you can provide MUHOME which should be an " + "absolute path to your mu home directory " + "-- typically, the default, ~/.cache/mu, should be just fine.") +#define FUNC_NAME s_mu_initialize +{ + char* muhome; + + SCM_ASSERT(scm_is_string(MUHOME) || MUHOME == SCM_BOOL_F || SCM_UNBNDP(MUHOME), + MUHOME, + SCM_ARG1, + FUNC_NAME); + + if (mu_guile_initialized()) + return mu_guile_error(FUNC_NAME, 0, "Already initialized", SCM_UNSPECIFIED); + + if (SCM_UNBNDP(MUHOME) || MUHOME == SCM_BOOL_F) + muhome = NULL; + else + muhome = scm_to_utf8_string(MUHOME); + + if (!mu_guile_init_instance(muhome ? muhome : "")) { + free(muhome); + mu_guile_error(FUNC_NAME, 0, "Failed to initialize mu", SCM_UNSPECIFIED); + return SCM_UNSPECIFIED; + } + + g_debug("mu-guile: initialized @ %s", muhome ? muhome : ""); + free(muhome); + + /* cleanup when we're exiting */ + atexit(mu_guile_uninit_instance); + + return SCM_UNSPECIFIED; +} +#undef FUNC_NAME + +SCM_DEFINE_PUBLIC(mu_initialized_p, + "mu:initialized?", + 0, + 0, + 0, + (void), + "Whether mu is initialized or not.\n") +#define FUNC_NAME s_mu_initialized_p +{ + return mu_guile_initialized() ? SCM_BOOL_T : SCM_BOOL_F; +} +#undef FUNC_NAME + +SCM_DEFINE(log_func, + "mu:c:log", + 1, + 0, + 1, + (SCM LEVEL, SCM FRM, SCM ARGS), + "log some message at LEVEL using a list of ARGS applied to FRM" + "(in 'simple-format' notation).\n") +#define FUNC_NAME s_log_func +{ + gchar* output; + SCM str; + int level; + + SCM_ASSERT(scm_integer_p(LEVEL), LEVEL, SCM_ARG1, FUNC_NAME); + SCM_ASSERT(scm_is_string(FRM), FRM, SCM_ARG2, ""); + SCM_VALIDATE_REST_ARGUMENT(ARGS); + + level = scm_to_int(LEVEL); + if (level != G_LOG_LEVEL_MESSAGE && level != G_LOG_LEVEL_WARNING && + level != G_LOG_LEVEL_CRITICAL) + return mu_guile_error(FUNC_NAME, 0, "invalid log level", SCM_UNSPECIFIED); + + str = scm_simple_format(SCM_BOOL_F, FRM, ARGS); + + if (!scm_is_string(str)) + return SCM_UNSPECIFIED; + + output = scm_to_utf8_string(str); + g_log(G_LOG_DOMAIN, (GLogLevelFlags)level, "%s", output); + free(output); + + return SCM_UNSPECIFIED; +} +#undef FUNC_NAME + +static struct { + const char* name; + unsigned val; +} VAR_PAIRS[] = { + + {"mu:message", G_LOG_LEVEL_MESSAGE}, + {"mu:warning", G_LOG_LEVEL_WARNING}, + {"mu:critical", G_LOG_LEVEL_CRITICAL}}; + +static void +define_vars(void) +{ + unsigned u; + for (u = 0; u != G_N_ELEMENTS(VAR_PAIRS); ++u) { + scm_c_define(VAR_PAIRS[u].name, scm_from_uint(VAR_PAIRS[u].val)); + scm_c_export(VAR_PAIRS[u].name, NULL); + } +} + +void* +mu_guile_init(void* data) +{ + define_vars(); + +#ifndef SCM_MAGIC_SNARFER +#include "mu-guile.x" +#endif /*SCM_MAGIC_SNARFER*/ + + return NULL; +} diff --git a/guile/mu-guile.hh b/guile/mu-guile.hh new file mode 100644 index 0000000..4954542 --- /dev/null +++ b/guile/mu-guile.hh @@ -0,0 +1,85 @@ +/* +** Copyright (C) 2011-2020 Dirk-Jan C. Binnema +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef __MU_GUILE_H__ +#define __MU_GUILE_H__ + +#include +#include + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wredundant-decls" +#include +#pragma GCC diagnostic pop + +/** + * get the singleton Store instance + */ +Mu::Store& mu_guile_store(); + +/** + * whether mu-guile is initialized + * + * @return TRUE if MuGuile is Initialized, FALSE otherwise + */ +gboolean mu_guile_initialized(); + +/** + * raise a guile error (based on a GError) + * + * @param func_name function name + * @param err the error + * + * @return SCM_UNSPECIFIED + */ +SCM mu_guile_g_error(const char* func_name, GError* err); + +/** + * raise a guile error + * + * @param func_name function + * @param status err code + * @param fmt format string for error msg + * @param args params for format string + * + * @return SCM_UNSPECIFIED + */ +SCM mu_guile_error(const char* func_name, int status, const char* fmt, SCM args); + +/** + * convert a string into an SCM -- . It assumes str is in UTF8 encoding, and + * replace characters with '?' if needed. + * + * @param str a string + * + * @return a guile string or #f for empty + */ +SCM mu_guile_scm_from_string(const std::string& str); + +/** + * Initialize this mu guile module. + * + * @param data + * + * @return + */ +extern "C" { +void* mu_guile_init(void* data); +} +#endif /*__MU_GUILE_H__*/ diff --git a/guile/mu-guile.texi b/guile/mu-guile.texi new file mode 100644 index 0000000..9eae2fe --- /dev/null +++ b/guile/mu-guile.texi @@ -0,0 +1,995 @@ +\input texinfo.tex @c -*-texinfo-*- +@c %**start of header +@setfilename mu-guile.info +@settitle mu-guile user manual + +@c Use proper quote and backtick for code sections in PDF output +@c Cf. Texinfo manual 14.2 +@set txicodequoteundirected +@set txicodequotebacktick + +@documentencoding UTF-8 +@c %**end of header + +@include version.texi + +@copying +Copyright @copyright{} 2012-@value{UPDATED-YEAR} Dirk-Jan C. Binnema + +@quotation +Permission is granted to copy, distribute and/or modify this document +under the terms of the GNU Free Documentation License, Version 1.3 or +any later version published by the Free Software Foundation; with no +Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. A +copy of the license is included in the section entitled ``GNU Free +Documentation License.'' +@end quotation +@end copying + +@titlepage +@title @t{mu-guile} - extending @t{mu} with Guile Scheme +@subtitle version @value{VERSION} +@author Dirk-Jan C. Binnema + +@c The following two commands start the copyright page. +@page +@vskip 0pt plus 1filll +@insertcopying +@end titlepage + +@dircategory The Algorithmic Language Scheme +@direntry +* Mu-guile: (mu-guile). Guile-bindings for the mu e-mail indexer/searcher +@end direntry + +@contents + +@ifnottex +@node Top +@top mu-guile manual +@end ifnottex + +@iftex +@node Welcome to mu-guile +@unnumbered Welcome to mu-guile +@end iftex + +Welcome to @t{mu-guile}! + +@t{mu} is a program for indexing and searching your e-mails. It can search +your messages in many different ways, but sometimes that may not be +enough. If you have very specific queries, or want do generate some +statistics, you need some more power. + +@t{mu-guile} is made for those cases. @t{mu-guile} exposes the internals of +@t{mu} and its database to the @t{guile} programming language. Guile is the +@emph{GNU Ubiquitous Intelligent Language for Extensions} - a version of the +@emph{Scheme} programming language and the official GNU extension language. + +Guile/Scheme is a member of the @emph{Lisp} family of programming languages -- +like emacs-lisp, @emph{Racket}, Common Lisp. If you're not familiar with +Scheme, @t{mu-guile} is an excellent opportunity to learn a bit about! + +Trust me, it's not very hard -- and it's @emph{fun}! + +@menu +* Getting started:: +* Initializing mu-guile:: +* Messages:: +* Contacts:: +* Attachments and other parts:: +* Statistics:: +* Plotting data:: +* Writing scripts:: + +Appendices + +* Recipes:: Snippets do specific things +* GNU Free Documentation License:: The license of this manual. +@end menu + +@node Getting started +@chapter Getting started + +@menu +* Installation:: +* Making sure it works:: +@end menu + +This chapter walks you through the installation and the basic setup. + +@node Installation +@section Installation + +@t{mu-guile} is part of @t{mu} - by installing the latter, the former is +necessarily installed as well. At the time of writing, there are no +distribution-provided packaged versions of @t{mu-guile}; so for now, you need +to follow the steps below. + +@subsection Guile 2.x + +@t{mu-guile} is built automatically when @t{mu} is built, if you have +@t{guile} version 2 or higher. (@t{mu} checks for this during +@t{configure}). Thus, the first step is to ensure you have @t{guile} +installed. + +On Debian/Ubuntu you can install @t{guile} 2.x using the @t{guile-2.0-dev} +package (and its dependencies): +@example +$ sudo apt-get install guile-2.0-dev +@end example + +At the time of writing, there are no official packages for +Fedora@footnote{@url{https://bugzilla.redhat.com/show_bug.cgi?id=678238}}. If +you are using Fedora or any other system that does not have packages, you need +to compile @t{guile} from +source@footnote{@url{http://www.gnu.org/software/guile/manual/html_node/Obtaining-and-Installing-Guile.html#Obtaining-and-Installing-Guile}}. + +@subsection gnuplot + +For creating graphs with @t{mu-guile}, you need the @t{gnuplot} program -- +most likely, there is a package available for your system; for example: + +@example +$ sudo apt-get install gnuplot +@end example + +and in Fedora: + +@example +$ sudo yum install gnuplot +@end example + +@subsection mu + +Assuming @t{guile} 2.x is installed correctly, @t{mu} finds it during its +@t{configure}-stage, and creates @t{mu-guile}. Building @t{mu} follows the +normal steps -- please see the @t{mu} documentation for the details. + +The output of @t{./configure} should end with a little text describing the +detected versions of various libraries @t{mu} depends on. In particular, it +should mention the @t{guile} version, e.g. + +@example +Guile version : 2.0.3.82-a2c66 +@end example + +If you don't see any line referring to @t{guile}, please install it, and run +@t{configure} again. After a successful @t{./configure}, we can make and +install the package: + +@example +$ make && sudo make install +@end example + +@subsection mu-guile + +After this, @t{mu} and @t{mu-guile} are installed -- usually somewhere under +@t{/usr/local}.You may need to update @t{guile}'s @code{%load-path} to find it +there. You can check the current @code{%load-path} with the following: + +@example +guile -c '(display %load-path)(newline)' +@end example + +If necessary, you can add the @t{%load-path} by adding to your +@file{~/.guile}: + +@lisp +(set! %load-path (cons "/usr/local/share/guile/site/2.0" %load-path)) +@end lisp + +Or, alternatively, you can set @t{GUILE_LOAD_PATH}: +@example +export GUILE_LOAD_PATH=/usr/local/share/guile/site/2.0 +@end example + +In both cases the directory should be the directory that contains the +installed @t{mu.scm}; if you installed @t{mu} under a different prefix, you +must change the @code{%load-path} accordingly. After this, you should be ready +to go! + +Furthermore, you need to ensure that @t{guile} can find the mu-guile +library; for this we can use @code{LTDL_LIBRARY_PATH}, e.g. +@example +export LTDL_LIBRARY_PATH=/usr/local/lib +@end example + +@node Making sure it works +@section Making sure it works + +Assuming @t{mu-guile} has been installed correctly (@ref{Installation}), and +also assuming that you have already indexed your e-mail messages (if +necessary, see the @t{mu-index} man-page), we are ready to start @t{mu-guile}; +a session may look something like this: + +@cartouche +@verbatim +GNU Guile 2.0.5.123-4bd53 +Copyright (C) 1995-2012 Free Software Foundation, Inc. + +Guile comes with ABSOLUTELY NO WARRANTY; for details type `,show w'. +This program is free software, and you are welcome to redistribute it +under certain conditions; type `,show c' for details. + +Enter `,help' for help. +scheme@(guile-user)> +@end verbatim +@end cartouche + +@noindent +Now, copy-paste the following after the prompt: + +@cartouche +@lisp +(use-modules (mu)) +(mu:initialize) +(for-each + (lambda(msg) + (format #t "Subject: ~a\n" (mu:subject msg))) + (mu:message-list "hello")) +@end lisp +@end cartouche + +@noindent +After pressing @key{Enter}, you should get a list of all subjects of messages +that match @t{hello}: + +@verbatim +... +Subject: RE: The Bird Serpent War Cataclysm +Subject: Hello! +Subject: Re: post-run tomorrow +Subject: When all is lost +... +@end verbatim + +@noindent +If all this works, congratulations! @t{mu-guile} is installed now, ready to +serve your every searching need! + +@node Initializing mu-guile +@chapter Initializing mu-guile + +We now have installed @t{mu-guile}, and in @ref{Making sure it works} +confirmed that things work by trying some simple script. In this and the +following chapters, we take a closer look at programming with @t{mu-guile}. + +It is possible to write separate programs with @t{mu-guile}, but for now we'll +do things @emph{interactively}, that is, from the Guile-prompt +(``@abbr{REPL}''). + +As we have seen, we start our @t{mu-guile} session by starting @t{guile}: + +@verbatim +$ guile +@end verbatim + +@cartouche +@verbatim +GNU Guile 2.0.5.123-4bd53 +Copyright (C) 1995-2012 Free Software Foundation, Inc. + +Guile comes with ABSOLUTELY NO WARRANTY; for details type `,show w'. +This program is free software, and you are welcome to redistribute it +under certain conditions; type `,show c' for details. + +Enter `,help' for help. +scheme@(guile-user)> +@end verbatim +@end cartouche + +The first thing we need to do is loading the modules. All the basics are in +the @t{(mu)} module, with some statistical extras in @t{(mu stats)}, and some +graph plotting functionality in @t{(mu plot)}@footnote{@code{(mu plot)} +requires the @t{gnuplot} program}. Let's load all of them: +@verbatim +scheme@(guile-user)> (use-modules (mu) (mu stats) (mu plot)) +@end verbatim + +The first time you do this, @t{guile} will probably respond by showing some +messages about compiling the modules, and then return to you with another +prompt. Before we can do anything with @t{mu guile}, we need to initialize the +system: + +@verbatim +scheme@(guile-user)> (mu:initialize) +@end verbatim + +This opens the database for reading, using the default location of +@file{~/.cache/mu}@footnote{If you keep your @t{mu} database in a non-standard +place, use @code{(mu:initialize "/path/to/my/mu/")}} + +Now, @t{mu-guile} is ready to go. In the next chapter, we go through the +modules and show what you can do with them. + +@node Messages +@chapter Messages + +In this chapter, we discuss searching messages and doing things with them. + +@menu +* Finding messages:: query for messages in the database +* Message methods:: what methods are available for messages? +* Example - the longest subject:: find the messages with the longest subject +@end menu + +@node Finding messages +@section Finding messages +Now we are ready to retrieve some messages from the system. There are two main +procedures to do this: + +@itemize +@item @code{(mu:message-list [])} +@item @code{(mu:for-each-message [])} +@end itemize + +@noindent +The first procedure, @code{mu:message-list} returns a list of all messages +matching @t{}; if you leave @t{} out, it +returns @emph{all} messages. For example, to get all messages with @t{coffee} +in the subject line: + +@verbatim +scheme@(guile-user)> (mu:message-list "subject:coffee") +$1 = (#< 9040640> #< 9040630> + #< 9040570>) +@end verbatim + +@noindent +Apparently, we have three messages matching @t{subject:coffee}, so we get a +list of three @code{} objects. Let's just use the +@code{mu:subject} procedure ('method') provided by @code{} objects +to retrieve the subject-field (more about methods in the next section). + +For your convenience, @t{guile} has saved the result of our last query in a +variable called @t{$1}, so to get the subject of the first message in the +list, we can do: + +@verbatim +scheme@(guile-user)> (mu:subject (car $1)) +$2 = "Re: best coffee ever!" +@end verbatim + +@noindent +The second procedure we mentioned, @code{mu:for-each-message}, executes some +procedure for each message matched by the search expression (or @emph{all} +messages if the search expression is omitted): + +@verbatim +scheme@(guile-user)> (mu:for-each-message + (lambda(msg) + (display (mu:subject msg)) + (newline)) + "subject:coffee") +Re: best coffee ever! +best coffee ever! +Coffee beans +scheme@(guile-user)> +@end verbatim + +@noindent +Using @code{mu:message-list} and/or +@code{mu:for-each-message}@footnote{Implementation node: +@code{mu:message-list} is implemented in terms of @code{mu:for-each-message}, +not the other way around. Due to the way @t{mu} works, +@code{mu:for-each-message} is rather more efficient than a combination of +@code{for-each} and @code{mu:message-list}} and a couple of @t{} +methods, together with what Guile/Scheme provides, should allow for many +interesting programs. + +@node Message methods +@section Message methods + +Now that we've seen how to retrieve lists of message objects +(@code{}), let's see what we can do with such an object. + +@code{} defines the following methods that all take a single +@code{} object as a parameter. We won't go into the exact meanings +for all of these procedures here - for the details about various flags / +properties, please refer to the @t{mu-find} man-page. + +@itemize +@item @code{(mu:bcc msg)}: the @t{Bcc} field of the message, or @t{#f} if there is none +@item @code{(mu:body-html msg)}: : the html body of the message, or @t{#f} if there is none +@item @code{(mu:body-txt msg)}: the plain-text body of the message, or @t{#f} if there is none +@item @code{(mu:cc msg)}: the @t{Bcc} field of the message, or @t{#f} if there is none +@item @code{(mu:date msg)}: the @t{Date} field of the message, or 0 if there is none +@item @code{(mu:flags msg)}: list of message-flags for this message +@item @code{(mu:from msg)}: the @t{From} field of the message, or @t{#f} if there is none +@item @code{(mu:maildir msg)}: the maildir this message lives in, or @t{#f} if there is none +@item @code{(mu:message-id msg)}: the @t{Message-Id} field of the message, or @t{#f} if there is none +@item @code{(mu:path msg)}: the file system path for this message +@item @code{(mu:priority msg)}: the priority of this message (either @t{mu:prio:low}, @t{mu:prio:normal} or @t{mu:prio:high} +@item @code{(mu:references msg)}: the list of messages (message-ids) this message +refers to in(mu: the @t{References:} header +@item @code{(mu:size msg)}: size of the message in bytes +@item @code{(mu:subject msg)}: the @t{Subject} field of the message, or @t{#f} if there is none. +@item @code{(mu:tags msg)}: list of tags for this message +@item @code{(mu:timestamp msg)}: the timestamp (mtime) of the message file, or +#f if there is none. +message file +@item @code{(mu:to msg)}: the sender of the message, or @t{#f} if there is none +@end itemize + +With these methods, we can query messages for their properties; for example: + +@verbatim +scheme@(guile-user)> (define msg (car (mu:message-list "snow"))) +scheme@(guile-user)> (mu:subject msg) +$1 = "Re: Running in the snow is beautiful" +scheme@(guile-user)> (mu:flags msg) +$2 = (mu:flag:replied mu:flag:seen) +scheme@(guile-user)> (strftime "%F" (localtime (mu:date msg))) +$3 = "2011-01-15" +@end verbatim + +There are a couple more methods: +@itemize +@item @code{(mu:header msg "")} returns an arbitrary message +header (or @t{#f} if not found) -- e.g. @code{(header msg "User-Agent")} +@item If you include the @t{mu contact} module, the @code{(mu:contacts +msg [contact-type])} method (to get a list of contacts) is +added. @xref{Contacts}. +@item If you include the @t{mu part} module, the @code{((mu:parts msg)} and +@code{(mu:attachments msg)} methods are added. @xref{Attachments and other parts}. +@end itemize + +@node Example - the longest subject +@section Example - the longest subject + +Now, let's write a little example -- let's find out what is the @emph{longest +subject} of any e-mail messages we received in the year 2011. You can try +this if you put the following in a separate file, make it executable, and run +it like any program. + +@lisp +#!/bin/sh +exec guile -s $0 $@ +!# + +(use-modules (mu)) +(use-modules (srfi srfi-1)) + +(mu:initialize) + +;; note: (subject msg) => #f if there is no subject +(define list-of-subjects + (map (lambda (msg) + (or (mu:subject msg) "")) (mu:message-list "date:2011..2011"))) +;; see the mu-find manpage for the date syntax + +(define longest-subject + (fold (lambda (subj1 subj2) + (if (> (string-length subj1) (string-length subj2)) + subj1 subj2)) + "" list-of-subjects)) + +(format #t "Longest subject: ~s\n" longest-subject) +@end lisp + +There are many other ways to solve the same problem, for example by using an +iterative approach with @code{mu:for-each-message}, but it should show how one +can easily write little programs to answer specific questions about your +e-mail corpus. + +@node Contacts +@chapter Contacts + +We can retrieve the sender and recipients of an e-mail message using methods +like @code{mu:from}, @code{mu:to} etc.; @xref{Message methods}. These +procedures return the list of recipients as a single string; however, often it +is more useful to deal with recipients as separate objects. + +@menu +* Contact procedures and objects:: +* All contacts:: +* Utility procedures:: +* Example - mutt export:: +@end menu + + +@node Contact procedures and objects +@section Contact procedures and objects + +Message objects (@pxref{Messages}) have a method @t{mu:contacts}: + + @code{(mu:contacts [])} + +The @t{} is a symbol, one of @code{mu:to}, @code{mu:from}, +@code{mu:cc} or @code{mu:bcc}. This will then get the contact objects for the +contacts of the corresponding type. If you leave out the contact-type (or +specify @t{#t} for it, you will get a list of @emph{all} contact objects for +the message. + +A contact object (@code{}) has two methods: +@itemize +@item @code{mu:name} returns the name of the contact, or #f if there is none +@item @code{mu:email} returns the e-mail address of the contact, or #f if there is none +@end itemize + +Let's get a list of all names and e-mail addresses in the 'To:' field, of +messages matching 'book': + +@lisp +(use-modules (mu)) +(mu:initialize) +(mu:for-each-message + (lambda (msg) + (for-each + (lambda (contact) + (format #t "~a => ~a\n" + (or (mu:email contact) "") (or (mu:name contact) "no-name"))) + (mu:contacts msg mu:contact:to))) + "book") +@end lisp + +This shows what the methods do, but for many uses, it would be more useful to +have each of the contacts only show up @emph{once} - for that, please refer to +@xref{All contacts}. + +@node All contacts +@section All contacts + +Sometimes you may want to inspect @emph{all} the different contacts in the +@t{mu} database. This is useful, for instance, when exporting contacts to some +external format that can then be important in an e-mail program. + +To enable this, there is the procedure @code{mu:for-each-contact}, defined as + + @code{(mu:for-each-contact procedure [search-expression])}. + +This will aggregate the unique contacts from @emph{all} messages matching +@t{} (when it is left empty, it will match all messages in +the database), and execute @t{procedure} for each of them. + +The @t{procedure} receives an object of the type @t{}, +which is a @emph{subclass} of the @t{} class discussed in +@xref{Contact procedures and objects}. @t{} objects +expose the following additional methods: + +@itemize +@item @code{(mu:frequency )}: returns the @emph{number of times} this contact occurred in +one of the address fields +@item @code{(mu:last-seen )}: returns the @emph{most recent time} the contact was +seen in one of the address fields, as a @t{time_t} value +@end itemize + +The method assumes an e-mail address is unique for a certain contact; if a +certain e-mail address occurs with different names, it uses the most recent +non-empty name. + +@node Utility procedures +@section Utility procedures + +To make dealing with contacts even easier, there are a number of utility +procedures that can save you a bit of typing. + +For converting contacts to some textual form, there is @code{(mu:contact->string + format)}, which takes a contact and returns a text string with +the given format. Currently supported formats are @t{"org-contact}, @t{"mutt-alias"}, +@t{"mutt-ab"}, @t{"wanderlust"} and @t{"plain"}. + + +@node Example - mutt export +@section Example - mutt export + +Let's see how we could export the addresses in the @t{mu} database to the +addressbook format of the venerable +@t{mutt}@footnote{@url{http://www.mutt.org/}} e-mail client. + +The addressbook format that @t{mutt} uses is a sequence of lines that look +something like: +@verbatim +alias [] "<" ">" +@end verbatim + +@t{mu guile} provides the procedure @code{(mu:contact->string +format)} that we can use to do the conversion. + +We may want to focus on people with whom we have frequent correspondence; so +we may want to limit ourselves to people we have seen at least 10 times in the +last year. + +It is a bit hard to @emph{guess} the nick name for e-mail contacts, but +@code{mu:contact->string} tries something based on the name. You can always +adjust them later by hand, obviously. + +@lisp +#!/bin/sh +exec guile -s $0 $@ +!# + +(use-modules (mu)) +(mu:initialize) + +;; Get a list of contacts that were seen at least 20 times since 2010 +(define (selected-contacts) + (let ((addrs '()) + (start (car (mktime (car (strptime "%F" "2010-01-01"))))) + (minfreq 20)) + (mu:for-each-contact + (lambda (contact) + (if (and (mu:email contact) + (>= (mu:frequency contact) minfreq) + (>= (mu:last-seen contact) start)) + (set! addrs (cons contact addrs))))) + addrs)) + +(for-each + (lambda (contact) + (format #t "~a\n" (mu:contact->string contact "mutt-alias"))) + (selected-contacts)) +@end lisp + +This simple program could be improved in many ways; this is left as an +exercise to the reader. + +@node Attachments and other parts +@chapter Attachments and other parts + +To deal with @emph{attachments}, or, more in general @emph{MIME-parts}, there +is the @t{mu part} module. + +@menu +* Parts and their methods:: +* Attachment example:: +@end menu + +@node Parts and their methods +@section Parts and their methods +The module defines the @code{} class, and adds two methods to +@code{} objects: +@itemize +@item @code{(mu:parts msg)} - returns a list @code{} objects, one for +each MIME-parts in the message. +@item @code{(mu:attachments msg)} - like @code{parts}, but only list those MIME-parts +that look like proper attachments. +@end itemize + +A @code{} object exposes a few methods to get information about the +part: +@itemize +@item @code{(mu:name )} - returns the file name of the mime-part, or @code{#f} if +there is none. +@item @code{(mu:mime-type )} - returns the mime-type of the mime-part, or @code{#f} +if there is none. +@item @code{(mu:size )} - returns the size in bytes of the mime-part +@end itemize + +@c Then, we may want to save the part to a file; this can be done using either: +@c @itemize +@c @item @code{(mu:save part )} - save a part to a temporary file, return the file +@c name@footnote{the temporary filename is a predictable procedure of (user-id, +@c msg-path, part-index)} +@c @item @code{(mu:save-as )} - save part to file at path +@c @end itemize + +@node Attachment example +@section Attachment example + +Let's look at some small example. Let's get a list of the biggest attachments +in messages about Luxemburg: + +@lisp +#!/bin/sh +exec guile -s $0 $@ +!# + +(use-modules (mu)) +(mu:initialize) + +(define (all-attachments expr) + "Return a list of (name . size) for all attachments in messages +matching EXPR." + (let ((pairs '())) + (mu:for-each-message + (lambda (msg) + (for-each + (lambda (att) ;; add (filename . size) to the list + (set! pairs (cons (cons (mu:name att) (or (mu:size att) 0)) pairs))) + (mu:attachments msg))) + expr) + pairs)) + +(for-each + (lambda (att) + (format #t "~a: ~,1fKb\n" + (car att) (exact->inexact (/ (cdr att) 1024)))) + (sort (all-attachments "Luxemburg") + (lambda (att1 att2) + (< (cdr att1) (cdr att2))))) +@end lisp + +As an exercise for the reader, you might want to re-rewrite the +@code{all-attachments} in terms of @code{mu:message-list}, which would +probably be a bit more elegant. + + +@node Statistics +@chapter Statistics + +@t{mu-guile} offers some convenience procedures to determine various statistics +about the messages in the database. + +@menu +* Basics:: @code{mu:count}, @code{mu:average}, ... +* Tabulating values:: @code{mu:tabulate} +* Most frequent values:: @code{mu:top-n-most-frequent} +@end menu + +@node Basics +@section Basics + +Let's look at some of the basic statistical operations available, in an +interactive session: +@example +GNU Guile 2.0.5.123-4bd53 +Copyright (C) 1995-2012 Free Software Foundation, Inc. + +Guile comes with ABSOLUTELY NO WARRANTY; for details type `,show w'. +This program is free software, and you are welcome to redistribute it +under certain conditions; type `,show c' for details. + +Enter `,help' for help. +scheme@@(guile-user)> ;; load modules, initialize mu +scheme@@(guile-user)> (use-modules (mu) (mu stats)) +scheme@@(guile-user)> (mu:initialize) +scheme@@(guile-user)> +scheme@@(guile-user)> ;; count the number of messages with 'hello' in their subject +scheme@@(guile-user)> (mu:count "subject:hello") +$1 = 162 +scheme@@(guile-user)> ;; average the size of messages with hello in their subject +scheme@@(guile-user)> (mu:average mu:size "subject:hello") +$2 = 34597733/81 +scheme@@(guile-user)> (exact->inexact $2) +$3 = 427132.506172839 +scheme@@(guile-user)> ;; calculate the correlation between message size and +scheme@@(guile-user)> ;; subject length +scheme@@(guile-user)> (mu:correl mu:size (lambda (msg) + (string-length (mu:subject msg))) "subject:hello") +$5 = -0.10804368622292 +scheme@@(guile-user)> +@end example + +@node Tabulating values +@section Tabulating values + +@code{(mu:tabulate [])} applies @t{} to each +message matching @t{} (leave empty to match @emph{all} messages), +and returns a associative list (a list of pairs) with each of the different +results of @t{} and their frequencies. For fields that contain lists +of values (such as address-fields), each of the values in the list is added +separately. + +@subsection Example: messages per weekday + +We demonstrate @code{mu:tabulate} with an example. Suppose we want to know how +many messages we receive per weekday: + +@lisp +#!/bin/sh +exec guile -s $0 $@ +!# + +(use-modules (mu) (mu stats) (mu plot)) +(mu:initialize) + +;; create a list like (("Sun" . 13) ("Mon" . 23) ...) +(define weekday-table + (mu:weekday-numbers->names + (sort + (mu:tabulate + (lambda (msg) + (tm:wday (localtime (mu:date msg))))) + (lambda (a b) (< (car a) (car b)))))) + +(for-each + (lambda (elm) + (format #t "~a: ~a\n" (car elm) (cdr elm))) + weekday-table) +@end lisp + + +The procedure @code{weekday-table} uses @code{mu:tabulate-message} to get the +frequencies per hour -- this returns a list of pairs: +@verbatim +((5 . 2339) (0 . 2278) (4 . 2800) (2 . 3184) (6 . 1856) (3 . 2833) (1 . 2993)) +@end verbatim + +We sort these pairs by the day number, and then apply +@code{mu:weekday-numbers->names}, which takes the list, and returns a list +where the day numbers are replace by there abbreviated name (in the current +locale). Note, there is also @code{mu:month-numbers->names}. + +The script then outputs these numbers in the following form: + +@verbatim +Sun: 2278 +Mon: 2993 +Tue: 3184 +Wed: 2833 +Thu: 2800 +Fri: 2339 +Sat: 1856 +@end verbatim + +Clearly, Saturday is a slow day for e-mail... + +@node Most frequent values +@section Most frequent values + +In the above example, the number of values is small (the seven weekdays); +however, in many cases there can be many different values (for example, all +different message subjects), many of which may not be very interesting -- all +we need to know is the top-10 of most frequently seen values. + +This is fairly easy to achieve using @code{mu:tabulate} -- to get the top-10 +subjects@footnote{this requires the @code{(srfi srfi-1)}-module}, we can use +something like this: +@lisp +(take + (sort + (mu:tabulate mu:subject) + (lambda (a b) (> (cdr a) (cdr b)))) + 10) +@end lisp + +If this is not short enough, @t{mu-guile} offers a convenience procedure to do +this: @code{mu:top-n-most-frequent}. For example, to get the top-10 people we +sent mail to most often: + +@lisp +(mu:top-n-most-frequent mu:to 10 "maildir:/sent") +@end lisp + +Can't make it much easier than that! + + +@node Plotting data +@chapter Plotting data + +You can plot the results in the format produced by @code{mu:tabulate} with the +@t{(mu plot)} module, an experimental module that requires the +@t{gnuplot}@footnote{@url{http://www.gnuplot.info/}} program to be installed +on your system. + +The @code{mu:plot-histogram} procedure takes the following arguments: + +@code{(mu:plot-histogram <x-label> <y-label> [<want-ascii>])} + +Here, @code{<data>} is a table of data in the format that @code{mu:tabulate} +produces. @code{<title>}, @code{<x-label>} and @code{<y-lablel>} are, +respectively, the title of the graph, and the labels for X- and +Y-axis. Finally, if you pass @t{#t} for the final @code{<want-ascii>} +parameter, a plain-text rendering of the graph will be produced; otherwise, a +graphical window will be shown. + +An example should clarify how this works in practice; let's plot the number of +message per hour: + +@lisp +#!/bin/sh +exec guile -s $0 $@ +!# + +(use-modules (mu) (mu stats) (mu plot)) +(mu:initialize) + +(define (mail-per-hour-table) + (sort + (mu:tabulate + (lambda (msg) + (tm:hour (localtime (mu:date msg))))) + (lambda (x y) (< (car x) (car y))))) + +(mu:plot-histogram (mail-per-hour-table) "Mail per hour" "Hour" "Frequency") +@end lisp + +@cartouche +@verbatim + Mail per hour +Frequency + 1200 ++--+--+--+--+-+--+--+--+--+-+--+--+--+-+--+--+--+--+-+--+--+--+--++ + |+ + + + + + + "/tmp/fileHz7D2u" using 2:xticlabels(1) ******** + 1100 ++ *** +* + **** * * * + 1000 *+ * **** * +* + * * ****** **** * ** * * + 900 *+ * * ** **** * **** ** * +* + * * * ** * * ********* * ** ** * * + 800 *+ * **** ** * * * * ** * * ** ** * +* + 700 *+ *** **** * ** * * * * ** **** * ** ** * +* + * * * **** * * ** * * * * ** * **** ** ** * * + 600 *+ * **** * * * * ** * * * * ** * * * ** ** * +* + * * ** * * * * * ** * * * * ** * * * ** ** * * + 500 *+ * ** * * * * * ** * * * * ** * * * ** ** * +* + * * ** **** *** * * * ** * * * * ** * * * ** ** * * + 400 *+ * ** ** **** * * * * * ** * * * * ** * * * ** ** * +* + *+ *+**+**+* +*******+* +* +*+ *+**+* +*+ *+ *+**+* +*+ *+**+**+* +* + 300 ******************************************************************** + 0 1 2 3 4 5 6 7 8 910 11 12 1314 15 16 17 1819 20 21 22 23 + Hour +@end verbatim +@end cartouche + +@node Writing scripts +@chapter Writing scripts + +The @t{mu} program has built-in support for running guile-scripts, and comes +with a number of examples. + +You can get a list of all scripts with the @t{mu script} command: +@verbatim +$ mu script +Available scripts (use --verbose for details): + * find-dups: find duplicate messages + * msgs-count: count the number of messages matching some query + * msgs-per-day: graph the number of messages per day + * msgs-per-hour: graph the number of messages per hour + * msgs-per-month: graph the number of messages per month + * msgs-per-year: graph the number of messages per year + * msgs-per-year-month: graph the number of messages per year-month +@end verbatim + +You can then execute such a script by its name: +@verbatim +$ mu msgs-per-month --textonly --query=hello + + + Messages per month matching hello + + 240 ++-+-----+----+-----+-----+-----+----+-----+-----+-----+----+-----+-++ + | + + + + "/tmp/filewi9H0N" using 2:xticlabels(1) ****** | + 220 ++ * * ****** + | * * * * + 200 ++ * * * +* + | * * * * + 180 ++ ****** * * * +* + | * * * * * * + 160 ****** * * * * * +* + * * * * * * * * + * ******* * * * * ****** * * + 140 *+ ** * * * * * * ******** +* + * ** ******* * * * * * ** ** * + 120 *+ ** ** ******* * * * * ** ** +* + * ** ** ** * * * ******* ** ** * + 100 *+ ** ** ** * * * * ** ** ** +* + * + ** + ** + ** + * + * + + * + * + ** + ** + ** + * + 80 ********************************************************************** + Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec + Month +@end verbatim + +Please refer to the @t{mu-script} man-page for some details on writing your +own scripts. + + +@node Recipes +@appendix Recipes + +@itemize +@item Calculating the average length of subject-lines +@lisp +;; the average length of all our +(let ((len 0) (n 0)) + (mu:for-each-message + (lambda (msg) + (set! len (+ len (string-length (or (mu:subject msg) "")))) + (set! n (+ n 1)))) + (if (= n 0) 0 (/ len n))) + ;; this gives a rational, exact result; + ;; use exact->inexact to get decimals + +;; we we can make this short with the mu:average (with (mu stats)) +(mu:average (lambda (msg) (string-length (or (mu:subject msg) "")))) + + +@end lisp +@end itemize + +@node GNU Free Documentation License +@appendix GNU Free Documentation License + +@include fdl.texi +@bye diff --git a/guile/mu-guile.x b/guile/mu-guile.x new file mode 100644 index 0000000..8aa8020 --- /dev/null +++ b/guile/mu-guile.x @@ -0,0 +1,4 @@ +/* cpp arguments: mu-guile.cc -DHAVE_CONFIG_H -I. -I.. -I../lib -I/usr/local/include/guile/3.0 -pthread -I/usr/include/glib-2.0 -I/usr/lib64/glib-2.0/include -I/usr/include/sysprof-4 -I/usr/include/libmount -I/usr/include/blkid -pthread -fno-strict-aliasing -Wall -Wextra -Wundef -Wwrite-strings -Wpointer-arith -Wmissing-declarations -Wredundant-decls -Wno-unused-parameter -Wno-missing-field-initializers -Wformat=2 -Wcast-align -Wformat-nonliteral -Wformat-security -Wsign-compare -Wstrict-aliasing -Wshadow -Winline -Wpacked -Wmissing-format-attribute -Wmissing-noreturn -Winit-self -Wmissing-include-dirs -Wunused-but-set-variable -Warray-bounds -Wreturn-type -Wno-overloaded-virtual -Wswitch-enum -Wswitch-default -Wno-error=unused-parameter -Wno-error=missing-field-initializers -Wno-error=overloaded-virtual -Wno-redundant-decls -Wno-missing-declarations -Wno-suggest-attribute=noreturn -O2 -Wno-inline */ +scm_c_define_gsubr (s_mu_initialize, 0, 1, 0, (scm_t_subr) mu_initialize); scm_c_export (s_mu_initialize, __null );; +scm_c_define_gsubr (s_mu_initialized_p, 0, 0, 0, (scm_t_subr) mu_initialized_p); scm_c_export (s_mu_initialized_p, __null );; +scm_c_define_gsubr (s_log_func, 1, 0, 1, (scm_t_subr) log_func);; diff --git a/guile/mu.scm b/guile/mu.scm new file mode 100644 index 0000000..08eae1f --- /dev/null +++ b/guile/mu.scm @@ -0,0 +1,318 @@ +;; Copyright (C) 2011-2013 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +(define-module (mu) + :use-module (oop goops) + :use-module (ice-9 optargs) + :use-module (texinfo string-utils) + :export + ( ;; classes + <mu:message> + <mu:contact> + <mu:part> + ;; general +;; mu:initialize + ;; mu:initialized? + mu:log-warning + mu:log-message + mu:log-critical + ;; search funcs + mu:for-each-message + mu:for-each-msg + mu:message-list + ;; message funcs + mu:header + ;; message accessors + mu:field:bcc + mu:field:body-html + mu:field:body-txt + mu:field:cc + mu:field:date + mu:field:flags + mu:field:from + mu:field:maildir + mu:field:message-id + mu:field:path + mu:field:prio + mu:field:refs + mu:field:size + mu:field:subject + mu:field:tags + mu:field:timestamp + mu:field:to + ;; contact funcs + mu:name + mu:email + mu:contact->string + ;; + mu:for-each-contact + ;; + mu:contacts + ;; + ;; <mu:contact-with-stats> + mu:frequency + mu:last-seen + ;; parts + + <mu:part> + ;; message function + mu:attachments + mu:parts + ;; <mu:part> methods + mu:name + mu:mime-type + ;; size + ;; mu:save + ;; mu:save-as + )) + +;; this is needed for guile < 2.0.4 +(setlocale LC_ALL "") + +;; load the binary +(load-extension "libguile-mu" "mu_guile_init") +(load-extension "libguile-mu" "mu_guile_message_init") + +;; define some dummies so we don't get errors during byte compilation +(eval-when (compile) + (define mu:c:get-field) + (define mu:c:get-contacts) + (define mu:c:for-each-message) + (define mu:c:get-header) + (define mu:critical) + (define mu:c:log) + (define mu:message) + (define mu:c:log) + (define mu:warning) + (define mu:c:log) + (define mu:c:get-parts)) + +(define (mu:log-warning frm . args) + "Log FRM with ARGS at warning." + (mu:c:log mu:warning frm args)) + +(define (mu:log-message frm . args) + "Log FRM with ARGS at warning." + (mu:c:log mu:message frm args)) + +(define (mu:log-critical frm . args) + "Log FRM with ARGS at warning." + (mu:c:log mu:critical frm args)) + +(define-class <mu:message> () + (msg #:init-keyword #:msg)) ;; the MuMsg-smob we're wrapping + +(define-syntax define-getter + (syntax-rules () + ((define-getter method-name field) + (begin + (define-method (method-name (msg <mu:message>)) + (mu:c:get-field (slot-ref msg 'msg) field)) + (export method-name))))) + +(define-getter mu:bcc mu:field:bcc) +(define-getter mu:body-html mu:field:body-html) +(define-getter mu:body-txt mu:field:body-txt) +(define-getter mu:cc mu:field:cc) +(define-getter mu:date mu:field:date) +(define-getter mu:flags mu:field:flags) +(define-getter mu:from mu:field:from) +(define-getter mu:maildir mu:field:maildir) +(define-getter mu:message-id mu:field:message-id) +(define-getter mu:path mu:field:path) +(define-getter mu:priority mu:field:prio) +(define-getter mu:references mu:field:refs) +(define-getter mu:size mu:field:size) +(define-getter mu:subject mu:field:subject) +(define-getter mu:tags mu:field:tags) +(define-getter mu:timestamp mu:field:timestamp) +(define-getter mu:to mu:field:to) + +(define-method (mu:header (msg <mu:message>) (hdr <string>)) + "Get an arbitrary header HDR from message MSG; return #f if it does +not exist." + (mu:c:get-header (slot-ref msg 'msg) hdr)) + +(define* (mu:for-each-message func #:optional (expr #t) (maxresults -1)) + "Execute function FUNC for each message that matches mu search expression EXPR. +If EXPR is not provided, match /all/ messages in the store. MAXRESULTS +specifies the maximum of messages to return, or -1 (the default) for +no limit." + (mu:c:for-each-message + (lambda (msg) + (func (make <mu:message> #:msg msg))) + expr + maxresults)) + +;; backward-compatibility alias +(define mu:for-each-msg mu:for-each-message) + +(define* (mu:message-list #:optional (expr #t) (maxresults -1)) + "Return a list of all messages matching mu search expression +EXPR. If EXPR is not provided, return a list of /all/ messages in the +store. MAXRESULTS specifies the maximum of messages to return, or +-1 (the default) for no limit." + (let ((lst '())) + (mu:for-each-message + (lambda (m) + (set! lst (append! lst (list m)))) expr maxresults) + lst)) + +;; contacts +(define-class <mu:contact> () + (name #:init-value #f #:accessor mu:name #:init-keyword #:name) + (email #:init-value #f #:accessor mu:email #:init-keyword #:email)) + +(define-method (mu:contacts (msg <mu:message>) contact-type) + "Get all contacts for MSG of the given CONTACT-TYPE. MSG is of type <mu-message>, +while contact type is either `mu:contact:to', `mu:contact:cc', +`mu:contact:from' or `mu:contact:bcc' to get the corresponding type of +contacts, or #t to get all. + +Returns a list of <mu-contact> objects." + (map (lambda (pair) ;; a pair (na . addr) + (make <mu:contact> #:name (car pair) #:email (cdr pair))) + (mu:c:get-contacts (slot-ref msg 'msg) contact-type))) + +(define-method (mu:contacts (msg <mu:message>)) + "Get contacts of all types for message MSG as a list of <mu-contact> +objects." + (mu:contacts msg #t)) + +(define-class <mu:contact-with-stats> (<mu:contact>) + (tstamp #:init-value 0 #:accessor mu:timestamp #:init-keyword #:timestamp) + (last-seen #:init-value 0 #:accessor mu:last-seen) + (freq #:init-value 1 #:accessor mu:frequency)) + +(define* (mu:for-each-contact proc #:optional (expr #t)) + "Execute PROC for each contact. PROC receives a <mu-contact> instance +as parameter. If EXPR is specified, only consider contacts in messages +matching EXPR." + (let ((c-hash (make-hash-table 4096))) + (mu:for-each-message + (lambda (msg) + (for-each + (lambda (ct) + (let ((ct-ws (make <mu:contact-with-stats> + #:name (mu:name ct) + #:email (mu:email ct) + #:timestamp (mu:date msg)))) + (update-contacts-hash c-hash ct-ws))) + (mu:contacts msg #t))) + expr) + (hash-for-each ;; c-hash now contains a map of email->contact + (lambda (email ct-ws) (proc ct-ws)) c-hash))) + +(define-method (update-contacts-hash c-hash (nc <mu:contact-with-stats>)) + "Update the contacts hash with a new and/or existing contact." + ;; xc: existing-contact, nc: new contact + (let ((xc (hash-ref c-hash (mu:email nc)))) + (if (not xc) ;; no existing contact with this email address? + (hash-set! c-hash (mu:email nc) nc) ;; store the new contact. + ;; otherwise: + (begin + ;; 1) update the frequency for the existing contact + (set! (mu:frequency xc) (1+ (mu:frequency xc))) + ;; 2) update the name if the new one is not empty and its timestamp is newer + ;; in that case, also update the timestamp + (if (and (mu:name nc) (> (string-length (mu:name nc))) + (> (mu:timestamp nc) (mu:timestamp xc))) + (set! (mu:name xc) (mu:name nc)) + (set! (mu:timestamp xc) (mu:timestamp nc))) + ;; 3) update last-seen with timestamp, if x's timestamp is newer + (if (> (mu:timestamp nc) (mu:last-seen xc)) + (set! (mu:last-seen xc) (mu:timestamp nc))) + ;; okay --> now xc has been updated; but it back in the hash + (hash-set! c-hash (mu:email xc) xc))))) + +(define-method (mu:contact->string (contact <mu:contact>) (form <string>)) + "Convert a contact to a string in format FORM, which is a string, +either \"org-contact\", \"mutt-alias\", \"mutt-ab\", +\"wanderlust\", \"quoted\" \"plain\"." + (let* ((name (mu:name contact)) (email (mu:email contact)) + (nick ;; simplistic nick guessing... + (string-map + (lambda(kar) + (if (char-alphabetic? kar) kar #\_)) + (string-downcase (or name email))))) + (cond + ((string= form "plain") + (format #f "~a~a~a" (or name "") (if name " " "") email)) + ((string= form "org-contact") + (format #f "* ~s\n:PROPERTIES:\n:EMAIL:~a\n:NICK:~a\n:END:" + (or name email) email nick)) + ((string= form "wanderlust") + (format #f "~a ~s ~s" + nick (or name email) email)) + ((string= form "mutt-alias") + (format #f "alias ~a ~a <~a>" + nick (or name email) email)) + ((string= form "mutt-ab") + (format #f "~a\t~a\t" + email (or name ""))) + ((string= form "quoted") + (string-append + "\"" + (escape-special-chars + (string-append + (if name + (format #f "\"~a\" " name) + "") + (format #f "<~a>" email)) + "\"" #\\) + "\"")) + (else (error "Unsupported format"))))) + +;; message parts + + +(define-class <mu:part> () + (msgpath #:init-value #f #:init-keyword #:msgpath) + (index #:init-value #f #:init-keyword #:index) + (name #:init-value #f #:getter mu:name #:init-keyword #:name) + (mime-type #:init-value #f #:getter mu:mime-type #:init-keyword #:mime-type) + (size #:init-value 0 #:getter mu:size #:init-keyword #:size)) + +(define-method (get-parts (msg <mu:message>) (files-only <boolean>)) + "Get the part for MSG as a list of <mu:part> objects; if FILES-ONLY is #t, +only get the part with file names." + (map (lambda (part) + (make <mu:part> + #:msgpath (list-ref part 0) + #:index (list-ref part 1) + #:name (list-ref part 2) + #:mime-type (list-ref part 3) + #:size (list-ref part 4))) + (mu:c:get-parts (slot-ref msg 'msg) files-only))) + +(define-method (mu:attachments (msg <mu:message>)) + "Get the attachments for MSG as a list of <mu:part> objects." + (get-parts msg #t)) + +(define-method (mu:parts (msg <mu:message>)) + "Get the MIME-parts for MSG as a list of <mu-part> objects." + (get-parts msg #f)) + +;; (define-method (mu:save (part <mu:part>)) +;; "Save PART to a temporary file, and return the file name. If the +;; part had a filename, the temporary file's file name will be just that; +;; otherwise a name is made up." +;; (mu:save-part (slot-ref part 'msgpath) (slot-ref part 'index))) + +;; (define-method (mu:save-as (part <mu:part>) (filepath <string>)) +;; "Save message-part PART to file system path PATH." +;; (copy-file (save part) filepath)) diff --git a/guile/mu/README b/guile/mu/README new file mode 100644 index 0000000..634ad8b --- /dev/null +++ b/guile/mu/README @@ -0,0 +1,207 @@ +* OUTDATED * + +* README + +** What is muile? + + `muile' is a little experiment/toy using the equally experimental mu guile + bindings, to be found in libmuguile/ in the top-level source directory. + + `guile'[1] is an interpreter/library for the Scheme programming language[2], + specifically meant for extending other programs. It is, in fact, the + official GNU language for doing so. 'muile' requires guile 2.x to get the full + support. + + Older versions may not support e.g. the 'mu-stats.scm' things discussed below. + + The combination of mu + guile is called `muile', and allows you to write + little Scheme-programs to query the mu-database, or inspect individual + messages. It is still in an experimental stage, but useful already. + +** How do I get it? + + The git-version and the future 0.9.7 version of mu will automatically build + muile if you have guile. I've been using guile 2.x from git, but installing + the 'guile-1.8-dev' package (Ubuntu/Debian) should do the trick. (I only did + very minimal testing with guile 1.8 though). + + Then, configure mu. The configure output should tell you about whether guile + was found (and where). If it's found, build mu, and toys/muile should be + created, as well. + +** What can I do with it? + + Go to toys/muile and start muile. You'll end up with a guile-shell where you + can type scheme [1], it looks something like this (for guile 2.x): + + ,---- + | scheme@(guile-user)> + `---- + + Now, let's load a message (of course, replace with a message on your system): + + ,---- + | scheme@(guile-user)> (define msg (mu:msg:make-from-file "/home/djcb/Maildir/cur/12131e7b20a2:2,S")) + `---- + + This defines a variable 'msg', which holds some message on your file + system. It's now easy to inspect this message: + + ,---- + | scheme@(guile-user)> (define msg (mu:msg:make-from-file "/home/djcb/Maildir/cur/12131e7b20a2:2,S")) + `---- + + Now, we can inspect this message a bit: + ,---- + | scheme@(guile-user)> (mu:msg:subject msg) + | $1 = "See me in bikini :-)" + | scheme@(guile-user)> (mu:msg:flags msg) + | $2 = (mu:attach mu:unread) + `---- + + and so on. Note, it's probably easiest to explore the various mu: methods + using autocompletion; to enable that make sure you have + + + ,---- + | (use-modules (ice-9 readline)) + | (activate-readline) + `---- + + in your ~/.guile configuration. + +** does this tool have some parameters? + + Yes, there is --muhome to set a non-default place for the message database + (see the documentation on --muhome in the mu-find manpage). + + And there is --msg=<path> where you specify some particular message file; + it will be available as 'mu:current-msg' in the guile (muile) environment. For + example: + + ,---- + | ./muile --msg=~/Maildir/inbox/cur/1311310172_1234:2,S + | [...] + | scheme@(guile-user)> mu:current-msg + | $1 = #<msg /home/djcb/Maildir/inbox/cur/1311310172_1234:2,S> + | scheme@(guile-user)> (mu:msg:size mu:current-msg) + | $2 = 7206 + `---- + +** What about searching messages in the database? + + That's easy, too - it does require a little more scheme knowledge. For + searching messages there is the mu:store:for-each function, which takes two + arguments; the first argument is a function that will be called for each + message found. The optional second argument is the search expression (following + 'mu find' syntax); if don't provide the argument, all messages match. + + So how does this work in practice? Let's see I want to see the subject and + sender for messages about milk: + + ,---- + | (mu:store:for-each (lambda(msg) (format #t "~s ~s\n" (mu:msg:from msg) (mu:msg:subject msg))) "milk") + `---- + + or slightly more readable: + + ,---- + | (mu:store:for-each + | (lambda(msg) + | (format #t "~s ~s\n" (mu:msg:from msg) (mu:msg:subject msg))) + | "milk") + `---- + + As you can see, I provide an anonymous ('lambda') function which will be + called for each message matching 'milk'. Admittedly, this requires a bit of + Scheme-knowledge... but this time is good as any to learn this nice + language. + + +** Can I do some statistics on my messages? + + Yes you can. In fact, it's pretty easy. If you load (in the muile/ directory) + the file 'mu-stats.scm': + + ,---- + | (load "mu-stats.scm") + `---- + + you'll get a bunch of functions (with names starting with 'mu:stats') to make + this very easy. Let's see, suppose I want to see how many messages I get per + weekday: + + ,---- + | scheme@(guile-user)> (mu:stats:per-weekday) + | $1 = ((0 . 2255) (1 . 2788) (2 . 2868) (3 . 2599) (4 . 2629) (5 . 2287) (6 . 1851)) + `---- + + Note, Sunday=0, Monday=1 and so on. Apparently, I get/send most of e-mail on + Tuesdays, and least on Saturday. + + And note that mu:stats:per-weekdays takes an optional search expression + argument, to limit the results to messages matching that, e.g., to only + consider messages related to emacs during this year: + + ,---- + | scheme@(guile-user)> (mu:stats:per-weekday "emacs date:2011..now") + | $8 = ((0 . 54) (1 . 22) (2 . 46) (3 . 47) (4 . 39) (5 . 54) (6 . 50)) + `---- + + There's also 'mu:stats:per-month', 'mu:stats:per-year', 'mu:stats:per-hour'. + I learnt that during 3-4am I sent/receive only about a third of what I sent + during 11-12pm. + +** What about getting the top-10 people in the To:-field? + + Easy. + + ,---- + | scheme@(guile-user)> (mu:stats:top-n-to) + | $1 = ((("Abc" "myself@example.com") . 4465) (("Def" "somebodyelse@example.com") . 2114) + | (and so on) + `---- + + I've changed the names a bit to protect the innocent, but what the function + does is return a list of pairs of + + (<name> <email>) . <frequency> + + descending in order of frequency. Note, 'mu:stats:top-n-to' takes two + optional arguments - first the 'n' in top-n (default is 10), and seconds as + search expression to limit the messages considered. + + There are also the functions 'mu:stats:top-n-subject' and + 'mu:stats:top-n-from' which do the same, mutatis mutandis, and it's quite + easy to add your own (see the mu-stats.scm for examples) + +** What about showing the results in a table? + + Even easier. Try: + + ,---- + | (mu:stats:table (mu:stats:top-n-to)) + `---- + + or + + ,---- + | (mu:stats:table (mu:stats:per-weekday)) + `---- + + You can also export the table: + + ,---- + | (mu:stats:export (mu:stats:per-weekday)) + `---- + + which will create a temporary file with the results, for further processing + in e.g. 'R' or 'gnuplot'. + + +[1] http://www.gnu.org/s/guile/ +[2] http://en.wikipedia.org/wiki/Scheme_(programming_language) + +# Local Variables: +# mode: org; org-startup-folded: nil +# End: diff --git a/guile/mu/contact.scm b/guile/mu/contact.scm new file mode 100644 index 0000000..843d9c4 --- /dev/null +++ b/guile/mu/contact.scm @@ -0,0 +1,4 @@ +(define-module (mu contact) :use-module(mu)) +(display "(mu contact) is deprecated, please remove from (use-modules ...)") +(newline) + diff --git a/guile/mu/message.scm b/guile/mu/message.scm new file mode 100644 index 0000000..bc9b27a --- /dev/null +++ b/guile/mu/message.scm @@ -0,0 +1,4 @@ +(define-module (mu message) :use-module (mu)) +(display "(mu message) is deprecated, please remove from (use-modules ...)") +(newline) + diff --git a/guile/mu/part.scm b/guile/mu/part.scm new file mode 100644 index 0000000..f9b9cd3 --- /dev/null +++ b/guile/mu/part.scm @@ -0,0 +1,4 @@ +(define-module (mu part) :use-module (mu)) +(display "(mu part) is deprecated, please remove from (use-modules ...)") +(newline) + diff --git a/guile/mu/plot.scm b/guile/mu/plot.scm new file mode 100644 index 0000000..cd09e22 --- /dev/null +++ b/guile/mu/plot.scm @@ -0,0 +1,83 @@ +;; +;; Copyright (C) 2011-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +(define-module (mu plot) + :use-module (mu) + :use-module (ice-9 popen) + :export ( mu:plot ;; alias for mu:plot-histogram + mu:plot-histogram + )) + +(define (export-pairs pairs) + "Write a temporary file with the list of PAIRS in table format, and +return the file name." + (let* ((output (mkstemp "/tmp/mu-guile-XXXXXX" "w")) + (datafile (port-filename output))) + (for-each + (lambda(pair) + (display (format #f "~a ~a\n" (car pair) (cdr pair)) output)) + pairs) + (close output) + datafile)) + +(define (find-program-in-path prog) + "Find exutable program PROG in PATH; return the full path, or #f if +not found." + (let* ((path (parse-path (getenv "PATH"))) + (progpath (search-path path prog))) + (if (not progpath) + #f + (if (access? progpath X_OK) ;; is + progpath + #f)))) + +(define* (mu:plot-histogram data title x-label y-label + #:optional (output "dumb") (extra-gnuplot-opts '())) + "Plot DATA with TITLE, X-LABEL and X-LABEL using the gnuplot +program. DATA is a list of cons-pairs (X . Y). + + OUTPUT is a string +that determines the type of output that gnuplot produces, depending on +the system. Which options are available depends on the particulars for +the gnuplot installation, but typical examples would be \"dumb\" for +text-only display, \"wxterm\" to write to a graphical window, or +\"png\" to write a PNG-image to stdout. + +EXTRA-GNUPLOT-OPTS is a list +of any additional options for gnuplot." + (if (not (find-program-in-path "gnuplot")) + (error "cannot find 'gnuplot' in path")) + (when (zero? (length data)) + (error "No data for plotting")) + (let* ((datafile (export-pairs data)) + (gnuplot (open-pipe "gnuplot -p" OPEN_WRITE)) + (recipe + (string-append + "reset\n" + "set term " (or output "dumb") "\n" + "set title \"" title "\"\n" + "set xlabel \"" x-label "\"\n" + "set ylabel \"" y-label "\"\n" + "set boxwidth 0.9\n" + (string-join extra-gnuplot-opts "\n") + "plot \"" datafile "\" using 2:xticlabels(1) with boxes fs solid title \"\"\n"))) + (display recipe gnuplot) + (close-pipe gnuplot))) + +;; backward compatibility +(define mu:plot mu:plot-histogram) diff --git a/guile/mu/script.scm b/guile/mu/script.scm new file mode 100644 index 0000000..45aad8a --- /dev/null +++ b/guile/mu/script.scm @@ -0,0 +1,58 @@ +;; Copyright (C) 2012-2013 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +(define-module (mu script) + :export (mu:run-stats)) + +(use-modules (ice-9 getopt-long) (ice-9 optargs) (ice-9 popen) (ice-9 format)) +(use-modules (mu) (mu stats) (mu plot)) + +(define (help-and-exit) + "Show some help." + (display + (string-append "usage: script [--help] [--textonly] " + "[--muhome=<muhome>] [--query=<query>") + (newline)) + (exit 0)) + +(define (mu:run-stats args func) + "Run some statistics function. +Interpret argument-list ARGS (like command-line +arguments). Possible arguments are: + --help (show some help and exit) + --muhome (path to alternative mu home directory) + --output (a string describing the output, e.g. \"dumb\", \"png\" \"wxt\") + searchexpr (a search query) +then call FUNC with args SEARCHEXPR and OUTPUT." + (setlocale LC_ALL "") + (let* ((optionspec '((muhome (value #t)) + (query (value #t)) + (output (value #f)) + (time-unit (value #t)) ;; Ignore. + (help (single-char #\h) (value #f)))) + (options (getopt-long args optionspec)) + (query (option-ref options 'query #f)) + (help (option-ref options 'help #f)) + (output (option-ref options 'output #f)) + (muhome (option-ref options 'muhome #f)) + (restargs (option-ref options '() #f))) + (if help (help-and-exit)) + (mu:initialize muhome) + (func (or query "") output))) + +;; Local Variables: +;; mode: scheme +;; End: diff --git a/guile/mu/stats.scm b/guile/mu/stats.scm new file mode 100644 index 0000000..1e73605 --- /dev/null +++ b/guile/mu/stats.scm @@ -0,0 +1,167 @@ +;; +;; Copyright (C) 2011-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +(define-module (mu stats) + :use-module (oop goops) + :use-module (mu) + :use-module (srfi srfi-1) + :use-module (ice-9 i18n) + :use-module (ice-9 r5rs) + :export ( mu:tabulate + mu:top-n-most-frequent + mu:count + mu:average + mu:stddev + mu:correl + mu:max + mu:min + mu:weekday-numbers->names + mu:month-numbers->names)) + + +(define* (mu:tabulate func #:optional (expr #t)) + "Execute FUNC for each message matching EXPR, and return an alist +with maps each result of FUNC to its frequency. If the result of FUNC +is a list, add each of its values separately. + FUNC is a function takes a <mu-message> instance as its argument. For +example, to tabulate messages by weekday, one could use: + (mu:tabulate (lambda(msg) (tm:wday (localtime (date msg))))), and +get back a list like + ((1 . 2) (2 . 5)(3 . 4)(4 . 4)(5 . 12)(6 . 7)(7. 2))." + (let* ((table '()) + ;; func to add a value to our table + (update-table + (lambda (val) + (let ((old-freq (or (assoc-ref table val) 0))) + (set! table (assoc-set! table val (1+ old-freq))))))) + (mu:for-each-message + (lambda(msg) + (let ((val (func msg))) + (if (list? val) + (for-each update-table val) + (update-table val)))) + expr) + table)) + +(define* (top-n func less n #:optional (expr #t)) + "Take the results of (mu:tabulate FUNC EXPR), sort using LESS (a +function taking two arguments A and B (cons cells, (VAL . FREQ)), and +returns #t if A < B, #f otherwise), and then take the first N." + (take (sort (mu:tabulate func expr) less) n)) + +(define* (mu:top-n-most-frequent func n #:optional (expr #t)) + "Take the results of (mu:tabulate FUNC EXPR), and return the N items +with the highest frequency." + (top-n func (lambda (a b) (> (cdr a) (cdr b))) n expr)) + +(define* (mu:count #:optional (expr #t)) + "Count the number of messages matching EXPR. If EXPR is not +provided, match /all/ messages." + (let ((num 0)) + (mu:for-each-message + (lambda (msg) (set! num (1+ num))) + expr) + num)) + +(define (average lst) + "Calculate the average of a list LST of numbers, or #f if undefined." + (if (null? lst) + #f + (/ (apply + lst) (length lst)))) + +(define (stddev lst) + "Calculate the standard deviation of a list LST of numbers or #f if +undefined." + (let* ((avg (average lst)) + (sosq (if avg + (apply + (map (lambda (x)(* (- x avg) (- x avg))) lst))))) + (if sosq + (sqrt (/ sosq (length lst)))))) + + +(define* (mu:average func #:optional (expr #t)) + "Get the average value of FUNC applied to all messages matching +EXPR (or #t for all). Returns #f if undefined." + (average (map func (mu:message-list expr)))) + +(define* (mu:stddev func #:optional (expr #t)) + "Get the standard deviation the the values of FUNC applied to all +messages matching EXPR (or #t for all). This is the 'population' stddev, +not the 'sample' stddev. Returns #f if undefined." + (stddev (map func (mu:message-list expr)))) + +(define* (mu:max func #:optional (expr #t)) + "Get the maximum value of FUNC applied to all messages matching +EXPR (or #t for all). Returns #f if undefined." + (apply max (map func (mu:message-list expr)))) + +(define* (mu:min func #:optional (expr #t)) + "Get the minimum value of FUNC applied to all messages matching +EXPR (or #t for all). Returns #f if undefined." + (apply min (map func (mu:message-list expr)))) + + +(define (correl lst) + "Calculate Pearson's correlation coefficient for a list LST of cons +pair, where the car and cdr of the pairs are values from data set 1 +and 2, respectively." + (let ((n (length lst)) + (sx (apply + (map car lst))) + (sy (apply + (map cdr lst))) + (sxy (apply + (map (lambda (cell) (* (car cell) (cdr cell))) lst))) + (sxx (apply + (map (lambda (cell) (* (car cell) (car cell))) lst))) + (syy (apply + (map (lambda (cell) (* (cdr cell) (cdr cell))) lst)))) + (/ (- (* n sxy) (* sx sy)) + (sqrt (* (- (* n sxx) (* sx sx)) (- (* n syy) (* sy sy))))))) + +(define* (mu:correl func1 func2 #:optional (expr #t)) + "Determine Pearson's correlation coefficient between the value for +functions FUNC1 and FUNC2 to all messages matching EXPR (or #t for +all). Returns #f if undefined." + (let ((data + (map (lambda (msg) + (cons (func1 msg) (func2 msg))) + (mu:message-list expr)))) + (if data (correl data) #f))) + + +;; a list of abbreviated, localized day names +(define day-names + (map locale-day-short (iota 7 1))) + +(define (mu:weekday-numbers->names table) + "Convert a list of pairs with the car denoting a day number (0-6) +into a list of pairs with the car replaced by the corresponding day +name (abbreviated) for the current locale." + (map + (lambda (pair) + (cons (list-ref day-names (car pair)) (cdr pair))) + table)) + +;; a list of abbreviated, localized month names +(define month-names + (map locale-month-short (iota 12 1))) + +(define (mu:month-numbers->names table) + "Convert a list of pairs with the car denoting a month number (0-11) +into a list of pairs with the car replaced by the corresponding day +name (abbreviated)." + (map + (lambda (pair) + (cons (list-ref month-names (car pair)) (cdr pair))) + table)) diff --git a/guile/scripts/find-dups.scm b/guile/scripts/find-dups.scm new file mode 100755 index 0000000..c4b6263 --- /dev/null +++ b/guile/scripts/find-dups.scm @@ -0,0 +1,119 @@ +#!/bin/sh +exec guile -e main -s $0 $@ +!# +;; +;; Copyright (C) 2013-2015 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +;; INFO: Find duplicate messages +;; INFO: options: +;; INFO: --muhome=<muhome>: path to mu home dir +;; INFO: --delete: delete all but the first one (experimental, be careful!) + +(use-modules (mu) (mu script) (mu stats)) +(use-modules (ice-9 getopt-long) (ice-9 optargs) + (ice-9 popen) (ice-9 format) (ice-9 rdelim)) + +(define (md5sum path) + (let* ((port (open-pipe* OPEN_READ "md5sum" path)) + (md5 (read-delimited " " port))) + (close-pipe port) + md5)) + +(define (find-dups delete expr) + (let ((id-table (make-hash-table 20000))) + ;; fill the hash with <msgid-size> => <list of paths> + (mu:for-each-message + (lambda (msg) + (let* ((id (format #f "~a-~d" (mu:message-id msg) + (mu:size msg))) + (lst (hash-ref id-table id))) + (if lst + (set! lst (cons (mu:path msg) lst)) + (set! lst (list (mu:path msg)))) + (hash-set! id-table id lst))) + expr) + ;; list all the paths with multiple elements; check the md5sum to + ;; make 100%-minus-ε sure they are really the same file. + (hash-for-each + (lambda (id paths) + (if (> (length paths) 1) + (let ((hash (make-hash-table 10))) + (for-each + (lambda (path) + (when (file-exists? path) + (let* ((md5 (md5sum path)) (lst (hash-ref hash md5))) + (if lst + (set! lst (cons path lst)) + (set! lst (list path))) + (hash-set! hash md5 lst)))) + paths) + ;; hash now maps the md5sum to the messages... + (hash-for-each + (lambda (md5 mpaths) + (if (> (length mpaths) 1) + (begin + ;;(format #t "md5sum: ~a:\n" md5) + (let ((num 1)) + (for-each + (lambda (path) + (if (equal? num 1) + (format #t "~a\n" path) + (begin + (format #t "~a: ~a\n" (if delete "deleting" "dup") path) + (if delete (delete-file path)))) + (set! num (+ 1 num))) + mpaths))))) + hash)))) + id-table))) + + + +(define (main args) + "Find duplicate messages and, potentially, delete the dups. + Be careful with that! +Interpret argument-list ARGS (like command-line +arguments). Possible arguments are: + --muhome (path to alternative mu home directory). + --delete (delete all but the first one). Run mu index afterwards. + --expr (expression to constrain search)." + (setlocale LC_ALL "") + (let* ((optionspec '( (muhome (value #t)) + (delete (value #f)) + (expr (value #t)) + (help (single-char #\h) (value #f)))) + (options (getopt-long args optionspec)) + (help (option-ref options 'help #f)) + (delete (option-ref options 'delete #f)) + (expr (option-ref options 'expr #t)) + (muhome (option-ref options 'muhome #f))) + (mu:initialize muhome) + (find-dups delete expr))) + + +;; Local Variables: +;; mode: scheme +;; End: + + + + + + + + + diff --git a/guile/scripts/histogram.scm b/guile/scripts/histogram.scm new file mode 100755 index 0000000..b845f28 --- /dev/null +++ b/guile/scripts/histogram.scm @@ -0,0 +1,127 @@ +#!/bin/sh +exec guile -e main -s $0 $@ +!# +;; Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. + +;; INFO: Histogram of the number of messages per time-unit +;; INFO: Options: +;; INFO: --query=<query>: limit to messages matching query +;; INFO: --muhome=<muhome>: path to mu home dir +;; INFO: --time-unit: hour|day|month|year|month-year +;; INFO: --output: the output format, such as "png", "wxt" +;; INFO: (depending on the environment) + +(use-modules (mu) (mu stats) (mu plot) + (ice-9 getopt-long) (ice-9 format)) + +(define (per-hour expr output) + "Count the total number of messages per hour that match EXPR. +OUTPUT corresponds to the output format, as per gnuplot's 'set terminal'." + (mu:plot-histogram + (sort + (mu:tabulate + (lambda (msg) + (tm:hour (localtime (mu:date msg)))) expr) + (lambda (x y) (< (car x) (car y)))) + (format #f "Messages per hour matching ~a" expr) + "Hour" "Messages" output)) + + +(define (per-day expr output) + "Count the total number of messages for each weekday (0-6 for +Sun..Sat) that match EXPR. OUTPUT corresponds to the output format, as +per gnuplot's 'set terminal'." + (mu:plot-histogram + (mu:weekday-numbers->names + (sort (mu:tabulate + (lambda (msg) + (tm:wday (localtime (mu:date msg)))) expr) + (lambda (x y) (< (car x) (car y))))) + (format #f "Messages per weekday matching ~a" expr) + "Day" "Messages" output)) + +(define (per-month expr output) + "Count the total number of messages per month that match EXPR. +OUTPUT corresponds to the output format, as per gnuplot's 'set terminal'." + (mu:plot-histogram + (mu:month-numbers->names + (sort + (mu:tabulate + (lambda (msg) + (tm:mon (localtime (mu:date msg)))) expr) + (lambda (x y) (< (car x) (car y))))) + (format #f "Messages per month matching ~a" expr) + "Month" "Messages" output)) + +(define (per-year expr output) + "Count the total number of messages per year that match EXPR. OUTPUT corresponds +to the output format, as per gnuplot's 'set terminal'." + (mu:plot-histogram + (sort (mu:tabulate + (lambda (msg) + (+ 1900 (tm:year (localtime (mu:date msg))))) expr) + (lambda (x y) (< (car x) (car y)))) + (format #f "Messages per year matching ~a" expr) + "Year" "Messages" output)) + + +(define (per-year-month expr output) + "Count the total number of messages for each year and month that match EXPR. +OUTPUT corresponds to the output format, as per gnuplot's 'set terminal'." + (mu:plot-histogram + (sort (mu:tabulate + (lambda (msg) + (string->number + (format #f "~d~2'0d" + (+ 1900 (tm:year (localtime (mu:date msg)))) + (tm:mon (localtime (mu:date msg)))))) + expr) + (lambda (x y) (< (car x) (car y)))) + (format #f "Messages per year/month matching ~a" expr) + "Year/Month" "Messages" output)) + +(define (main args) + (let* ((optionspec + '((time-unit (value #t)) + (query (value #t)) + (muhome (value #t)) + (output (value #t)) + (help (single-char #\h) (value #f)))) + (options (getopt-long args optionspec)) + (help (option-ref options 'help #f)) + (time-unit (option-ref options 'time-unit "year")) + (muhome (option-ref options 'muhome #f)) + (query (option-ref options 'query "")) + (output (option-ref options 'output "dumb")) + (rest (option-ref options '() #f)) + (func + (cond + ((equal? time-unit "hour") per-hour) + ((equal? time-unit "day") per-day) + ((equal? time-unit "month") per-month) + ((equal? time-unit "year") per-year) + ((equal? time-unit "year-month") per-year-month) + (else #f)))) + (setlocale LC_ALL "") + (unless func + (display "error: unknown time-unit\n") + (set! help #t)) + (if help + (begin + (display + (string-append "parameters: [--help] [--output=dumb|png|wxt] " + "[--muhome=<muhome>] [--query=<query>]" + "[--time-unit=hour|day|month|year|year-month]")) + (newline)) + (begin + (mu:initialize muhome) + (func query output))))) + +;; Local Variables: +;; mode: scheme +;; End: diff --git a/guile/scripts/msgs-count.scm b/guile/scripts/msgs-count.scm new file mode 100755 index 0000000..9a73efe --- /dev/null +++ b/guile/scripts/msgs-count.scm @@ -0,0 +1,40 @@ +#!/bin/sh +exec guile -e main -s $0 $@ +!# +;; Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + +;; INFO: Count the number of messages matching some query +;; INFO: options: +;; INFO: --query=<query>: limit to messages matching query +;; INFO: --muhome=<muhome>: path to mu home dir (optional) + +(use-modules (mu) (mu script) (mu stats)) + +(define (count expr output) + "Print the total number of messages matching the query EXPR. +OUTPUT is ignored." + (display (mu:count expr)) + (newline)) + +(define (main args) + (mu:run-stats args count)) + +;; Local Variables: +;; mode: scheme +;; End: diff --git a/guile/tests/meson.build b/guile/tests/meson.build new file mode 100644 index 0000000..07b3790 --- /dev/null +++ b/guile/tests/meson.build @@ -0,0 +1,38 @@ +## Copyright (C) 2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# +# guile test; they don't work with ASAN. +# +if get_option('b_sanitize') == 'none' + guile_load_path = join_paths(meson.project_source_root(), 'guile') + guile_extensions_path = ':'.join([ + join_paths(meson.project_build_root(), 'guile'), + meson.current_build_dir()]) + + test('test-mu-guile', + executable('test-mu-guile', + 'test-mu-guile.cc', + install: false, + cpp_args: [ + '-DABS_SRCDIR="' + meson.current_source_dir() + '"', + '-DGUILE_LOAD_PATH="' + guile_load_path + '"', + '-DGUILE_EXTENSIONS_PATH="' + guile_extensions_path + '"' + ], + dependencies: [glib_dep, lib_mu_dep])) +else + message('sanitizer build; skip guile test') +endif diff --git a/guile/tests/test-mu-guile.cc b/guile/tests/test-mu-guile.cc new file mode 100644 index 0000000..09a53d0 --- /dev/null +++ b/guile/tests/test-mu-guile.cc @@ -0,0 +1,130 @@ +/* +** Copyright (C) 2012-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include <glib.h> +#include <glib/gstdio.h> + +#include <lib/mu-query.hh> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> + +#include "utils/mu-test-utils.hh" +#include <lib/mu-store.hh> +#include <utils/mu-utils.hh> + +using namespace Mu; + +static std::string test_dir; + +static std::string +fill_database(void) +{ + const auto cmdline = mu_format( + "/bin/sh -c '" + "{} init --muhome={} --maildir={} --quiet; " + "{} index --muhome={} --quiet'", + MU_PROGRAM, + test_dir, + MU_TESTMAILDIR2, + MU_PROGRAM, + test_dir); + + if (g_test_verbose()) + mu_println("{}", cmdline); + + GError *err{}; + if (!g_spawn_command_line_sync(cmdline.c_str(), NULL, NULL, NULL, &err)) { + mu_printerrln("Error: {}", err ? err->message : "?"); + g_clear_error(&err); + g_assert(0); + } + + return test_dir; +} + +static void +test_something(const char* what) +{ + g_setenv("GUILE_AUTO_COMPILE", "0", TRUE); + g_setenv("GUILE_LOAD_PATH", GUILE_LOAD_PATH, TRUE); + g_setenv("GUILE_EXTENSIONS_PATH",GUILE_EXTENSIONS_PATH, TRUE); + + if (g_test_verbose()) + g_print("GUILE_LOAD_PATH: %s\n", GUILE_LOAD_PATH); + + const auto dir = fill_database(); + const auto cmdline = mu_format("{} -q -e main {}/test-mu-guile.scm " + "--muhome={} --test={}", + GUILE_BINARY, ABS_SRCDIR, + dir, what); + + if (g_test_verbose()) + mu_println("cmdline: {}", cmdline); + + GError *err{}; + int status{}; + if (!g_spawn_command_line_sync(cmdline.c_str(), NULL, NULL, &status, &err) || + status != 0) { + mu_printerrln("Error: {}", err ? err->message : "something went wrong"); + g_clear_error(&err); + g_assert(0); + } +} + +static void +test_mu_guile_queries(void) +{ + test_something("queries"); +} + +static void +test_mu_guile_messages(void) +{ + test_something("message"); +} + +static void +test_mu_guile_stats(void) +{ + test_something("stats"); +} + +int +main(int argc, char* argv[]) +{ + int rv; + TempDir tempdir; + test_dir = tempdir.path(); + + mu_test_init(&argc, &argv); + + if (!set_en_us_utf8_locale()) + return 0; /* don't error out... */ + + g_test_add_func("/guile/queries", test_mu_guile_queries); + g_test_add_func("/guile/message", test_mu_guile_messages); + g_test_add_func("/guile/stats", test_mu_guile_stats); + + rv = g_test_run(); + + return rv; +} diff --git a/guile/tests/test-mu-guile.scm b/guile/tests/test-mu-guile.scm new file mode 100755 index 0000000..afa4f48 --- /dev/null +++ b/guile/tests/test-mu-guile.scm @@ -0,0 +1,124 @@ +#!/bin/sh +exec guile -e main -s $0 $@ +!# + +;; Copyright (C) 2012-2013 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; +;; This program is free software; you can redistribute it and/or modify it +;; under the terms of the GNU General Public License as published by the +;; Free Software Foundation; either version 3, or (at your option) any +;; later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; + +;; You should have received a copy of the GNU General Public License +;; along with this program; if not, write to the Free Software Foundation, +;; Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +(setlocale LC_ALL "") + +(use-modules (srfi srfi-1)) +(use-modules (ice-9 getopt-long) (ice-9 optargs) (ice-9 popen) (ice-9 format)) +(use-modules (mu) (mu stats)) + +(define (n-results-or-exit query n) + "Run QUERY, and exit 1 if the number of results != N." + (let ((lst (mu:message-list query))) + (if (not (= (length lst) n)) + (begin + (simple-format (current-error-port) "Query: \"~A\"; expected ~A, got ~A\n" + query n (length lst)) + (exit 1))))) + +(define (test-queries) + "Test a bunch of queries (or die trying)." + (n-results-or-exit "hello" 1) + (n-results-or-exit "f:john fruit" 1) + (n-results-or-exit "f:soc@example.com" 1) + (n-results-or-exit "t:alki@example.com" 1) + (n-results-or-exit "t:alcibiades" 1) + (n-results-or-exit "f:soc@example.com OR f:john" 2) + (n-results-or-exit "f:soc@example.com OR f:john OR t:edmond" 3) + (n-results-or-exit "t:julius" 1) + (n-results-or-exit "s:dude" 1) + (n-results-or-exit "t:dantès" 1) + (n-results-or-exit "file:sittingbull.jpg" 1) + (n-results-or-exit "file:custer.jpg" 1) + (n-results-or-exit "file:custer.*" 1) + (n-results-or-exit "j:sit*" 1) + (n-results-or-exit "mime:image/jpeg" 1) + (n-results-or-exit "mime:text/plain" 14) + (n-results-or-exit "y:text*" 14) + (n-results-or-exit "y:image*" 1) + (n-results-or-exit "mime:message/rfc822" 2)) + +(define (error-exit msg . args) + "Print error and exit." + (let ((msg (apply format #f msg args))) + (simple-format (current-error-port) "*ERROR*: ~A\n" msg) + (exit 1))) + +(define (str-equal-or-exit got exp) + "S1 == S2 or exit 1." + ;; (format #t "'~A' <=> '~A'\n" s1 s2) + (if (not (string= exp got)) + (error-exit "Expected \"~A\", got \"~A\"\n" exp got))) + +(define (test-message) + "Test functions for a particular message." + + (let ((msg (car (mu:message-list "hello")))) + (str-equal-or-exit (mu:subject msg) "Fwd: rfc822") + (str-equal-or-exit (mu:to msg) "martin") + (str-equal-or-exit (mu:from msg) "foobar <foo@example.com>") + (str-equal-or-exit (mu:header msg "X-Mailer") "Ximian Evolution 1.4.5") + + (if (not (equal? (mu:priority msg) mu:prio:normal)) + (error-exit "Expected ~A, got ~A" (mu:priority msg) mu:prio:normal))) + + (let ((msg (car (mu:message-list "atoms")))) + (str-equal-or-exit (mu:subject msg) "atoms") + (str-equal-or-exit (mu:to msg) "Democritus <demo@example.com>") + (str-equal-or-exit (mu:from msg) "Richard P. Feynman <rpf@example.com>") + ;;(str-equal-or-exit (mu:header msg "Content-transfer-encoding") "7BIT") + + (if (not (equal? (mu:priority msg) mu:prio:high)) + (error-exit "Expected ~a, got ~a" (mu:priority msg) mu:prio:high)))) + +(define (num-equal-or-exit got exp) + "S1 == S2 or exit 1." + ;; (format #t "'~A' <=> '~A'\n" s1 s2) + (if (not (= exp got)) + (error-exit "Expected \"~S\", got \"~S\"\n" exp got))) + +(define (test-stats) + "Test statistical functions." + ;; average + (num-equal-or-exit (mu:average mu:size) 82601/14) + (num-equal-or-exit (floor (mu:stddev mu:size)) 12637.0) + (num-equal-or-exit (mu:max mu:size) 46308) + (num-equal-or-exit (mu:min mu:size) 111)) + +(define (main args) + (let* ((optionspec '((muhome (value #t)) + (test (value #t)))) + (options (getopt-long args optionspec)) + (muhome (option-ref options 'muhome #f)) + (test (option-ref options 'test #f))) + + (mu:initialize muhome) + + (if test + (cond + ((string= test "queries") (test-queries)) + ((string= test "message") (test-message)) + ((string= test "stats") (test-stats)) + (#t (exit 1)))))) + + +;; Local Variables: +;; mode: scheme +;; End: diff --git a/lib/meson.build b/lib/meson.build new file mode 100644 index 0000000..b3b519d --- /dev/null +++ b/lib/meson.build @@ -0,0 +1,95 @@ +## Copyright (C) 2021-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + +subdir('utils') +subdir('message') + +lib_mu=static_library( + 'mu', + [ + # db + 'mu-config.cc', + 'mu-contacts-cache.cc', + 'mu-store.cc', + 'mu-xapian-db.cc', + # querying + 'mu-query-macros.cc', + 'mu-query-match-deciders.cc', + 'mu-query-parser.cc', + 'mu-query-processor.cc', + 'mu-query-threads.cc', + 'mu-query-xapianizer.cc', + 'mu-query.cc', + # indexing + 'mu-indexer.cc', + 'mu-scanner.cc', + # mu4e + 'mu-server.cc', + # misc + 'mu-maildir.cc', + 'mu-script.cc', + ], + dependencies: [ + glib_dep, + gio_dep, + gmime_dep, + xapian_dep, + guile_dep, + config_h_dep, + lib_mu_utils_dep, + lib_mu_message_dep], + install: false) + +lib_mu_dep = declare_dependency( + link_with: lib_mu, + dependencies: [ lib_mu_message_dep, thread_dep ], + include_directories: + include_directories(['.', '..'])) + +# +# dev helpers +# + +process_query = executable('process-query', [ 'mu-query-processor.cc'], + install: false, + cpp_args: ['-DBUILD_PROCESS_QUERY'], + dependencies: [glib_dep, lib_mu_dep]) + +parse_query = executable( 'parse-query', [ 'mu-query-parser.cc' ], + install: false, + cpp_args: ['-DBUILD_PARSE_QUERY'], + dependencies: [glib_dep, lib_mu_dep]) + +parse_query_expand = executable( 'parse-query-expand', [ 'mu-query-parser.cc' ], + install: false, + cpp_args: ['-DBUILD_PARSE_QUERY_EXPAND'], + dependencies: [glib_dep, lib_mu_dep]) + +xapian_query = executable('xapianize-query', [ 'mu-query-xapianizer.cc' ], + install: false, + cpp_args: ['-DBUILD_XAPIANIZE_QUERY'], + dependencies: [glib_dep, lib_mu_dep]) + +list_maildirs = executable('list-maildirs', 'mu-scanner.cc', + install: false, + cpp_args: ['-DBUILD_LIST_MAILDIRS'], + dependencies: [glib_dep, config_h_dep, + lib_mu_utils_dep]) + +if not get_option('tests').disabled() + subdir('tests') +endif diff --git a/lib/message/meson.build b/lib/message/meson.build new file mode 100644 index 0000000..006bb18 --- /dev/null +++ b/lib/message/meson.build @@ -0,0 +1,47 @@ +## Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + +lib_mu_message=static_library( + 'mu-message', + [ + 'mu-message.cc', + 'mu-message-file.cc', + 'mu-message-part.cc', + 'mu-contact.cc', + 'mu-document.cc', + 'mu-fields.cc', + 'mu-flags.cc', + 'mu-priority.cc', + 'mu-mime-object.cc', + ], + dependencies: [ + glib_dep, + gmime_dep, + xapian_dep, + config_h_dep, + lib_mu_utils_dep], + install: false) + +lib_mu_message_dep = declare_dependency( + link_with: lib_mu_message, + dependencies: [ xapian_dep, gmime_dep, lib_mu_utils_dep, config_h_dep ], + include_directories: + include_directories(['.', '..'])) + +if not get_option('tests').disabled() + subdir('tests') +endif diff --git a/lib/message/mu-contact.cc b/lib/message/mu-contact.cc new file mode 100644 index 0000000..c6439b0 --- /dev/null +++ b/lib/message/mu-contact.cc @@ -0,0 +1,207 @@ +/* +** Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-contact.hh" +#include "mu-message.hh" +#include "utils/mu-utils.hh" +#include "mu-mime-object.hh" + +#include <gmime/gmime.h> +#include <glib.h> +#include <string> + +using namespace Mu; + +std::string +Contact::display_name() const +{ + const auto needs_quoting= [](const std::string& n) { + for (auto& c: n) + if (c == ',' || c == '"' || c == '@') + return true; + return false; + }; + + if (name.empty()) + return email; + else if (!needs_quoting(name)) + return name + " <" + email + '>'; + else + return Mu::quote(name) + " <" + email + '>'; +} + +std::string +Mu::to_string(const Mu::Contacts& contacts) +{ + std::string res; + + seq_for_each(contacts, [&](auto&& contact) { + if (res.empty()) + res = contact.display_name(); + else + res += ", " + contact.display_name(); + }); + + return res; +} + +size_t +Mu::lowercase_hash(const std::string& s) +{ + std::size_t djb = 5381; // djb hash + for (const auto c : s) + djb = ((djb << 5) + djb) + + static_cast<size_t>(g_ascii_tolower(c)); + return djb; +} + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +static void +test_ctor_foo() +{ + Contact c{ + "foo@example.com", + "Foo Bar", + Contact::Type::Bcc, + 1645214647 + }; + + assert_equal(c.email, "foo@example.com"); + assert_equal(c.name, "Foo Bar"); + g_assert_true(*c.field_id() == Field::Id::Bcc); + g_assert_cmpuint(c.message_date,==,1645214647); + + assert_equal(c.display_name(), "Foo Bar <foo@example.com>"); +} + + +static void +test_ctor_blinky() +{ + Contact c{ + "bar@example.com", + "Blinky", + 1645215014, + true, /* personal */ + 13, /*freq*/ + 12345 /* tstamp */ + }; + + assert_equal(c.email, "bar@example.com"); + assert_equal(c.name, "Blinky"); + g_assert_true(c.personal); + g_assert_cmpuint(c.frequency,==,13); + g_assert_cmpuint(c.tstamp,==,12345); + g_assert_cmpuint(c.message_date,==,1645215014); + + assert_equal(c.display_name(), "Blinky <bar@example.com>"); +} + +static void +test_ctor_cleanup() +{ + Contact c{ + "bar@example.com", + "Bli\nky", + 1645215014, + true, /* personal */ + 13, /*freq*/ + 12345 /* tstamp */ + }; + + assert_equal(c.email, "bar@example.com"); + assert_equal(c.name, "Bli ky"); + g_assert_true(c.personal); + g_assert_cmpuint(c.frequency,==,13); + g_assert_cmpuint(c.tstamp,==,12345); + g_assert_cmpuint(c.message_date,==,1645215014); + + assert_equal(c.display_name(), "Bli ky <bar@example.com>"); +} + +static void +test_encode() +{ + Contact c{ + "cassius@example.com", + "Ali, Muhammad \"The Greatest\"", + 345, + false, /* personal */ + 333, /*freq*/ + 768 /* tstamp */ + }; + + assert_equal(c.email, "cassius@example.com"); + assert_equal(c.name, "Ali, Muhammad \"The Greatest\""); + g_assert_false(c.personal); + g_assert_cmpuint(c.frequency,==,333); + g_assert_cmpuint(c.tstamp,==,768); + g_assert_cmpuint(c.message_date,==,345); + + assert_equal(c.display_name(), + "\"Ali, Muhammad \\\"The Greatest\\\"\" <cassius@example.com>"); +} + + +static void +test_sender() +{ + Contact c{"aa@example.com", "Anders Ångström", + Contact::Type::Sender, 54321}; + + assert_equal(c.email, "aa@example.com"); + assert_equal(c.name, "Anders Ångström"); + g_assert_false(c.personal); + g_assert_cmpuint(c.frequency,==,1); + g_assert_cmpuint(c.message_date,==,54321); + + g_assert_false(!!c.field_id()); +} + + +static void +test_misc() +{ + g_assert_false(!!contact_type_from_field_id(Field::Id::Subject)); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + g_mime_init(); + + g_test_add_func("/message/contact/ctor-foo", test_ctor_foo); + g_test_add_func("/message/contact/ctor-blinky", test_ctor_blinky); + g_test_add_func("/message/contact/ctor-cleanup", test_ctor_cleanup); + g_test_add_func("/message/contact/encode", test_encode); + + g_test_add_func("/message/contact/sender", test_sender); + g_test_add_func("/message/contact/misc", test_misc); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/message/mu-contact.hh b/lib/message/mu-contact.hh new file mode 100644 index 0000000..d417d4e --- /dev/null +++ b/lib/message/mu-contact.hh @@ -0,0 +1,219 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_MESSAGE_CONTACT_HH__ +#define MU_MESSAGE_CONTACT_HH__ + +#include <functional> +#include <string> +#include <vector> +#include <functional> +#include <cctype> +#include <cstring> +#include <cstdlib> +#include <ctime> + +#include <utils/mu-option.hh> +#include "mu-fields.hh" + +struct _InternetAddressList; + +namespace Mu { + +/** + * Get the hash value for a lowercase value of s; useful for email-addresses + * + * @param s a string + * + * @return a hash value. + */ +size_t lowercase_hash(const std::string& s); + +struct Contact { + enum struct Type { + None, Sender, From, ReplyTo, To, Cc, Bcc + }; + + /** + * Construct a new Contact + * + * @param email_ email address + * @param name_ name or empty + * @param type_ contact field type + * @param message_date_ data for the message for this contact + */ + Contact(const std::string& email_, const std::string& name_ = "", + Type type_ = Type::None, ::time_t message_date_ = 0) + : email{email_}, name{name_}, type{type_}, + message_date{message_date_}, personal{}, frequency{1}, tstamp{} + { cleanup_name(); } + + /** + * Construct a new Contact + * + * @param email_ email address + * @param name_ name or empty + * @param message_date_ date of message this contact originate from + * @param personal_ is this a personal contact? + * @param freq_ how often was this contact seen? + * @param tstamp_ timestamp for last change + */ + Contact(const std::string& email_, const std::string& name_, + time_t message_date_, bool personal_, size_t freq_, + int64_t tstamp_) + : email{email_}, name{name_}, type{Type::None}, + message_date{message_date_}, personal{personal_}, frequency{freq_}, + tstamp{tstamp_} + { cleanup_name();} + + /** + * Get the "display name" for this contact: + * + * If there's a non-empty name, it's Jane Doe <email@example.com> + * otherwise it's just the e-mail address. Names with commas are quoted + * (with the quotes escaped). + * + * @return the display name + */ + std::string display_name() const; + + + /** + * Does the contact contain a valid email address as per + * https://html.spec.whatwg.org/multipage/input.html#valid-e-mail-address + * ? + * + * @return true or false + */ + bool has_valid_email() const; + + /** + * Operator==; based on the hash values (ie. lowercase e-mail address) + * + * @param rhs some other Contact + * + * @return true orf false. + */ + bool operator== (const Contact& rhs) const noexcept { + return hash() == rhs.hash(); + } + + /** + * Get a hash-value for this contact, which gets lazily calculated. This + * * is for use with container classes. This uses the _lowercase_ email + * address. + * + * @return the hash + */ + size_t hash() const { + static size_t cached_hash; + if (cached_hash == 0) { + cached_hash = lowercase_hash(email); + } + return cached_hash; + } + + /** + * Get the corresponding Field::Id (if any) + * for this contact. + * + * @return the field-id or Nothing. + */ + constexpr Option<Field::Id> field_id() const noexcept { +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" + switch(type) { + case Type::Bcc: + return Field::Id::Bcc; + case Type::Cc: + return Field::Id::Cc; + case Type::From: + return Field::Id::From; + case Type::To: + return Field::Id::To; + default: + return Nothing; + } +#pragma GCC diagnostic pop + } + + + /* + * data members + */ + + std::string email; /**< Email address for this contact.Not empty */ + std::string name; /**< Name for this contact; can be empty. */ + Type type; /**< Type of contact */ + int64_t message_date; /**< Date of the contact's message */ + bool personal; /**< A personal message? */ + size_t frequency; /**< Frequency of this contact */ + int64_t tstamp; /**< Timestamp for this contact (internal use) */ + +private: + void cleanup_name() { // replace control characters by spaces. + for (auto& c: name) + if (iscntrl(c)) + c = ' '; + } +}; + +constexpr Option<Contact::Type> +contact_type_from_field_id(Field::Id id) noexcept { + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" + switch(id) { + case Field::Id::Bcc: + return Contact::Type::Bcc; + case Field::Id::Cc: + return Contact::Type::Cc; + case Field::Id::From: + return Contact::Type::From; + case Field::Id::To: + return Contact::Type::To; + default: + return Nothing; + } +#pragma GCC diagnostic pop +} + +using Contacts = std::vector<Contact>; + +/** + * Get contacts as a comma-separated list. + * + * @param contacts contacs + * + * @return string with contacts. + */ +std::string to_string(const Contacts& contacts); + +} // namespace Mu + +/** + * Implement our hash int std:: + */ +template<> struct std::hash<Mu::Contact> { + std::size_t operator()(const Mu::Contact& c) const noexcept { + return c.hash(); + } +}; + +#endif /* MU_CONTACT_HH__ */ diff --git a/lib/message/mu-document.cc b/lib/message/mu-document.cc new file mode 100644 index 0000000..428b946 --- /dev/null +++ b/lib/message/mu-document.cc @@ -0,0 +1,497 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include "config.h" + +#include "mu-document.hh" +#include "mu-message.hh" +#include "utils/mu-sexp.hh" + +#include <cstdint> +#include <glib.h> +#include <numeric> +#include <algorithm> +#include <charconv> +#include <cinttypes> + +#include <string> +#include <utils/mu-utils.hh> + +using namespace Mu; + +// backward compat +#ifndef HAVE_XAPIAN_FLAG_NGRAMS +#define FLAG_NGRAMS FLAG_CJK_NGRAM +#endif /*HAVE_XAPIAN_FLAG_NGRAMS*/ + + +const Xapian::Document& +Document::xapian_document() const +{ + if (dirty_sexp_) { + xdoc_.set_data(sexp().to_string()); + dirty_sexp_ = false; + } + return xdoc_; +} + +template<typename SexpType> void +Document::put_prop(const std::string& pname, SexpType&& val) +{ + cached_sexp().put_props(pname, std::forward<SexpType>(val)); + dirty_sexp_ = true; +} + +template<typename SexpType> void +Document::put_prop(const Field& field, SexpType&& val) +{ + put_prop(std::string(":") + std::string{field.name}, + std::forward<SexpType>(val)); +} + +static Xapian::TermGenerator +make_term_generator(Xapian::Document& doc, Document::Options opts) +{ + Xapian::TermGenerator termgen; + + if (any_of(opts & Document::Options::SupportNgrams)) + termgen.set_flags(Xapian::TermGenerator::FLAG_NGRAMS); + + termgen.set_document(doc); + + return termgen; +} + +static void +add_search_term(Xapian::Document& doc, const Field& field, const std::string& val, + Document::Options opts) +{ + if (field.is_normal_term() || field.is_phrasable_term()) { + const auto flat{utf8_flatten(val)}; + if (field.is_normal_term()) + doc.add_term(field.xapian_term(flat)); + if (field.is_phrasable_term()) { + auto termgen{make_term_generator(doc, opts)}; + termgen.index_text(flat, 1, field.xapian_term()); + } + } else if (field.is_boolean_term()) { + doc.add_boolean_term(field.xapian_term(val)); + } else + throw std::logic_error("not a search term"); +} + +void +Document::add(Field::Id id, const std::string& val) +{ + const auto field{field_from_id(id)}; + + if (field.is_value()) + xdoc_.add_value(field.value_no(), val); + + if (field.is_searchable()) + add_search_term(xdoc_, field, val, options_); + + if (field.include_in_sexp()) + put_prop(field, val); +} + +void +Document::add(Field::Id id, const std::vector<std::string>& vals) +{ + if (vals.empty()) + return; + + const auto field{field_from_id(id)}; + if (field.is_value()) + xdoc_.add_value(field.value_no(), Mu::join(vals, SepaChar1)); + + if (field.is_searchable()) + std::for_each(vals.begin(), vals.end(), + [&](const auto& val) { + add_search_term(xdoc_, field, val, options_); }); + + if (field.include_in_sexp()) { + Sexp elms{}; + for(auto&& val: vals) + elms.add(val); + put_prop(field, std::move(elms)); + } +} + + +std::vector<std::string> +Document::string_vec_value(Field::Id field_id) const noexcept +{ + return Mu::split(string_value(field_id), SepaChar1); +} + +static Sexp +make_contacts_sexp(const Contacts& contacts) +{ + Sexp contacts_sexp; + + seq_for_each(contacts, [&](auto&& c) { + Sexp contact(":email"_sym, c.email); + if (!c.name.empty()) + contact.add(":name"_sym, c.name); + contacts_sexp.add(std::move(contact)); + }); + + return contacts_sexp; +} + +void +Document::add(Field::Id id, const Contacts& contacts) +{ + if (contacts.empty()) + return; + + const auto field{field_from_id(id)}; + std::vector<std::string> cvec; + + const std::string sepa2(1, SepaChar2); + auto&& termgen{make_term_generator(xdoc_, options_)}; + + for (auto&& contact: contacts) { + + const auto cfield_id{contact.field_id()}; + if (!cfield_id || *cfield_id != id) + continue; + + const auto e{contact.email}; + xdoc_.add_term(field.xapian_term(e)); + + /* allow searching for address components, too */ + const auto atpos = e.find('@'); + if (atpos != std::string::npos && atpos < e.size() - 1) { + xdoc_.add_term(field.xapian_term(e.substr(0, atpos))); + xdoc_.add_term(field.xapian_term(e.substr(atpos + 1))); + } + + if (!contact.name.empty()) + termgen.index_text(utf8_flatten(contact.name), 1, + field.xapian_term()); + cvec.emplace_back(contact.email + sepa2 + contact.name); + } + + if (!cvec.empty()) + xdoc_.add_value(field.value_no(), join(cvec, SepaChar1)); + + if (field.include_in_sexp()) + put_prop(field, make_contacts_sexp(contacts)); +} + +Contacts +Document::contacts_value(Field::Id id) const noexcept +{ + const auto vals{string_vec_value(id)}; + Contacts contacts; + contacts.reserve(vals.size()); + + const auto ctype{contact_type_from_field_id(id)}; + if (G_UNLIKELY(!ctype)) { + mu_critical("invalid field-id for contact-type: <{}>", + static_cast<size_t>(id)); + return {}; + } + + for (auto&& s: vals) { + + const auto pos = s.find(SepaChar2); + if (G_UNLIKELY(pos == std::string::npos)) { + mu_critical("invalid contact data '{}'", s); + break; + } + + contacts.emplace_back(s.substr(0, pos), s.substr(pos + 1), *ctype); + } + + return contacts; +} + +void +Document::add_extra_contacts(const std::string& propname, const Contacts& contacts) +{ + if (!contacts.empty()) { + put_prop(propname, make_contacts_sexp(contacts)); + dirty_sexp_ = true; + } +} + + +static Sexp +make_emacs_time_sexp(::time_t t) +{ + return Sexp().add(static_cast<unsigned>(t >> 16), + static_cast<unsigned>(t & 0xffff), + 0); +} + +void +Document::add(Field::Id id, int64_t val) +{ + /* + * Xapian stores everything (incl. numbers) as strings. + * + * we comply, by storing a number a base-16 and prefixing with 'f' + + * length; such that the strings are sorted in the numerical order. + */ + const auto field{field_from_id(id)}; + + if (field.is_value()) + xdoc_.add_value(field.value_no(), to_lexnum(val)); + + if (field.include_in_sexp()) { + if (field.is_time_t()) + put_prop(field, make_emacs_time_sexp(val)); + else + put_prop(field, val); + } +} + +int64_t +Document::integer_value(Field::Id field_id) const noexcept +{ + if (auto&& v{string_value(field_id)}; v.empty()) + return 0; + else + return from_lexnum(v); +} + +void +Document::add(Priority prio) +{ + constexpr auto field{field_from_id(Field::Id::Priority)}; + + xdoc_.add_value(field.value_no(), std::string(1, to_char(prio))); + xdoc_.add_boolean_term(field.xapian_term(to_char(prio))); + + if (field.include_in_sexp()) + put_prop(field, Sexp::Symbol(priority_name(prio))); +} + +Priority +Document::priority_value() const noexcept +{ + const auto val{string_value(Field::Id::Priority)}; + return priority_from_char(val.empty() ? 'n' : val[0]); +} + +void +Document::add(Flags flags) +{ + constexpr auto field{field_from_id(Field::Id::Flags)}; + + Sexp flaglist; + xdoc_.add_value(field.value_no(), to_lexnum(static_cast<int64_t>(flags))); + flag_infos_for_each([&](auto&& flag_info) { + auto term=[&](){return field.xapian_term(flag_info.shortcut_lower());}; + if (any_of(flag_info.flag & flags)) { + xdoc_.add_boolean_term(term()); + flaglist.add(Sexp::Symbol(flag_info.name)); + } + }); + + if (field.include_in_sexp()) + put_prop(field, std::move(flaglist)); +} + + +Flags +Document::flags_value() const noexcept +{ + return static_cast<Flags>(integer_value(Field::Id::Flags)); +} + +void +Document::remove(Field::Id field_id) +{ + const auto field{field_from_id(field_id)}; + const auto pfx{field.xapian_prefix()}; + + xapian_try([&]{ + + if (auto&& val{xdoc_.get_value(field.value_no())}; !val.empty()) { + // g_debug("removing value<%u>: '%s'", field.value_no(), + // val.c_str()); + xdoc_.remove_value(field.value_no()); + } + + std::vector<std::string> kill_list; + for (auto&& it = xdoc_.termlist_begin(); + it != xdoc_.termlist_end(); ++it) { + const auto term{*it}; + if (!term.empty() && term.at(0) == pfx) + kill_list.emplace_back(term); + } + + for (auto&& term: kill_list) { + // g_debug("removing term '%s'", term.c_str()); + try { + xdoc_.remove_term(term); + } catch(const Xapian::InvalidArgumentError& xe) { + mu_critical("failed to remove '{}'", term); + } + } + }); +} + + +#ifdef BUILD_TESTS + +#include "utils/mu-test-utils.hh" + +#define assert_same_contact(C1,C2) do { \ + g_assert_cmpstr(C1.email.c_str(),==,C2.email.c_str()); \ + g_assert_cmpstr(C2.name.c_str(),==,C2.name.c_str()); \ + } while (0) + +#define assert_same_contacts(CV1,CV2) do { \ + g_assert_cmpuint(CV1.size(),==,CV2.size()); \ + for (auto i = 0U; i != CV1.size(); ++i) \ + assert_same_contact(CV1[i], CV2[i]); \ + } while(0) + + + +static const Contacts test_contacts = {{ + Contact{"john@example.com", "John", Contact::Type::Bcc}, + Contact{"ringo@example.com", "Ringo", Contact::Type::Bcc}, + Contact{"paul@example.com", "Paul", Contact::Type::Cc}, + Contact{"george@example.com", "George", Contact::Type::Cc}, + Contact{"james@example.com", "James", Contact::Type::From}, + Contact{"lars@example.com", "Lars", Contact::Type::To}, + Contact{"kirk@example.com", "Kirk", Contact::Type::To}, + Contact{"jason@example.com", "Jason", Contact::Type::To} + }}; + +static void +test_bcc() +{ + { + Document doc; + doc.add(Field::Id::Bcc, test_contacts); + + Contacts expected_contacts = {{ + Contact{"john@example.com", "John", + Contact::Type::Bcc}, + Contact{"ringo@example.com", "Ringo", + Contact::Type::Bcc}, + }}; + const auto actual_contacts = doc.contacts_value(Field::Id::Bcc); + assert_same_contacts(expected_contacts, actual_contacts); + } + + { + Document doc; + Contacts contacts = {{ + Contact{"john@example.com", "John Lennon", + Contact::Type::Bcc}, + Contact{"ringo@example.com", "Ringo", + Contact::Type::Bcc}, + }}; + doc.add(Field::Id::Bcc, contacts); + + TempDir tempdir; + auto db = Xapian::WritableDatabase(tempdir.path()); + db.add_document(doc.xapian_document()); + + auto contacts2 = doc.contacts_value(Field::Id::Bcc); + assert_same_contacts(contacts, contacts2); + } + +} + +static void +test_cc() +{ + Document doc; + doc.add(Field::Id::Cc, test_contacts); + + Contacts expected_contacts = {{ + Contact{"paul@example.com", "Paul", Contact::Type::Cc}, + Contact{"george@example.com", "George", Contact::Type::Cc} + }}; + const auto actual_contacts = doc.contacts_value(Field::Id::Cc); + + assert_same_contacts(expected_contacts, actual_contacts); +} + + +static void +test_from() +{ + Document doc; + doc.add(Field::Id::From, test_contacts); + + Contacts expected_contacts = {{ + Contact{"james@example.com", "James", Contact::Type::From}, + }}; + const auto actual_contacts = doc.contacts_value(Field::Id::From); + + assert_same_contacts(expected_contacts, actual_contacts); +} + +static void +test_to() +{ + Document doc; + doc.add(Field::Id::To, test_contacts); + + Contacts expected_contacts = {{ + Contact{"lars@example.com", "Lars", Contact::Type::To}, + Contact{"kirk@example.com", "Kirk", Contact::Type::To}, + Contact{"jason@example.com", "Jason", Contact::Type::To} + }}; + const auto actual_contacts = doc.contacts_value(Field::Id::To); + + assert_same_contacts(expected_contacts, actual_contacts); +} + + +static void +test_size() +{ + { + Document doc; + doc.add(Field::Id::Size, 12345); + g_assert_cmpuint(doc.integer_value(Field::Id::Size),==,12345); + } + + { + Document doc; + g_assert_cmpuint(doc.integer_value(Field::Id::Size),==,0); + } +} + + +int +main(int argc, char* argv[]) +{ + g_test_init(&argc, &argv, NULL); + + g_test_add_func("/message/document/bcc", test_bcc); + g_test_add_func("/message/document/cc", test_cc); + g_test_add_func("/message/document/from", test_from); + g_test_add_func("/message/document/to", test_to); + + g_test_add_func("/message/document/size", test_size); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/message/mu-document.hh b/lib/message/mu-document.hh new file mode 100644 index 0000000..5119044 --- /dev/null +++ b/lib/message/mu-document.hh @@ -0,0 +1,261 @@ +/** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_DOCUMENT_HH__ +#define MU_DOCUMENT_HH__ + +#include <utility> +#include <string> +#include <vector> + +#include "mu-xapian-db.hh" +#include "mu-fields.hh" +#include "mu-priority.hh" +#include "mu-flags.hh" +#include "mu-contact.hh" +#include <utils/mu-option.hh> +#include <utils/mu-sexp.hh> + +namespace Mu { + +/** + * A Document describes the information about a message that is + * or can be stored in the database. + * + */ +class Document { +public: + enum struct Options { + None = 0, + SupportNgrams = 1 << 0, /**< Support ngrams, as used in + * CJK and other languages. */ + }; + + /** + * Construct a message for a new Xapian Document + * + * @param flags behavioral flags + */ + Document(Options opts = Options::None): options_{opts} {} + + /** + * Construct a message document based on an existing Xapian document. + * + * @param doc + * @param flags behavioral flags + */ + Document(const Xapian::Document& doc, Options opts = Options::None): + xdoc_{doc}, options_{opts} {} + + /** + * DTOR + */ + ~Document() { + xapian_document(); // for side-effect up updating sexp. + } + + /** + * Get a reference to the underlying Xapian document. + * + */ + const Xapian::Document& xapian_document() const; + + /** + * Get the doc-id for this document + * + * @return the docid + */ + Xapian::docid docid() const { return xdoc_.get_docid(); } + + /* + * updating a document with terms & values + */ + + /** + * Add a string value to the document + * + * @param field_id field id + * @param val string value + */ + void add(Field::Id field_id, const std::string& val); + + /** + * Add a string-vec value to the document, if non-empty + * + * @param field_id field id + * @param val string-vec value + */ + void add(Field::Id field_id, const std::vector<std::string>& vals); + + + /** + * Add message-contacts to the document, if non-empty + * + * @param field_id field id + * @param contacts message contacts + */ + void add(Field::Id id, const Contacts& contacts); + + /** + * Add some extra contacts with the given propname; this is useful for + * ":reply-to" and ":list-post" which don't have a Field::Id and are + * only present in the sexp, not in the terms/values + * + * @param propname property name (e.g.,. ":reply-to") + * @param contacts contacts for this property. + */ + void add_extra_contacts(const std::string& propname, + const Contacts& contacts); + + /** + * Add an integer value to the document + * + * @param field_id field id + * @param val integer value + */ + void add(Field::Id field_id, int64_t val); + + /** + * Add a message priority to the document + * + * @param prio priority + */ + void add(Priority prio); + + + /** + * Add message flags to the document + * + * @param flags mesage flags. + */ + void add(Flags flags); + + /** + * Remove values and terms for some field. + * + * @param field_id + */ + void remove(Field::Id field_id); + + /** + * Get the cached s-expression + * + * @return the cached s-expression + */ + const Sexp& sexp() const { return cached_sexp(); } + + /** + * Get the message s-expression as a string + * + * @return message s-expression string + */ + std::string sexp_str() const { return xdoc_.get_data(); } + + /** + * Generically adds an optional value, if set, to the document + * + * @param id the field 0d + * @param an optional value + */ + template<typename T> void add(Field::Id id, const Option<T>& val) { + if (val) + add(id, val.value()); + } + + /* + * Retrieving values + */ + + /** + * Get a message-field as a string-value + * + * @param field_id id of the field to get. + * + * @return a string (empty if not found) + */ + std::string string_value(Field::Id field_id) const noexcept { + return xapian_try([&]{ + return xdoc_.get_value(field_from_id(field_id).value_no()); + }, std::string{}); + } + + /** + * Get a vec of string values. + * + * @param field_id id of the field to get + * + * @return a string list + */ + std::vector<std::string> string_vec_value(Field::Id field_id) const noexcept; + + + /** + * Get an integer value + * + * @param field_id id of the field to get + * + * @return an integer or 0 if not found. + */ + int64_t integer_value(Field::Id field_id) const noexcept; + + + /** + * Get contacts + * + * @param field_id id of the contacts field to get + * + * @return a contacts list + */ + Contacts contacts_value(Field::Id id) const noexcept; + + /** + * Get the priority + * + * @return the message priority + */ + Priority priority_value() const noexcept; + + /** + * Get the message flags + * + * + * @return flags + */ + Flags flags_value() const noexcept; + +private: + template<typename SexpType> void put_prop(const Field& field, SexpType&& val); + template<typename SexpType> void put_prop(const std::string& pname, SexpType&& val); + + Sexp& cached_sexp() const { + if (cached_sexp_.empty()) + if (auto&& s{Sexp::parse(xdoc_.get_data())}; s) + cached_sexp_ = std::move(*s); + return cached_sexp_; + } + + mutable Xapian::Document xdoc_; + Options options_; + mutable Sexp cached_sexp_; + mutable bool dirty_sexp_{}; /* xdoc's sexp is outdated */ +}; +MU_ENABLE_BITOPS(Document::Options); + +} // namepace Mu + +#endif /* MU_DOCUMENT_HH__ */ diff --git a/lib/message/mu-fields.cc b/lib/message/mu-fields.cc new file mode 100644 index 0000000..f64df5f --- /dev/null +++ b/lib/message/mu-fields.cc @@ -0,0 +1,194 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-fields.hh" +#include "mu-flags.hh" + +#include "utils/mu-test-utils.hh" + +using namespace Mu; + +std::string +Field::xapian_term(const std::string& s) const +{ + const auto start{std::string(1U, xapian_prefix())}; + if (const auto& size = s.size(); size == 0) + return start; + + std::string res{start}; + res.reserve(s.size() + 10); + + /* slightly optimized common pure-ascii. */ + if (G_LIKELY(g_str_is_ascii(s.c_str()))) { + res += s; + for (auto i = 1; res[i]; ++i) + res[i] = g_ascii_tolower(res[i]); + } else + res += utf8_flatten(s); + + if (G_UNLIKELY(res.size() > MaxTermLength)) + res.erase(MaxTermLength); + + return res; +} + +/** + * compile-time checks + */ +constexpr bool +validate_field_ids() +{ + for (auto id = 0U; id != Field::id_size(); ++id) { + const auto field_id = static_cast<Field::Id>(id); + if (field_from_id(field_id).id != field_id) + return false; + } + return true; +} + +constexpr bool +validate_field_shortcuts() +{ +#ifdef BUILD_TESTS + std::array<size_t, 26> no_dups = {0}; +#endif /*BUILD_TESTS*/ + for (auto id = 0U; id != Field::id_size(); ++id) { + const auto field_id = static_cast<Field::Id>(id); + const auto shortcut = field_from_id(field_id).shortcut; + if (shortcut != 0 && + (shortcut < 'a' || shortcut > 'z')) + return false; +#ifdef BUILD_TESTS + if (shortcut != 0) { + if (++no_dups[static_cast<size_t>(shortcut-'a')] > 1) { + mu_critical("shortcut '{}' is duplicated", shortcut); + return false; + } + } +#endif + } + + return true; +} + + +constexpr /*static*/ bool +validate_field_flags() +{ + for (auto&& field: Fields) { + /* - A field has at most one of Phrasable, Boolean */ + size_t flagnum{}; + + if (field.is_phrasable_term()) + ++flagnum; + if (field.is_boolean_term()) + ++flagnum; + + if (flagnum > 1) { + //mu_warning("invalid field {}", field.name); + return false; + } + } + + return true; +} + +/* + * tests... also build as runtime-tests, so we can get coverage info + */ +#ifdef BUILD_TESTS +#define static_assert g_assert_true +#endif /*BUILD_TESTS*/ + + +[[maybe_unused]] +static void +test_ids() +{ + static_assert(validate_field_ids()); +} + +[[maybe_unused]] +static void +test_shortcuts() +{ + static_assert(validate_field_shortcuts()); +} + +[[maybe_unused]] +static void +test_prefix() +{ + static_assert(field_from_id(Field::Id::Subject).xapian_prefix() == 'S'); +} + +[[maybe_unused]] +static void +test_field_flags() +{ + static_assert(validate_field_flags()); +} + +#ifdef BUILD_TESTS + + +static void +test_field_from_name() +{ + g_assert_true(field_from_name("s")->id == Field::Id::Subject); + g_assert_true(field_from_name("subject")->id == Field::Id::Subject); + g_assert_false(!!field_from_name("8")); + g_assert_false(!!field_from_name("")); + + g_assert_true(field_from_name("").value_or(field_from_id(Field::Id::Bcc)).id == + Field::Id::Bcc); +} + +static void +test_xapian_term() +{ + using namespace std::string_literals; + using namespace std::literals; + + assert_equal(field_from_id(Field::Id::Subject).xapian_term(""s), "S"); + assert_equal(field_from_id(Field::Id::Subject).xapian_term("boo"s), "Sboo"); + + assert_equal(field_from_id(Field::Id::From).xapian_term('x'), "Fx"); + assert_equal(field_from_id(Field::Id::To).xapian_term("boo"sv), "Tboo"); + + auto s1 = field_from_id(Field::Id::Subject).xapian_term(std::string(MaxTermLength - 1, 'x')); + auto s2 = field_from_id(Field::Id::Subject).xapian_term(std::string(MaxTermLength, 'x')); + g_assert_cmpuint(s1.length(), ==, s2.length()); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/message/fields/ids", test_ids); + g_test_add_func("/message/fields/shortcuts", test_shortcuts); + g_test_add_func("/message/fields/from-name", test_field_from_name); + g_test_add_func("/message/fields/prefix", test_prefix); + g_test_add_func("/message/fields/xapian-term", test_xapian_term); + g_test_add_func("/message/fields/flags", test_field_flags); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/message/mu-fields.hh b/lib/message/mu-fields.hh new file mode 100644 index 0000000..19a222b --- /dev/null +++ b/lib/message/mu-fields.hh @@ -0,0 +1,605 @@ +/* +** Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_FIELDS_HH__ +#define MU_FIELDS_HH__ + +#include <cstdint> +#include <string_view> +#include <algorithm> +#include <array> +#include <mu-xapian-db.hh> +#include <utils/mu-utils.hh> +#include <utils/mu-option.hh> + +namespace Mu { + +// Xapian does not like terms much longer than this +constexpr auto MaxTermLength = 240; +// http://article.gmane.org/gmane.comp.search.xapian.general/3656 */ + +struct Field { + /** + * Field Ids. + * + * Note, the Ids are also used as indices in the Fields array, + * so their numerical values must be 0...Count. + * + */ + enum struct Id { + Bcc = 0, /**< Blind Carbon-Copy */ + BodyText, /**< Text body */ + Cc, /**< Carbon-Copy */ + Changed, /**< Last change time (think 'ctime') */ + Date, /**< Message date */ + EmbeddedText, /**< Embedded text in message */ + File, /**< Filename */ + Flags, /**< Message flags */ + From, /**< Message sender */ + Language, /**< Body language */ + Maildir, /**< Maildir path */ + MailingList, /**< Mailing list */ + MessageId, /**< Message Id */ + MimeType, /**< MIME-Type */ + Path, /**< File-system Path */ + Priority, /**< Message priority */ + References, /**< All references (incl. Reply-To:) */ + Size, /**< Message size (in bytes) */ + Subject, /**< Message subject */ + Tags, /**< Message Tags */ + ThreadId, /**< Thread Id */ + To, /**< To: recipient */ + // + _count_ /**< Number of Ids */ + }; + + /** + * Get the number of Id values. + * + * @return the number. + */ + static constexpr size_t id_size() + { + return static_cast<size_t>(Id::_count_); + } + + constexpr Xapian::valueno value_no() const { + return static_cast<Xapian::valueno>(id); + } + + /** + * Field types + * + */ + enum struct Type { + String, /**< String */ + StringList, /**< List of strings */ + ContactList, /**< List of contacts */ + ByteSize, /**< Size in bytes */ + TimeT, /**< A time_t value */ + Integer, /**< An integer */ + }; + + constexpr bool is_string() const { return type == Type::String; } + constexpr bool is_string_list() const { return type == Type::StringList; } + constexpr bool is_byte_size() const { return type == Type::ByteSize; } + constexpr bool is_time_t() const { return type == Type::TimeT; } + constexpr bool is_integer() const { return type == Type::Integer; } + constexpr bool is_numerical() const { return is_byte_size() || is_time_t() || is_integer(); } + + /** + * Field flags + * note: the differences for our purposes between a xapian field and a + * term: - there is only a single value for some item in per document + * (msg), ie. one value containing the list of To: addresses - there + * can be multiple terms, each containing e.g. one of the To: + * addresses - searching uses terms, but to display some field, it + * must be in the value + * + * Rules (build-time enforced): + * - A field has at most one of PhrasableTerm, BooleanTerm, ContactTerm. + */ + + enum struct Flag { + /* + * Different kind of terms; at most one is true, and cannot be combined with + * Contact. Compile-time enforced. + */ + NormalTerm = 1 << 0, + /**< Field is a searchable term */ + BooleanTerm = 1 << 1, + /**< Field is a boolean search-term (i.e. at most one per message); + * wildcards do not work */ + PhrasableTerm = 1 << 2, + /**< Field has phrasable/indexable text as term */ + /* + * Contact flag cannot be combined with any of the term flags. + * This is compile-time enforced. + */ + Contact = 1 << 10, + /**< field contains one or more e-mail-addresses */ + Value = 1 << 11, + /**< Field value is stored (so the literal value can be retrieved) */ + + Range = 1 << 21, + + IncludeInSexp = 1 << 24, + /**< whether to include this field in the cached sexp. */ + + /**< whether this is a range field (e.g., date, size)*/ + Internal = 1 << 26 + }; + + constexpr bool any_of(Flag some_flag) const{ + return (static_cast<int>(some_flag) & static_cast<int>(flags)) != 0; + } + + constexpr bool is_phrasable_term() const { return any_of(Flag::PhrasableTerm); } + constexpr bool is_boolean_term() const { return any_of(Flag::BooleanTerm); } + constexpr bool is_normal_term() const { return any_of(Flag::NormalTerm); } + constexpr bool is_searchable() const { return is_phrasable_term() || + is_boolean_term() || + is_normal_term(); } + constexpr bool is_sortable() const { return is_value(); } + + + constexpr bool is_value() const { return any_of(Flag::Value); } + constexpr bool is_internal() const { return any_of(Flag::Internal); } + + constexpr bool is_contact() const { return any_of(Flag::Contact); } + constexpr bool is_range() const { return any_of(Flag::Range); } + + constexpr bool include_in_sexp() const { return any_of(Flag::IncludeInSexp);} + + /** + * Field members + * + */ + Id id; /**< Id of the message field */ + Type type; /**< Type of the message field */ + std::string_view name; /**< Name of the message field */ + std::string_view alias; /**< Alternative name for the message field */ + std::string_view description; /**< Decription of the message field */ + std::string_view example_query; /**< Example query */ + char shortcut; /**< Shortcut for the message field; a..z */ + Flag flags; /**< Flags */ + + /** + * Convenience / helpers + * + */ + + constexpr char xapian_prefix() const { + /* xapian uses uppercase shortcuts; toupper is not constexpr */ + return shortcut == 0 ? 0 : shortcut - ('a' - 'A'); + } + + /** + * Get the xapian term; truncated to MaxTermLength and + * utf8-flattened. + * + * @param s + * + * @return the xapian term + */ + std::string xapian_term(const std::string& s="") const; + std::string xapian_term(std::string_view sv) const { + return xapian_term(std::string{sv}); + } + std::string xapian_term(char c) const { + return xapian_term(std::string(1, c)); + } +}; + +// equality +static inline constexpr bool operator==(const Field& f1, const Field& f2) { return f1.id == f2.id; } +static inline constexpr bool operator==(const Field& f1, const Field::Id id) { return f1.id == id; } + + +MU_ENABLE_BITOPS(Field::Flag); + +/** + * Sequence of _all_ message fields + */ +static constexpr std::array<Field, Field::id_size()> + Fields = { + { + { + Field::Id::Bcc, + Field::Type::ContactList, + "bcc", {}, + "Blind carbon-copy recipient", + "bcc:foo@example.com", + 'h', + Field::Flag::Contact | + Field::Flag::Value | + Field::Flag::IncludeInSexp | + Field::Flag::NormalTerm | + Field::Flag::PhrasableTerm, + }, + { + Field::Id::BodyText, + Field::Type::String, + "body", {}, + "Message plain-text body", + "body:capybara", + 'b', + Field::Flag::PhrasableTerm, + }, + { + Field::Id::Cc, + Field::Type::ContactList, + "cc", {}, + "Carbon-copy recipient", + "cc:quinn@example.com", + 'c', + Field::Flag::Contact | + Field::Flag::Value | + Field::Flag::IncludeInSexp | + Field::Flag::NormalTerm | + Field::Flag::PhrasableTerm, + }, + { + Field::Id::Changed, + Field::Type::TimeT, + "changed", {}, + "Last change time", + "changed:30M..", + 'k', + Field::Flag::Value | + Field::Flag::Range | + Field::Flag::IncludeInSexp + }, + { + Field::Id::Date, + Field::Type::TimeT, + "date", {}, + "Message date", + "date:20220101..20220505", + 'd', + Field::Flag::Value | + Field::Flag::Range | + Field::Flag::IncludeInSexp + }, + { + Field::Id::EmbeddedText, + Field::Type::String, + "embed", {}, + "Embedded text", + "embed:war OR embed:peace", + 'e', + Field::Flag::PhrasableTerm + }, + { + Field::Id::File, + Field::Type::String, + "file", {}, + "Attachment file name", + "file:/image\\.*.jpg/", + 'j', + Field::Flag::BooleanTerm + }, + { + Field::Id::Flags, + Field::Type::Integer, + "flags", "flag", + "Message properties", + "flag:unread AND flag:personal", + 'g', + Field::Flag::BooleanTerm | + Field::Flag::Value | + Field::Flag::IncludeInSexp + }, + { + Field::Id::From, + Field::Type::ContactList, + "from", {}, + "Message sender", + "from:jimbo", + 'f', + Field::Flag::Contact | + Field::Flag::Value | + Field::Flag::IncludeInSexp | + Field::Flag::NormalTerm | + Field::Flag::PhrasableTerm, + }, + { + Field::Id::Language, + Field::Type::String, + "language", "lang", + "ISO 639-1 language code for body", + "lang:nl", + 'a', + Field::Flag::BooleanTerm | + Field::Flag::Value | + Field::Flag::IncludeInSexp + }, + { + Field::Id::Maildir, + Field::Type::String, + "maildir", {}, + "Maildir path for message", + "maildir:/private/archive", + 'm', + Field::Flag::BooleanTerm | + Field::Flag::Value | + Field::Flag::IncludeInSexp + }, + { + Field::Id::MailingList, + Field::Type::String, + "list", {}, + "Mailing list (List-Id:)", + "list:mu-discuss.example.com", + 'v', + Field::Flag::BooleanTerm | + Field::Flag::Value | + Field::Flag::IncludeInSexp + }, + { + Field::Id::MessageId, + Field::Type::String, + "message-id", "msgid", + "Message-Id", + "msgid:abc@123", + 'i', + Field::Flag::BooleanTerm | + Field::Flag::Value | + Field::Flag::IncludeInSexp + }, + { + Field::Id::MimeType, + Field::Type::String, + "mime", "mime-type", + "Attachment MIME-type", + "mime:image/jpeg", + 'y', + Field::Flag::BooleanTerm + }, + { + Field::Id::Path, + Field::Type::String, + "path", {}, + "File system path to message", + "path:/a/b/Maildir/cur/msg:2,S", + 'l', + Field::Flag::BooleanTerm | + Field::Flag::Value | + Field::Flag::IncludeInSexp + }, + { + Field::Id::Priority, + Field::Type::Integer, + "priority", "prio", + "Priority", + "prio:high", + 'p', + Field::Flag::BooleanTerm | + Field::Flag::Value | + Field::Flag::IncludeInSexp + }, + { + Field::Id::References, + Field::Type::StringList, + "references", {}, + "References to related messages", + {}, + 'r', + Field::Flag::Value | + Field::Flag::IncludeInSexp + }, + { + Field::Id::Size, + Field::Type::ByteSize, + "size", {}, + "Message size in bytes", + "size:1M..5M", + 'z', + Field::Flag::Value | + Field::Flag::Range | + Field::Flag::IncludeInSexp + }, + { + Field::Id::Subject, + Field::Type::String, + "subject", {}, + "Message subject", + "subject:wombat", + 's', + Field::Flag::Value | + Field::Flag::IncludeInSexp | + Field::Flag::NormalTerm | + Field::Flag::PhrasableTerm + }, + { + Field::Id::Tags, + Field::Type::StringList, + "tags", "tag", + "Message tags", + "tag:projectx", + 'x', + Field::Flag::BooleanTerm | + Field::Flag::Value | + Field::Flag::IncludeInSexp + }, + { + Field::Id::ThreadId, + Field::Type::String, + "thread", {}, + "Thread a message belongs to", + {}, + 'w', + Field::Flag::BooleanTerm | + Field::Flag::Value + }, + { + Field::Id::To, + Field::Type::ContactList, + "to", {}, + "Message recipient", + "to:flimflam@example.com", + 't', + Field::Flag::Contact | + Field::Flag::Value | + Field::Flag::IncludeInSexp | + Field::Flag::NormalTerm | + Field::Flag::PhrasableTerm, + }, + }}; + +/* + * Convenience + */ + +/** + * Get the message field for the given Id. + * + * @param id of the message field + * + * @return ref of the message field. + */ +constexpr const Field& +field_from_id(Field::Id id) +{ + return Fields.at(static_cast<size_t>(id)); +} + +/** + * Invoke func for each message-field + * + * @param func some callable + */ +template <typename Func> +constexpr void field_for_each(Func&& func) { + for (const auto& field: Fields) + func(field); +} + +/** + * Find a message field that satisfies some predicate + * + * @param pred the predicate (a callable) + * + * @return a message-field id, or nullopt if not found. + */ +template <typename Pred> +constexpr Option<Field> field_find_if(Pred&& pred) { + for (auto&& field: Fields) + if (pred(field)) + return field; + return Nothing; +} + +/** + * Get the the message-field for the given name or shortcut + * + * @param name_or_shortcut + * + * @return the message-field or Nothing + */ +static inline +Option<Field> field_from_shortcut(char shortcut) { + return field_find_if([&](auto&& field){ + return field.shortcut == shortcut; + }); +} +static inline +Option<Field> field_from_name(const std::string& name) { + switch(name.length()) { + case 0: + return Nothing; + case 1: + return field_from_shortcut(name[0]); + default: + return field_find_if([&](auto&& field){ + return name == field.name || name == field.alias; + }); + } +} + +/** + * Return combination-fields such + * as "contact", "recip" and "" (empty) + * + * @param name combination field name + * + * @return list of matching fields + */ +using FieldsVec = std::vector<Field>; +static inline +const FieldsVec& fields_from_name(const std::string& name) { + + static const FieldsVec none{}; + static const FieldsVec recip_fields ={ + field_from_id(Field::Id::To), + field_from_id(Field::Id::Cc), + field_from_id(Field::Id::Bcc)}; + + static const FieldsVec contact_fields = { + field_from_id(Field::Id::To), + field_from_id(Field::Id::Cc), + field_from_id(Field::Id::Bcc), + field_from_id(Field::Id::From), + }; + static const FieldsVec empty_fields= { + field_from_id(Field::Id::To), + field_from_id(Field::Id::Cc), + field_from_id(Field::Id::Bcc), + field_from_id(Field::Id::From), + field_from_id(Field::Id::Subject), + field_from_id(Field::Id::BodyText), + field_from_id(Field::Id::EmbeddedText), + }; + + if (name == "recip") + return recip_fields; + else if (name == "contact") + return contact_fields; + else if (name.empty()) + return empty_fields; + else + return none; +} + +static inline bool +field_is_combi (const std::string& name) +{ + return name == "recip" || name == "contact"; +} + + +/** + * Get the Field::Id for some number, or nullopt if it does not match + * + * @param id an id number + * + * @return Field::Id or nullopt + */ +static inline +Option<Field> field_from_number(size_t id) +{ + if (id >= static_cast<size_t>(Field::Id::_count_)) + return Nothing; + else + return field_from_id(static_cast<Field::Id>(id)); +} + + +} // namespace Mu +#endif /* MU_FIELDS_HH__ */ diff --git a/lib/message/mu-flags.cc b/lib/message/mu-flags.cc new file mode 100644 index 0000000..7ff340e --- /dev/null +++ b/lib/message/mu-flags.cc @@ -0,0 +1,196 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +/* + * implementation is almost completely in the header; here we just add some + * compile-time tests. + */ + +#include "mu-flags.hh" + +using namespace Mu; + +std::string +Mu::to_string(Flags flags) +{ + std::string str; + + for (auto&& info: AllMessageFlagInfos) + if (any_of(info.flag & flags)) + str+=info.shortcut; + + return str; +} + + +/* + * flags & flag-info + */ +constexpr bool +validate_message_info_flags() +{ + for (auto id = 0U; id != AllMessageFlagInfos.size(); ++id) { + const auto flag = static_cast<Flags>(1 << id); + if (flag != AllMessageFlagInfos[id].flag) + return false; + } + return true; +} + + +/* + * tests... also build as runtime-tests, so we can get coverage info + */ +#ifdef BUILD_TESTS +#define static_assert g_assert_true +#endif /*BUILD_TESTS*/ + +[[maybe_unused]] static void +test_basic() +{ + static_assert(AllMessageFlagInfos.size() == + __builtin_ctz(static_cast<unsigned>(Flags::_final_))); + static_assert(validate_message_info_flags()); + + static_assert(!!flag_info(Flags::Encrypted)); + static_assert(!flag_info(Flags::None)); + static_assert(!flag_info(static_cast<Flags>(0))); + static_assert(!flag_info(static_cast<Flags>(1<<AllMessageFlagInfos.size()))); +} + +/* + * flag_info + */ +[[maybe_unused]] static void +test_flag_info() +{ + static_assert(flag_info('D')->flag == Flags::Draft); + static_assert(flag_info('l')->flag == Flags::MailingList); + static_assert(!flag_info('y')); + + static_assert(flag_info("trashed")->flag == Flags::Trashed); + static_assert(flag_info("attach")->flag == Flags::HasAttachment); + static_assert(!flag_info("fnorb")); + + + static_assert(flag_info('D')->shortcut_lower() == 'd'); + static_assert(flag_info('u')->shortcut_lower() == 'u'); +} + +/* + * flags_from_expr + */ +[[maybe_unused]] static void +test_flags_from_expr() +{ + static_assert(flags_from_absolute_expr("SRP").value() == + (Flags::Seen | Flags::Replied | Flags::Passed)); + static_assert(flags_from_absolute_expr("Faul").value() == + (Flags::Flagged | Flags::Unread | + Flags::HasAttachment | Flags::MailingList)); + + /* note: unread is a special flag, _implied_ from "new or not seen" */ + static_assert(flags_from_absolute_expr("N").value() == (Flags::New|Flags::Unread)); + + static_assert(!flags_from_absolute_expr("DRT?")); + static_assert(flags_from_absolute_expr("DRT?", true/*ignore invalid*/).value() == + (Flags::Draft | Flags::Replied | + Flags::Trashed | Flags::Unread)); + static_assert(flags_from_absolute_expr("DFPNxulabcdef", true/*ignore invalid*/).value() == + (Flags::Draft|Flags::Flagged|Flags::Passed| + Flags::New | Flags::Encrypted | + Flags::Unread | Flags::MailingList | Flags::Calendar | + Flags::HasAttachment)); +} + + +/* + * flags_from_delta_expr + */ +[[maybe_unused]] static void +test_flags_from_delta_expr() +{ + static_assert(flags_from_delta_expr( + "+S-u-N", Flags::New|Flags::Unread).value() == + Flags::Seen); + + /* note: unread is a special flag, _implied_ from "new or not seen" */ + static_assert(flags_from_delta_expr( + "+S-N", Flags::New|Flags::Unread).value() == + Flags::Seen); + static_assert(flags_from_delta_expr( + "-S", Flags::Seen).value() == + Flags::Unread); + + static_assert(flags_from_delta_expr("+R+P-F", Flags::Seen).value() == + (Flags::Seen|Flags::Passed|Flags::Replied)); + /* '-B' is invalid */ + static_assert(!flags_from_delta_expr("+R+P-B", Flags::Seen)); + /* '-B' is invalid, but ignore invalid */ + static_assert(flags_from_delta_expr("+R+P-B", Flags::Seen, true) == + (Flags::Replied|Flags::Passed|Flags::Seen)); + static_assert(flags_from_delta_expr("+F+T-S", Flags::None, true).value() == + (Flags::Flagged|Flags::Trashed|Flags::Unread)); +} + +/* + * flags_filter + */ +[[maybe_unused]] static void +test_flags_filter() +{ + static_assert(flags_filter(flags_from_absolute_expr( + "DFPNxulabcdef", true/*ignore invalid*/).value(), + MessageFlagCategory::Mailfile) == + (Flags::Draft|Flags::Flagged|Flags::Passed)); +} + + + +[[maybe_unused]] static void +test_flags_keep_unmutable() +{ + static_assert(flags_keep_unmutable((Flags::Seen|Flags::Passed), + (Flags::Flagged|Flags::Draft), + Flags::Replied) == + (Flags::Flagged|Flags::Draft)); +} + + + +#ifdef BUILD_TESTS +int +main(int argc, char* argv[]) +{ + g_test_init(&argc, &argv, NULL); + + g_test_add_func("/message/flags/basic", test_basic); + g_test_add_func("/message/flags/flag-info", test_flag_info); + g_test_add_func("/message/flags/flags-from-absolute-expr", + test_flags_from_expr); + g_test_add_func("/message/flags/flags-from-delta-expr", + test_flags_from_delta_expr); + g_test_add_func("/message/flags/flags-filter", + test_flags_filter); + g_test_add_func("/message/flags/flags-keep-unmutable", + test_flags_keep_unmutable); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/message/mu-flags.hh b/lib/message/mu-flags.hh new file mode 100644 index 0000000..8e424dd --- /dev/null +++ b/lib/message/mu-flags.hh @@ -0,0 +1,422 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_FLAGS_HH__ +#define MU_FLAGS_HH__ + +#include <algorithm> +#include <string_view> +#include <array> +#include <utils/mu-utils.hh> +#include <utils/mu-option.hh> + +namespace Mu { + +enum struct Flags { + None = 0, /**< No flags */ + /** + * next 6 are seen in the file-info part of maildir message file + * names, ie., in a name like "1234345346:2,<fileinfo>", + * <fileinfo> consists of zero or more of the following + * characters (in ascii order) + */ + Draft = 1 << 0, /**< A draft message */ + Flagged = 1 << 1, /**< A flagged message */ + Passed = 1 << 2, /**< A passed (forwarded) message */ + Replied = 1 << 3, /**< A replied message */ + Seen = 1 << 4, /**< A seen (read) message */ + Trashed = 1 << 5, /**< A trashed message */ + + /** + * decides on cur/ or new/ in the maildir + */ + New = 1 << 6, /**< A new message */ + + /** + * content flags -- not visible in the filename, but used for + * searching + */ + Signed = 1 << 7, /**< Cryptographically signed */ + Encrypted = 1 << 8, /**< Encrypted */ + HasAttachment = 1 << 9, /**< Has an attachment */ + + Unread = 1 << 10, /**< Unread; pseudo-flag, only for queries, so we can + * search for flag:unread, which is equivalent to + * 'flag:new OR NOT flag:seen' */ + /** + * other content flags + */ + MailingList = 1 << 11, /**< A mailing-list message */ + Personal = 1 << 12, /**< A personal message (i.e., at least one of the + * contact fields contains a personal address) */ + Calendar = 1 << 13, /**< A calendar invitation */ + /* + * <private> + */ + _final_ = 1 << 14 +}; +MU_ENABLE_BITOPS(Flags); + +/** + * Message flags category + * + */ +enum struct MessageFlagCategory { + None, /**< Nothing */ + Mailfile, /**< Flag for a message file */ + Maildir, /**< Flag for message file's location */ + Content, /**< Message content flag */ + Pseudo /**< Pseudo flag */ +}; + +/** + * Info about invidual message flags + * + */ +struct MessageFlagInfo { + + Flags flag; /**< The message flag */ + char shortcut; /**< Shortcut character; + * tolower(shortcut) must be + * unique for all flags */ + std::string_view name; /**< Name of the flag */ + MessageFlagCategory category; /**< Flag category */ + std::string_view description; /**< Description */ + + /** + * Get the lower-case version of shortcut + * + * @return lower-case shortcut + */ + constexpr char shortcut_lower() const { + return shortcut >= 'A' && shortcut <= 'Z' ? + shortcut + ('a' - 'A') : shortcut; + } +}; + +/** + * Array of all flag information. + */ +constexpr std::array<MessageFlagInfo, 14> AllMessageFlagInfos = {{ + MessageFlagInfo{Flags::Draft, 'D', "draft", MessageFlagCategory::Mailfile, + "Draft (in progress)" + }, + MessageFlagInfo{Flags::Flagged, 'F', "flagged", MessageFlagCategory::Mailfile, + "User-flagged" + }, + MessageFlagInfo{Flags::Passed, 'P', "passed", MessageFlagCategory::Mailfile, + "Forwarded message" + }, + MessageFlagInfo{Flags::Replied, 'R', "replied", MessageFlagCategory::Mailfile, + "Replied-to" + }, + MessageFlagInfo{Flags::Seen, 'S', "seen", MessageFlagCategory::Mailfile, + "Viewed at least once" + }, + MessageFlagInfo{Flags::Trashed, 'T', "trashed", MessageFlagCategory::Mailfile, + "Marked for deletion" + }, + MessageFlagInfo{Flags::New, 'N', "new", MessageFlagCategory::Maildir, + "New message" + }, + MessageFlagInfo{Flags::Signed, 'z', "signed", MessageFlagCategory::Content, + "Cryptographically signed" + }, + MessageFlagInfo{Flags::Encrypted, 'x', "encrypted", MessageFlagCategory::Content, + "Encrypted" + }, + MessageFlagInfo{Flags::HasAttachment,'a', "attach", MessageFlagCategory::Content, + "Has at least one attachment" + }, + MessageFlagInfo{Flags::Unread, 'u', "unread", MessageFlagCategory::Pseudo, + "New or not seen message" + }, + MessageFlagInfo{Flags::MailingList, 'l', "list", MessageFlagCategory::Content, + "Mailing list message" + }, + MessageFlagInfo{Flags::Personal, 'q', "personal", MessageFlagCategory::Content, + "Personal message" + }, + MessageFlagInfo{Flags::Calendar, 'c', "calendar", MessageFlagCategory::Content, + "Calendar invitation" + }, +}}; + + +/** + * Invoke some callable Func for each flag info + * + * @param func some callable + */ +template<typename Func> +constexpr void flag_infos_for_each(Func&& func) +{ + for (auto&& info: AllMessageFlagInfos) + func(info); +} + +/** + * Get flag info for some flag + * + * @param flag a singular flag + * + * @return the MessageFlagInfo, or Nothing in case of error. + */ +constexpr const Option<MessageFlagInfo> +flag_info(Flags flag) +{ + constexpr auto upper = static_cast<unsigned>(Flags::_final_); + const auto val = static_cast<unsigned>(flag); + + if (__builtin_popcount(val) != 1 || val >= upper) + return Nothing; + + return AllMessageFlagInfos[static_cast<unsigned>(__builtin_ctz(val))]; +} + +/** + * Get flag info for some flag + * + * @param shortcut shortcut character + * + * @return the MessageFlagInfo + */ +constexpr const Option<MessageFlagInfo> +flag_info(char shortcut) +{ + for (auto&& info : AllMessageFlagInfos) + if (info.shortcut == shortcut) + return info; + + return Nothing; +} + +/** + * Get flag info for some flag, either by its name of is shortcut + * + * @param name the name of the message-flag, or its shortcut + * + * @return the MessageFlagInfo or Nothing if not found + */ +constexpr const Option<MessageFlagInfo> +flag_info(std::string_view name) +{ + if (name.empty()) + return Nothing; + + for (auto&& info : AllMessageFlagInfos) + if (info.name == name) + return info; + + return flag_info(name.at(0)); +} + +/** + * 'unread' is a pseudo-flag that means 'new or not seen' + * + * @param flags + * + * @return flags with unread added or removed. + */ +constexpr Flags +imply_unread(Flags flags) +{ + /* unread is a pseudo flag equivalent to 'new or not seen' */ + if (any_of(flags & Flags::New) || none_of(flags & Flags::Seen)) + return flags | Flags::Unread; + else + return flags & ~Flags::Unread; +} + +/** + * There are two string-based expression types for flags: + * 1) 'absolute': replace the existing flags + * 2) 'delta' : flags as a delta of existing flags. + */ + +/** + * Get the (OR'ed) flags corresponding to an expression. + * + * @param expr the expression (a sequence of flag shortcut characters) + * @param ignore_invalid if @true, ignore invalid flags, otherwise return + * nullopt if an invalid flag is encountered + * + * @return the (OR'ed) flags or Flags::None + */ +constexpr Option<Flags> +flags_from_absolute_expr(std::string_view expr, bool ignore_invalid = false) +{ + Flags flags{Flags::None}; + + for (auto&& kar : expr) { + if (const auto& info{flag_info(kar)}; !info) { + if (!ignore_invalid) + return Nothing; + } else + flags |= info->flag; + } + + return imply_unread(flags); +} + +/** + * Calculate flags from existing flags and a delta expression + * + * Update @p flags with the flags in @p expr, where @p exprt consists of the the + * normal flag shortcut characters, prefixed with either '+' or '-', which means + * resp. "add this flag" or "remove this flag". + * + * So, e.g. "-N+S" would unset the NEW flag and set the SEEN flag, without + * affecting other flags. + * + * @param expr delta expression + * @param flags existing flags + * @param ignore_invalid if @true, ignore invalid flags, otherwise return + * Nothing if an invalid flag is encountered + * + * @return new flags, or Nothing in case of error + */ +constexpr Option<Flags> +flags_from_delta_expr(std::string_view expr, Flags flags, + bool ignore_invalid = false) +{ + if (expr.size() % 2 != 0) + return Nothing; + + for (auto u = 0U; u != expr.size(); u += 2) { + if (const auto& info{flag_info(expr[u + 1])}; !info) { + if (!ignore_invalid) + return Nothing; + } else { + switch (expr[u]) { + case '+': flags |= info->flag; break; + case '-': flags &= ~info->flag; break; + default: + if (!ignore_invalid) + return Nothing; + break; + } + } + } + + return imply_unread(flags); +} + +/** + * Calculate the flags from either 'absolute' or 'delta' expressions + * + * @param expr a flag expression, either 'delta' or 'absolute' + * @param flags optional: existing flags or none. Required for delta. + * + * @return either messages flags or Nothing in case of error. + */ +constexpr Option<Flags> +flags_from_expr(std::string_view expr, Option<Flags> flags = Nothing) +{ + if (expr.empty()) + return Nothing; + + if (expr[0] == '+' || expr[0] == '-') + return flags_from_delta_expr( + expr, flags.value_or(Flags::None), true); + else + return flags_from_absolute_expr(expr, true); +} + +/** + * Filter out flags which are not in the given category + * + * @param flags flags + * @param cat category + * + * @return filtered flags + */ +constexpr Flags +flags_filter(Flags flags, MessageFlagCategory cat) +{ + for (auto&& info : AllMessageFlagInfos) + if (info.category != cat) + flags &= ~info.flag; + return flags; +} + +/** + * Filter out any flags which are _not_ Maildir / Mailfile flags + * + * @param flags flags + * + * @return filtered flags + */ +constexpr Flags +flags_maildir_file(Flags flags) +{ + for (auto&& info : AllMessageFlagInfos) + if (info.category != MessageFlagCategory::Maildir && + info.category != MessageFlagCategory::Mailfile) + flags &= ~info.flag; + return flags; +} + + + + +/** + * Return flags, where flags = new_flags but with unmutable_flag in the + * result the same as in old_flags + * + * @param old_flags + * @param new_flags + * @param immutable_flag + * + * @return + */ +constexpr Flags +flags_keep_unmutable(Flags old_flags, Flags new_flags, Flags immutable_flag) +{ + if (any_of(old_flags & immutable_flag)) + return new_flags | immutable_flag; + else + return new_flags & ~immutable_flag; +} + + +/** + * Get a string representation of flags + * + * @param flags flags + * + * @return string as a sequence of message-flag shortcuts + */ +std::string to_string(Flags flags); + + +/** + * Get a string representation of Flags for fmt + * + * @param flags flags + * + * @return string as a sequence of message-flag shortcuts + */ +static inline auto format_as(const Flags& flags) { + return to_string(flags); +} + +} // namespace Mu + +#endif /* MU_FLAGS_HH__ */ diff --git a/lib/message/mu-message-file.cc b/lib/message/mu-message-file.cc new file mode 100644 index 0000000..b077c3b --- /dev/null +++ b/lib/message/mu-message-file.cc @@ -0,0 +1,198 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb.bulk@gmail.com> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-message-file.hh" +#include "utils/mu-utils-file.hh" + +using namespace Mu; + +Result<std::string> +Mu::maildir_from_path(const std::string& path, const std::string& root) +{ + const auto pos = path.find(root); + if (pos != 0 || path[root.length()] != '/') + return Err(Error{Error::Code::InvalidArgument, + "root '{}' is not a root for path '{}'", root, path}); + + auto mdir{path.substr(root.length())}; + auto slash{mdir.rfind('/')}; + + if (G_UNLIKELY(slash == std::string::npos) || slash < 4) + return Err(Error{Error::Code::InvalidArgument, + "invalid path: {}", path}); + mdir.erase(slash); + auto subdir = mdir.data() + slash - 4; + if (G_UNLIKELY(strncmp(subdir, "/cur", 4) != 0 && strncmp(subdir, "/new", 4))) + return Err(Error::Code::InvalidArgument, + "cannot find '/new' or '/cur' - invalid path: {}", path); + + if (mdir.length() == 4) + return "/"; + + mdir.erase(mdir.length() - 4); + + return Ok(std::move(mdir)); +} + +Mu::FileParts +Mu::message_file_parts(const std::string& file) +{ + const auto pos{file.find_last_of(":!;")}; + + /* no suffix at all? */ + if (pos == std::string::npos || + pos > file.length() - 3 || + file[pos + 1] != '2' || + file[pos + 2] != ',') + return FileParts{ file, ':', {}}; + + return FileParts { + file.substr(0, pos), + file[pos], + file.substr(pos + 3) + }; +} + +Mu::Result<DirFile> +Mu::base_message_dir_file(const std::string& path) +{ + constexpr auto newdir{"/new"}; + + const auto dname{dirname(path)}; + bool is_new{!!g_str_has_suffix(dname.c_str(), newdir)}; + + std::string mdir{dname.substr(0, dname.size() - 4)}; + return Ok(DirFile{std::move(mdir), basename(path), is_new}); +} + +Mu::Result<Mu::Flags> +Mu::flags_from_path(const std::string& path) +{ /* + * this gets us the source maildir filesystem path, the directory + * in which new/ & cur/ lives, and the source file + */ + auto dirfile{base_message_dir_file(path)}; + if (!dirfile) + return Err(std::move(dirfile.error())); + + /* a message under new/ is just.. New. Filename is not considered */ + if (dirfile->is_new) + return Ok(Flags::New); + + /* it's cur/ message, so parse the file name */ + const auto parts{message_file_parts(dirfile->file)}; + auto flags{flags_from_absolute_expr(parts.flags_suffix, + true/*ignore invalid*/)}; + if (!flags) { + /* LCOV_EXCL_START*/ + return Err(Error{Error::Code::InvalidArgument, + "invalid flags ('{}')", parts.flags_suffix}); + /* LCOV_EXCL_STOP*/ + } + + /* of course, only _file_ flags are allowed */ + return Ok(flags_filter(flags.value(), MessageFlagCategory::Mailfile)); +} + + +#ifdef BUILD_TESTS + +#include "utils/mu-test-utils.hh" + +static void +test_maildir_from_path() +{ + std::array<std::tuple<std::string, std::string, std::string>, 1> test_cases = {{ + { "/home/foo/Maildir/hello/cur/msg123", "/home/foo/Maildir", "/hello" } + }}; + + for(auto&& tcase: test_cases) { + const auto res{maildir_from_path(std::get<0>(tcase), std::get<1>(tcase))}; + assert_valid_result(res); + assert_equal(*res, std::get<2>(tcase)); + } + + g_assert_false(!!maildir_from_path("/home/foo/Maildir/cur/test1", "/home/bar")); + g_assert_false(!!maildir_from_path("/x", "/x/y")); + g_assert_false(!!maildir_from_path("/home/a/Maildir/b/xxx/test", "/home/a/Maildir")); +} + +static void +test_base_message_dir_file() +{ + struct TestCase { + const std::string path; + DirFile expected; + }; + std::array<TestCase, 1> test_cases = {{ + { "/home/djcb/Maildir/foo/cur/msg:2,S", + { "/home/djcb/Maildir/foo", "msg:2,S", false } } + }}; + for(auto&& tcase: test_cases) { + const auto res{base_message_dir_file(tcase.path)}; + assert_valid_result(res); + assert_equal(res->dir, tcase.expected.dir); + assert_equal(res->file, tcase.expected.file); + g_assert_cmpuint(res->is_new, ==, tcase.expected.is_new); + } +} + +static void +test_flags_from_path() +{ + std::array<std::pair<std::string, Flags>, 5> test_cases = {{ + {"/home/foo/Maildir/test/cur/123456:2,FSR", + (Flags::Replied | Flags::Seen | Flags::Flagged)}, + {"/home/foo/Maildir/test/new/123456", Flags::New}, + {/* NOTE: when in new/, the :2,.. stuff is ignored */ + "/home/foo/Maildir/test/new/123456:2,FR", + Flags::New}, + {"/home/foo/Maildir/test/cur/123456:2,DTP", + (Flags::Draft | Flags::Trashed | Flags::Passed)}, + {"/home/foo/Maildir/test/cur/123456:2,S", Flags::Seen} + }}; + + for (auto&& tcase: test_cases) { + auto res{flags_from_path(tcase.first)}; + assert_valid_result(res); + /* LCOV_EXCL_START*/ + if (g_test_verbose()) { + mu_println("{} -> <{}>", tcase.first, + to_string(res.value())); + g_assert_true(res.value() == tcase.second); + } + /*LCOV_EXCL_STOP*/ + } +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/message/file/maildir-from-path", + test_maildir_from_path); + g_test_add_func("/message/file/base-message-dir-file", + test_base_message_dir_file); + g_test_add_func("/message/file/flags-from-path", test_flags_from_path); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/message/mu-message-file.hh b/lib/message/mu-message-file.hh new file mode 100644 index 0000000..09a9ed3 --- /dev/null +++ b/lib/message/mu-message-file.hh @@ -0,0 +1,98 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb.bulk@gmail.com> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_MESSAGE_FILE_HH__ +#define MU_MESSAGE_FILE_HH__ + +#include "mu-flags.hh" +#include <utils/mu-result.hh> + +namespace Mu { + +/* + * The file-components, ie. + * 1631819685.fb7b279bbb0a7b66.evergrey:2,RS + * => { + * "1631819685.fb7b279bbb0a7b66.evergrey", + * ':', + * "2,", + * "RS" + * } + */ +struct FileParts { + std::string base; /**< basename */ + char separator; /**< separator */ + std::string flags_suffix; /**< suffix (with flags) */ +}; + +/** + * Get the file-parts for some message-file + * + * @param file path to some message file (does not have to exist) + * + * @return FileParts for the message file + */ +FileParts message_file_parts(const std::string& file); + + +struct DirFile { + std::string dir; + std::string file; + bool is_new; +}; + +/** + * Get information about the message file componemts + * + * @param path message path + * + * @return the components for the message file or an error. + */ +Result<DirFile> base_message_dir_file(const std::string& path); + + + +/** + * Get the Maildir flags from the full path of a mailfile. The flags are as + * specified in http://cr.yp.to/proto/maildir.html, plus Flag::New for new + * messages, ie the ones that live in new/. The flags are logically OR'ed. Note + * that the file does not have to exist; the flags are based on the path only. + * + * @param pathname of a mailfile; it does not have to refer to an + * actual message + * + * @return the message flags or an error + */ +Result<Flags> flags_from_path(const std::string& pathname); + +/** + * get the maildir for a certain message path, ie, the path *before* + * cur/ or new/ and *after* the root. + * + * @param path path for some message + * @param root filesystem root for the maildir + * + * @return the maildir or an Error + */ +Result<std::string> maildir_from_path(const std::string& path, + const std::string& root); +} // Mu + + +#endif /* MU_MESSAGE_FILE_HH__ */ diff --git a/lib/message/mu-message-part.cc b/lib/message/mu-message-part.cc new file mode 100644 index 0000000..d1c7ac5 --- /dev/null +++ b/lib/message/mu-message-part.cc @@ -0,0 +1,256 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + + +#include "mu-message-part.hh" +#include "mu-mime-object.hh" +#include "utils/mu-utils.hh" +#include "utils/mu-utils-file.hh" +#include <string> + +using namespace Mu; + +MessagePart::MessagePart(const Mu::MimeObject& obj): + mime_obj{std::make_unique<Mu::MimeObject>(obj)} +{} + +MessagePart::MessagePart(const MessagePart& other): + MessagePart(*other.mime_obj) +{} + +MessagePart::~MessagePart() = default; + +const MimeObject& +MessagePart::mime_object() const noexcept +{ + return *mime_obj; +} + +static std::string +cook(const std::string& fname, const std::vector<char>& forbidden) +{ + std::string clean; + clean.reserve(fname.length()); + + for (auto& c: basename(fname)) + if (seq_some(forbidden,[&](char fc){return ::iscntrl(c) || c == fc;})) + clean += '-'; + else + clean += c; + + if (clean[0] == '.' && (clean == "." || clean == "..")) + return "-"; + else + return clean; +} + +static std::string +cook_minimal(const std::string& fname) +{ + return cook(fname, { '/' }); +} + +static std::string +cook_full(const std::string& fname) +{ + auto cooked = cook(fname, { '/', ' ', '\\', ':' }); + if (cooked.size() > 1 && cooked[0] == '-') + cooked.erase(0, 1); + + return cooked; +} + +Option<std::string> +MessagePart::cooked_filename(bool minimal) const noexcept +{ + auto&& cooker{minimal ? cook_minimal : cook_full}; + + // a MimePart... use the name if there is one. + if (mime_object().is_part()) + return MimePart{mime_object()}.filename().map(cooker); + + // MimeMessagepart. Construct a name based on subject. + if (mime_object().is_message_part()) { + auto msg{MimeMessagePart{mime_object()}.get_message()}; + if (!msg) + return Nothing; + else + return msg->subject() + .map(cooker) + .value_or("no-subject") + ".eml"; + } + + return Nothing; +} + +Option<std::string> +MessagePart::raw_filename() const noexcept +{ + if (!mime_object().is_part()) + return Nothing; + else + return MimePart{mime_object()}.filename(); +} + + + +Option<std::string> +MessagePart::mime_type() const noexcept +{ + if (const auto ctype{mime_object().content_type()}; ctype) + return ctype->media_type() + "/" + ctype->media_subtype(); + else + return Nothing; +} + +Option<std::string> +MessagePart::content_description() const noexcept +{ + if (!mime_object().is_part()) + return Nothing; + else + return MimePart{mime_object()}.content_description(); +} + +size_t +MessagePart::size() const noexcept +{ + if (!mime_object().is_part()) + return 0; + else + return MimePart{mime_object()}.size(); +} + +bool +MessagePart::is_attachment() const noexcept +{ + if (!mime_object().is_part()) + return false; + else + return MimePart{mime_object()}.is_attachment(); +} + + +Option<std::string> +MessagePart::to_string() const noexcept +{ + if (mime_object().is_part()) + return MimePart{mime_object()}.to_string(); + else + return mime_object().to_string_opt(); +} + +Result<size_t> +MessagePart::to_file(const std::string& path, bool overwrite) const noexcept +{ + if (mime_object().is_part()) + return MimePart{mime_object()}.to_file(path, overwrite); + else if (mime_object().is_message_part()) { + if (auto&& msg{MimeMessagePart{mime_object()}.get_message()}; !msg) + return Err(Error::Code::Message, "failed to get message-part"); + else + return msg->to_file(path, overwrite); + } else + return mime_object().to_file(path, overwrite); +} + +bool +MessagePart::is_signed() const noexcept +{ + return mime_object().is_multipart_signed(); +} + +bool +MessagePart::is_encrypted() const noexcept +{ + return mime_object().is_multipart_encrypted(); +} + +bool /* heuristic */ +MessagePart::looks_like_attachment() const noexcept +{ + auto matches=[](const MimeContentType& ctype, + const std::initializer_list<std::pair<const char*, const char*>>& ctypes) { + return std::find_if(ctypes.begin(), ctypes.end(), [&](auto&& item){ + return ctype.is_type(item.first, item.second); }) != ctypes.end(); + }; + + const auto ctype{mime_object().content_type()}; + if (!ctype) + return false; // no content-type: not an attachment. + + // we consider some parts _not_ to be attachments regardless of disposition + if (matches(*ctype,{{"application", "pgp-keys"}})) + return false; + + // we consider some parts to be attachments regardless of disposition + if (matches(*ctype,{{"image", "*"}, + {"audio", "*"}, + {"application", "*"}, + {"application", "x-patch"}})) + return true; + + // otherwise, rely on the disposition + return is_attachment(); +} + + + +#ifdef BUILD_TESTS +#include "utils/mu-test-utils.hh" + +static void +test_cooked_full() +{ + std::array<std::pair<std::string, std::string>, 4> cases = {{ + { "/hello/world/foo", "foo" }, + { "foo:/\n/bar", "bar"}, + { "Aap Noot Mies", "Aap-Noot-Mies"}, + { "..", "-"} + }}; + + for (auto&& test: cases) + assert_equal(cook_full(test.first), test.second); +} + +static void +test_cooked_minimal() +{ + std::array<std::pair<std::string, std::string>, 4> cases = {{ + { "/hello/world/foo", "foo" }, + { "foo:/\n/bar", "bar"}, + { "Aap Noot Mies.doc", "Aap Noot Mies.doc"}, + { "..", "-"} + }}; + + for (auto&& test: cases) + assert_equal(cook_minimal(test.first), test.second); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/message/message-part/cooked-full", test_cooked_full); + g_test_add_func("/message/message-part/cooked-minimal", test_cooked_minimal); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/message/mu-message-part.hh b/lib/message/mu-message-part.hh new file mode 100644 index 0000000..1d31e0e --- /dev/null +++ b/lib/message/mu-message-part.hh @@ -0,0 +1,167 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + + +#ifndef MU_MESSAGE_PART_HH__ +#define MU_MESSAGE_PART_HH__ + +#include <string> +#include <memory> +#include <utils/mu-option.hh> +#include <utils/mu-result.hh> + +namespace Mu { + +class MimeObject; // forward declaration; don't want to include for build-time + // reasons. + +class MessagePart { +public: + /** + * Construct MessagePart from a MimeObject + * + * @param obj + */ + MessagePart(const MimeObject& obj); + + /** + * Copy CTOR + * + * @param other + */ + MessagePart(const MessagePart& other); + + /** + * DTOR + * + */ + ~MessagePart(); + + /** + * Get the underlying MimeObject; you need to include mu-mime-object.hh + * to do anything useful with it. + * + * @return reference to the mime-object + */ + const MimeObject& mime_object() const noexcept; + + /** + * Filename for the mime-part file. This is a "cooked" filename with + * unallowed characters removed. If there's no filename specified, + * construct one (such as in the case of a MimeMessagePart). + * + * @param minimal if true, only perform *minimal* cookiing, where we + * only remove forward-slashes. + * + * @see raw_filename() + * + * @return the name + */ + Option<std::string> cooked_filename(bool minimal=false) const noexcept; + + /** + * Name for the mime-part file, i.e., MimePart::filename + * + * @return the filename or Nothing if there is none + */ + Option<std::string> raw_filename() const noexcept; + + /** + * Mime-type for the mime-part (e.g. "text/plain") + * + * @return the mime-part or Nothing if there is none + */ + Option<std::string> mime_type() const noexcept; + + + /** + * Get the content description for this part, or Nothing + * + * @return the content description + */ + Option<std::string> content_description() const noexcept; + + /** + * Get the length of the (unencoded) MIME-part. + * + * @return the size + */ + size_t size() const noexcept; + + /** + * Does this part have an "attachment" disposition? Otherwise it is + * "inline". Note that does *not* map 1:1 to a message's HasAttachment + * flag (which uses looks_like_attachment()) + * + * @return true or false. + */ + bool is_attachment() const noexcept; + + + /** + * Does this part appear to be an attachment from an end-users point of + * view? This uses some heuristics to guess. Some parts for which + * is_attachment() is true may not "really" be attachments, and + * vice-versa + * + * @return true or false. + */ + bool looks_like_attachment() const noexcept; + + /** + * Is this part signed? + * + * @return true or false + */ + bool is_signed() const noexcept; + + + /** + * Is this part encrypted? + * + * @return true or false + */ + bool is_encrypted() const noexcept; + + + /** + * Write (decoded) mime-part contents to string + * + * @return a string or nothing if there is no contemt + */ + Option<std::string> to_string() const noexcept; + + /** + * Write (decoded) mime part to a file + * + * @param path path to file + * @param overwrite whether to possibly overwrite + * + * @return size of file or or an error. + */ + Result<size_t> to_file(const std::string& path, bool overwrite) const noexcept; + + struct Private; +private: + const std::unique_ptr<MimeObject> mime_obj; +}; + +} // namespace Mu + +#endif /* MU_MESSAGE_PART_HH__ */ diff --git a/lib/message/mu-message.cc b/lib/message/mu-message.cc new file mode 100644 index 0000000..6ddd1f3 --- /dev/null +++ b/lib/message/mu-message.cc @@ -0,0 +1,863 @@ +/* +** Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include "config.h" + +#include "mu-message.hh" +#include "gmime/gmime-references.h" +#include "gmime/gmime-stream-mem.h" +#include "mu-maildir.hh" + +#include <array> +#include <string> +#include <regex> +#include <utils/mu-utils.hh> +#include <utils/mu-error.hh> +#include <utils/mu-option.hh> +#include <utils/mu-lang-detector.hh> + +#include <atomic> +#include <mutex> +#include <cstdlib> + +#include <glib.h> +#include <glib/gstdio.h> +#include <gmime/gmime.h> + +#include "gmime/gmime-message.h" +#include "mu-mime-object.hh" + +using namespace Mu; + +struct Message::Private { + Private(Message::Options options): + opts{options}, doc{doc_opts(opts)} {} + Private(Message::Options options, Xapian::Document&& xdoc): + opts{options}, doc{std::move(xdoc), doc_opts(opts)} {} + + Message::Options opts; + Document doc; + mutable Option<MimeMessage> mime_msg; + + Flags flags{}; + Option<std::string> mailing_list; + std::vector<Part> parts; + + ::time_t ctime{}; + + std::string cache_path; + /* + * we only need to index these, so we don't + * really need these copy if we re-arrange things + * a bit + */ + Option<std::string> body_txt; + Option<std::string> body_html; + Option<std::string> embedded; + + Option<std::string> language; /* body ISO language code */ + +private: + Document::Options doc_opts(Message::Options mopts) { + return any_of(opts & Message::Options::SupportNgrams) ? + Document::Options::SupportNgrams : + Document::Options::None; + } +}; + + +static void fill_document(Message::Private& priv); + +static Result<struct stat> +get_statbuf(const std::string& path, Message::Options opts = Message::Options::None) +{ + if (none_of(opts & Message::Options::AllowRelativePath) && + !g_path_is_absolute(path.c_str())) + return Err(Error::Code::File, "path '{}' is not absolute", path); + + if (::access(path.c_str(), R_OK) != 0) + return Err(Error::Code::File, "file @ '{}' is not readable", path); + + struct stat statbuf{}; + if (::stat(path.c_str(), &statbuf) < 0) + return Err(Error::Code::File, "cannot stat {}: {}", path, + g_strerror(errno)); + + if (!S_ISREG(statbuf.st_mode)) + return Err(Error::Code::File, "not a regular file: {}", path); + + return Ok(std::move(statbuf)); +} + + +Message::Message(const std::string& path, Message::Options opts): + priv_{std::make_unique<Private>(opts)} +{ + const auto statbuf{get_statbuf(path, opts)}; + if (!statbuf) + throw statbuf.error(); + + priv_->ctime = statbuf->st_ctime; + + init_gmime(); + if (auto msg{MimeMessage::make_from_file(path)}; !msg) + throw msg.error(); + else + priv_->mime_msg = std::move(msg.value()); + + auto xpath{to_string_opt_gchar(g_canonicalize_filename(path.c_str(), NULL))}; + if (xpath) + priv_->doc.add(Field::Id::Path, std::move(xpath.value())); + + priv_->doc.add(Field::Id::Size, static_cast<int64_t>(statbuf->st_size)); + + // rest of the fields + fill_document(*priv_); +} + +Message::Message(const std::string& text, const std::string& path, + Message::Options opts): + priv_{std::make_unique<Private>(opts)} +{ + if (text.empty()) + throw Error{Error::Code::InvalidArgument, "text must not be empty"}; + + if (!path.empty()) { + auto xpath{to_string_opt_gchar(g_canonicalize_filename(path.c_str(), {}))}; + if (xpath) + priv_->doc.add(Field::Id::Path, std::move(xpath.value())); + } + + priv_->ctime = ::time({}); + + priv_->doc.add(Field::Id::Size, static_cast<int64_t>(text.size())); + + init_gmime(); + if (auto msg{MimeMessage::make_from_text(text)}; !msg) + throw msg.error(); + else + priv_->mime_msg = std::move(msg.value()); + + fill_document(*priv_); +} + + +Message::Message(Message&& other) noexcept +{ + *this = std::move(other); +} + +Message& +Message::operator=(Message&& other) noexcept +{ + if (this != &other) + priv_ = std::move(other.priv_); + + return *this; +} + +Message::Message(Xapian::Document&& doc): + priv_{std::make_unique<Private>(Message::Options::None, std::move(doc))} +{} + + +Message::~Message() = default; + +const Mu::Document& +Message::document() const +{ + return priv_->doc; +} + +Message::Options +Message::options() const +{ + return priv_->opts; +} + +unsigned +Message::docid() const +{ + return priv_->doc.xapian_document().get_docid(); +} + + +const Mu::Sexp& +Message::sexp() const +{ + return priv_->doc.sexp(); +} + +Result<void> +Message::set_maildir(const std::string& maildir) +{ + /* sanity check a little bit */ + if (maildir.empty() || + maildir.at(0) != '/' || + (maildir.size() > 1 && maildir.at(maildir.length()-1) == '/')) + return Err(Error::Code::Message, + "'{}' is not a valid maildir", maildir.c_str()); + + const auto path{document().string_value(Field::Id::Path)}; + if (path == maildir || path.find(maildir) == std::string::npos) + return Err(Error::Code::Message, + "'{}' is not a valid maildir for message @ {}", + maildir, path); + + priv_->doc.remove(Field::Id::Maildir); + priv_->doc.add(Field::Id::Maildir, maildir); + + return Ok(); +} + +void +Message::set_flags(Flags flags) +{ + priv_->doc.remove(Field::Id::Flags); + priv_->doc.add(flags); +} + +bool +Message::load_mime_message(bool reload) const +{ + if (priv_->mime_msg && !reload) + return true; + + const auto path{document().string_value(Field::Id::Path)}; + if (auto mime_msg{MimeMessage::make_from_file(path)}; !mime_msg) { + mu_warning("failed to load '{}': {}", + path, mime_msg.error().what()); + return false; + } else { + priv_->mime_msg = std::move(mime_msg.value()); + fill_document(*priv_); + return true; + } +} + +void +Message::unload_mime_message() const +{ + priv_->mime_msg = Nothing; +} + +bool +Message::has_mime_message() const +{ + return !!priv_->mime_msg; +} + + +static Priority +get_priority(const MimeMessage& mime_msg) +{ + constexpr std::array<std::pair<std::string_view, Priority>, 10> + prio_alist = {{ + {"high", Priority::High}, + {"1", Priority::High}, + {"2", Priority::High}, + + {"normal", Priority::Normal}, + {"3", Priority::Normal}, + + {"low", Priority::Low}, + {"list", Priority::Low}, + {"bulk", Priority::Low}, + {"4", Priority::Low}, + {"5", Priority::Low} + }}; + + const auto opt_str = mime_msg.header("Precedence") + .disjunction(mime_msg.header("X-Priority")) + .disjunction(mime_msg.header("Importance")); + + if (!opt_str) + return Priority::Normal; + + const auto it = seq_find_if(prio_alist, [&](auto&& item) { + return g_ascii_strncasecmp(item.first.data(), opt_str->c_str(), + item.first.size()) == 0; }); + + return it == prio_alist.cend() ? Priority::Normal : it->second; +} + + +/* see: http://does-not-exist.org/mail-archives/mutt-dev/msg08249.html */ +static std::vector<std::string> +extract_tags(const MimeMessage& mime_msg) +{ + constexpr std::array<std::pair<const char*, char>, 3> tag_headers = {{ + {"X-Label", ' '}, {"X-Keywords", ','}, {"Keywords", ' '} + }}; + + std::vector<std::string> tags; + seq_for_each(tag_headers, [&](auto&& item) { + if (auto&& hdr = mime_msg.header(item.first); hdr) { + for (auto&& tagval : split(*hdr, item.second)) { + tagval.erase(0, tagval.find_first_not_of(' ')); + tagval.erase(tagval.find_last_not_of(' ')+1); + tags.emplace_back(std::move(tagval)); + } + } + }); + + return tags; +} + +static Option<std::string> +get_mailing_list(const MimeMessage& mime_msg) +{ + char *dechdr, *res; + const char *b, *e; + + const auto hdr{mime_msg.header("List-Id")}; + if (!hdr) { + /* some marketing messages don't have a List-Id, but _do_ have a + * List-Unsubscribe; if so, return an empty string here, so this + * message is still flagged as "MailingList" + */ + if (const auto lu = mime_msg.header("List-Unsubscribe"); !!lu) + return ""; + else + return Nothing; + } + + dechdr = g_mime_utils_header_decode_phrase(NULL, hdr->c_str()); + if (!dechdr) + return {}; + + e = NULL; + b = ::strchr(dechdr, '<'); + if (b) + e = strchr(b, '>'); + + if (b && e) + res = g_strndup(b + 1, e - b - 1); + else + res = g_strdup(dechdr); + + g_free(dechdr); + + return to_string_opt_gchar(std::move(res)); +} + +static void +append_text(Option<std::string>& str, Option<std::string>&& app) +{ + if (!str && app) + str = std::move(*app); + else if (str && app) + str.value() += app.value(); +} + +static void +accumulate_text(const MimePart& part, Message::Private& info, + const MimeContentType& ctype) +{ + if (!ctype.is_type("text", "*")) + return; /* not a text type */ + + if (part.is_attachment()) + append_text(info.embedded, part.to_string()); + else if (ctype.is_type("text", "plain")) + append_text(info.body_txt, part.to_string()); + else if (ctype.is_type("text", "html")) + append_text(info.body_html, part.to_string()); +} + + +static bool /* heuristic */ +looks_like_attachment(const MimeObject& parent, const MessagePart& mpart) +{ + if (parent) { /* crypto multipart children are not considered attachments */ + if (const auto parent_ctype{parent.content_type()}; parent_ctype) { + if (parent_ctype->is_type("multipart", "signed") || + parent_ctype->is_type("multipart", "encrypted")) + return false; + } + } + + return mpart.looks_like_attachment(); +} + + +static void +process_part(const MimeObject& parent, const MimePart& part, + Message::Private& info, const MessagePart& mpart) +{ + const auto ctype{part.content_type()}; + if (!ctype) + return; + + // flag as calendar, if not already + if (none_of(info.flags & Flags::Calendar) && + ctype->is_type("text", "calendar")) + info.flags |= Flags::Calendar; + + // flag as attachment, if not already. + if (none_of(info.flags & Flags::HasAttachment) && + looks_like_attachment(parent, mpart)) + info.flags |= Flags::HasAttachment; + + // if there are text parts, gather. + accumulate_text(part, info, *ctype); +} + + +static void +process_message_part(const MimeMessagePart& msg_part, + Message::Private& info) +{ + auto submsg{msg_part.get_message()}; + if (!submsg) + return; + + submsg->for_each([&](auto&& parent, auto&& child_obj) { + /* NOTE: we only handle one level; ideally, we'd apply the whole + parsing machinery recursively; so this a little crude. */ + if (!child_obj.is_part()) + return; + if (const auto ctype{child_obj.content_type()}; !ctype) + return; + else if (ctype->is_type("text", "plain")) + append_text(info.embedded, MimePart{child_obj}.to_string()); + else if (ctype->is_type("text", "html")) { + if (auto&& str{MimePart{child_obj}.to_string()}; str) + append_text(info.embedded, html_to_text(*str)); + } + }); +} + +static void +handle_object(const MimeObject& parent, + const MimeObject& obj, Message::Private& info); + + +static void +handle_encrypted(const MimeMultipartEncrypted& part, Message::Private& info) +{ + if (!any_of(info.opts & Message::Options::Decrypt)) { + /* just added to the list */ + info.parts.emplace_back(part); + return; + } + + const auto proto{part.content_type_parameter("protocol").value_or("unknown")}; + const auto ctx = MimeCryptoContext::make(proto); + if (!ctx) { + mu_warning("failed to create context for protocol <{}>", proto); + return; + } + + auto res{part.decrypt(*ctx)}; + if (!res) { + mu_warning("failed to decrypt: {}", res.error().what()); + return; + } + + if (res->first.is_multipart()) { + MimeMultipart{res->first}.for_each( + [&](auto&& parent, auto&& child_obj) { + handle_object(parent, child_obj, info); + }); + + } else + handle_object(part, res->first, info); +} + + +static void +handle_object(const MimeObject& parent, + const MimeObject& obj, Message::Private& info) +{ + /* if it's an encrypted part we should decrypt, recurse */ + if (obj.is_multipart_encrypted()) + handle_encrypted(MimeMultipartEncrypted{obj}, info); + else if (obj.is_part() || + obj.is_message_part() || + obj.is_multipart_signed() || + obj.is_multipart_encrypted()) + info.parts.emplace_back(obj); + + if (obj.is_part()) + process_part(parent, obj, info, info.parts.back()); + else if (obj.is_message_part()) + process_message_part(obj, info); + else if (obj.is_multipart_signed()) + info.flags |= Flags::Signed; + else if (obj.is_multipart_encrypted()) { + /* FIXME: An encrypted part might be signed at the same time. + * In that case the signed flag is lost. */ + info.flags |= Flags::Encrypted; + } else if (obj.is_mime_application_pkcs7_mime()) { + MimeApplicationPkcs7Mime smime(obj); +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" + // CompressedData, CertsOnly, Unknown + switch (smime.smime_type()) { + case Mu::MimeApplicationPkcs7Mime::SecureMimeType::SignedData: + info.flags |= Flags::Signed; + break; + case Mu::MimeApplicationPkcs7Mime::SecureMimeType::EnvelopedData: + info.flags |= Flags::Encrypted; + break; + default: + break; + } +#pragma GCC diagnostic pop + } +} + +/** + * This message -- recursively walk through message, and initialize some + * other values that depend on another. + * + * @param mime_msg + * @param path + * @param info + */ +static void +process_message(const MimeMessage& mime_msg, const std::string& path, + Message::Private& info) +{ + /* only have file-flags when there's a path. */ + if (!path.empty()) { + info.flags = flags_from_path(path).value_or(Flags::None); + /* pseudo-flag --> unread means either NEW or NOT SEEN, just + * for searching convenience */ + if (any_of(info.flags & Flags::New) || none_of(info.flags & Flags::Seen)) + info.flags |= Flags::Unread; + } + + // parts + mime_msg.for_each([&](auto&& parent, auto&& child_obj) { + handle_object(parent, child_obj, info); + }); + + // get the mailing here, and use it do update flags, too. + info.mailing_list = get_mailing_list(mime_msg); + if (info.mailing_list) + info.flags |= Flags::MailingList; + +#ifdef HAVE_CLD2 + /* language detection requires the cld2 lib */ + if (info.body_txt) { /* attempt to get the body-language */ + if (const auto lang{detect_language(info.body_txt.value())}; lang) { + info.language = lang->code; + mu_debug("detected language: {}", lang->code); + } else + mu_debug("could not detect language"); + } +#endif /*HAVE_CLD2*/ +} + +static Mu::Result<std::string> +calculate_sha256(const std::string& path) +{ + g_autoptr(GChecksum) checksum{g_checksum_new(G_CHECKSUM_SHA256)}; + + FILE *file{::fopen(path.c_str(), "r")}; + if (!file) + return Err(Error{Error::Code::File, "failed to open {}: {}", + path, ::strerror(errno)}); + + std::array<uint8_t, 4096> buf{}; + while (true) { + const auto n = ::fread(buf.data(), 1, buf.size(), file); + if (n == 0) + break; + g_checksum_update(checksum, buf.data(), n); + } + + bool has_err = ::ferror(file) != 0; + ::fclose(file); + + if (has_err) + return Err(Error{Error::Code::File, "failed to read {}", path}); + + return Ok(g_checksum_get_string(checksum)); +} + +/** + * Get a fake-message-id for a message without one. + * + * @param path message path + * + * @return a fake message-id + */ +static std::string +fake_message_id(const std::string& path) +{ + constexpr auto mu_suffix{"@mu.id"}; + + // not a very good message-id, only for testing. + if (path.empty() || ::access(path.c_str(), R_OK) != 0) + return mu_format("{:08x}{}", g_str_hash(path.c_str()), mu_suffix); + if (const auto sha256_res{calculate_sha256(path)}; !sha256_res) + return mu_format("{:08x}{}", g_str_hash(path.c_str()), mu_suffix); + else + return mu_format("{}{}", sha256_res.value(), mu_suffix); +} + +/* many of the doc.add(fields ....) automatically update the sexp-list as well; + * however, there are some _extra_ values in the sexp-list that are not + * based on a field. So we add them here. + */ + + +static void +doc_add_list_post(Document& doc, const MimeMessage& mime_msg) +{ + /* some mailing lists do not set the reply-to; see pull #1278. So for + * those cases, check the List-Post address and use that instead */ + + GMatchInfo* minfo; + GRegex* rx; + const auto list_post{mime_msg.header("List-Post")}; + if (!list_post) + return; + + rx = g_regex_new("<?mailto:([a-z0-9!@#$%&'*+-/=?^_`{|}~]+)>?", + G_REGEX_CASELESS, (GRegexMatchFlags)0, {}); + g_return_if_fail(rx); + + Contacts contacts; + if (g_regex_match(rx, list_post->c_str(), (GRegexMatchFlags)0, &minfo)) { + auto address = (char*)g_match_info_fetch(minfo, 1); + contacts.push_back(Contact(address)); + g_free(address); + } + + g_match_info_free(minfo); + g_regex_unref(rx); + + doc.add_extra_contacts(":list-post", contacts); +} + +static void +doc_add_reply_to(Document& doc, const MimeMessage& mime_msg) +{ + doc.add_extra_contacts(":reply-to", mime_msg.contacts(Contact::Type::ReplyTo)); +} + +static void +fill_document(Message::Private& priv) +{ + /* hunt & gather info from message tree */ + Document& doc{priv.doc}; + MimeMessage& mime_msg{priv.mime_msg.value()}; + + const auto path{doc.string_value(Field::Id::Path)}; + const auto refs{mime_msg.references()}; + const auto& raw_message_id = mime_msg.message_id(); + const auto message_id = raw_message_id.has_value() && !raw_message_id->empty() + ? *raw_message_id + : fake_message_id(path); + + process_message(mime_msg, path, priv); + + doc_add_list_post(doc, mime_msg); /* only in sexp */ + doc_add_reply_to(doc, mime_msg); /* only in sexp */ + + field_for_each([&](auto&& field) { + /* insist on explicitly handling each */ +#pragma GCC diagnostic push +#pragma GCC diagnostic error "-Wswitch" + switch(field.id) { + case Field::Id::Bcc: + doc.add(field.id, mime_msg.contacts(Contact::Type::Bcc)); + break; + case Field::Id::BodyText: + doc.add(field.id, priv.body_txt); + if (priv.body_html) + doc.add(field.id, html_to_text(*priv.body_html)); + break; + case Field::Id::Cc: + doc.add(field.id, mime_msg.contacts(Contact::Type::Cc)); + break; + case Field::Id::Changed: + doc.add(field.id, priv.ctime); + break; + case Field::Id::Date: + doc.add(field.id, mime_msg.date()); + break; + case Field::Id::EmbeddedText: + doc.add(field.id, priv.embedded); + break; + case Field::Id::File: + for (auto&& part: priv.parts) + doc.add(field.id, part.raw_filename()); + break; + case Field::Id::Flags: + doc.add(priv.flags); + break; + case Field::Id::From: + doc.add(field.id, mime_msg.contacts(Contact::Type::From)); + break; + case Field::Id::Language: + doc.add(field.id, priv.language); + break; + case Field::Id::Maildir: /* already */ + break; + case Field::Id::MailingList: + doc.add(field.id, priv.mailing_list); + break; + case Field::Id::MessageId: + doc.add(field.id, message_id); + break; + case Field::Id::MimeType: + for (auto&& part: priv.parts) + doc.add(field.id, part.mime_type()); + break; + case Field::Id::Path: /* already */ + break; + case Field::Id::Priority: + doc.add(get_priority(mime_msg)); + break; + case Field::Id::References: + if (!refs.empty()) + doc.add(field.id, refs); + break; + case Field::Id::Size: /* already */ + break; + case Field::Id::Subject: + doc.add(field.id, mime_msg.subject().map(remove_ctrl)); + break; + case Field::Id::Tags: + if (auto&& tags{extract_tags(mime_msg)}; !tags.empty()) + doc.add(field.id, tags); + break; + case Field::Id::ThreadId: + // either the oldest reference, or otherwise the message id + doc.add(field.id, refs.empty() ? message_id : refs.at(0)); + break; + case Field::Id::To: + doc.add(field.id, mime_msg.contacts(Contact::Type::To)); + break; + /* LCOV_EXCL_START */ + case Field::Id::_count_: + default: + break; + /* LCOV_EXCL_STOP */ + } +#pragma GCC diagnostic pop + + }); +} + +Option<std::string> +Message::header(const std::string& header_field) const +{ + load_mime_message(); + return priv_->mime_msg->header(header_field); +} + +Option<std::string> +Message::body_text() const +{ + load_mime_message(); + return priv_->body_txt; +} + +Option<std::string> +Message::body_html() const +{ + load_mime_message(); + return priv_->body_html; +} + +Contacts +Message::all_contacts() const +{ + Contacts contacts; + + if (!load_mime_message()) + return contacts; /* empty */ + + return priv_->mime_msg->contacts(Contact::Type::None); /* get all types */ +} + +const std::vector<Message::Part>& +Message::parts() const +{ + if (!load_mime_message()) { + static std::vector<Message::Part> empty; + return empty; + } + + return priv_->parts; +} + +Result<std::string> +Message::cache_path(Option<size_t> index) const +{ + /* create tmpdir for this message, if needed */ + if (priv_->cache_path.empty()) { + GError *err{}; + auto tpath{to_string_opt_gchar(g_dir_make_tmp("mu-cache-XXXXXX", &err))}; + if (!tpath) + return Err(Error::Code::File, &err, "failed to create temp dir"); + + priv_->cache_path = std::move(tpath.value()); + } + + if (index) { + GError *err{}; + auto tpath = mu_format("{}/{}", priv_->cache_path, *index); + if (g_mkdir(tpath.c_str(), 0700) != 0) + return Err(Error::Code::File, &err, + "failed to create cache dir '{}'; err={}", tpath, errno); + return Ok(std::move(tpath)); + } else + + return Ok(std::string{priv_->cache_path}); +} + +// for now this only remove stray '/' at the end +std::string +Message::sanitize_maildir(const std::string& mdir) +{ + if (mdir.size() > 1 && mdir.at(mdir.length()-1) == '/') + return mdir.substr(0, mdir.length() - 1); + else + return mdir; +} + +Result<void> +Message::update_after_move(const std::string& new_path, + const std::string& new_maildir, + Flags new_flags) +{ + if (auto statbuf{get_statbuf(new_path)}; !statbuf) + return Err(statbuf.error()); + else + priv_->ctime = statbuf->st_ctime; + + priv_->doc.remove(Field::Id::Path); + priv_->doc.remove(Field::Id::Changed); + + priv_->doc.add(Field::Id::Path, new_path); + priv_->doc.add(Field::Id::Changed, priv_->ctime); + + set_flags(new_flags); + + if (const auto res = set_maildir(sanitize_maildir(new_maildir)); !res) + return res; + + return Ok(); +} diff --git a/lib/message/mu-message.hh b/lib/message/mu-message.hh new file mode 100644 index 0000000..0f029f4 --- /dev/null +++ b/lib/message/mu-message.hh @@ -0,0 +1,478 @@ +/* +** Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_MESSAGE_HH__ +#define MU_MESSAGE_HH__ + +#include <memory> +#include <string> +#include <vector> +#include <iostream> + +#include "mu-xapian-db.hh" + +#include "mu-contact.hh" +#include "mu-priority.hh" +#include "mu-flags.hh" +#include "mu-fields.hh" +#include "mu-document.hh" +#include "mu-message-part.hh" +#include "mu-message-file.hh" + +#include "utils/mu-utils.hh" +#include "utils/mu-option.hh" +#include "utils/mu-result.hh" +#include "utils/mu-sexp.hh" + +namespace Mu { + +class Message { +public: + enum struct Options { + None = 0, /**< Defaults */ + Decrypt = 1 << 0, /**< Attempt to decrypt */ + RetrieveKeys = 1 << 1, /**< Auto-retrieve crypto keys (implies network + * access) */ + AllowRelativePath = 1 << 2, /**< Allow relative paths for filename + * in make_from_path */ + SupportNgrams = 1 << 3, /**< Support ngrams, as used in + * CJK and other languages. */ + }; + + /** + * Move CTOR + * + * @param some other message + */ + Message(Message&& other) noexcept; + + /** + * operator= + * + * @param other move some object object + * + * @return + */ + Message& operator=(Message&& other) noexcept; + + /** + * Construct a message based on a path + * + * @param path path to message + * @param opts options + * + * @return a message or an error + */ + static Result<Message> make_from_path(const std::string& path, + Options opts={}) try { + return Ok(Message{path,opts}); + } catch (Error& err) { + return Err(err); + } + /* LCOV_EXCL_START */ + catch (...) { + return Err(Mu::Error(Error::Code::Message, + "failed to create message from path")); + } + /* LCOV_EXCL_STOP */ + + /** + * Construct a message based on a Xapian::Document + * + * @param doc a Mu Document + * + * @return a message or an error + */ + static Result<Message> make_from_document(Xapian::Document&& doc) try { + return Ok(Message{std::move(doc)}); + } catch (Error& err) { + return Err(err); + } + /* LCOV_EXCL_START */ + catch (...) { + return Err(Mu::Error(Error::Code::Message, + "failed to create message from document")); + } + /* LCOV_EXCL_STOP */ + + /** + * Construct a message from a string. This is mostly useful for testing. + * + * @param text message text + * @param path path to message - optional; path does not have to exist. + * @param opts options + * + * @return a message or an error + */ + static Result<Message> make_from_text(const std::string& text, + const std::string& path={}, + Options opts={}) try { + return Ok(Message{text, path, opts}); + } catch (Error& err) { + return Err(err); + } + /* LCOV_EXCL_START */ + catch (...) { + return Err(Mu::Error(Error::Code::Message, + "failed to create message from text")); + } + /* LCOV_EXCL_STOP */ + + /** + * DTOR + */ + ~Message(); + + /** + * Get the document. + * + * + * @return document + */ + const Document& document() const; + + + /** + * The message options for this message + * + * @return message options + */ + Options options() const; + + + /** + * Get the document-id, or 0 if non-existent. + * + * @return document id + */ + unsigned docid() const; + + /** + * Get the file system path of this message + * + * @return the path of this Message or NULL in case of error. + * the returned string should *not* be modified or freed. + */ + std::string path() const { return document().string_value(Field::Id::Path); } + + /** + * Get the sender (From:) of this message + * + * @return the sender(s) of this Message + */ + Contacts from() const { return document().contacts_value(Field::Id::From); } + + /** + * Get the recipient(s) (To:) for this message + * + * @return recipients + */ + Contacts to() const { return document().contacts_value(Field::Id::To); } + + /** + * Get the recipient(s) (Cc:) for this message + * + * @return recipients + */ + Contacts cc() const { return document().contacts_value(Field::Id::Cc); } + + /** + * Get the recipient(s) (Bcc:) for this message + * + * @return recipients + */ + Contacts bcc() const { return document().contacts_value(Field::Id::Bcc); } + + /** + * Get the maildir this message resides in; i.e., if the path is + * ~/Maildir/foo/bar/cur/msg, the maildir would typically be foo/bar + * + * This is determined when _storing_ the message (which uses + * set_maildir()) + * + * @return the maildir requested or empty */ + std::string maildir() const { return document().string_value(Field::Id::Maildir); } + + /** + * Set the maildir for this message. This is for use by the _store_ when + * it has determined the maildir for this message from the message's path and + * the root-maildir known by the store. + * + * @param maildir the maildir for this message + * + * @return Ok() or some error if the maildir is invalid + */ + Result<void> set_maildir(const std::string& maildir); + + /** + * Clean up the maildir. This is for internal use, but exposed for testing. + * For now cleaned-up means "stray trailing / removed". + * + * @param maildir some maildir + * + * @return a cleaned-up version + */ + static std::string sanitize_maildir(const std::string& maildir); + + /** + * Get the subject of this message + * + * @return the subject of this Message + */ + std::string subject() const { return document().string_value(Field::Id::Subject); } + + /** + * Get the Message-Id of this message + * + * @return the Message-Id of this message (without the enclosing <>), or + * a fake message-id for messages that don't have them. + * + * For file-backed message, this fake message-id is based on a hash of the + * message contents. For non-file-backed (test) messages, some other value + * is concocted. + */ + std::string message_id() const { return document().string_value(Field::Id::MessageId);} + + /** + * get the mailing list for a message, i.e. the mailing-list + * identifier in the List-Id header. + * + * @return the mailing list id for this message (without the enclosing <>) + * or NULL in case of error or if there is none. + */ + std::string mailing_list() const { return document().string_value(Field::Id::MailingList);} + + /** + * get the message date/time (the Date: field) as time_t + * + * @return message date/time or 0 in case of error or if there + * is no such header. + */ + ::time_t date() const { + return static_cast<::time_t>(document().integer_value(Field::Id::Date)); + } + + /** + * get the last change-time this message. For path/document-based + * messages this corresponds with the ctime of the underlying file; for + * the text-based ones (as used for testing) it is the creation time. + * + * @return last-change time or 0 if unknown + */ + ::time_t changed() const { + return static_cast<::time_t>(document().integer_value(Field::Id::Changed)); + } + + /** + * get the flags for this message. + * + * @return the file/content flags + */ + Flags flags() const { return document().flags_value(); } + + + /** + * Update the flags for this message. This is useful for flags + * that can only be determined after the message has been created already, + * such as the 'personal' flag. + * + * @param flags new flags. + */ + void set_flags(Flags flags); + + /** + * get the message priority for this message. The X-Priority, X-MSMailPriority, + * Importance and Precedence header are checked, in that order. if no known or + * explicit priority is set, Priority::Id::Normal is assumed + * + * @return the message priority + */ + Priority priority() const { return document().priority_value(); } + + /** + * get the file size in bytes of this message + * + * @return the filesize + */ + size_t size() const { return static_cast<size_t>(document().integer_value(Field::Id::Size)); } + + /** + * Get the (possibly empty) list of references (consisting of both the + * References and In-Reply-To fields), with the oldest first and the + * direct parent as the last one. Note, any reference (message-id) will + * appear at most once, duplicates and fake-message-id (see impls) are + * filtered out. + * + * @return a vec with the references for this msg. + */ + std::vector<std::string> references() const { + return document().string_vec_value(Field::Id::References); + } + + /** + * Get the thread-id for this message. This is the message-id of the + * oldest-known (grand) parent, or the message-id of this message if + * none. + * + * @return the thread id. + */ + std::string thread_id() const { + return document().string_value(Field::Id::ThreadId); + } + + /** + * get the list of tags (ie., X-Label) + * + * @param msg a valid MuMsg + * + * @return a list with the tags for this msg. Don't modify/free + */ + std::vector<std::string> tags() const { + return document() + .string_vec_value(Field::Id::Tags); + } + + /* + * Convert to Sexp + */ + + /** + * Get the s-expression for this message. Stays valid as long as this + * message is. + * + * @return an Sexp representing the message. + */ + const Sexp& sexp() const; + + /* + * And some non-const message, for updating an existing + * message after a file-system move. + * + * @return Ok or an error. + */ + Result<void> update_after_move(const std::string& new_path, + const std::string& new_maildir, + Flags new_flags); + /* + * Below require a file-backed message, which is a relatively slow + * if there isn't one already; see load_mime_message() + */ + + /** + * Get the text body + * + * @return text body + */ + Option<std::string> body_text() const; + + /** + * Get the HTML body + * + * @return text body + */ + Option<std::string> body_html() const; + + /** + * Get some message-header + * + * @param header_field name of the header + * + * @return the value (UTF-8), or Nothing. + */ + Option<std::string> header(const std::string& header_field) const; + + + /** + * Get all contacts for this message. + * + * @return contacts + */ + Contacts all_contacts() const; + + /** + * Get information about MIME-parts in this message. + * + * @return mime-part info. + */ + using Part = MessagePart; + const std::vector<Part>& parts() const; + + /** + * Get the path to a cache directory for this message, which is useful + * for temporarily saving attachments + * + * @param index optionally, create <cache-path>/<index> instead; + * this is useful for having part-specific subdirectories. + * + * @return path to a (created) cache directory, or an error. + */ + Result<std::string> cache_path(Option<size_t> index={}) const; + + + /** + * Load the GMime (file) message (for a database-backed message), + * if not already (but see @param reload). + * + * Affects cached-state only, so we still mark this as 'const' + * + * @param reload whether to force reloading (even if already) + * + * @return true if loading worked; false otherwise. + */ + bool load_mime_message(bool reload=false) const; + + /** + * Clear the GMime message. + * + * Affects cached-state only, so we still mark this as 'const' + */ + void unload_mime_message() const; + + /** + * Has a (file-base) GMime message been loaded? + * + * + * @return true or false + */ + bool has_mime_message() const; + + struct Private; + + /* + * Usually the make_ builders are better to create a message, but in + * some special cases, we need a heap-allocated message... */ + + Message(Xapian::Document&& xdoc); + Message(const std::string& path, Options opts); + +private: + Message(const std::string& str, const std::string& path, Options opt); + + std::unique_ptr<Private> priv_; + +}; // Message +MU_ENABLE_BITOPS(Message::Options); + +static inline auto +format_as(const Message& msg) { + return msg.path(); +} + +} // Mu +#endif /* MU_MESSAGE_HH__ */ diff --git a/lib/message/mu-mime-object.cc b/lib/message/mu-mime-object.cc new file mode 100644 index 0000000..a75da5b --- /dev/null +++ b/lib/message/mu-mime-object.cc @@ -0,0 +1,798 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + + +#include "mu-mime-object.hh" +#include "gmime/gmime-message.h" +#include "utils/mu-utils.hh" +#include "utils/mu-utils-file.hh" +#include <mutex> +#include <regex> +#include <fcntl.h> +#include <sys/stat.h> +#include <errno.h> + +using namespace Mu; + + + +/* note, we do the gmime initialization here rather than in mu-runtime, because this way + * we don't need mu-runtime for simple cases -- such as our unit tests. Also note that we + * need gmime init even for the doc backend, as we use the address parsing functions also + * there. */ + +void +Mu::init_gmime(void) +{ + // fast path. + static bool gmime_initialized = false; + if (gmime_initialized) + return; + + static std::mutex gmime_lock; + std::lock_guard lock (gmime_lock); + if (gmime_initialized) + return; // already + + mu_debug("initializing gmime {}.{}.{}", + gmime_major_version, + gmime_minor_version, + gmime_micro_version); + + g_mime_init(); + gmime_initialized = true; + + std::atexit([] { + mu_debug("shutting down gmime"); + g_mime_shutdown(); + gmime_initialized = false; + }); +} + + +std::string +Mu::address_rfc2047(const Contact& contact) +{ + init_gmime(); + + InternetAddress *addr = + internet_address_mailbox_new(contact.name.c_str(), + contact.email.c_str()); + + std::string encoded = to_string_gchar( + internet_address_to_string(addr, {}, true)); + + g_object_unref(addr); + + return encoded; +} + + +/* + * MimeObject + */ + +Option<std::string> +MimeObject::header(const std::string& hdr) const noexcept +{ + if (auto val{g_mime_object_get_header(self(), hdr.c_str())}; !val) + return Nothing; + else if (!g_utf8_validate(val, -1, {})) + return utf8_clean(val); + else + return std::string{val}; +} + + +std::vector<std::pair<std::string, std::string>> +MimeObject::headers() const noexcept +{ + GMimeHeaderList *lst; + + lst = g_mime_object_get_header_list(self()); /* _not_ owned */ + if (!lst) + return {}; + + std::vector<std::pair<std::string, std::string>> hdrs; + const auto hdr_num{g_mime_header_list_get_count(lst)}; + + for (int i = 0; i != hdr_num; ++i) { + GMimeHeader *hdr{g_mime_header_list_get_header_at(lst, i)}; + if (!hdr) /* ^^^ _not_ owned */ + continue; + const auto name{g_mime_header_get_name(hdr)}; + const auto val{g_mime_header_get_value(hdr)}; + if (!name || !val) + continue; + hdrs.emplace_back(name, val); + } + + return hdrs; +} + +Result<size_t> +MimeObject::write_to_stream(const MimeFormatOptions& f_opts, + MimeStream& stream) const +{ + auto written = g_mime_object_write_to_stream(self(), f_opts.get(), + GMIME_STREAM(stream.object())); + if (written < 0) + return Err(Error::Code::File, "failed to write mime-object to stream"); + else + return Ok(static_cast<size_t>(written)); +} + +Result<size_t> +MimeObject::to_file(const std::string& path, bool overwrite) const noexcept +{ + GError *err{}; + auto strm{g_mime_stream_fs_open(path.c_str(), + O_WRONLY | O_CREAT | O_TRUNC |(overwrite ? 0 : O_EXCL), + S_IRUSR|S_IWUSR, + &err)}; + if (!strm) + return Err(Error::Code::File, &err, "failed to open '{}'", path); + + MimeStream stream{MimeStream::make_from_stream(strm)}; + return write_to_stream({}, stream); +} + + +Option<std::string> +MimeObject::to_string_opt() const noexcept +{ + auto stream{MimeStream::make_mem()}; + if (!stream) { + mu_warning("failed to create mem stream"); + return Nothing; + } + + const auto written = g_mime_object_write_to_stream( + self(), {}, GMIME_STREAM(stream.object())); + if (written < 0) { + mu_warning("failed to write object to stream"); + return Nothing; + } + + std::string buffer; + buffer.resize(written + 1); + stream.reset(); + + auto bytes{g_mime_stream_read(GMIME_STREAM(stream.object()), + buffer.data(), written)}; + if (bytes < 0) + return Nothing; + + buffer.data()[written]='\0'; + buffer.resize(written); + + return buffer; +} + + +/* + * MimeCryptoContext + */ + +Result<size_t> +MimeCryptoContext::import_keys(MimeStream& stream) +{ + GError *err{}; + auto res = g_mime_crypto_context_import_keys( + self(), GMIME_STREAM(stream.object()), &err); + + if (res < 0) + return Err(Error::Code::File, &err, + "error importing keys"); + + return Ok(static_cast<size_t>(res)); +} + +void +MimeCryptoContext::set_request_password(PasswordRequestFunc pw_func) +{ + static auto request_func = pw_func; + + g_mime_crypto_context_set_request_password( + self(), + [](GMimeCryptoContext *ctx, + const char *user_id, + const char *prompt, + gboolean reprompt, + GMimeStream *response, + GError **err) -> gboolean { + MimeStream mstream{MimeStream::make_from_stream(response)}; + + auto res = request_func(MimeCryptoContext(ctx), + std::string{user_id ? user_id : ""}, + std::string{prompt ? prompt : ""}, + !!reprompt, + mstream); + if (res) + return TRUE; + + res.error().fill_g_error(err); + return FALSE; + }); + +} + +Result<void> +MimeCryptoContext::setup_gpg_test(const std::string& testpath) +{ + /* setup clean environment for testing; inspired by gmime */ + + g_setenv ("GNUPGHOME", join_paths(testpath, ".gnupg").c_str(), 1); + + /* disable environment variables that gpg-agent uses for pinentry */ + g_unsetenv ("DBUS_SESSION_BUS_ADDRESS"); + g_unsetenv ("DISPLAY"); + g_unsetenv ("GPG_TTY"); + + if (g_mkdir_with_parents((testpath + "/.gnupg").c_str(), 0700) != 0) + return Err(Error::Code::File, + "failed to create gnupg dir; err={}", errno); + + auto write_gpgfile=[&](const std::string& fname, const std::string& data) + -> Result<void> { + + GError *err{}; + std::string path{mu_format("{}/{}", testpath, fname)}; + if (!g_file_set_contents(path.c_str(), data.c_str(), data.size(), &err)) + return Err(Error::Code::File, &err, "failed to write {}", path); + else + return Ok(); + }; + + // some more elegant way? + if (auto&& res = write_gpgfile("gpg.conf", "pinentry-mode loopback\n"); !res) + return res; + if (auto&& res = write_gpgfile("gpgsm.conf", "disable-crl-checks\n")) + return res; + + return Ok(); +} + + +/* + * MimeMessage + */ + + + +static Result<MimeMessage> +make_from_stream(GMimeStream* &&stream/*consume*/) +{ + init_gmime(); + GMimeParser *parser{g_mime_parser_new_with_stream(stream)}; + g_object_unref(stream); + if (!parser) + return Err(Error::Code::Message, "cannot create mime parser"); + + GMimeMessage *gmime_msg{g_mime_parser_construct_message(parser, NULL)}; + g_object_unref(parser); + if (!gmime_msg) + return Err(Error::Code::Message, "message seems invalid"); + + auto mime_msg{MimeMessage{std::move(G_OBJECT(gmime_msg))}}; + g_object_unref(gmime_msg); + + return Ok(std::move(mime_msg)); +} + +Result<MimeMessage> +MimeMessage::make_from_file(const std::string& path) +{ + GError* err{}; + init_gmime(); + if (auto&& stream{g_mime_stream_file_open(path.c_str(), "r", &err)}; !stream) + return Err(Error::Code::Message, &err, + "failed to open stream for {}", path); + else + return make_from_stream(std::move(stream)); +} + +Result<MimeMessage> +MimeMessage::make_from_text(const std::string& text) +{ + init_gmime(); + if (auto&& stream{g_mime_stream_mem_new_with_buffer( + text.c_str(), text.length())}; !stream) + return Err(Error::Code::Message, + "failed to open stream for string"); + else + return make_from_stream(std::move(stream)); +} + +Option<int64_t> +MimeMessage::date() const noexcept +{ + GDateTime *dt{g_mime_message_get_date(self())}; + if (!dt) + return Nothing; + else + return g_date_time_to_unix(dt); +} + +constexpr Option<GMimeAddressType> +address_type(Contact::Type ctype) +{ + switch(ctype) { + case Contact::Type::Bcc: + return GMIME_ADDRESS_TYPE_BCC; + case Contact::Type::Cc: + return GMIME_ADDRESS_TYPE_CC; + case Contact::Type::From: + return GMIME_ADDRESS_TYPE_FROM; + case Contact::Type::To: + return GMIME_ADDRESS_TYPE_TO; + case Contact::Type::ReplyTo: + return GMIME_ADDRESS_TYPE_REPLY_TO; + case Contact::Type::Sender: + return GMIME_ADDRESS_TYPE_SENDER; + case Contact::Type::None: + default: + return Nothing; + } +} + +static Mu::Contacts +all_contacts(const MimeMessage& msg) +{ + Contacts contacts; + + for (auto&& cctype: { + Contact::Type::Sender, + Contact::Type::From, + Contact::Type::ReplyTo, + Contact::Type::To, + Contact::Type::Cc, + Contact::Type::Bcc + }) { + auto addrs{msg.contacts(cctype)}; + std::move(addrs.begin(), addrs.end(), + std::back_inserter(contacts)); + } + + return contacts; +} + +Mu::Contacts +MimeMessage::contacts(Contact::Type ctype) const noexcept +{ + /* special case: get all */ + if (ctype == Contact::Type::None) + return all_contacts(*this); + + const auto atype{address_type(ctype)}; + if (!atype) + return {}; + + auto addrs{g_mime_message_get_addresses(self(), *atype)}; + if (!addrs) + return {}; + + const auto msgtime{date().value_or(0)}; + + Contacts contacts; + auto lst_len{internet_address_list_length(addrs)}; + contacts.reserve(lst_len); + for (auto i = 0; i != lst_len; ++i) { + + auto&& addr{internet_address_list_get_address(addrs, i)}; + const auto name{internet_address_get_name(addr)}; + + if (G_UNLIKELY(!INTERNET_ADDRESS_IS_MAILBOX(addr))) + continue; + + const auto email{internet_address_mailbox_get_addr ( + INTERNET_ADDRESS_MAILBOX(addr))}; + if (G_UNLIKELY(!email)) + continue; + + contacts.emplace_back(email, name ? name : "", ctype, msgtime); + } + + return contacts; +} + +/* + * references() returns the concatenation of the References and In-Reply-To + * message-ids (in that order). Duplicates are removed. + * + * The _first_ one in the list determines the thread-id for the message. + */ +std::vector<std::string> +MimeMessage::references() const noexcept +{ + // is ref already in the list? O(n) but with small n. + auto is_dup = [](auto&& seq, const std::string& ref) { + return seq_some(seq, [&](auto&& str) { return ref == str; }); + }; + + auto is_fake = [](auto&& msgid) { + // this is bit ugly; protonmail injects fake References which + // can otherwise screw up threading. + if (g_str_has_suffix(msgid, "protonmail.internalid")) + return true; + /* ... */ + return false; + }; + + std::vector<std::string> refs; + for (auto&& ref_header: { "References", "In-reply-to" }) { + + auto hdr{header(ref_header)}; + if (!hdr) + continue; + + GMimeReferences *mime_refs{g_mime_references_parse({}, hdr->c_str())}; + refs.reserve(refs.size() + g_mime_references_length(mime_refs)); + + for (auto i = 0; i != g_mime_references_length(mime_refs); ++i) { + const auto msgid{g_mime_references_get_message_id(mime_refs, i)}; + if (msgid && !is_dup(refs, msgid) && !is_fake(msgid)) + refs.emplace_back(msgid); + } + g_mime_references_free(mime_refs); + } + + return refs; +} + +void +MimeMessage::for_each(const ForEachFunc& func) const noexcept +{ + struct CallbackData { const ForEachFunc& func; }; + CallbackData cbd{func}; + + g_mime_message_foreach( + self(), + [] (GMimeObject *parent, GMimeObject *part, gpointer user_data) { + auto cb_data{reinterpret_cast<CallbackData*>(user_data)}; + cb_data->func(MimeObject{parent}, MimeObject{part}); + }, &cbd); +} + + + +/* + * MimePart + */ +size_t +MimePart::size() const noexcept +{ + auto wrapper{g_mime_part_get_content(self())}; + if (!wrapper) { + mu_warning("failed to get content wrapper"); + return 0; + } + + auto stream{g_mime_data_wrapper_get_stream(wrapper)}; + if (!stream) { + mu_warning("failed to get stream"); + return 0; + } + + return static_cast<size_t>(g_mime_stream_length(stream)); +} +Option<std::string> +MimePart::to_string() const noexcept +{ + /* + * easy case: text. this automatically handles conversion to utf-8. + */ + if (GMIME_IS_TEXT_PART(self())) { + if (char* txt{g_mime_text_part_get_text(GMIME_TEXT_PART(self()))}; !txt) + return Nothing; + else + return to_string_gchar(std::move(txt)/*consumes*/); + } + + /* + * harder case: read from stream manually + */ + GMimeDataWrapper *wrapper{g_mime_part_get_content(self())}; + if (!wrapper) { /* this happens with invalid mails */ + mu_warning("failed to create data wrapper"); + return Nothing; + } + + GMimeStream *stream{g_mime_stream_mem_new()}; + if (!stream) { + mu_warning("failed to create mem stream"); + return Nothing; + } + + ssize_t buflen{g_mime_data_wrapper_write_to_stream(wrapper, stream)}; + if (buflen <= 0) { /* empty buffer, not an error */ + g_object_unref(stream); + return Nothing; + } + + std::string buffer; + buffer.resize(buflen + 1); + g_mime_stream_reset(stream); + + auto bytes{g_mime_stream_read(stream, buffer.data(), buflen)}; + g_object_unref(stream); + if (bytes < 0) + return Nothing; + + buffer.resize(bytes + 1); + + return buffer; +} + +Result<size_t> +MimePart::to_file(const std::string& path, bool overwrite) const noexcept +{ + MimeDataWrapper wrapper{g_mime_part_get_content(self())}; + if (!wrapper) /* this happens with invalid mails */ + return Err(Error::Code::File, "failed to create data wrapper"); + + GError *err{}; + auto strm{g_mime_stream_fs_open(path.c_str(), + O_WRONLY | O_CREAT | O_TRUNC |(overwrite ? 0 : O_EXCL), + S_IRUSR|S_IWUSR, + &err)}; + if (!strm) + return Err(Error::Code::File, &err, "failed to open '{}'", path); + + MimeStream stream{MimeStream::make_from_stream(strm)}; + ssize_t written{g_mime_data_wrapper_write_to_stream( + GMIME_DATA_WRAPPER(wrapper.object()), + GMIME_STREAM(stream.object()))}; + + if (written < 0) { + return Err(Error::Code::File, &err, + "failed to write to '{}'", path); + } + + return Ok(static_cast<size_t>(written)); +} + +void +MimeMultipart::for_each(const ForEachFunc& func) const noexcept +{ + struct CallbackData { const ForEachFunc& func; }; + CallbackData cbd{func}; + + g_mime_multipart_foreach( + self(), + [] (GMimeObject *parent, GMimeObject *part, gpointer user_data) { + auto cb_data{reinterpret_cast<CallbackData*>(user_data)}; + cb_data->func(MimeObject{parent}, MimeObject{part}); + }, &cbd); +} + + +/* + * we need to be able to pass a crypto-context to the verify(), but + * g_mime_multipart_signed_verify() doesn't offer that anymore in GMime 3.x. + * + * So, add that by reimplementing it a bit (follow the upstream impl) + */ + + +static bool +mime_types_equal (const std::string& mime_type, const std::string& official_type) +{ + if (g_ascii_strcasecmp(mime_type.c_str(), official_type.c_str()) == 0) + return true; + + const auto slash_pos = official_type.find("/"); + if (slash_pos == std::string::npos || slash_pos == 0) + return false; + + /* If the official mime-type's subtype already begins with "x-", then there's + * nothing else to check. */ + const auto subtype{official_type.substr(slash_pos + 1)}; + if (g_ascii_strncasecmp (subtype.c_str(), "x-", 2) == 0) + return false; + const auto supertype{official_type.substr(0, slash_pos - 1)}; + const auto xtype{official_type.substr(0, slash_pos - 1) + "x-" + subtype}; + + /* Check if the "x-" version of the official mime-type matches the + * supplied mime-type. For example, if the official mime-type is + * "application/pkcs7-signature", then we also want to match + * "application/x-pkcs7-signature". */ + return g_ascii_strcasecmp(mime_type.c_str(), xtype.c_str()) == 0; +} + + +/** + * A bit of a monster, this impl. + * + * It's the transliteration of the g_mime_multipart_signed_verify() which + * adds the feature of passing in the CryptoContext. + * + */ +Result<std::vector<MimeSignature>> +MimeMultipartSigned::verify(const MimeCryptoContext& ctx, VerifyFlags vflags) const noexcept +{ + if (g_mime_multipart_get_count(GMIME_MULTIPART(self())) < 2) + return Err(Error::Code::Crypto, "cannot verify, not enough subparts"); + + const auto proto{content_type_parameter("protocol")}; + const auto sign_proto{ctx.signature_protocol()}; + + if (!proto || !sign_proto || !mime_types_equal(*proto, *sign_proto)) + return Err(Error::Code::Crypto, "unsupported protocol {}", + proto.value_or("<unknown>")); + + const auto sig{signed_signature_part()}; + const auto content{signed_content_part()}; + if (!sig || !content) + return Err(Error::Code::Crypto, "cannot find part"); + + const auto sig_mime_type{sig->mime_type()}; + if (!sig || !mime_types_equal(sig_mime_type.value_or("<none>"), *sign_proto)) + return Err(Error::Code::Crypto, "failed to find matching signature part"); + + MimeFormatOptions fopts{g_mime_format_options_new()}; + g_mime_format_options_set_newline_format(fopts.get(), GMIME_NEWLINE_FORMAT_DOS); + + MimeStream stream{MimeStream::make_mem()}; + if (auto&& res = content->write_to_stream(fopts, stream); !res) + return Err(res.error()); + stream.reset(); + + MimeDataWrapper wrapper{g_mime_part_get_content(GMIME_PART(sig->object()))}; + MimeStream sigstream{MimeStream::make_mem()}; + if (auto&& res = wrapper.write_to_stream(sigstream); !res) + return Err(res.error()); + sigstream.reset(); + + GError *err{}; + GMimeSignatureList *siglist{g_mime_crypto_context_verify( + GMIME_CRYPTO_CONTEXT(ctx.object()), + static_cast<GMimeVerifyFlags>(vflags), + GMIME_STREAM(stream.object()), + GMIME_STREAM(sigstream.object()), + {}, + &err)}; + if (!siglist) + return Err(Error::Code::Crypto, &err, "failed to verify"); + + std::vector<MimeSignature> sigs; + for (auto i = 0; + i != g_mime_signature_list_length(siglist); ++i) { + GMimeSignature *msig = g_mime_signature_list_get_signature(siglist, i); + sigs.emplace_back(MimeSignature(msig)); + } + g_object_unref(siglist); + + return sigs; +} + + +std::vector<MimeCertificate> +MimeDecryptResult::recipients() const noexcept +{ + GMimeCertificateList *lst{g_mime_decrypt_result_get_recipients(self())}; + if (!lst) + return {}; + + std::vector<MimeCertificate> certs; + for (int i = 0; i != g_mime_certificate_list_length(lst); ++i) + certs.emplace_back( + MimeCertificate( + g_mime_certificate_list_get_certificate(lst, i))); + + return certs; +} + +std::vector<MimeSignature> +MimeDecryptResult::signatures() const noexcept +{ + GMimeSignatureList *lst{g_mime_decrypt_result_get_signatures(self())}; + if (!lst) + return {}; + + std::vector<MimeSignature> sigs; + for (auto i = 0; i != g_mime_signature_list_length(lst); ++i) { + GMimeSignature *sig = g_mime_signature_list_get_signature(lst, i); + sigs.emplace_back(MimeSignature(sig)); + } + + return sigs; +} +/** + * Like verify, a bit of a monster, this impl. + * + * It's the transliteration of the g_mime_multipart_encrypted_decrypt() which + * adds the feature of passing in the CryptoContext. + * + */ + +Mu::Result<MimeMultipartEncrypted::Decrypted> +MimeMultipartEncrypted::decrypt(const MimeCryptoContext& ctx, DecryptFlags dflags, + const std::string& session_key) const noexcept +{ + if (g_mime_multipart_get_count(GMIME_MULTIPART(self())) < 2) + return Err(Error::Code::Crypto, "cannot decrypted, not enough subparts"); + + const auto proto{content_type_parameter("protocol")}; + const auto enc_proto{ctx.encryption_protocol()}; + + if (!proto || !enc_proto || !mime_types_equal(*proto, *enc_proto)) + return Err(Error::Code::Crypto, "unsupported protocol {}", + proto.value_or("<unknown>")); + + const auto version{encrypted_version_part()}; + const auto encrypted{encrypted_content_part()}; + if (!version || !encrypted) + return Err(Error::Code::Crypto, "cannot find part"); + + if (!mime_types_equal(version->mime_type().value_or(""), proto.value())) + return Err(Error::Code::Crypto, + "cannot decrypt; unexpected version content-type '{}' != '{}'", + version->mime_type().value_or(""), proto.value()); + + if (!mime_types_equal(encrypted->mime_type().value_or(""), + "application/octet-stream")) + return Err(Error::Code::Crypto, + "cannot decrypt; unexpected encrypted content-type '{}'", + encrypted->mime_type().value_or("")); + + const auto content{encrypted->content()}; + auto ciphertext{MimeStream::make_mem()}; + content.write_to_stream(ciphertext); + ciphertext.reset(); + + auto stream{MimeStream::make_mem()}; + auto filtered{MimeStream::make_filtered(stream)}; + auto filter{g_mime_filter_dos2unix_new(FALSE)}; + g_mime_stream_filter_add(GMIME_STREAM_FILTER(filtered.object()), + filter); + g_object_unref(filter); + + GError *err{}; + GMimeDecryptResult *dres = + g_mime_crypto_context_decrypt(GMIME_CRYPTO_CONTEXT(ctx.object()), + static_cast<GMimeDecryptFlags>(dflags), + session_key.empty() ? + NULL : session_key.c_str(), + GMIME_STREAM(ciphertext.object()), + GMIME_STREAM(filtered.object()), + &err); + if (!dres) + return Err(Error::Code::Crypto, &err, "decryption failed"); + + filtered.flush(); + stream.reset(); + + auto parser{g_mime_parser_new()}; + g_mime_parser_init_with_stream(parser, GMIME_STREAM(stream.object())); + + auto decrypted{g_mime_parser_construct_part(parser, NULL)}; + g_object_unref(parser); + if (!decrypted) { + g_object_unref(dres); + return Err(Error::Code::Crypto, "failed to parse decrypted part"); + } + + Decrypted result = { MimeObject{decrypted}, MimeDecryptResult{dres} }; + + g_object_unref(decrypted); + g_object_unref(dres); + + return Ok(std::move(result)); +} diff --git a/lib/message/mu-mime-object.hh b/lib/message/mu-mime-object.hh new file mode 100644 index 0000000..bfb2867 --- /dev/null +++ b/lib/message/mu-mime-object.hh @@ -0,0 +1,1389 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_MIME_OBJECT_HH__ +#define MU_MIME_OBJECT_HH__ + +#include <stdexcept> +#include <string> +#include <functional> +#include <array> +#include <vector> +#include <gmime/gmime.h> +#include "gmime/gmime-application-pkcs7-mime.h" +#include "gmime/gmime-crypto-context.h" +#include "utils/mu-option.hh" +#include "utils/mu-result.hh" +#include "utils/mu-utils.hh" +#include "mu-contact.hh" + +namespace Mu { + +/* non-GObject types */ + +using MimeFormatOptions = deletable_unique_ptr<GMimeFormatOptions, g_mime_format_options_free>; + +/** + * Initialize gmime (idempotent) + * + */ +void init_gmime(void); + + +/** + * Get a RFC2047-compatible address for the given contact + * + * @param contact a contact + * + * @return an address string + */ +std::string address_rfc2047(const Contact& contact); + +class Object { +public: + /** + * Default CTOR + * + */ + Object() noexcept: self_{} {} + + /** + * Create an object from a GObject + * + * @param obj a gobject. A ref is added. + */ + Object(GObject* &&obj): self_{G_OBJECT(g_object_ref(obj))} { + if (!G_IS_OBJECT(obj)) + throw std::runtime_error("not a g-object"); + } + + /** + * Copy CTOR + * + * @param other some other Object + */ + Object(const Object& other) noexcept { *this = other; } + + /** + * Move CTOR + * + * @param other some other Object + */ + Object(Object&& other) noexcept { *this = std::move(other); } + + /** + * operator= + * + * @param other copy some other object + * + * @return *this + */ + Object& operator=(const Object& other) noexcept { + + if (this != &other) { + auto oldself = self_; + self_ = other.self_ ? + G_OBJECT(g_object_ref(other.self_)) : nullptr; + if (oldself) + g_object_unref(oldself); + } + return *this; + } + + /** + * operator= + * + * @param other move some object object + * + * @return + */ + Object& operator=(Object&& other) noexcept { + + if (this != &other) { auto oldself = self_; + self_ = other.self_; + other.self_ = nullptr; + if (oldself) + g_object_unref(oldself); + } + return *this; + } + + /** + * DTOR + */ + virtual ~Object() { + if (self_) { + g_object_unref(self_); + } + } + + /** + * operator bool + * + * @return true if object wraps a GObject, false otherwise + */ + operator bool() const noexcept { return !!self_; } + + /** + * Get a ptr to the underlying GObject + * + * @return GObject or NULL + */ + GObject* object() const { return self_; } + + + /** + * Unref the object + * + */ + void unref() noexcept { + g_object_unref(self_); + } + + + /** + * Ref the object + * + */ + void ref() noexcept { + g_object_ref(self_); + } + + +private: + mutable GObject *self_{}; +}; + + + + + + +/** + * Thin wrapper around a GMimeContentType + * + */ +struct MimeContentType: public Object { + + MimeContentType(GMimeContentType *ctype) : Object{G_OBJECT(ctype)} { + if (!GMIME_IS_CONTENT_TYPE(self())) + throw std::runtime_error("not a content-type"); + } + std::string media_type() const noexcept { + return g_mime_content_type_get_media_type(self()); + } + std::string media_subtype() const noexcept { + return g_mime_content_type_get_media_subtype(self()); + } + + Option<std::string> mime_type() const noexcept { + return to_string_opt_gchar(g_mime_content_type_get_mime_type(self())); + } + + bool is_type(const std::string& type, const std::string& subtype) const { + return g_mime_content_type_is_type(self(), type.c_str(), + subtype.c_str()); + } +private: + GMimeContentType* self() const { + return reinterpret_cast<GMimeContentType*>(object()); + } +}; + + + + + +/** + * Thin wrapper around a GMimeStream + * + */ +struct MimeStream: public Object { + + ssize_t write(const char* buf, ::size_t size) { + return g_mime_stream_write(self(), buf, size); + } + + bool reset() { + return g_mime_stream_reset(self()) < 0 ? false : true; + } + + bool flush() { + return g_mime_stream_flush(self()) < 0 ? false : true; + } + + static MimeStream make_mem() { + MimeStream mstream{g_mime_stream_mem_new()}; + mstream.unref(); /* remove extra ref */ + return mstream; + } + + static MimeStream make_filtered(MimeStream& stream) { + MimeStream mstream{g_mime_stream_filter_new(stream.self())}; + mstream.unref(); /* remove extra refs */ + return mstream; + } + + static MimeStream make_from_stream(GMimeStream *strm) { + MimeStream mstream{strm}; + mstream.unref(); /* remove extra ref */ + return mstream; + } + +private: + MimeStream(GMimeStream *stream): Object(G_OBJECT(stream)) { + if (!GMIME_IS_STREAM(self())) + throw std::runtime_error("not a mime-stream"); + }; + + GMimeStream* self() const { + return reinterpret_cast<GMimeStream*>(object()); + } +}; + +template<typename S, typename T> +constexpr Option<std::string_view> to_string_view_opt(const S& seq, T t) { + auto&& it = seq_find_if(seq, [&](auto&& item){return item.first == t;}); + if (it == seq.cend()) + return Nothing; + else + return it->second; +} + + +/** + * Thin wrapper around a GMimeDataWrapper + * + */ +struct MimeDataWrapper: public Object { + MimeDataWrapper(GMimeDataWrapper *wrapper): Object(G_OBJECT(wrapper)) { + if (!GMIME_IS_DATA_WRAPPER(self())) + throw std::runtime_error("not a data-wrapper"); + }; + + Result<size_t> write_to_stream(MimeStream& stream) const { + if (auto&& res = g_mime_data_wrapper_write_to_stream( + self(), GMIME_STREAM(stream.object())) ; res < 0) + return Err(Error::Code::Message, "failed to write to stream"); + else + return Ok(static_cast<size_t>(res)); + } + +private: + GMimeDataWrapper* self() const { + return reinterpret_cast<GMimeDataWrapper*>(object()); + } +}; + + + +/** + * Thin wrapper around a GMimeCertifcate + * + */ +struct MimeCertificate: public Object { + MimeCertificate(GMimeCertificate *cert) : Object{G_OBJECT(cert)} { + if (!GMIME_IS_CERTIFICATE(self())) + throw std::runtime_error("not a certificate"); + } + + enum struct PubkeyAlgo { + Default = GMIME_PUBKEY_ALGO_DEFAULT, + Rsa = GMIME_PUBKEY_ALGO_RSA, + RsaE = GMIME_PUBKEY_ALGO_RSA_E, + RsaS = GMIME_PUBKEY_ALGO_RSA_S, + ElgE = GMIME_PUBKEY_ALGO_ELG_E, + Dsa = GMIME_PUBKEY_ALGO_DSA, + Ecc = GMIME_PUBKEY_ALGO_ECC, + Elg = GMIME_PUBKEY_ALGO_ELG, + EcDsa = GMIME_PUBKEY_ALGO_ECDSA, + EcDh = GMIME_PUBKEY_ALGO_ECDH, + EdDsa = GMIME_PUBKEY_ALGO_EDDSA, + }; + + enum struct DigestAlgo { + Default = GMIME_DIGEST_ALGO_DEFAULT, + Md5 = GMIME_DIGEST_ALGO_MD5, + Sha1 = GMIME_DIGEST_ALGO_SHA1, + RipEmd160 = GMIME_DIGEST_ALGO_RIPEMD160, + Md2 = GMIME_DIGEST_ALGO_MD2, + Tiger192 = GMIME_DIGEST_ALGO_TIGER192, + Haval5160 = GMIME_DIGEST_ALGO_HAVAL5160, + Sha256 = GMIME_DIGEST_ALGO_SHA256, + Sha384 = GMIME_DIGEST_ALGO_SHA384, + Sha512 = GMIME_DIGEST_ALGO_SHA512, + Sha224 = GMIME_DIGEST_ALGO_SHA224, + Md4 = GMIME_DIGEST_ALGO_MD4, + Crc32 = GMIME_DIGEST_ALGO_CRC32, + Crc32Rfc1510 = GMIME_DIGEST_ALGO_CRC32_RFC1510, + Crc32Rfc2440 = GMIME_DIGEST_ALGO_CRC32_RFC2440, + }; + + enum struct Trust { + Unknown = GMIME_TRUST_UNKNOWN, + Undefined = GMIME_TRUST_UNDEFINED, + Never = GMIME_TRUST_NEVER, + Marginal = GMIME_TRUST_MARGINAL, + TrustFull = GMIME_TRUST_FULL, + TrustUltimate = GMIME_TRUST_ULTIMATE, + }; + + enum struct Validity { + Unknown = GMIME_VALIDITY_UNKNOWN, + Undefined = GMIME_VALIDITY_UNDEFINED, + Never = GMIME_VALIDITY_NEVER, + Marginal = GMIME_VALIDITY_MARGINAL, + Full = GMIME_VALIDITY_FULL, + Ultimate = GMIME_VALIDITY_ULTIMATE, + }; + + PubkeyAlgo pubkey_algo() const { + return static_cast<PubkeyAlgo>( + g_mime_certificate_get_pubkey_algo(self())); + } + + DigestAlgo digest_algo() const { + return static_cast<DigestAlgo>( + g_mime_certificate_get_digest_algo(self())); + } + + Validity id_validity() const { + return static_cast<Validity>( + g_mime_certificate_get_id_validity(self())); + } + + Trust trust() const { + return static_cast<Trust>( + g_mime_certificate_get_trust(self())); + } + + Option<std::string> issuer_serial() const { + return to_string_opt(g_mime_certificate_get_issuer_serial(self())); + } + Option<std::string> issuer_name() const { + return to_string_opt(g_mime_certificate_get_issuer_name(self())); + } + + Option<std::string> fingerprint() const { + return to_string_opt(g_mime_certificate_get_fingerprint(self())); + } + + Option<std::string> key_id() const { + return to_string_opt(g_mime_certificate_get_key_id(self())); + } + + + Option<std::string> name() const { + return to_string_opt(g_mime_certificate_get_name(self())); + } + + Option<std::string> user_id() const { + return to_string_opt(g_mime_certificate_get_user_id(self())); + } + + Option<::time_t> created() const { + if (auto t = g_mime_certificate_get_created(self()); t >= 0) + return t; + else + return Nothing; + } + + Option<::time_t> expires() const { + if (auto t = g_mime_certificate_get_expires(self()); t >= 0) + return t; + else + return Nothing; + } + +private: + GMimeCertificate* self() const { + return reinterpret_cast<GMimeCertificate*>(object()); + } +}; + +constexpr std::array<std::pair<MimeCertificate::PubkeyAlgo, std::string_view>, 11> +AllPubkeyAlgos = {{ + { MimeCertificate::PubkeyAlgo::Default, "default"}, + { MimeCertificate::PubkeyAlgo::Rsa, "rsa"}, + { MimeCertificate::PubkeyAlgo::RsaE, "rsa-encryption-only"}, + { MimeCertificate::PubkeyAlgo::RsaS, "rsa-signing-only"}, + { MimeCertificate::PubkeyAlgo::ElgE, "el-gamal-encryption-only"}, + { MimeCertificate::PubkeyAlgo::Dsa, "dsa"}, + { MimeCertificate::PubkeyAlgo::Ecc, "elliptic curve"}, + { MimeCertificate::PubkeyAlgo::Elg, "el-gamal"}, + { MimeCertificate::PubkeyAlgo::EcDsa, "elliptic-curve+dsa"}, + { MimeCertificate::PubkeyAlgo::EcDh, "elliptic-curve+diffie-helman"}, + { MimeCertificate::PubkeyAlgo::EdDsa, "elliptic-curve+dsa-2"} + }}; + +constexpr Option<std::string_view> to_string_view_opt(MimeCertificate::PubkeyAlgo algo) { + return to_string_view_opt(AllPubkeyAlgos, algo); +} + +constexpr std::array<std::pair<MimeCertificate::DigestAlgo, std::string_view>, 15> +AllDigestAlgos = {{ + { MimeCertificate::DigestAlgo::Default, "default"}, + { MimeCertificate::DigestAlgo::Md5, "md5"}, + { MimeCertificate::DigestAlgo::Sha1, "sha1"}, + { MimeCertificate::DigestAlgo::RipEmd160, "ripemd-160"}, + { MimeCertificate::DigestAlgo::Md2, "md2"}, + { MimeCertificate::DigestAlgo::Tiger192, "tiger-192"}, + { MimeCertificate::DigestAlgo::Haval5160, "haval-5-160"}, + { MimeCertificate::DigestAlgo::Sha256, "sha-256"}, + { MimeCertificate::DigestAlgo::Sha384, "sha-384"}, + { MimeCertificate::DigestAlgo::Sha512, "sha-512"}, + { MimeCertificate::DigestAlgo::Sha224, "sha-224"}, + { MimeCertificate::DigestAlgo::Md4, "md4"}, + { MimeCertificate::DigestAlgo::Crc32, "crc32"}, + { MimeCertificate::DigestAlgo::Crc32Rfc1510, "crc32-rfc1510"}, + { MimeCertificate::DigestAlgo::Crc32Rfc2440, "crc32-rfc2440"}, + }}; + +constexpr Option<std::string_view> to_string_view_opt(MimeCertificate::DigestAlgo algo) { + return to_string_view_opt(AllDigestAlgos, algo); +} + +constexpr std::array<std::pair<MimeCertificate::Trust, std::string_view>, 6> +AllTrusts = {{ + { MimeCertificate::Trust::Unknown, "unknown" }, + { MimeCertificate::Trust::Undefined, "undefined" }, + { MimeCertificate::Trust::Never, "never" }, + { MimeCertificate::Trust::Marginal, "marginal" }, + { MimeCertificate::Trust::TrustFull, "trust-full" }, + { MimeCertificate::Trust::TrustUltimate,"trust-ultimate" }, + }}; + +constexpr Option<std::string_view> to_string_view_opt(MimeCertificate::Trust trust) { + return to_string_view_opt(AllTrusts, trust); +} + +constexpr std::array<std::pair<MimeCertificate::Validity, std::string_view>, 6> +AllValidities = {{ + { MimeCertificate::Validity::Unknown, "unknown" }, + { MimeCertificate::Validity::Undefined, "undefined" }, + { MimeCertificate::Validity::Never, "never" }, + { MimeCertificate::Validity::Marginal, "marginal" }, + { MimeCertificate::Validity::Full, "full" }, + { MimeCertificate::Validity::Ultimate, "ultimate" }, + }}; + +constexpr Option<std::string_view> to_string_view_opt(MimeCertificate::Validity val) { + return to_string_view_opt(AllValidities, val); +} + + + +/** + * Thin wrapper around a GMimeSignature + * + */ +struct MimeSignature: public Object { + MimeSignature(GMimeSignature *sig) : Object{G_OBJECT(sig)} { + if (!GMIME_IS_SIGNATURE(self())) + throw std::runtime_error("not a signature"); + } + + /** + * Signature status + * + */ + enum struct Status { + Valid = GMIME_SIGNATURE_STATUS_VALID, + Green = GMIME_SIGNATURE_STATUS_GREEN, + Red = GMIME_SIGNATURE_STATUS_RED, + KeyRevoked = GMIME_SIGNATURE_STATUS_KEY_REVOKED, + KeyExpired = GMIME_SIGNATURE_STATUS_KEY_EXPIRED, + SigExpired = GMIME_SIGNATURE_STATUS_SIG_EXPIRED, + KeyMissing = GMIME_SIGNATURE_STATUS_KEY_MISSING, + CrlMissing = GMIME_SIGNATURE_STATUS_CRL_MISSING, + CrlTooOld = GMIME_SIGNATURE_STATUS_CRL_TOO_OLD, + BadPolicy = GMIME_SIGNATURE_STATUS_BAD_POLICY, + SysError = GMIME_SIGNATURE_STATUS_SYS_ERROR, + TofuConflict = GMIME_SIGNATURE_STATUS_TOFU_CONFLICT + }; + + Status status() const { return static_cast<Status>( + g_mime_signature_get_status(self())); } + + ::time_t created() const { return g_mime_signature_get_created(self()); } + ::time_t expires() const { return g_mime_signature_get_expires(self()); } + + + const MimeCertificate certificate() const { + return MimeCertificate{g_mime_signature_get_certificate(self())}; + } + +private: + GMimeSignature* self() const { + return reinterpret_cast<GMimeSignature*>(object()); + } +}; + +constexpr std::array<std::pair<MimeSignature::Status, std::string_view>, 12> +AllMimeSignatureStatuses= {{ + { MimeSignature::Status::Valid, "valid" }, + { MimeSignature::Status::Green, "green" }, + { MimeSignature::Status::Red, "red" }, + { MimeSignature::Status::KeyRevoked, "key-revoked" }, + { MimeSignature::Status::KeyExpired, "key-expired" }, + { MimeSignature::Status::SigExpired, "sig-expired" }, + { MimeSignature::Status::KeyMissing, "key-missing" }, + { MimeSignature::Status::CrlMissing, "crl-missing" }, + { MimeSignature::Status::CrlTooOld, "crl-too-old" }, + { MimeSignature::Status::BadPolicy, "bad-policy" }, + { MimeSignature::Status::SysError, "sys-error" }, + { MimeSignature::Status::TofuConflict, "tofu-confict" }, + }}; +MU_ENABLE_BITOPS(MimeSignature::Status); + +static inline std::string to_string(MimeSignature::Status status) { + std::string str; + for (auto&& item: AllMimeSignatureStatuses) { + if (none_of(item.first & status)) + continue; + if (!str.empty()) + str += ", "; + str += item.second; + } + if (str.empty()) + str = "none"; + + return str; +} + + + + +/** +* Thin wrapper around a GMimeDecryptResult + * + */ +struct MimeDecryptResult: public Object { + MimeDecryptResult (GMimeDecryptResult *decres) : Object{G_OBJECT(decres)} { + if (!GMIME_IS_DECRYPT_RESULT(self())) + throw std::runtime_error("not a decrypt-result"); + } + + std::vector<MimeCertificate> recipients() const noexcept; + std::vector<MimeSignature> signatures() const noexcept; + + enum struct CipherAlgo { + Default = GMIME_CIPHER_ALGO_DEFAULT, + Idea = GMIME_CIPHER_ALGO_IDEA, + Des3 = GMIME_CIPHER_ALGO_3DES, + Cast5 = GMIME_CIPHER_ALGO_CAST5, + Blowfish = GMIME_CIPHER_ALGO_BLOWFISH, + Aes = GMIME_CIPHER_ALGO_AES, + Aes192 = GMIME_CIPHER_ALGO_AES192, + Aes256 = GMIME_CIPHER_ALGO_AES256, + TwoFish = GMIME_CIPHER_ALGO_TWOFISH, + Camellia128 = GMIME_CIPHER_ALGO_CAMELLIA128, + Camellia192 = GMIME_CIPHER_ALGO_CAMELLIA192, + Camellia256 = GMIME_CIPHER_ALGO_CAMELLIA256 + }; + + CipherAlgo cipher() const noexcept { + return static_cast<CipherAlgo>( + g_mime_decrypt_result_get_cipher(self())); + } + + using DigestAlgo = MimeCertificate::DigestAlgo; + DigestAlgo mdc() const noexcept { + return static_cast<DigestAlgo>( + g_mime_decrypt_result_get_mdc(self())); + } + + Option<std::string> session_key() const noexcept { + return to_string_opt(g_mime_decrypt_result_get_session_key(self())); + } + +private: + GMimeDecryptResult* self() const { + return reinterpret_cast<GMimeDecryptResult*>(object()); + } +}; + +constexpr std::array<std::pair<MimeDecryptResult::CipherAlgo, std::string_view>, 12> +AllCipherAlgos= {{ + {MimeDecryptResult::CipherAlgo::Default, "default"}, + {MimeDecryptResult::CipherAlgo::Idea, "idea"}, + {MimeDecryptResult::CipherAlgo::Des3, "3des"}, + {MimeDecryptResult::CipherAlgo::Cast5, "cast5"}, + {MimeDecryptResult::CipherAlgo::Blowfish, "blowfish"}, + {MimeDecryptResult::CipherAlgo::Aes, "aes"}, + {MimeDecryptResult::CipherAlgo::Aes192, "aes192"}, + {MimeDecryptResult::CipherAlgo::Aes256, "aes256"}, + {MimeDecryptResult::CipherAlgo::TwoFish, "twofish"}, + {MimeDecryptResult::CipherAlgo::Camellia128, "camellia128"}, + {MimeDecryptResult::CipherAlgo::Camellia192, "camellia192"}, + {MimeDecryptResult::CipherAlgo::Camellia256, "camellia256"}, + }}; + +constexpr Option<std::string_view> to_string_view_opt(MimeDecryptResult::CipherAlgo algo) { + return to_string_view_opt(AllCipherAlgos, algo); +} + + +/** + * Thin wrapper around a GMimeCryptoContext + * + */ +struct MimeCryptoContext : public Object { + + /** + * Make a new PGP crypto context. + * + * For 'test-mode', pass a test-path; in this mode GPG will be setup + * in an isolated mode so it does not affect normal usage. + * + * @param testpath (for unit-tests) pass a path to an existing dir to + * create a pgp setup. For normal use, leave empty. + * + * @return A MimeCryptoContext or an error + */ + static Result<MimeCryptoContext> + make_gpg(const std::string& testpath={}) try { + if (!testpath.empty()) { + if (auto&& res = setup_gpg_test(testpath); !res) + return Err(res.error()); + } + MimeCryptoContext ctx(g_mime_gpg_context_new()); + ctx.unref(); /* remove extra ref */ + return Ok(std::move(ctx)); + } catch (...) { + return Err(Error::Code::Crypto, "failed to create crypto context"); + } + + static Result<MimeCryptoContext> + make(const std::string& protocol) { + auto ctx = g_mime_crypto_context_new(protocol.c_str()); + if (!ctx) + return Err(Error::Code::Crypto, + "unsupported protocol {}", protocol); + MimeCryptoContext mctx{ctx}; + mctx.unref(); /* remove extra ref */ + return Ok(std::move(mctx)); + } + + Option<std::string> encryption_protocol() const noexcept { + return to_string_opt(g_mime_crypto_context_get_encryption_protocol(self())); + } + Option<std::string> signature_protocol() const noexcept { + return to_string_opt(g_mime_crypto_context_get_signature_protocol(self())); + } + Option<std::string> key_exchange_protocol() const noexcept { + return to_string_opt(g_mime_crypto_context_get_key_exchange_protocol(self())); + } + + /** + * Imports a stream of keys/certificates contained within stream into + * the key/certificate database controlled by @this. + * + * @param stream + * + * @return number of keys imported, or an error. + */ + Result<size_t> import_keys(MimeStream& stream); + + /** + * Prototype for a request-password function. + * + * @param ctx the MimeCryptoContext making the request + * @param user_id the user_id of the password being requested + * @param prompt a string containing some helpful context for the prompt + * @param reprompt true if this password request is a reprompt due to a + * previously bad password response + * @param response a stream for the application to write the password to + * (followed by a newline '\n' character) + * + * @return nothing (Ok) or an error, + */ + using PasswordRequestFunc = + std::function<Result<void>( + const MimeCryptoContext& ctx, + const std::string& user_id, + const std::string& prompt, + bool reprompt, + MimeStream& response)>; + /** + * Set a function to request a password. + * + * @param pw_func password function. + */ + void set_request_password(PasswordRequestFunc pw_func); + + +private: + MimeCryptoContext(GMimeCryptoContext *ctx): Object{G_OBJECT(ctx)} { + if (!GMIME_IS_CRYPTO_CONTEXT(self())) + throw std::runtime_error("not a crypto-context"); + } + + static Result<void> setup_gpg_test(const std::string& testpath); + + GMimeCryptoContext* self() const { + return reinterpret_cast<GMimeCryptoContext*>(object()); + } +}; + + +/** + * Thin wrapper around a GMimeObject + * + */ +class MimeObject: public Object { +public: + /** + * Construct a new MimeObject. Take a ref on the obj + * + * @param mime_part mime-part pointer + */ + MimeObject(const Object& obj): Object{obj} { + if (!GMIME_IS_OBJECT(self())) + throw std::runtime_error("not a mime-object"); + } + MimeObject(GMimeObject *mobj): Object{G_OBJECT(mobj)} { + if (mobj && !GMIME_IS_OBJECT(self())) + throw std::runtime_error("not a mime-object"); + } + + /** + * Get a header from the MimeObject + * + * @param header the header to retrieve + * + * @return header value (UTF-8) or Nothing + */ + Option<std::string> header(const std::string& header) const noexcept; + + + /** + * Get all headers as pairs of name, value + * + * @return all headers + */ + std::vector<std::pair<std::string, std::string>> headers() const noexcept; + + + /** + * Get the content type + * + * @return the content-type or Nothing + */ + Option<MimeContentType> content_type() const noexcept { + auto ct{g_mime_object_get_content_type(self())}; + if (!ct) + return Nothing; + else + return MimeContentType(ct); + } + + Option<std::string> mime_type() const noexcept { + if (auto ct = content_type(); !ct) + return Nothing; + else + return ct->mime_type(); + } + + /** + * Get the content-type parameter + * + * @param param name of parameter + * + * @return the value of the parameter, or Nothing + */ + Option<std::string> content_type_parameter(const std::string& param) const noexcept { + return Mu::to_string_opt( + g_mime_object_get_content_type_parameter(self(), param.c_str())); + } + + /** + * Write this MimeObject to some stream + * + * @param f_opts formatting options + * @param stream the stream + * + * @return the number or bytes written or an error + */ + Result<size_t> write_to_stream(const MimeFormatOptions& f_opts, + MimeStream& stream) const; + /** + * Write the object to a string. + * + * @return + */ + Option<std::string> to_string_opt() const noexcept; + + /** + * Write object to a file + * + * @param path path to file + * @param overwrite if true, overwrite existing file, if it bqexists + * + * @return size of the wrtten file, or an error. + */ + Result<size_t> to_file(const std::string& path, bool overwrite) const noexcept; + + /* + * subtypes. + */ + + /** + * Is this a MimePart? + * + * @return true or false + */ + bool is_part() const { return GMIME_IS_PART(self()); } + + /** + * Is this a MimeMultiPart? + * + * @return true or false + */ + bool is_multipart() const { return GMIME_IS_MULTIPART(self());} + + /** + * Is this a MimeMultiPart? + * + * @return true or false + */ + bool is_multipart_encrypted() const { + return GMIME_IS_MULTIPART_ENCRYPTED(self()); + } + + /** + * Is this a MimeMultiPart? + * + * @return true or false + */ + bool is_multipart_signed() const { + return GMIME_IS_MULTIPART_SIGNED(self()); + } + + /** + * Is this a MimeMessage? + * + * @return true or false + */ + bool is_message() const { return GMIME_IS_MESSAGE(self());} + + /** + * Is this a MimeMessagePart? + * + * @return true orf alse + */ + bool is_message_part() const { return GMIME_IS_MESSAGE_PART(self());} + + /** + * Is this a MimeApplicationpkcs7Mime? + * + * @return true orf alse + */ + bool is_mime_application_pkcs7_mime() const { + return GMIME_IS_APPLICATION_PKCS7_MIME(self()); + } + + /** + * Callback for for_each(). See GMimeObjectForEachFunc. + * + */ + using ForEachFunc = std::function<void(const MimeObject& parent, + const MimeObject& part)>; + +private: + GMimeObject* self() const { + return reinterpret_cast<GMimeObject*>(object()); + } +}; + + +/** + * Thin wrapper around a GMimeMessage + * + */ +class MimeMessage: public MimeObject { +public: + /** + * Construct a MimeMessage + * + * @param obj an Object of the right type + */ + MimeMessage(const Object& obj): MimeObject(obj) { + if (!is_message()) + throw std::runtime_error("not a mime-message"); + } + + /** + * Make a MimeMessage from a file + * + * @param path path to the file + * + * @return a MimeMessage or an error. + */ + static Result<MimeMessage> make_from_file (const std::string& path); + + /** + * Make a MimeMessage from a string + * + * @param path path to the file + * + * @return a MimeMessage or an error. + */ + static Result<MimeMessage> make_from_text (const std::string& text); + + /** + * Get the contacts of a given type, or None for _all_ + * + * @param ctype contact type + * + * @return contacts + */ + Contacts contacts(Contact::Type ctype) const noexcept; + + /** + * Gets the message-id if it exists, or nullopt otherwise. + * + * @return string or nullopt + */ + Option<std::string> message_id() const noexcept { + return Mu::to_string_opt(g_mime_message_get_message_id(self())); + } + + /** + * Gets the message-id if it exists, or nullopt otherwise. + * + * @return string or nullopt + */ + Option<std::string> subject() const noexcept { + return Mu::to_string_opt(g_mime_message_get_subject(self())); + } + + /** + * Gets the date if it exists, or nullopt otherwise. + * + * @return a time_t value (expressed as a 64-bit number) or nullopt + */ + Option<int64_t> date() const noexcept; + + + /** + * Get the references for this message (including in-reply-to), in the + * order of older..newer; the first one would the oldest parent, and + * in-reply-to would be the last one (if any). These are de-duplicated, + * and known-fake references removed (see implementation) + * + * @return references. + */ + std::vector<std::string> references() const noexcept; + + + /** + * Recursively apply func tol all parts of this message + * + * @param func a function + */ + void for_each(const ForEachFunc& func) const noexcept; + +private: + GMimeMessage* self() const { + return reinterpret_cast<GMimeMessage*>(object()); + } +}; + +/** + * Thin wrapper around a GMimePart. + * + */ +class MimePart: public MimeObject { +public: + /** + * Construct a MimePart + * + * @param obj an Object of the right type + */ + MimePart(const Object& obj): MimeObject(obj) { + if (!is_part()) + throw std::runtime_error("not a mime-part"); + } + + /** + * Determines whether or not the part is an attachment based on the + * value of the Content-Disposition header. + * + * @return true or false + */ + bool is_attachment() const noexcept { + return g_mime_part_is_attachment(self()); + } + + /** + * Gets the value of the Content-Description for this mime part + * if it exists, or nullopt otherwise. + * + * @return string or nullopt + */ + Option<std::string> content_description() const noexcept { + return Mu::to_string_opt(g_mime_part_get_content_description(self())); + } + + /** + * Gets the value of the Content-Id for this mime part + * if it exists, or nullopt otherwise. + * + * @return string or nullopt + */ + Option<std::string> content_id() const noexcept { + return Mu::to_string_opt(g_mime_part_get_content_id(self())); + } + + /** + * Gets the value of the Content-Md5 header for this mime part + * if it exists, or nullopt otherwise. + * + * @return string or nullopt + */ + Option<std::string> content_md5() const noexcept { + return Mu::to_string_opt(g_mime_part_get_content_md5(self())); + + } + + /** + * Verify the content md5 for the specified mime part. Returns false if + * the mime part does not contain a Content-MD5. + * + * @return true or false + */ + bool verify_content_md5() const noexcept { + return g_mime_part_verify_content_md5(self()); + } + + /** + * Gets the value of the Content-Location for this mime part if it + * exists, or nullopt otherwise. + * + * @return string or nullopt + */ + Option<std::string> content_location() const noexcept { + return Mu::to_string_opt(g_mime_part_get_content_location(self())); + } + + + MimeDataWrapper content() const noexcept { + return MimeDataWrapper{g_mime_part_get_content(self())}; + } + + /** + * Gets the filename for this mime part if it exists, or nullopt + * otherwise. + * + * @return string or nullopt + */ + Option<std::string> filename() const noexcept { + return Mu::to_string_opt(g_mime_part_get_filename(self())); + } + + /** + * Size of content, in bytes + * + * @return size + */ + size_t size() const noexcept; + + /** + * Get as UTF-8 string + * + * @return a string, or NULL. + */ + Option<std::string> to_string() const noexcept; + + /** + * Write part to a file + * + * @param path path to file + * @param overwrite if true, overwrite existing file, if it bqexists + * + * @return size of the wrtten file, or an error. + */ + Result<size_t> to_file(const std::string& path, bool overwrite) + const noexcept; + + /** + * Types of Content Encoding. + * + */ + enum struct ContentEncoding { + Default = GMIME_CONTENT_ENCODING_DEFAULT, + SevenBit = GMIME_CONTENT_ENCODING_7BIT, + EightBit = GMIME_CONTENT_ENCODING_8BIT, + Binary = GMIME_CONTENT_ENCODING_BINARY, + Base64 = GMIME_CONTENT_ENCODING_BASE64, + QuotedPrintable = GMIME_CONTENT_ENCODING_QUOTEDPRINTABLE, + UuEncode = GMIME_CONTENT_ENCODING_UUENCODE + }; + + /** + * Gets the content encoding of the mime part. + * + * @return the content encoding + */ + ContentEncoding content_encoding() const noexcept { + const auto enc{g_mime_part_get_content_encoding(self())}; + g_return_val_if_fail(enc <= GMIME_CONTENT_ENCODING_UUENCODE, + ContentEncoding::Default); + return static_cast<ContentEncoding>(enc); + } + + + /** + * Types of OpenPGP data + * + */ + enum struct OpenPGPData { + None = GMIME_OPENPGP_DATA_NONE, + Encrypted = GMIME_OPENPGP_DATA_ENCRYPTED, + Signed = GMIME_OPENPGP_DATA_SIGNED, + PublicKey = GMIME_OPENPGP_DATA_PUBLIC_KEY, + PrivateKey = GMIME_OPENPGP_DATA_PRIVATE_KEY, + }; + + /** + * Gets whether or not (and what type) of OpenPGP data is contained + * + * @return OpenGPGData + */ + OpenPGPData openpgp_data() const noexcept { + const auto data{g_mime_part_get_openpgp_data(self())}; + g_return_val_if_fail(data <= GMIME_OPENPGP_DATA_PRIVATE_KEY, + OpenPGPData::None); + return static_cast<OpenPGPData>(data); + } + +private: + GMimePart* self() const { + return reinterpret_cast<GMimePart*>(object()); + } +}; + + + +/** + * Thin wrapper around a GMimeMessagePart. + * + */ +class MimeMessagePart: public MimeObject { +public: + /** + * Construct a MimeMessagePart + * + * @param obj an Object of the right type + */ + MimeMessagePart(const Object& obj): MimeObject(obj) { + if (!is_message_part()) + throw std::runtime_error("not a mime-message-part"); + } + + /** + * Get the MimeMessage for this MimeMessagePart. + * + * @return the MimeMessage or Nothing + */ + Option<MimeMessage> get_message() const { + auto msg{g_mime_message_part_get_message(self())}; + if (msg) + return MimeMessage(Object(G_OBJECT(msg))); + else + return Nothing; + } +private: + GMimeMessagePart* self() const { + return reinterpret_cast<GMimeMessagePart*>(object()); + } + +}; + /** + * Thin wrapper around a GMimeApplicationPkcs7Mime + * + */ +class MimeApplicationPkcs7Mime: public MimePart { +public: + /** + * Construct a MimeApplicationPkcs7Mime + * + * @param obj an Object of the right type + */ + MimeApplicationPkcs7Mime(const Object& obj): MimePart(obj) { + if (!is_mime_application_pkcs7_mime()) + throw std::runtime_error("not a mime-application-pkcs7-mime"); + } + + enum struct SecureMimeType { + CompressedData = GMIME_SECURE_MIME_TYPE_COMPRESSED_DATA, + EnvelopedData = GMIME_SECURE_MIME_TYPE_ENVELOPED_DATA, + SignedData = GMIME_SECURE_MIME_TYPE_SIGNED_DATA, + CertsOnly = GMIME_SECURE_MIME_TYPE_CERTS_ONLY, + Unknown = GMIME_SECURE_MIME_TYPE_UNKNOWN + }; + + SecureMimeType smime_type() const { + return static_cast<SecureMimeType>( + g_mime_application_pkcs7_mime_get_smime_type(self())); + } + +private: + GMimeApplicationPkcs7Mime* self() const { + return reinterpret_cast<GMimeApplicationPkcs7Mime*>(object()); + } +}; + + +/** + * Thin wrapper around a GMimeMultiPart + * + */ +class MimeMultipart: public MimeObject { +public: + /** + * Construct a MimeMultipart + * + * @param obj an Object of the right type + */ + MimeMultipart(const Object& obj): MimeObject(obj) { + if (!is_multipart()) + throw std::runtime_error("not a mime-multipart"); + } + + Option<MimePart> signed_content_part() const { + return part(GMIME_MULTIPART_SIGNED_CONTENT); + } + + Option<MimePart> signed_signature_part() const { + return part(GMIME_MULTIPART_SIGNED_SIGNATURE); + } + + Option<MimePart> encrypted_version_part() const { + return part(GMIME_MULTIPART_ENCRYPTED_VERSION); + } + + Option<MimePart> encrypted_content_part() const { + return part(GMIME_MULTIPART_ENCRYPTED_CONTENT); + } + + /** + * Recursively apply func to all parts + * + * @param func a function + */ + void for_each(const ForEachFunc& func) const noexcept; + +private: + // Note: the part may not be available if the message was marked as + // _signed_ or _encrypted_ because it contained a forwarded signed or + // encrypted message. + Option<MimePart> part(int index) const { + if (auto&& p{g_mime_multipart_get_part(self() ,index)}; + !GMIME_IS_PART(p)) + return Nothing; + else + return Some(MimeObject{p}); + } + + GMimeMultipart* self() const { + return reinterpret_cast<GMimeMultipart*>(object()); + } +}; + + +/** + * Thin wrapper around a GMimeMultiPartEncrypted + * + */ +class MimeMultipartEncrypted: public MimeMultipart { +public: + /** + * Construct a MimeMultipartEncrypted + * + * @param obj an Object of the right type + */ + MimeMultipartEncrypted(const Object& obj): MimeMultipart(obj) { + if (!is_multipart_encrypted()) + throw std::runtime_error("not a mime-multipart-encrypted"); + } + + enum struct DecryptFlags { + None = GMIME_DECRYPT_NONE, + ExportSessionKey = GMIME_DECRYPT_EXPORT_SESSION_KEY, + NoVerify = GMIME_DECRYPT_NO_VERIFY, + EnableKeyserverLookups = GMIME_DECRYPT_ENABLE_KEYSERVER_LOOKUPS, + EnableOnlineCertificateChecks = GMIME_DECRYPT_ENABLE_ONLINE_CERTIFICATE_CHECKS + }; + + using Decrypted = std::pair<MimeObject, MimeDecryptResult>; + Result<Decrypted> decrypt(const MimeCryptoContext& ctx, + DecryptFlags flags=DecryptFlags::None, + const std::string& session_key = {}) const noexcept; + +private: + GMimeMultipartEncrypted* self() const { + return reinterpret_cast<GMimeMultipartEncrypted*>(object()); + } +}; + +MU_ENABLE_BITOPS(MimeMultipartEncrypted::DecryptFlags); + + +/** + * Thin wrapper around a GMimeMultiPartSigned + * + */ +class MimeMultipartSigned: public MimeMultipart { +public: + /** + * Construct a MimeMultipartSigned + * + * @param obj an Object of the right type + */ + MimeMultipartSigned(const Object& obj): MimeMultipart(obj) { + if (!is_multipart_signed()) + throw std::runtime_error("not a mime-multipart-signed"); + } + + enum struct VerifyFlags { + None = GMIME_VERIFY_NONE, + EnableKeyserverLookups = GMIME_VERIFY_ENABLE_KEYSERVER_LOOKUPS, + EnableOnlineCertificateChecks = GMIME_VERIFY_ENABLE_ONLINE_CERTIFICATE_CHECKS + }; + + // Result<std::vector<MimeSignature>> verify(VerifyFlags vflags=VerifyFlags::None) const noexcept; + + Result<std::vector<MimeSignature>> verify(const MimeCryptoContext& ctx, + VerifyFlags vflags=VerifyFlags::None) const noexcept; + +private: + GMimeMultipartSigned* self() const { + return reinterpret_cast<GMimeMultipartSigned*>(object()); + } +}; + + +MU_ENABLE_BITOPS(MimeMultipartSigned::VerifyFlags); + +} // namespace Mu + + +#endif /* MU_MIME_OBJECT_HH__ */ diff --git a/lib/message/mu-priority.cc b/lib/message/mu-priority.cc new file mode 100644 index 0000000..9b57cea --- /dev/null +++ b/lib/message/mu-priority.cc @@ -0,0 +1,76 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-priority.hh" + +using namespace Mu; + +std::string +Mu::to_string(Priority prio) +{ + return std::string{priority_name(prio)}; +} + +/* + * tests... also build as runtime-tests, so we can get coverage info + */ +#ifdef BUILD_TESTS +#include <glib.h> +#define static_assert g_assert_true +#endif /*BUILD_TESTS*/ + +[[maybe_unused]] static void +test_priority_to_char() +{ + static_assert(to_char(Priority::Low) == 'l'); + static_assert(to_char(Priority::Normal) == 'n'); + static_assert(to_char(Priority::High) == 'h'); +} + +[[maybe_unused]] static void +test_priority_from_char() +{ + static_assert(priority_from_char('l') == Priority::Low); + static_assert(priority_from_char('n') == Priority::Normal); + static_assert(priority_from_char('h') == Priority::High); + static_assert(priority_from_char('x') == Priority::Normal); +} + +[[maybe_unused]] static void +test_priority_name() +{ + static_assert(priority_name(Priority::Low) == "low"); + static_assert(priority_name(Priority::Normal) == "normal"); + static_assert(priority_name(Priority::High) == "high"); +} + + +#ifdef BUILD_TESTS +int +main(int argc, char* argv[]) +{ + g_test_init(&argc, &argv, NULL); + + g_test_add_func("/message/priority/to-char", test_priority_to_char); + g_test_add_func("/message/priority/from-char", test_priority_from_char); + g_test_add_func("/message/priority/name", test_priority_name); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/message/mu-priority.hh b/lib/message/mu-priority.hh new file mode 100644 index 0000000..a4bded3 --- /dev/null +++ b/lib/message/mu-priority.hh @@ -0,0 +1,154 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_PRIORITY_HH__ +#define MU_PRIORITY_HH__ + +#include <array> +#include <string> +#include <string_view> +#include "mu-fields.hh" + +namespace Mu { +/** + * Message priorities + * + */ + +/** + * The priority ids + * + */ +enum struct Priority : char { + Low = 'l', /**< Low priority */ + Normal = 'n', /**< Normal priority */ + High = 'h', /**< High priority */ +}; + +/** + * Sequence of all message priorities. + */ +static constexpr std::array<Priority, 3> AllMessagePriorities = { + Priority::Low, Priority::Normal, Priority::High}; + +/** + * Get the char for some priority + * + * @param id an id + * + * @return the char + */ +constexpr char +to_char(Priority prio) +{ + return static_cast<char>(prio); +} + +/** + * Get the priority for some character; unknown ones + * become Normal. + * + * @param c some character + */ +constexpr Priority +priority_from_char(char c) +{ + switch (c) { + case 'l': + return Priority::Low; + case 'h': + return Priority::High; + case 'n': + default: + return Priority::Normal; + } +} + +/** + * Get the priority from their (internal) name, i.e., low/normal/high + * or shortcut. + * + * @param pname + * + * @return the priority or none + */ +static inline Option<Priority> +priority_from_name(std::string_view pname) +{ + if (pname == "low" || pname == "l") + return Priority::Low; + else if (pname == "high" || pname == "h") + return Priority::High; + else if (pname == "normal" || pname == "n") + return Priority::Normal; + else + return Nothing; +} + + +/** + * Get the name for a given priority + * + * @return the name + */ +constexpr std::string_view +priority_name(Priority prio) +{ + switch (prio) { + case Priority::Low: + return "low"; + case Priority::High: + return "high"; + case Priority::Normal: + default: + return "normal"; + } +} + +/** + * Get the name for a given priority (backward compatibility) + * + * @return the name + */ +constexpr const char* +priority_name_c_str(Priority prio) +{ + switch (prio) { + case Priority::Low: + return "low"; + case Priority::High: + return "high"; + case Priority::Normal: + default: + return "normal"; + } +} + +/** + * Get a the message priority as a string + * + * @param prio priority + * + * @return a string + */ +std::string to_string(Priority prio); + +} // namespace Mu + +#endif /*MU_PRIORITY_HH_*/ diff --git a/lib/message/test-mu-message.cc b/lib/message/test-mu-message.cc new file mode 100644 index 0000000..1e0962e --- /dev/null +++ b/lib/message/test-mu-message.cc @@ -0,0 +1,1125 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include "utils/mu-test-utils.hh" +#include "mu-message.hh" +#include "mu-mime-object.hh" +#include <glib.h> +#include <regex> + +using namespace Mu; + +/* + * test message 1 + */ + +static void +test_message_mailing_list() +{ + constexpr const char *test_message_1 = +R"(Return-Path: <sqlite-dev-bounces@sqlite.org> +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 32F276963F + for <xxxx@localhost>; Mon, 4 Aug 2008 21:49:34 +0300 (EEST) +Message-Id: <83B5AF40-DBFA-4578-A043-04C80276E195@sqlabs.net> +From: anon@example.com +To: sqlite-dev@sqlite.org +Mime-Version: 1.0 (Apple Message framework v926) +Date: Mon, 4 Aug 2008 11:40:49 +0200 +X-Mailer: Apple Mail (2.926) +Subject: [sqlite-dev] VM optimization inside sqlite3VdbeExec +Precedence: list +Reply-To: sqlite-dev@sqlite.org +List-Id: <sqlite-dev.sqlite.org> +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit +Sender: sqlite-dev-bounces@sqlite.org +Content-Length: 639 + +Inside sqlite3VdbeExec there is a very big switch statement. +In order to increase performance with few modifications to the +original code, why not use this technique ? +http://docs.freebsd.org/info/gcc/gcc.info.Labels_as_Values.html + +With a properly defined "instructions" array, instead of the switch +statement you can use something like: +goto * instructions[pOp->opcode]; +)"; + auto message{Message::make_from_text( + test_message_1, + "/home/test/Maildir/inbox/cur/1649279256.107710_1.evergrey:2,S")}; + g_assert_true(!!message); + assert_equal(message->path(), + "/home/test/Maildir/inbox/cur/1649279256.107710_1.evergrey:2,S"); + g_assert_true(message->maildir().empty()); + + g_assert_true(message->bcc().empty()); + + g_assert_true(!message->body_html()); + assert_equal(message->body_text().value_or(""), +R"(Inside sqlite3VdbeExec there is a very big switch statement. +In order to increase performance with few modifications to the +original code, why not use this technique ? +http://docs.freebsd.org/info/gcc/gcc.info.Labels_as_Values.html + +With a properly defined "instructions" array, instead of the switch +statement you can use something like: +goto * instructions[pOp->opcode]; +)"); + g_assert_true(message->cc().empty()); + g_assert_cmpuint(message->date(), ==, 1217842849); + g_assert_true(message->flags() == (Flags::MailingList | Flags::Seen)); + + const auto from{message->from()}; + g_assert_cmpuint(from.size(),==,1); + assert_equal(from.at(0).name, ""); + assert_equal(from.at(0).email, "anon@example.com"); + + assert_equal(message->mailing_list(), "sqlite-dev.sqlite.org"); + assert_equal(message->message_id(), + "83B5AF40-DBFA-4578-A043-04C80276E195@sqlabs.net"); + + g_assert_true(message->priority() == Priority::Low); + g_assert_cmpuint(message->size(),==,::strlen(test_message_1)); + + /* text-based message use time({}) as their changed-time */ + g_assert_cmpuint(::time({}) - message->changed(), >=, 0); + g_assert_cmpuint(::time({}) - message->changed(), <=, 2); + + g_assert_true(message->references().empty()); + + assert_equal(message->subject(), + "[sqlite-dev] VM optimization inside sqlite3VdbeExec"); + + const auto to{message->to()}; + g_assert_cmpuint(to.size(),==,1); + assert_equal(to.at(0).name, ""); + assert_equal(to.at(0).email, "sqlite-dev@sqlite.org"); + + assert_equal(message->header("X-Mailer").value_or(""), "Apple Mail (2.926)"); + + auto all_contacts{message->all_contacts()}; + g_assert_cmpuint(all_contacts.size(), ==, 4); + seq_sort(all_contacts, [](auto&& c1, auto&& c2){return c1.email < c2.email; }); + assert_equal(all_contacts[0].email, "anon@example.com"); + assert_equal(all_contacts[1].email, "sqlite-dev-bounces@sqlite.org"); + assert_equal(all_contacts[2].email, "sqlite-dev@sqlite.org"); + assert_equal(all_contacts[3].email, "sqlite-dev@sqlite.org"); +} + + +static void +test_message_attachments(void) +{ + constexpr const char* msg_text = +R"(Return-Path: <foo@example.com> +Received: from pop.gmail.com [256.85.129.309] + by evergrey with POP3 (fetchmail-6.4.29) + for <djcb@localhost> (single-drop); Thu, 24 Mar 2022 20:12:40 +0200 (EET) +Sender: "Foo, Example" <foo@example.com> +User-agent: mu4e 1.7.11; emacs 29.0.50 +From: "Foo Example" <foo@example.com> +To: bar@example.com +Subject: =?utf-8?B?w6R0dMOkY2htZcOxdHM=?= +Date: Thu, 24 Mar 2022 20:04:39 +0200 +Organization: ACME Inc. +Message-Id: <3144HPOJ0VC77.3H1XTAG2AMTLH@"@WILSONB.COM> +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="=-=-=" + +--=-=-= +Content-Type: text/plain + +Hello, +--=-=-= +Content-Type: image/jpeg +Content-Disposition: attachment; filename=file-01.bin +Content-Transfer-Encoding: base64 + +AAECAw== +--=-=-= +Content-Type: audio/ogg +Content-Disposition: inline; filename=/tmp/file-02.bin +Content-Transfer-Encoding: base64 + +BAUGBw== +--=-=-= +Content-Type: message/rfc822 +Content-Disposition: attachment; + filename="message.eml" + +From: "Fnorb" <fnorb@example.com> +To: Bob <bob@example.com> +Subject: news for you +Date: Mon, 28 Mar 2022 22:53:26 +0300 + +Attached message! + +--=-=-= +Content-Type: text/plain + +World! +--=-=-=-- +)"; + + auto message{Message::make_from_text(msg_text)}; + g_assert_true(!!message); + g_assert_true(message->has_mime_message()); + g_assert_true(message->path().empty()); + + g_assert_true(message->bcc().empty()); + g_assert_true(!message->body_html()); + assert_equal(message->body_text().value_or(""), R"(Hello,World!)"); + + g_assert_true(message->cc().empty()); + g_assert_cmpuint(message->date(), ==, 1648145079); + /* no Flags::Unread since it's a message without path */ + g_assert_true(message->flags() == (Flags::HasAttachment)); + + const auto from{message->from()}; + g_assert_cmpuint(from.size(),==,1); + assert_equal(from.at(0).name, "Foo Example"); + assert_equal(from.at(0).email, "foo@example.com"); + + // problem case: https://github.com/djcb/mu/issues/2232o + assert_equal(message->message_id(), + "3144HPOJ0VC77.3H1XTAG2AMTLH@\"@WILSONB.COM"); + + g_assert_true(message->path().empty()); + g_assert_true(message->priority() == Priority::Normal); + g_assert_cmpuint(message->size(),==,::strlen(msg_text)); + + /* text-based message use time({}) as their changed-time */ + g_assert_cmpuint(::time({}) - message->changed(), >=, 0); + g_assert_cmpuint(::time({}) - message->changed(), <=, 2); + + assert_equal(message->subject(), "ättächmeñts"); + + const auto cache_path{message->cache_path()}; + g_assert_true(!!cache_path); + + g_assert_cmpuint(message->parts().size(),==,5); + { + auto&& part{message->parts().at(0)}; + g_assert_false(!!part.raw_filename()); + assert_equal(part.mime_type().value(), "text/plain"); + assert_equal(part.to_string().value(), "Hello,"); + } + { + auto&& part{message->parts().at(1)}; + assert_equal(part.raw_filename().value(), "file-01.bin"); + assert_equal(part.mime_type().value(), "image/jpeg"); + // file consists of 4 bytes 0...3 + g_assert_cmpuint(part.to_string()->at(0), ==, 0); + g_assert_cmpuint(part.to_string()->at(1), ==, 1); + g_assert_cmpuint(part.to_string()->at(2), ==, 2); + g_assert_cmpuint(part.to_string()->at(3), ==, 3); + } + { + auto&& part{message->parts().at(2)}; + assert_equal(part.raw_filename().value(), "/tmp/file-02.bin"); + assert_equal(part.cooked_filename().value(), "file-02.bin"); + assert_equal(part.mime_type().value(), "audio/ogg"); + // file consistso of 4 bytes 4..7 + assert_equal(part.to_string().value(), "\004\005\006\007"); + const auto fpath{*cache_path + part.cooked_filename().value()}; + const auto res = part.to_file(fpath, true); + + g_assert_cmpuint(*res,==,4); + g_assert_cmpuint(::access(fpath.c_str(), R_OK), ==, 0); + } + + { + auto&& part{message->parts().at(3)}; + g_assert_true(part.mime_type() == "message/rfc822"); + + const auto fname{*cache_path + "/msgpart"}; + g_assert_cmpuint(part.to_file(fname, true).value_or(123), ==, 139); + g_assert_true(::access(fname.c_str(), F_OK) == 0); + } + + { + auto&& part{message->parts().at(4)}; + g_assert_false(!!part.raw_filename()); + g_assert_true(!!part.mime_type()); + assert_equal(part.mime_type().value(), "text/plain"); + assert_equal(part.to_string().value(), "World!"); + } +} + + +/* + * some test keys. + */ + +constexpr std::string_view pub_key = +R"(-----BEGIN PGP PUBLIC KEY BLOCK----- + +mDMEYlbaNhYJKwYBBAHaRw8BAQdAEgxZnlN3mIwqV89zchjFlEby8OgrbrkT+yRN +hQhc+A+0LU11IFRlc3QgKG11IHRlc3Rpbmcga2V5KSA8bXVAZGpjYnNvZnR3YXJl +Lm5sPoiUBBMWCgA8FiEE/HZRT+2bPjARz29Cw7FsU49t3vAFAmJW2jYCGwMFCwkI +BwIDIgIBBhUKCQgLAgQWAgMBAh4HAheAAAoJEMOxbFOPbd7wJ2kBAIGmUDWYEPtn +qYTwhZIdZtTa4KJ3UdtTqey9AnxJ9mzAAQDRJOoVppj5wW2xRhgYP+ysN2iBUYGE +MhahOcNgxodbCLg4BGJW2jYSCisGAQQBl1UBBQEBB0D4Sp+GTVre7Cx5a8D3SwLJ +/bRAVGDwqI7PL9B/cMmCTwMBCAeIeAQYFgoAIBYhBPx2UU/tmz4wEc9vQsOxbFOP +bd7wBQJiVto2AhsMAAoJEMOxbFOPbd7w1tYA+wdfYCcwOP0QoNZZz2Yk12YkDk2R +FsRrZZpb0GKC/a2VAP4qFceeSegcUCBTQaoeFE9vq9XiUVOO98QI8r9C8QwvBw== +=jM/g +-----END PGP PUBLIC KEY BLOCK----- +)"; + +constexpr std::string_view priv_key = // "test1234" +R"(-----BEGIN PGP PRIVATE KEY BLOCK----- + +lIYEYlbaNhYJKwYBBAHaRw8BAQdAEgxZnlN3mIwqV89zchjFlEby8OgrbrkT+yRN +hQhc+A/+BwMCz6T2uBpk6a7/rXyE7C1bRbGjP6YSFcyRFz8VRV3Xlm7z6rdbdKZr +8R15AtLvXA4DOK5GiZRB2VbIxi8B9CtZ9qQx6YbQPkAmRzISGAjECrQtTXUgVGVz +dCAobXUgdGVzdGluZyBrZXkpIDxtdUBkamNic29mdHdhcmUubmw+iJQEExYKADwW +IQT8dlFP7Zs+MBHPb0LDsWxTj23e8AUCYlbaNgIbAwULCQgHAgMiAgEGFQoJCAsC +BBYCAwECHgcCF4AACgkQw7FsU49t3vAnaQEAgaZQNZgQ+2ephPCFkh1m1NrgondR +21Op7L0CfEn2bMABANEk6hWmmPnBbbFGGBg/7Kw3aIFRgYQyFqE5w2DGh1sInIsE +YlbaNhIKKwYBBAGXVQEFAQEHQPhKn4ZNWt7sLHlrwPdLAsn9tEBUYPCojs8v0H9w +yYJPAwEIB/4HAwI9MZDWcsoiJ/9oV5DRiAedeo3Ta/1M+aKfeNV36Ch1VGLwQF3E +V77qIrJlsT8CwOZHWUksUBENvG3ak3vd84awHHaHoTmoFwtISfvQrFK0iHgEGBYK +ACAWIQT8dlFP7Zs+MBHPb0LDsWxTj23e8AUCYlbaNgIbDAAKCRDDsWxTj23e8NbW +APsHX2AnMDj9EKDWWc9mJNdmJA5NkRbEa2WaW9Bigv2tlQD+KhXHnknoHFAgU0Gq +HhRPb6vV4lFTjvfECPK/QvEMLwc= +=w1Nc +-----END PGP PRIVATE KEY BLOCK----- +)"; + + +static void +test_message_signed(void) +{ + constexpr const char *msgtext = +R"(Return-Path: <diggler@gmail.com> +From: Mu Test <mu@djcbsoftware.nl> +To: Mu Test <mu@djcbsoftware.nl> +Subject: boo +Date: Wed, 13 Apr 2022 17:19:08 +0300 +Message-ID: <878rs9ysin.fsf@djcbsoftware.nl> +MIME-Version: 1.0 +Content-Type: multipart/signed; boundary="=-=-="; + micalg=pgp-sha512; protocol="application/pgp-signature" + +--=-=-= +Content-Type: text/plain + +Sapperdeflap + +--=-=-= +Content-Type: application/pgp-signature; name="signature.asc" + +-----BEGIN PGP SIGNATURE----- + +iIkEARYKADEWIQT8dlFP7Zs+MBHPb0LDsWxTj23e8AUCYlbcLhMcbXVAZGpjYnNv +ZnR3YXJlLm5sAAoJEMOxbFOPbd7waIkA/jK1oY7OL8vrDoubNYxamy8HHmwtvO01 +Q46aYjxe0As6AP90bcAZ3dcn5RcTJaM0UhZssguawZ+tnriD3+5DPkMMCg== +=e32+ +-----END PGP SIGNATURE----- +--=-=-=-- +)"; + TempDir tempdir; + auto ctx{MimeCryptoContext::make_gpg(tempdir.path())}; + g_assert_true(!!ctx); + + auto stream{MimeStream::make_mem()}; + stream.write(pub_key.data(), pub_key.size()); + stream.reset(); + + auto imported = ctx->import_keys(stream); + g_assert_cmpuint(*imported, ==, 1); + + auto message{Message::make_from_text( + msgtext, + "/home/test/Maildir/inbox/cur/1649279777.107710_1.mindcrime:2,RS")}; + g_assert_true(!!message); + + g_assert_true(message->bcc().empty()); + assert_equal(message->body_text().value_or(""), "Sapperdeflap\n"); + g_assert_true(message->flags() == (Flags::Signed|Flags::Seen|Flags::Replied)); + + size_t n{}; + for (auto&& part: message->parts()) { + if (!part.is_signed()) + continue; + + const auto& mobj{part.mime_object()}; + if (!mobj.is_multipart_signed()) + continue; + + const auto mpart{MimeMultipartSigned(mobj)}; + const auto sigs{mpart.verify(*ctx)}; + if (!sigs) + mu_warning("{}", sigs.error().what()); + + g_assert_true(!!sigs); + g_assert_cmpuint(sigs->size(), ==, 1); + ++n; + } + + g_assert_cmpuint(n, ==, 1); +} + + +static void +test_message_signed_encrypted(void) +{ + constexpr const char *msgtext = +R"(From: "Mu Test" <mu@djcbsoftware.nl> +To: mu@djcbsoftware.nl +Subject: encrypted and signed +Date: Wed, 13 Apr 2022 17:32:30 +0300 +Message-ID: <87lew9xddt.fsf@djcbsoftware.nl> +MIME-Version: 1.0 +Content-Type: multipart/encrypted; boundary="=-=-="; + protocol="application/pgp-encrypted" + +--=-=-= +Content-Type: application/pgp-encrypted + +Version: 1 + +--=-=-= +Content-Type: application/octet-stream + +-----BEGIN PGP MESSAGE----- + +hF4DeEerj6WhdZASAQdAKdZwmugAlQA8c06Q5iQw4rwSADgfEWBTWlI6tDw7hEAw +0qSSeeQbA802qjG5TesaDVbFoPp1gOESt67HkJBABj9niwZLnjbzVRXKFoPTYabu +1MBWAQkCEO6kS0N73XQeJ9+nDkUacRX6sSgVM0j+nRdCGcrCQ8MOfLd9KUUBxpXy +r/rIBMpZGOIpKJnoZ2x75VsQIp/ADHLe9zzXVe0tkahXJqvLo26w3gn4NSEIEDp6 +4T/zMZImqGrENaixNmRiRSAnwPkLt95qJGOIqYhuW3X6hMRZyU4zDNwkAvnK+2Fv +Wjd+EmiFzh5tvCmPOSj556YFMV7UpFWO9VznXX/T5+f4i+95Lsm9Uotv/SiNtNQG +DPU3wiL347SzmPFXckknjlzSzDL1XbdbHdmoJs0uNnbaZxRwhkuTYbLHdpBZrBgR +C0bdoCx44QVU8HaZ2x91h3GoM/0q5bqM/rvCauwbokiJgAUrznecNPY= +=Ado7 +-----END PGP MESSAGE----- +--=-=-=-- +)"; + TempDir tempdir; + auto ctx{MimeCryptoContext::make_gpg(tempdir.path())}; + g_assert_true(!!ctx); + + /// test1234 + // ctx->set_request_password([](const MimeCryptoContext& ctx, + // const std::string& user_id, + // const std::string& prompt, + // bool reprompt, + // MimeStream& response)->Result<void> { + // return Err(Error::Code::Internal, "boo"); + // //return Ok(); + // }); + + { + auto stream{MimeStream::make_mem()}; + stream.write(priv_key.data(), priv_key.size()); + stream.write(pub_key.data(), pub_key.size()); + stream.reset(); + + + g_assert_cmpint(ctx->import_keys(stream).value_or(-1),==,1); + } + + auto message{Message::make_from_text( + msgtext, + "/home/test/Maildir/inbox/cur/1649279888.107710_1.mindcrime:2,FS")}; + g_assert_true(!!message); + g_assert_true(message->flags() == (Flags::Encrypted|Flags::Seen|Flags::Flagged)); + + size_t n{}; + for (auto&& part: message->parts()) { + + if (!part.is_encrypted()) + continue; + + g_assert_false(!!part.content_description()); + g_assert_false(part.is_attachment()); + g_assert_cmpuint(part.size(),==,0); + + const auto& mobj{part.mime_object()}; + if (!mobj.is_multipart_encrypted()) + continue; + + /* FIXME: make this work without user having to + * type password */ + + // const auto mpart{MimeMultipartEncrypted(mobj)}; + // const auto decres = mpart.decrypt(*ctx); + // assert_valid_result(decres); + + ++n; + } + + g_assert_cmpuint(n, ==, 1); +} + + +static void +test_message_multipart_mixed_rfc822(void) +{ + constexpr const char *msgtext = +R"(Content-Type: multipart/mixed; + boundary="Multipart_Tue_Sep__2_15:42:35_2014-1" + +--Multipart_Tue_Sep__2_15:42:35_2014-1 +Content-Type: message/rfc822 +)"; + auto message{Message::make_from_text(msgtext)}; + g_assert_true(!!message); + //g_assert_true(message->sexp().empty()); +} + + +static void +test_message_detect_attachment(void) +{ + constexpr const char *msgtext = +R"(From: "DUCK, Donald" <donald@example.com> +Date: Tue, 3 May 2022 10:26:26 +0300 +Message-ID: <SADKLAJCLKDJLAS-xheQjE__+hS-3tff=pTYpMUyGiJwNGF_DA@mail.gmail.com> +Subject: =?Windows-1252?Q?Purkuty=F6urakka?= +To: Hello <moika@example.com> +Cc: =?iso-8859-1?q?M=FCller=2C?= Mickey <Mickey.Mueller@example.com> +Content-Type: multipart/mixed; boundary="000000000000e687ed05de166d71" + +--000000000000e687ed05de166d71 +Content-Type: multipart/alternative; boundary="000000000000e687eb05de166d6f" + +--000000000000e687eb05de166d6f +Content-Type: text/plain; charset="UTF-8" +Content-Transfer-Encoding: quoted-printable + +fyi + +---------- Forwarded message --------- +From: Fooish Bar <foobar@example.com> +Date: Tue, 3 May 2022 at 08:59 +Subject: Ty=C3=B6t +To: "DUCK, Donald" <donald@example.com> + +Moi, + +-- + +--000000000000e687eb05de166d6f +Content-Type: text/html; charset="UTF-8" +Content-Transfer-Encoding: quoted-printable + +abc + +--000000000000e687eb05de166d6f-- +--000000000000e687ed05de166d71 +Content-Type: application/pdf; + name="test1.pdf" +Content-Disposition: attachment; + filename="test2.pdf" +Content-Transfer-Encoding: base64 +Content-ID: <18088cfd4bc5517c6321> +X-Attachment-Id: 18088cfd4bc5517c6321 + +JVBERi0xLjcKJeLjz9MKNyAwIG9iago8PCAvVHlwZSAvUGFnZSAvUGFyZW50IDEgMCBSIC9MYXN0 +TW9kaWZpZWQgKEQ6MjAyMjA1MDMwODU3MzYrMDMnMDAnKSAvUmVzb3VyY2VzIDIgMCBSIC9NZWRp +cmVmCjM1NjE4CiUlRU9GCg== +--000000000000e687ed05de166d71-- +)"; + auto message{Message::make_from_text(msgtext)}; + g_assert_true(!!message); + + g_assert_true(message->path().empty()); + + /* https://groups.google.com/g/mu-discuss/c/kCtrlxMXBjo */ + g_assert_cmpuint(message->cc().size(),==, 1); + assert_equal(message->cc().at(0).email, "Mickey.Mueller@example.com"); + assert_equal(message->cc().at(0).name, "Müller, Mickey"); + assert_equal(message->cc().at(0).display_name(), "\"Müller, Mickey\" <Mickey.Mueller@example.com>"); + + g_assert_true(message->bcc().empty()); + assert_equal(message->subject(), "Purkutyöurakka"); + assert_equal(message->body_html().value_or(""), "abc\n"); + assert_equal(message->body_text().value_or(""), + R"(fyi + +---------- Forwarded message --------- +From: Fooish Bar <foobar@example.com> +Date: Tue, 3 May 2022 at 08:59 +Subject: Työt +To: "DUCK, Donald" <donald@example.com> + +Moi, + +-- +)"); + g_assert_cmpuint(message->date(), ==, 1651562786); + g_assert_true(message->flags() == (Flags::HasAttachment)); + + g_assert_cmpuint(message->parts().size(), ==, 3); + + for (auto&& part: message->parts()) + g_info("%s %s", + part.is_attachment() ? "yes" : "no", + part.mime_type().value_or("boo").c_str()); +} + + +static void +test_message_calendar(void) +{ + constexpr const char *msgtext = +R"(MIME-Version: 1.0 +From: William <william@example.com> +To: Billy <billy@example.com> +Date: Thu, 9 Jan 2014 11:09:34 +0100 +Subject: Invitation: HELLO, @ Thu 9 Jan 2014 08:30 - 09:30 + (william@example.com) +Thread-Topic: Invitation: HELLO, @ Thu 9 Jan 2014 08:30 - 09:30 + (william@example.com) +Thread-Index: Ac8NIuske7OtG01VRpukb/bHE7SVHg== +Message-ID: <001a11c3440066ee0b04ef86cea8@google.com> +Accept-Language: en-US +Content-Language: en-US +X-MS-Exchange-Organization-AuthAs: Anonymous +X-MS-Has-Attach: yes +Content-Type: multipart/mixed; + boundary="_004_001a11c3440066ee0b04ef86cea8googlecom_" + +--_004_001a11c3440066ee0b04ef86cea8googlecom_ +Content-Type: multipart/alternative; + boundary="_002_001a11c3440066ee0b04ef86cea8googlecom_" + +--_002_001a11c3440066ee0b04ef86cea8googlecom_ +Content-Type: text/html; charset="utf-8" +Content-Transfer-Encoding: base64 + +PGh0bWw+DQo8aGVhZD4NCjxtZXRhIGh0dHAtZXF1aXY9IkNvbnRlbnQtVHlwZSIgY29udGVudD0i +dGV4dC9odG1sOyBjaGFyc2V0PXV0Zi04Ij4NCjxtZXRhIG5hbWU9IkdlbmVyYXRvciIgY29udGVu +dD0iTWljcm9zb2Z0IEV4Y2hhbmdlIFNlcnZlciI+DQo8IS0tIGNvbnZlcnRlZCBmcm9tIHJ0ZiAt +LT4NCjxzdHlsZT48IS0tIC5FbWFpbFF1b3RlIHsgbWFyZ2luLWxlZnQ6IDFwdDsgcGFkZGluZy1s +ZWZ0OiA0cHQ7IGJvcmRlci1sZWZ0OiAjODAwMDAwIDJweCBzb2xpZDsgfSAtLT48L3N0eWxlPg0K +PC9oZWFkPg0KPGJvZHk+DQo8Zm9udCBmYWNlPSJUaW1lcyBOZXcgUm9tYW4iIHNpemU9IjMiPjxh +IG5hbWU9IkJNX0JFR0lOIj48L2E+DQo8dGFibGUgYm9yZGVyPSIxIiB3aWR0aD0iNzM0IiBzdHls +ZT0iYm9yZGVyOjEgc29saWQ7IGJvcmRlci1jb2xsYXBzZTpjb2xsYXBzZTsgbWFyZ2luLWxlZnQ6 +IDJwdDsgIj4NCjx0cj4NCjx0ZD48Zm9udCBzaXplPSIxIj48YSBocmVmPSJodHRwczovL3d3dy5n +b29nbGUuY29tL2NhbGVuZGFyL2V2ZW50P2FjdGlvbj1WSUVXJmFtcDtlaWQ9YzNOemNXUXhjRGxs +Ym1VeU0ySnZNbWsyYjNOeU56ZG5jRzhnWkdwallrQmthbU5pYzI5bWRIZGhjbVV1Ym13JmFtcDt0 +b2s9TWpZamQybHNiR2xoYlhOZlpESXdRR2RzYjJKdmJXRnBiQzVqYjIxak16YzJZVGhrWW1Ga016 +QTJaRFV3TldVMlltWXhOamRqTm1ZMVlUVXhObUpqTWpFNU4yWTMmYW1wO2N0ej1BbWVyaWNhL1Nh +b19QYXVsbyZhbXA7aGw9ZW5fR0IiPjxmb250IGNvbG9yPSIjMjIwMENDIj48dT5tb3JlDQpkZXRh +aWxzIMK7PC91PjwvZm9udD48L2E+PGJyPg0KDQo8ZGl2IHN0eWxlPSJtYXJnaW4tYm90dG9tOiAx +NHB0OyAiPjxmb250IGZhY2U9IkFyaWFsLCBzYW5zLXNlcmlmIiBzaXplPSIyIiBjb2xvcj0iIzIy +MjIyMiI+PGI+SEVMTE8sPC9iPjwvZm9udD48L2Rpdj4NCjxkaXY+PGZvbnQgc2l6ZT0iMSIgY29s +b3I9IiMyMjIyMjIiPjxicj4NCg0KSSBBTSBERVNNT05EIFdJTExJQU1TIEFORCBNWSBMSVRUTEUg +U0lTVEVSIElTIEdMT1JJQSwgT1VSIEZBVEhFUiBPV05TIEEgTElNSVRFRCBPRiBDT0NPQSBBTkQg +R09MRCBCVVNJTkVTUyBJTiBSRVBVQkxJUVVFIERVIENPTkdPLiBBRlRFUiBISVMgVFJJUCBUTyBD +T1RFIERJVk9JUkUgVE8gTkVHT1RJQVRFIE9OIENPQ09BIEFORCBHT0xEIEJVU0lORVNTIEhFIFdB +TlRFRCBUTyBJTlZFU1QgSU4gQUJST0FELiA8L2ZvbnQ+PC9kaXY+DQo8ZGl2IHN0eWxlPSJtYXJn +aW4tdG9wOiAxNHB0OyBtYXJnaW4tYm90dG9tOiAxNHB0OyAiPjxmb250IHNpemU9IjMiPk9ORSBX +RUVLIEhFIENBTUUgQkFDSyBGUk9NIEhJUyBUUklQIFRPIEFCSURKQU4gSEUgSEFEIEEgTU9UT1Ig +QUNDSURFTlQgV0lUSCBPVVIgTU9USEVSIFdISUNIIE9VUiBNT1RIRVIgRElFRCBJTlNUQU5UTFkg +QlVUIE9VUiBGQVRIRVIgRElFRCBBRlRFUiBGSVZFIERBWVMgSU4gQSBQUklWQVRFIEhPU1BJVEFM +IElOIE9VUiBDT1VOVFJZLg0KSVQgV0FTIExJS0UgT1VSIEZBVEhFUiBLTkVXIEhFIFdBUyBHT0lO +RyBUTyBESUUgTUFZIEhJUyBHRU5UTEUgU09VTCBSRVNUIElOIFBSRUZFQ1QgUEVBQ0UuIDwvZm9u +dD48L2Rpdj4NCjxkaXYgc3R5bGU9Im1hcmdpbi10b3A6IDE0cHQ7IG1hcmdpbi1ib3R0b206IDE0 +cHQ7ICI+PGZvbnQgc2l6ZT0iMyI+SEUgRElTQ0xPU0VEIFRPIE1FIEFTIFRIRSBPTkxZIFNPTiBU +SEFUIEhFIERFUE9TSVRFRCBUSEUgU1VNIE9GIChVU0QgJCAxMCw1MDAsMDAwKSBJTlRPIEEgQkFO +SyBJTiBBQklESkFOIFRIQVQgVEhFIE1PTkVZIFdBUyBNRUFOVCBGT1IgSElTIENPQ09BIEFORCBH +T0xEIEJVU0lORVNTIEhFIFdBTlRFRCBUTyBFU1RBQkxJU0ggSU4NCkFCUk9BRC5XRSBBUkUgU09M +SUNJVElORyBGT1IgWU9VUiBIRUxQIFRPIFRSQU5TRkVSIFRISVMgTU9ORVkgSU5UTyBZT1VSIEFD +Q09VTlQgSU4gWU9VUiBDT1VOVFJZIEZPUiBPVVIgSU5WRVNUTUVOVC4gPC9mb250PjwvZGl2Pg0K +PGRpdiBzdHlsZT0ibWFyZ2luLXRvcDogMTRwdDsgbWFyZ2luLWJvdHRvbTogMTRwdDsgIj48Zm9u +dCBzaXplPSIzIj5QTEVBU0UgRk9SIFNFQ1VSSVRZIFJFQVNPTlMsSSBBRFZJQ0UgWU9VIFJFUExZ +IFVTIFRIUk9VR0ggT1VSIFBSSVZBVEUgRU1BSUw6IDxhIGhyZWY9Im1haWx0bzp3aWxsaWFtc2Rl +c21vbmQxMDdAeWFob28uY29tLnZuIj48Zm9udCBjb2xvcj0iIzAwMDBGRiI+PHU+d2lsbGlhbXNk +ZXNtb25kMTA3QHlhaG9vLmNvbS52bjwvdT48L2ZvbnQ+PC9hPg0KRk9SIE1PUkUgREVUQUlMUy4g +PC9mb250PjwvZGl2Pg0KPGRpdiBzdHlsZT0ibWFyZ2luLXRvcDogMTRwdDsgbWFyZ2luLWJvdHRv +bTogMTRwdDsgIj48Zm9udCBzaXplPSIzIj5SRUdBUkRTLiA8L2ZvbnQ+PC9kaXY+DQo8ZGl2IHN0 +eWxlPSJtYXJnaW4tdG9wOiAxNHB0OyBtYXJnaW4tYm90dG9tOiAxNHB0OyAiPjxmb250IHNpemU9 +IjMiPkRFU01PTkQgL0dMT1JJQSBXSUxMSUFNUy48L2ZvbnQ+PC9kaXY+DQo8ZGl2Pjxmb250IHNp +emU9IjMiIGNvbG9yPSIjMjIyMjIyIj4mbmJzcDs8L2ZvbnQ+PC9kaXY+DQo8ZGl2Pjxmb250IHNp +emU9IjMiIGNvbG9yPSIjMjIyMjIyIj4mbmJzcDs8L2ZvbnQ+PC9kaXY+DQo8ZGl2Pjxmb250IHNp +emU9IjMiIGNvbG9yPSIjMjIyMjIyIj4mbmJzcDs8L2ZvbnQ+PC9kaXY+DQo8ZGl2Pjxmb250IHNp +emU9IjMiIGNvbG9yPSIjMjIyMjIyIj4mbmJzcDs8L2ZvbnQ+PC9kaXY+DQo8dGFibGUgYm9yZGVy +PSIxIiB3aWR0aD0iNzM0IiBzdHlsZT0iYm9yZGVyOjEgc29saWQ7IGJvcmRlci1jb2xsYXBzZTpj +b2xsYXBzZTsgbWFyZ2luLWxlZnQ6IDJwdDsgIj4NCjxjb2wgd2lkdGg9IjM2NSI+DQo8Y29sIHdp +ZHRoPSIzNjkiPg0KPHRyPg0KPHRkPjxmb250IHNpemU9IjMiPjxpPldoZW48L2k+PC9mb250Pjwv +dGQ+DQo8dGQ+PGZvbnQgZmFjZT0iQXJpYWwsIHNhbnMtc2VyaWYiIHNpemU9IjEiIGNvbG9yPSIj +MjIyMjIyIj5UaHUgOSBKYW4gMjAxNCAwODozMCDigJMgMDk6MzAgPGZvbnQgY29sb3I9IiM4ODg4 +ODgiPlNhbyBQYXVsbzwvZm9udD48L2ZvbnQ+PC90ZD4NCjwvdHI+DQo8dHI+DQo8dGQ+PGZvbnQg +c2l6ZT0iMyI+PGk+Q2FsZW5kYXI8L2k+PC9mb250PjwvdGQ+DQo8dGQ+PGZvbnQgZmFjZT0iQXJp +YWwsIHNhbnMtc2VyaWYiIHNpemU9IjEiIGNvbG9yPSIjMjIyMjIyIj53aWxsaWFtc19kMjBAZ2xv +Ym9tYWlsLmNvbTwvZm9udD48L3RkPg0KPC90cj4NCjx0cj4NCjx0ZD48Zm9udCBzaXplPSIzIj48 +aT5XaG88L2k+PC9mb250PjwvdGQ+DQo8dGQ+PGZvbnQgZmFjZT0iQXJpYWwsIHNhbnMtc2VyaWYi +IHNpemU9IjEiIGNvbG9yPSIjMjIyMjIyIj4oR3Vlc3QgbGlzdCBoYXMgYmVlbiBoaWRkZW4gYXQg +b3JnYW5pc2VyJ3MgcmVxdWVzdCk8L2ZvbnQ+PC90ZD4NCjwvdHI+DQo8L3RhYmxlPg0KPGRpdiBz +dHlsZT0ibWFyZ2luLWJvdHRvbTogMTRwdDsgIj48Zm9udCBzaXplPSIxIiBjb2xvcj0iIzg4ODg4 +OCI+R29pbmc/Jm5ic3A7Jm5ic3A7IDxhIGhyZWY9Imh0dHBzOi8vd3d3Lmdvb2dsZS5jb20vY2Fs +ZW5kYXIvZXZlbnQ/YWN0aW9uPVJFU1BPTkQmYW1wO2VpZD1jM056Y1dReGNEbGxibVV5TTJKdk1t +azJiM055TnpkbmNHOGdaR3BqWWtCa2FtTmljMjltZEhkaGNtVXVibXcmYW1wO3JzdD0xJmFtcDt0 +b2s9TWpZamQybHNiR2xoYlhOZlpESXdRR2RzYjJKdmJXRnBiQzVqYjIxak16YzJZVGhrWW1Ga016 +QTJaRFV3TldVMlltWXhOamRqTm1ZMVlUVXhObUpqTWpFNU4yWTMmYW1wO2N0ej1BbWVyaWNhL1Nh +b19QYXVsbyZhbXA7aGw9ZW5fR0IiPjxmb250IGNvbG9yPSIjMjIwMENDIj48dT48Yj5ZZXM8L2I+ +PC91PjwvZm9udD48L2E+PGZvbnQgY29sb3I9IiMyMjIyMjIiPjxiPg0KLSA8L2I+PC9mb250Pjxh +IGhyZWY9Imh0dHBzOi8vd3d3Lmdvb2dsZS5jb20vY2FsZW5kYXIvZXZlbnQ/YWN0aW9uPVJFU1BP +TkQmYW1wO2VpZD1jM056Y1dReGNEbGxibVV5TTJKdk1tazJiM055TnpkbmNHOGdaR3BqWWtCa2Ft +TmljMjltZEhkaGNtVXVibXcmYW1wO3JzdD0zJmFtcDt0b2s9TWpZamQybHNiR2xoYlhOZlpESXdR +R2RzYjJKdmJXRnBiQzVqYjIxak16YzJZVGhrWW1Ga016QTJaRFV3TldVMlltWXhOamRqTm1ZMVlU +VXhObUpqTWpFNU4yWTMmYW1wO2N0ej1BbWVyaWNhL1Nhb19QYXVsbyZhbXA7aGw9ZW5fR0IiPjxm +b250IGNvbG9yPSIjMjIwMENDIj48dT48Yj5NYXliZTwvYj48L3U+PC9mb250PjwvYT48Zm9udCBj +b2xvcj0iIzIyMjIyMiI+PGI+DQotIDwvYj48L2ZvbnQ+PGEgaHJlZj0iaHR0cHM6Ly93d3cuZ29v +Z2xlLmNvbS9jYWxlbmRhci9ldmVudD9hY3Rpb249UkVTUE9ORCZhbXA7ZWlkPWMzTnpjV1F4Y0Rs +bGJtVXlNMkp2TW1rMmIzTnlOemRuY0c4Z1pHcGpZa0JrYW1OaWMyOW1kSGRoY21VdWJtdyZhbXA7 +cnN0PTImYW1wO3Rvaz1NallqZDJsc2JHbGhiWE5mWkRJd1FHZHNiMkp2YldGcGJDNWpiMjFqTXpj +MllUaGtZbUZrTXpBMlpEVXdOV1UyWW1ZeE5qZGpObVkxWVRVeE5tSmpNakU1TjJZMyZhbXA7Y3R6 +PUFtZXJpY2EvU2FvX1BhdWxvJmFtcDtobD1lbl9HQiI+PGZvbnQgY29sb3I9IiMyMjAwQ0MiPjx1 +PjxiPk5vPC9iPjwvdT48L2ZvbnQ+PC9hPjxmb250IGNvbG9yPSIjMjIyMjIyIj4mbmJzcDsmbmJz +cDsmbmJzcDsNCjwvZm9udD48YSBocmVmPSJodHRwczovL3d3dy5nb29nbGUuY29tL2NhbGVuZGFy +L2V2ZW50P2FjdGlvbj1WSUVXJmFtcDtlaWQ9YzNOemNXUXhjRGxsYm1VeU0ySnZNbWsyYjNOeU56 +ZG5jRzhnWkdwallrQmthbU5pYzI5bWRIZGhjbVV1Ym13JmFtcDt0b2s9TWpZamQybHNiR2xoYlhO +ZlpESXdRR2RzYjJKdmJXRnBiQzVqYjIxak16YzJZVGhrWW1Ga016QTJaRFV3TldVMlltWXhOamRq +Tm1ZMVlUVXhObUpqTWpFNU4yWTMmYW1wO2N0ej1BbWVyaWNhL1Nhb19QYXVsbyZhbXA7aGw9ZW5f +R0IiPjxmb250IGNvbG9yPSIjMjIwMENDIj48dT5tb3JlDQpvcHRpb25zIMK7PC91PjwvZm9udD48 +L2E+PC9mb250PjwvZGl2Pg0KPC9mb250PjwvdGQ+DQo8L3RyPg0KPHRyPg0KPHRkIHN0eWxlPSJi +YWNrZ3JvdW5kLWNvbG9yOiAjRjZGNkY2OyAiPjxmb250IHNpemU9IjMiPkludml0YXRpb24gZnJv +bSA8YSBocmVmPSJodHRwczovL3d3dy5nb29nbGUuY29tL2NhbGVuZGFyLyI+PGZvbnQgY29sb3I9 +IiMwMDAwRkYiPjx1Pkdvb2dsZSBDYWxlbmRhcjwvdT48L2ZvbnQ+PC9hPg0KPGRpdiBzdHlsZT0i +bWFyZ2luLXRvcDogMTRwdDsgbWFyZ2luLWJvdHRvbTogMTRwdDsgIj48Zm9udCBzaXplPSIzIj5Z +b3UgYXJlIHJlY2VpdmluZyB0aGlzIGNvdXJ0ZXN5IGVtYWlsIGF0IHRoZSBhY2NvdW50IGRqY2JA +ZGpjYnNvZnR3YXJlLm5sIGJlY2F1c2UgeW91IGFyZSBhbiBhdHRlbmRlZSBvZiB0aGlzIGV2ZW50 +LjwvZm9udD48L2Rpdj4NCjxkaXYgc3R5bGU9Im1hcmdpbi10b3A6IDE0cHQ7IG1hcmdpbi1ib3R0 +b206IDE0cHQ7ICI+PGZvbnQgc2l6ZT0iMyI+VG8gc3RvcCByZWNlaXZpbmcgZnV0dXJlIG5vdGlm +aWNhdGlvbnMgZm9yIHRoaXMgZXZlbnQsIGRlY2xpbmUgdGhpcyBldmVudC4gQWx0ZXJuYXRpdmVs +eSwgeW91IGNhbiBzaWduIHVwIGZvciBhIEdvb2dsZSBhY2NvdW50IGF0DQo8YSBocmVmPSJodHRw +czovL3d3dy5nb29nbGUuY29tL2NhbGVuZGFyLyI+aHR0cHM6Ly93d3cuZ29vZ2xlLmNvbS9jYWxl +bmRhci88L2E+IGFuZCBjb250cm9sIHlvdXIgbm90aWZpY2F0aW9uIHNldHRpbmdzIGZvciB5b3Vy +IGVudGlyZSBjYWxlbmRhci48L2ZvbnQ+PC9kaXY+DQo8L2ZvbnQ+PC90ZD4NCjwvdHI+DQo8L3Rh +YmxlPg0KPC9mb250Pg0KPC9ib2R5Pg0KPC9odG1sPg0K + +--_002_001a11c3440066ee0b04ef86cea8googlecom_ +Content-Type: text/calendar; charset="UTF-8"; method=REQUEST +Content-Transfer-Encoding: 7bit + +BEGIN:VCALENDAR +PRODID:-//Google Inc//Google Calendar 70.9054//EN +VERSION:2.0 +CALSCALE:GREGORIAN +METHOD:REQUEST +BEGIN:VEVENT +DTSTART:20140109T103000Z +DTEND:20140109T113000Z +DTSTAMP:20140109T100934Z +ORGANIZER;CN=William:mailto:william@example.com +UID:sssqd1p9ene23bo2i6osr77gpo@google.com +ATTENDEE;CUTYPE=INDIVIDUAL;ROLE=REQ-PARTICIPANT;PARTSTAT=NEEDS-ACTION;RSVP= + TRUE;CN=billy@example.com;X-NUM-GUESTS=0:mailto:billy@example.com +CREATED:20140109T100932Z +DESCRIPTION:\nI AM DESMOND WILLIAMS AND MY LITTLE SISTER IS GLORIA\, OUR FA + THER OWNS A LIMITED OF COCOA AND GOLD BUSINESS IN REPUBLIQUE DU CONGO. AFTE + R HIS TRIP TO COTE DIVOIRE TO NEGOTIATE ON COCOA AND GOLD BUSINESS HE WANTE + D TO INVEST IN ABROAD. \n\nONE WEEK HE CAME BACK FROM HIS TRIP TO ABIDJAN H + E HAD A MOTOR ACCIDENT WITH OUR MOTHER WHICH OUR MOTHER DIED INSTANTLY BUT + OUR FATHER DIED AFTER FIVE DAYS IN A PRIVATE HOSPITAL IN OUR COUNTRY. IT WA + S LIKE OUR FATHER KNEW HE WAS GOING TO DIE MAY HIS GENTLE SOUL REST IN PREF + ECT PEACE. \n\nHE DISCLOSED TO ME AS THE ONLY SON THAT HE DEPOSITED THE SUM + OF (USD $ 10\,500\,000) INTO A BANK IN ABIDJAN THAT THE MONEY WAS MEANT FO + R HIS COCOA AND GOLD BUSINESS HE WANTED TO ESTABLISH IN ABROAD.WE ARE SOLIC + ITING FOR YOUR HELP TO TRANSFER THIS MONEY INTO YOUR ACCOUNT IN YOUR COUNTR + Y FOR OUR INVESTMENT. \n\nPLEASE FOR SECURITY REASONS\,I ADVICE YOU REPLY U + S THROUGH OUR PRIVATE EMAIL FOR MORE DETAI + LS. \n\nREGARDS. \n\nDESMOND /GLORIA WILLIAMS.\nView your event at http://w + ww.google.com/calendar/event?action=VIEW&eid=c3NzcWQxcDllbmUyM2JvMmk2b3NyNz + dncG8gZGpjYkBkamNic29mdHdhcmUubmw&tok=MjYjd2lsbGlhbXNfZDIwQGdsb2JvbWFpbC5jb + 21jMzc2YThkYmFkMzA2ZDUwNWU2YmYxNjdjNmY1YTUxNmJjMjE5N2Y3&ctz=America/Sao_Pau + lo&hl=en_GB. +LAST-MODIFIED:20140109T100932Z +LOCATION: +SEQUENCE:0 +STATUS:CONFIRMED +SUMMARY:HELLO\, +TRANSP:OPAQUE +END:VEVENT +END:VCALENDAR + +--_002_001a11c3440066ee0b04ef86cea8googlecom_-- + +--_004_001a11c3440066ee0b04ef86cea8googlecom_ +Content-Type: application/ics; name="invite.ics" +Content-Description: invite.ics +Content-Disposition: attachment; filename="invite.ics"; size=2029; + creation-date="Thu, 09 Jan 2014 10:09:44 GMT"; + modification-date="Thu, 09 Jan 2014 10:09:44 GMT" +Content-Transfer-Encoding: base64 + +QkVHSU46VkNBTEVOREFSDQpQUk9ESUQ6LS8vR29vZ2xlIEluYy8vR29vZ2xlIENhbGVuZGFyIDcw +LjkwNTQvL0VODQpWRVJTSU9OOjIuMA0KQ0FMU0NBTEU6R1JFR09SSUFODQpNRVRIT0Q6UkVRVUVT +VA0KQkVHSU46VkVWRU5UDQpEVFNUQVJUOjIwMTQwMTA5VDEwMzAwMFoNCkRURU5EOjIwMTQwMTA5 +VDExMzAwMFoNCkRUU1RBTVA6MjAxNDAxMDlUMTAwOTM0Wg0KT1JHQU5JWkVSO0NOPVdpbGxpYW1z +IFdpbGxpYW1zOm1haWx0bzp3aWxsaWFtc19kMjBAZ2xvYm9tYWlsLmNvbQ0KVUlEOnNzc3FkMXA5 +ZW5lMjNibzJpNm9zcjc3Z3BvQGdvb2dsZS5jb20NCkFUVEVOREVFO0NVVFlQRT1JTkRJVklEVUFM +O1JPTEU9UkVRLVBBUlRJQ0lQQU5UO1BBUlRTVEFUPU5FRURTLUFDVElPTjtSU1ZQPQ0KIFRSVUU7 +Q049ZGpjYkBkamNic29mdHdhcmUubmw7WC1OVU0tR1VFU1RTPTA6bWFpbHRvOmRqY2JAZGpjYnNv +ZnR3YXJlLm5sDQpDUkVBVEVEOjIwMTQwMTA5VDEwMDkzMloNCkRFU0NSSVBUSU9OOlxuSSBBTSBE +RVNNT05EIFdJTExJQU1TIEFORCBNWSBMSVRUTEUgU0lTVEVSIElTIEdMT1JJQVwsIE9VUiBGQQ0K +IFRIRVIgT1dOUyBBIExJTUlURUQgT0YgQ09DT0EgQU5EIEdPTEQgQlVTSU5FU1MgSU4gUkVQVUJM +SVFVRSBEVSBDT05HTy4gQUZURQ0KIFIgSElTIFRSSVAgVE8gQ09URSBESVZPSVJFIFRPIE5FR09U +SUFURSBPTiBDT0NPQSBBTkQgR09MRCBCVVNJTkVTUyBIRSBXQU5URQ0KIEQgVE8gSU5WRVNUIElO +IEFCUk9BRC4gXG5cbk9ORSBXRUVLIEhFIENBTUUgQkFDSyBGUk9NIEhJUyBUUklQIFRPIEFCSURK +QU4gSA0KIEUgSEFEIEEgTU9UT1IgQUNDSURFTlQgV0lUSCBPVVIgTU9USEVSIFdISUNIIE9VUiBN +T1RIRVIgRElFRCBJTlNUQU5UTFkgQlVUIA0KIE9VUiBGQVRIRVIgRElFRCBBRlRFUiBGSVZFIERB +WVMgSU4gQSBQUklWQVRFIEhPU1BJVEFMIElOIE9VUiBDT1VOVFJZLiBJVCBXQQ0KIFMgTElLRSBP +VVIgRkFUSEVSIEtORVcgSEUgV0FTIEdPSU5HIFRPIERJRSBNQVkgSElTIEdFTlRMRSBTT1VMIFJF +U1QgSU4gUFJFRg0KIEVDVCBQRUFDRS4gXG5cbkhFIERJU0NMT1NFRCBUTyBNRSBBUyBUSEUgT05M +WSBTT04gVEhBVCBIRSBERVBPU0lURUQgVEhFIFNVTQ0KICBPRiAoVVNEICQgMTBcLDUwMFwsMDAw +KSBJTlRPIEEgQkFOSyBJTiBBQklESkFOIFRIQVQgVEhFIE1PTkVZIFdBUyBNRUFOVCBGTw0KIFIg +SElTIENPQ09BIEFORCBHT0xEIEJVU0lORVNTIEhFIFdBTlRFRCBUTyBFU1RBQkxJU0ggSU4gQUJS +T0FELldFIEFSRSBTT0xJQw0KIElUSU5HIEZPUiBZT1VSIEhFTFAgVE8gVFJBTlNGRVIgVEhJUyBN +T05FWSBJTlRPIFlPVVIgQUNDT1VOVCBJTiBZT1VSIENPVU5UUg0KIFkgRk9SIE9VUiBJTlZFU1RN +RU5ULiBcblxuUExFQVNFIEZPUiBTRUNVUklUWSBSRUFTT05TXCxJIEFEVklDRSBZT1UgUkVQTFkg +VQ0KIFMgVEhST1VHSCBPVVIgUFJJVkFURSBFTUFJTDogd2lsbGlhbXNkZXNtb25kMTA3QHlhaG9v +LmNvbS52biBGT1IgTU9SRSBERVRBSQ0KIExTLiBcblxuUkVHQVJEUy4gXG5cbkRFU01PTkQgL0dM +T1JJQSBXSUxMSUFNUy5cblZpZXcgeW91ciBldmVudCBhdCBodHRwOi8vdw0KIHd3Lmdvb2dsZS5j +b20vY2FsZW5kYXIvZXZlbnQ/YWN0aW9uPVZJRVcmZWlkPWMzTnpjV1F4Y0RsbGJtVXlNMkp2TW1r +MmIzTnlOeg0KIGRuY0c4Z1pHcGpZa0JrYW1OaWMyOW1kSGRoY21VdWJtdyZ0b2s9TWpZamQybHNi +R2xoYlhOZlpESXdRR2RzYjJKdmJXRnBiQzVqYg0KIDIxak16YzJZVGhrWW1Ga016QTJaRFV3TldV +MlltWXhOamRqTm1ZMVlUVXhObUpqTWpFNU4yWTMmY3R6PUFtZXJpY2EvU2FvX1BhdQ0KIGxvJmhs +PWVuX0dCLg0KTEFTVC1NT0RJRklFRDoyMDE0MDEwOVQxMDA5MzJaDQpMT0NBVElPTjoNClNFUVVF +TkNFOjANClNUQVRVUzpDT05GSVJNRUQNClNVTU1BUlk6SEVMTE9cLA0KVFJBTlNQOk9QQVFVRQ0K +RU5EOlZFVkVOVA0KRU5EOlZDQUxFTkRBUg0K + +--_004_001a11c3440066ee0b04ef86cea8googlecom_-- + +)"; + auto message{Message::make_from_text( + msgtext, + "/home/test/Maildir/inbox/cur/162342449279256.107710_1.evergrey:2,PSp")}; + g_assert_true(!!message); + assert_equal(message->subject(), + "Invitation: HELLO, @ Thu 9 Jan 2014 08:30 - 09:30 (william@example.com)"); + g_assert_true(message->flags() == (Flags::Passed|Flags::Seen| + Flags::HasAttachment|Flags::Calendar)); + g_assert_cmpuint(message->body_html().value_or("").find("DETAILS"), ==, 2271); +} + + +static void +test_message_references() +{ + constexpr auto msgtext = +R"(Content-Transfer-Encoding: quoted-printable +Content-Type: text/plain; charset=utf-8 +References: <YuvYh1JbE3v+abd5@kili> + <90a760c4-6e88-07b4-1f20-8b10414e49aa@arm.com> + <T4CDWjUrgtI5n4mh1JEdW6RLYzqbPE9-yDrhEVwDM22WX-198fBwcnLd-4_xR1gvsVSHQps9fp_pZevTF0ZmaA==@protonmail.internalid> +To: "Robin Murphy" <robin.murphy@arm.com> +Reply-To: "Dan Carpenter" <dan.carpenter@oracle.com> +From: "Dan Carpenter" <dan.carpenter@oracle.com> +Subject: Re: [PATCH] iommu/omap: fix buffer overflow in debugfs +List-Id: <kernel-janitors.vger.kernel.org> +Date: Fri, 5 Aug 2022 09:37:02 +0300 +In-Reply-To: <90a760c4-6e88-07b4-1f20-8b10414e49aa@arm.com> +Precedence: bulk +Message-Id: <20220805063702.GH3438@kadam> + +On Thu, Aug 04, 2022 at 05:31:39PM +0100, Robin Murphy wrote: +> On 04/08/2022 3:32 pm, Dan Carpenter wrote: +> > There are two issues here: +)"; + auto message{Message::make_from_text( + msgtext, + "/home/test/Maildir/inbox/cur/162342449279256.88888_1.evergrey:2,S")}; + g_assert_true(!!message); + assert_equal(message->subject(), + "Re: [PATCH] iommu/omap: fix buffer overflow in debugfs"); + g_assert_true(message->priority() == Priority::Low); + + /* + * "90a760c4-6e88-07b4-1f20-8b10414e49aa@arm.com" is seen both in + * references and in-reply-to; in the de-duplication, the first one wins. + */ + std::vector<std::string> expected_refs = { + "YuvYh1JbE3v+abd5@kili", + "90a760c4-6e88-07b4-1f20-8b10414e49aa@arm.com", + /* protonmail.internalid is fake and removed */ + // "T4CDWjUrgtI5n4mh1JEdW6RLYzqbPE9-yDrhEVwDM22WX-198fBwcnLd-4_" + // "xR1gvsVSHQps9fp_pZevTF0ZmaA==@protonmail.internalid" + }; + + assert_equal_seq_str(expected_refs, message->references()); +} + + +static void +test_message_outlook_body() +{ + constexpr auto msgtext = +R"x(Received: from vu-ex2.activedir.vu.lt (172.16.159.219) by + vu-ex1.activedir.vu.lt (172.16.159.218) with Microsoft SMTP Server + (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id 15.2.1118.9 + via Mailbox Transport; Fri, 27 May 2022 11:40:05 +0300 +Received: from vu-ex2.activedir.vu.lt (172.16.159.219) by + vu-ex2.activedir.vu.lt (172.16.159.219) with Microsoft SMTP Server + (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) id + 15.2.1118.9; Fri, 27 May 2022 11:40:05 +0300 +Received: from vu-ex2.activedir.vu.lt ([172.16.159.219]) by + vu-ex2.activedir.vu.lt ([172.16.159.219]) with mapi id 15.02.1118.009; Fri, + 27 May 2022 11:40:05 +0300 +From: =?windows-1257?Q?XXXXXXXXXX= <XXXXXXXXXX> +To: <XXXXXXXXXX@XXXXXXXXXX.com> +Subject: =?windows-1257?Q?Pra=F0ymas?= +Thread-Topic: =?windows-1257?Q?Pra=F0ymas?= +Thread-Index: AQHYcaRi3ejPSLxkl0uTFDto7z2OcA== +Date: Fri, 27 May 2022 11:40:05 +0300 +Message-ID: <5c2cd378af634e929a6cc69da1e66b9d@XX.vu.lt> +Accept-Language: en-US, lt-LT +Content-Language: en-US +X-MS-Has-Attach: +Content-Type: text/html; charset="windows-1257" +Content-Transfer-Encoding: quoted-printable +MIME-Version: 1.0 +X-TUID: 1vFQ9RPwwg/u + +<html> +<head> +<meta http-equiv=3D"Content-Type" content=3D"text/html; charset=3Dwindows-1= +257"> +<style type=3D"text/css" style=3D"display:none;"><!-- P {margin-top:0;margi= +n-bottom:0;} --></style> +</head> +<body dir=3D"ltr"> +<div id=3D"divtagdefaultwrapper" style=3D"font-size:12pt;color:#000000;font= +-family:Calibri,Helvetica,sans-serif;" dir=3D"ltr"> +<p>Laba diena visiems,</p> +<p>Trumpai.</p> +<p>D=EBl leidimo ar neleidimo ginti darb=E0: ed=EBstytojo paskyroje spaud= +=FEiate ikon=E0 "ra=F0to darbai", atidar=E6 susiraskite =E1ra=F0= +=E0 "tvirtinti / netvirtinti", pa=FEym=EBkite vien=E0 i=F0 j=F8.&= +nbsp;</p> +<p><br> +</p> +<p>=D0=E1 darb=E0 privalu atlikti, kad paskui nekilt=F8 problem=F8 studentu= +i =E1vedant =E1vertinim=E0.</p> +<p><br> +</p> +<p>Jei neleid=FEiate ginti darbo, pra=F0au informuoti mane ir komisijos sek= +retori=F8.  </p> +<p><br> +</p> +<p>Vis=E0 tolesn=E6 informacij=E0 atsi=F8siu artimiausiu metu (stengsiuosi = +=F0iandien vakare).</p> +<p><br> +</p> +<p>Pagarbiai.</p> +<p><br> +</p> +<p><br> +</p> +<div id=3D"Signature"> +<div id=3D"divtagdefaultwrapper" dir=3D"ltr" style=3D"font-family: Calibri,= + Helvetica, sans-serif, EmojiFont, "Apple Color Emoji", "Seg= +oe UI Emoji", NotoColorEmoji, "Segoe UI Symbol", "Andro= +id Emoji", EmojiSymbols;"> +<p style=3D"color:rgb(0,0,0); font-size:12pt"><br> +</p> +<p style=3D"color:rgb(0,0,0); font-size:12pt"><br> +</p> +<p style=3D"color:rgb(0,0,0); font-size:12pt"><br> +</p> +<p style=3D"color:rgb(0,0,0); font-size:12pt"><span style=3D"font-size:10pt= +; background-color:rgb(255,255,255); color:rgb(0,111,201)"><br> +</span></p> +<p style=3D"color:rgb(0,0,0); font-size:12pt"><span style=3D"font-size:10pt= +; background-color:rgb(255,255,255); color:rgb(0,111,201)">XXXXXXXXXX</span></p> +<p style=3D""><font color=3D"#006fc9"><span style=3D"font-size:13.3333px"><= +/span></font></p> +<span style=3D"font-size:10pt; background-color:rgb(255,255,255); color:rgb= +(0,111,201); font-size:10pt"></span> +<p style=3D""><font color=3D"#006fc9"><span style=3D"font-size:13.3333px">XXXXXXXXXX</span></font></p> +<p style=3D""><font color=3D"#006fc9"><span style=3D"font-size:13.3333px">XXXXXXXXXX</span></font></p> +<p style=3D""><font color=3D"#006fc9"><span style=3D"font-size:13.3333px">XXXXXXXXXX</span></font></p> +<p style=3D""><br> +</p> +<p style=3D""><br> +</p> +</div> +</div> +</div> +</body> +</html> +)x"; + g_test_bug("2349"); + + auto message{Message::make_from_text( + msgtext, + "/home/test/Maildir/inbox/cur/162342449279256.77777_1.evergrey:2,S")}; + g_assert_true(!!message); + + assert_equal(message->subject(), "PraÅ¡ymas"); + g_assert_true(message->priority() == Priority::Normal); + + g_assert_false(!!message->body_text()); + g_assert_true(!!message->body_html()); + g_assert_cmpuint(message->body_html()->find("<p>Pagarbiai.</p>"), ==, 935); +} + + +static void +test_message_message_id() +{ + constexpr const auto msg1 = +R"(From: "Mu Test" <mu@djcbsoftware.nl> +To: mu@djcbsoftware.nl +Message-ID: <87lew9xddt.fsf@djcbsoftware.nl> + +abc +)"; + + constexpr const auto msg2 = +R"(From: "Mu Test" <mu@djcbsoftware.nl> +To: mu@djcbsoftware.nl + +abc +)"; + + constexpr const auto msg3 = +R"(From: "Mu Test" <mu@djcbsoftware.nl> +To: mu@djcbsoftware.nl +Message-ID: + +abc +)"; + const auto m1{Message::make_from_text(msg1, "/foo/cur/m123:2,S")}; + assert_valid_result(m1); + + const auto m2{Message::make_from_text(msg2, "/foo/cur/m456:2,S")}; + assert_valid_result(m2); + const auto m3{Message::make_from_text(msg3, "/foo/cur/m789:2,S")}; + assert_valid_result(m3); + + assert_equal(m1->message_id(), "87lew9xddt.fsf@djcbsoftware.nl"); + + /* both with absent and empty message-id, generate "random" fake one, + * which must end in @mu.id */ + const auto id2{m2->message_id()}; + const auto id3{m3->message_id()}; + + g_assert_true(g_str_has_suffix(id2.c_str(), "@mu.id")); + g_assert_true(g_str_has_suffix(id3.c_str(), "@mu.id")); +} + + +static void +test_message_fail () +{ + { + const auto msg = Message::make_from_path("/root/non-existent-path-12345"); + g_assert_false(!!msg); + } + + { + const auto msg = Message::make_from_text("", ""); + g_assert_false(!!msg); + } +} + +static void +test_message_sanitize_maildir() +{ + assert_equal(Message::sanitize_maildir("/"), "/"); + assert_equal(Message::sanitize_maildir("/foo/bar"), "/foo/bar"); + assert_equal(Message::sanitize_maildir("/foo/bar/cuux/"), "/foo/bar/cuux"); +} + +static void +test_message_subject_with_newline() +{ +constexpr const auto txt = +R"(To: foo@example.com +Subject: =?utf-8?q?Le_poids_=C3=A9conomique_de_la_chasse_:_=0A=0Ala_dette_cach?= =?utf-8?q?=C3=A9e_de_la_chasse_!?= +Date: Mon, 24 Apr 2023 07:32:43 +0000 + +Hello! +)"; + g_test_bug("2477"); + + const auto msg{Message::make_from_text(txt, "/foo/cur/m123:2,S")}; + assert_valid_result(msg); + + assert_equal(msg->subject(), // newlines are filtered-out + "Le poids économique de la chasse : la dette cachée de la chasse !"); + assert_equal(msg->header("Subject").value_or(""), + "Le poids économique de la chasse : \n\nla dette cachée de la chasse !"); + g_assert_true(none_of(msg->flags() & Flags::MailingList)); +} + +static void +test_message_list_unsubscribe() +{ + constexpr const auto txt = +R"(From: "Mu Test" <mu@djcbsoftware.nl> +To: mu@djcbsoftware.nl +Subject: Test +Message-ID: <87lew9xddt.fsf@djcbsoftware.nl> +List-Unsubscribe: <mailto:unsubscribe-T7BC8RRQMK-booking-email-9@booking.com> + +abcdef +)"; + const auto msg{Message::make_from_text(txt, "/xxx/m123:2,S")}; + assert_valid_result(msg); + + assert_equal(msg->mailing_list(), ""); + g_assert_true(any_of(msg->flags() & Flags::MailingList)); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/message/message/mailing-list", + test_message_mailing_list); + g_test_add_func("/message/message/attachments", + test_message_attachments); + g_test_add_func("/message/message/signed", + test_message_signed); + g_test_add_func("/message/message/signed-encrypted", + test_message_signed_encrypted); + g_test_add_func("/message/message/multipart-mixed-rfc822", + test_message_multipart_mixed_rfc822); + g_test_add_func("/message/message/detect-attachment", + test_message_detect_attachment); + g_test_add_func("/message/message/calendar", + test_message_calendar); + g_test_add_func("/message/message/references", + test_message_references); + g_test_add_func("/message/message/outlook-body", + test_message_outlook_body); + g_test_add_func("/message/message/message-id", + test_message_message_id); + g_test_add_func("/message/message/subject-with-newline", + test_message_subject_with_newline); + g_test_add_func("/message/message/fail", + test_message_fail); + g_test_add_func("/message/message/sanitize-maildir", + test_message_sanitize_maildir); + g_test_add_func("/message/message/message-list-unsubscribe", + test_message_list_unsubscribe); + + return g_test_run(); +} diff --git a/lib/message/tests/meson.build b/lib/message/tests/meson.build new file mode 100644 index 0000000..94d0de9 --- /dev/null +++ b/lib/message/tests/meson.build @@ -0,0 +1,74 @@ +## Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# +# tests +# + +test('test-contact', + executable('test-contact', + '../mu-contact.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, gmime_dep, lib_mu_message_dep])) + +test('test-document', + executable('test-document', + '../mu-document.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, gmime_dep, lib_mu_message_dep])) + +test('test-fields', + executable('test-fields', + '../mu-fields.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, gmime_dep, lib_mu_message_dep])) + +test('test-flags', + executable('test-flags', + '../mu-flags.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, gmime_dep, lib_mu_message_dep])) + +test('test-message', + executable('test-message', + '../test-mu-message.cc', + install: false, + dependencies: [glib_dep, gmime_dep, lib_mu_message_dep])) + +test('test-priority', + executable('test-priority', + '../mu-priority.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, gmime_dep, lib_mu_message_dep])) + +test('test-message-file', + executable('test-message-file', + '../mu-message-file.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_message_dep])) + +test('test-message-part', + executable('test-message-part', + '../mu-message-part.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_message_dep])) diff --git a/lib/mu-config.cc b/lib/mu-config.cc new file mode 100644 index 0000000..6ce5c84 --- /dev/null +++ b/lib/mu-config.cc @@ -0,0 +1,126 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-config.hh" + +using namespace Mu; + +constexpr /*static*/ bool +validate_props() +{ + size_t id{0}; + for (auto&& prop: Config::properties) { + + // ids must match + if (static_cast<size_t>(prop.id) != id) + return false; + ++id; + } + + return true; +} + +#ifdef BUILD_TESTS +#define static_assert g_assert_true +#endif /*BUILD_TESTS*/ + +[[maybe_unused]] +static void +test_props() +{ + static_assert(validate_props()); +} + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +static void +test_basic() +{ + MemDb db; + Config conf_db{db}; + + g_assert_false(conf_db.read_only()); + + using Id = Config::Id; + + { + const auto rmd = conf_db.get<Id::RootMaildir>(); + g_assert_true(rmd.empty()); + } + + { + auto res = conf_db.set<Id::RootMaildir>("/home/djcb/Maildir"); + assert_valid_result(res); + + const auto rmd = conf_db.get<Id::RootMaildir>(); + assert_equal(rmd, "/home/djcb/Maildir"); + } + + { + g_assert_true(Config::property<Id::BatchSize>().default_val == "50000"); + g_assert_cmpuint(conf_db.get<Id::BatchSize>(),==,50000); + + assert_valid_result(conf_db.set<Id::BatchSize>(123456)); + g_assert_cmpuint(conf_db.get<Id::BatchSize>(),==,123456); + } + + + { + MemDb db2; + Config conf_db2{db2}; + + g_assert_cmpuint(conf_db2.get<Id::BatchSize>(),==,50000); + g_assert_true(conf_db2.get<Id::RootMaildir>().empty()); + + // BatchSize is configurable; RootMaildir is not. + conf_db2.import_configurable(conf_db); + + g_assert_cmpuint(conf_db2.get<Id::BatchSize>(),==,123456); + g_assert_true(conf_db2.get<Id::RootMaildir>().empty()); + } +} + +static void +test_read_only() +{ + MemDb db{true/*read-only*/}; + Config conf_db{db}; + + auto res = conf_db.set<Config::Id::MaxMessageSize>(12345); + g_assert_false(!!res); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/config-db/props", test_props); + g_test_add_func("/config-db/basic", test_basic); + g_test_add_func("/config-db/read-only", test_read_only); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/mu-config.hh b/lib/mu-config.hh new file mode 100644 index 0000000..17924c7 --- /dev/null +++ b/lib/mu-config.hh @@ -0,0 +1,316 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_CONFIG_HH__ +#define MU_CONFIG_HH__ + +#include <cstdint> +#include <cinttypes> +#include <string_view> +#include <string> +#include <array> +#include <vector> +#include <variant> +#include <unordered_map> + +#include "mu-xapian-db.hh" + +#include <utils/mu-utils.hh> +#include <utils/mu-result.hh> +#include <utils/mu-option.hh> + +namespace Mu { + +struct Property { + enum struct Id { + BatchSize, /**< Xapian batch-size */ + Contacts, /**< Cache of contact information */ + Created, /**< Time of creation */ + IgnoredAddresses,/**< Email addresses ignored for the contacts-cache */ + LastChange, /**< Time of last change */ + LastIndex, /**< Time of last index */ + MaxMessageSize, /**< Maximum message size (in bytes) */ + PersonalAddresses, /**< List of personal e-mail addresses */ + RootMaildir, /**< Root maildir path */ + SchemaVersion, /**< Xapian DB schema version */ + SupportNgrams, /**< Support ngrams for indexing & querying + * for e.g. CJK languages */ + /* <private> */ + _count_ /* Number of Ids */ + }; + + static constexpr size_t id_size = static_cast<size_t>(Id::_count_); + /**< Number of Property::Ids */ + + enum struct Flags { + None = 0, /**< Nothing in particular */ + ReadOnly = 1 << 0, /**< Property is read-only for external use + * (but can change from within the store) */ + Configurable = 1 << 1, /**< A user-configurable parameter; name + * starts with 'conf-' */ + Internal = 1 << 2, /**< Mu-internal field */ + }; + enum struct Type { + Boolean, /**< Some boolean value */ + Number, /**< Some number */ + Timestamp, /**< Timestamp number */ + Path, /**< Path string */ + String, /**< A string */ + StringList, /**< A list of strings */ + }; + + using Value = std::variant<int64_t, std::string, std::vector<std::string> >; + + Id id; + Type type; + Flags flags; + std::string_view name; + std::string_view default_val; + std::string_view description; +}; + +MU_ENABLE_BITOPS(Property::Flags); + +class Config { +public: + using Id = Property::Id; + using Type = Property::Type; + using Flags = Property::Flags; + using Value = Property::Value; + + static constexpr std::array<Property, Property::id_size> + properties = {{ + { + Id::BatchSize, + Type::Number, + Flags::Configurable, + "batch-size", + "50000", + "Number of changes in a database transaction" + }, + { + Id::Contacts, + Type::String, + Flags::Internal, + "contacts", + {}, + "Serialized contact information" + }, + { + Id::Created, + Type::Timestamp, + Flags::ReadOnly, + MetadataIface::created_key, + {}, + "Database creation time" + }, + { + Id::IgnoredAddresses, + Type::StringList, + Flags::Configurable, + "ignored-addresses", + {}, + "E-mail addresses ignored for the contacts-cache, " + "literal or /regexp/" + }, + { + Id::LastChange, + Type::Timestamp, + Flags::ReadOnly, + MetadataIface::last_change_key, + {}, + "Time when last change occurred" + }, + { + Id::LastIndex, + Type::Timestamp, + Flags::ReadOnly, + "last-index", + {}, + "Time when last indexing operation was completed" + }, + { + Id::MaxMessageSize, + Type::Number, + Flags::Configurable, + "max-message-size", + "100000000", // default max: 100M bytes + "Maximum message size (in bytes); bigger messages are skipped" + }, + { + Id::PersonalAddresses, + Type::StringList, + Flags::Configurable, + "personal-addresses", + {}, + "Personal e-mail addresses, literal or /regexp/" + }, + { + Id::RootMaildir, + Type::Path, + Flags::ReadOnly, + "root-maildir", + {}, + "Absolute path of the top of the Maildir tree" + }, + { + Id::SchemaVersion, + Type::Number, + Flags::ReadOnly, + "schema-version", + {}, + "Version of the Xapian database schema" + }, + { + Id::SupportNgrams, + Type::Boolean, + Flags::Configurable, + "support-ngrams", + {}, + "Support n-grams for working with CJK and other languages" + }, + }}; + + /** + * Construct a new Config object. + * + * @param db The config-store (database); must stay valid for the + * lifetime of this config. + */ + Config(MetadataIface& cstore): cstore_{cstore}{} + + /** + * Get the property by its id + * + * @param id a property id (!= Id::_count_) + * + * @return the property + */ + template <Id ID> + constexpr static const Property& property() { + return properties[static_cast<size_t>(ID)]; + } + + /** + * Get a Property by its name. + * + * @param name The name + * + * @return the property or Nothing if not found + */ + static Option<const Property&> property(const std::string& name) { + const auto pname{std::string_view(name.data(), name.size())}; + for(auto&& prop: properties) + if (prop.name == pname) + return prop; + return Nothing; + } + + /** + * Get the property value of the correct type + * + * @param prop_id a property id + * + * @return the value or Nothing + */ + template<Id ID> + auto get() const { + constexpr auto&& prop{property<ID>()}; + const auto str = std::invoke([&]()->std::string { + const auto str = cstore_.metadata(std::string{prop.name}); + return str.empty() ? std::string{prop.default_val} : str; + }); + if constexpr (prop.type == Type::Number) + return static_cast<size_t>(str.empty() ? 0 : std::atoll(str.c_str())); + if constexpr (prop.type == Type::Boolean) + return static_cast<size_t>(str.empty() ? false : + std::atol(str.c_str()) != 0); + else if constexpr (prop.type == Type::Timestamp) + return static_cast<time_t>(str.empty() ? 0 : std::atoll(str.c_str())); + else if constexpr (prop.type == Type::Path || prop.type == Type::String) + return str; + else if constexpr (prop.type == Type::StringList) + return split(str, SepaChar1); + + throw std::logic_error("invalid prop " + std::string{prop.name}); + } + + /** + * Set a new value for some property + * + * @param prop_id property-id + * @param val the new value (of the correct type) + * + * @return Ok() or some error + */ + template<Id ID, typename T> + Result<void> set(const T& val) { + constexpr auto&& prop{property<ID>()}; + if (read_only()) + return Err(Error::Code::AccessDenied, + "cannot write to read-only db"); + + const auto strval = std::invoke([&]{ + if constexpr (prop.type == Type::Number || prop.type == Type::Timestamp) + return mu_format("{}", static_cast<int64_t>(val)); + if constexpr (prop.type == Type::Boolean) + return val ? "1" : "0"; + else if constexpr (prop.type == Type::Path || prop.type == Type::String) + return std::string{val}; + else if constexpr (prop.type == Type::StringList) + return join(val, SepaChar1); + else + throw std::logic_error("invalid prop " + std::string{prop.name}); + }); + + cstore_.set_metadata(std::string{prop.name}, strval); + return Ok(); + } + + /** + * Is this a read-only Config? + * + * + * @return true or false + */ + bool read_only() const { return cstore_.read_only();}; + + /** + * Import configurable settings to some other MetadataIface + * + * @param target some other metadata interface + */ + void import_configurable(const Config& src) const { + for (auto&& prop: properties) { + if (any_of(prop.flags & Flags::Configurable)) { + const auto&& key{std::string{prop.name}}; + if (auto&& val{src.cstore_.metadata(key)}; !val.empty()) + cstore_.set_metadata(key, std::string{val}); + } + } + } + +private: + MetadataIface& cstore_; +}; + + +} // namespace Mu + +#endif /* MU_CONFIG_DB_HH__ */ diff --git a/lib/mu-contacts-cache.cc b/lib/mu-contacts-cache.cc new file mode 100644 index 0000000..b9b9b50 --- /dev/null +++ b/lib/mu-contacts-cache.cc @@ -0,0 +1,609 @@ +/* +** Copyright (C) 2019-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-contacts-cache.hh" + +#include <mutex> +#include <unordered_map> +#include <set> +#include <sstream> +#include <functional> +#include <algorithm> +#include <ctime> + +#include <utils/mu-utils.hh> +#include <utils/mu-regex.hh> +#include <glib.h> + +using namespace Mu; + +struct EmailHash { + std::size_t operator()(const std::string& email) const { + return lowercase_hash(email); + } +}; +struct EmailEqual { + bool operator()(const std::string& email1, const std::string& email2) const { + return lowercase_hash(email1) == lowercase_hash(email2); + } +}; + +using ContactUMap = std::unordered_map<const std::string, Contact, EmailHash, EmailEqual>; +struct ContactsCache::Private { + Private(Config& config_db) + :config_db_{config_db}, + contacts_{deserialize(config_db_.get<Config::Id::Contacts>())}, + personal_plain_{make_matchers<Config::Id::PersonalAddresses>()}, + personal_rx_{make_rx_matchers<Config::Id::PersonalAddresses>()}, + ignored_plain_{make_matchers<Config::Id::IgnoredAddresses>()}, + ignored_rx_{make_rx_matchers<Config::Id::IgnoredAddresses>()}, + dirty_{0}, + email_rx_{unwrap(Regex::make(email_rx_str, G_REGEX_OPTIMIZE))} + {} + + ~Private() { serialize(); } + + ContactUMap deserialize(const std::string&) const; + void serialize() const; + + bool is_valid_email(const std::string& email) const { + return email_rx_.matches(email); + } + + Config& config_db_; + ContactUMap contacts_; + mutable std::mutex mtx_; + + const StringVec personal_plain_; + const std::vector<Regex> personal_rx_; + + const StringVec ignored_plain_; + const std::vector<Regex> ignored_rx_; + + mutable size_t dirty_; + Regex email_rx_; + +private: + static bool is_rx(const std::string& p) { + return p.size() >= 2 && p.at(0) == '/' && p.at(p.length() - 1) == '/'; + } + + template<Config::Id Id> StringVec make_matchers() const { + return seq_remove(config_db_.get<Id>(), is_rx); + } + template<Config::Id Id> std::vector<Regex> make_rx_matchers() const { + std::vector<Regex> rxvec; + for (auto&& p: config_db_.get<Id>()) { + + if (!is_rx(p)) + continue; + constexpr auto opts{static_cast<GRegexCompileFlags>(G_REGEX_OPTIMIZE|G_REGEX_CASELESS)}; + + const auto rxstr{p.substr(1, p.length() - 2)}; + try { + rxvec.push_back(unwrap(Regex::make(rxstr, opts))); + mu_debug("match {}: '{}' {}", Config::property<Id>().name, + p, rxvec.back()); + } catch (const Error& rex) { + mu_warning("invalid personal address regexp '{}': {}", + p, rex.what()); + } + } + return rxvec; + } + + /* regexp as per: + * https://html.spec.whatwg.org/multipage/input.html#valid-e-mail-address + * + * "This requirement is a willful violation of RFC 5322, which defines a + * syntax for email addresses that is simultaneously too strict (before + * the "@" character), too vague (after the "@" character), and too lax + * (allowing comments, whitespace characters, and quoted strings in + * manners unfamiliar to most users) to be of practical use here." + */ + static constexpr auto email_rx_str = + R"(^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$)"; + +}; + +ContactUMap +ContactsCache::Private::deserialize(const std::string& serialized) const +{ + ContactUMap contacts; + std::stringstream ss{serialized, std::ios_base::in}; + std::string line; + + while (getline(ss, line)) { + const auto parts = Mu::split(line, SepaChar2); + if (G_UNLIKELY(parts.size() != 5)) { + mu_warning("error: '{}'", line); + continue; + } + Contact ci(parts[0], // email + std::move(parts[1]), // name + (time_t)g_ascii_strtoll(parts[3].c_str(), NULL, 10), // message_date + parts[2][0] == '1' ? true : false, // personal + (std::size_t)g_ascii_strtoll(parts[4].c_str(), NULL, 10), // frequency + g_get_monotonic_time()); // tstamp + contacts.emplace(std::move(parts[0]), std::move(ci)); + } + + return contacts; +} + + +void +ContactsCache::Private::serialize() const +{ + if (config_db_.read_only()) { + if (dirty_ > 0) + mu_critical("dirty data in read-only ccache!"); // bug + return; + } + + std::string s; + std::unique_lock lock(mtx_); + + if (dirty_ == 0) + return; // nothing to do. + + for (auto& item : contacts_) { + const auto& ci{item.second}; + s += mu_format("{}{}{}{}{}{}{}{}{}\n", + ci.email, SepaChar2, + ci.name, SepaChar2, + ci.personal ? 1 : 0, SepaChar2, + ci.message_date, SepaChar2, + ci.frequency); + } + config_db_.set<Config::Id::Contacts>(s); + dirty_ = 0; +} + +ContactsCache::ContactsCache(Config& config_db) + : priv_{std::make_unique<Private>(config_db)} +{} + +ContactsCache::~ContactsCache() = default; + +void +ContactsCache::serialize() const +{ + if (priv_->config_db_.read_only()) + throw std::runtime_error("cannot serialize read-only contacts-cache"); + + priv_->serialize(); +} + +void +ContactsCache::add(Contact&& contact) +{ + /* we do _not_ cache invalid email addresses, so we won't offer them in completions etc. It + * should be _rare_, but we've seen cases ( broken local messages) */ + if (!is_valid(contact.email)) { + mu_warning("not caching invalid e-mail address '{}'", contact.email); + return; + } + + if (is_ignored(contact.email)) { + /* ignored this address, e.g. 'noreply@example.com */ + return; + } + + std::lock_guard<std::mutex> l_{priv_->mtx_}; + + ++priv_->dirty_; + + auto it = priv_->contacts_.find(contact.email); + + if (it == priv_->contacts_.end()) { // completely new contact + + contact.name = contact.name; + if (!contact.personal) + contact.personal = is_personal(contact.email); + contact.tstamp = g_get_monotonic_time(); + + auto email{contact.email}; + // return priv_->contacts_.emplace(ContactUMap::value_type(email, std::move(contact))) + // .first->second; + mu_debug("adding contact {} <{}>", contact.name.c_str(), contact.email.c_str()); + priv_->contacts_.emplace(ContactUMap::value_type(email, std::move(contact))); + + } else { // existing contact. + auto& existing{it->second}; + ++existing.frequency; + if (contact.message_date > existing.message_date) { // update? + existing.email = std::move(contact.email); + // update name only if new one is not empty. + if (!contact.name.empty()) + existing.name = std::move(contact.name); + existing.tstamp = g_get_monotonic_time(); + existing.message_date = contact.message_date; + } + mu_debug("updating contact {} <{}> ({})", + contact.name, contact.email, existing.frequency); + } +} + + +void +ContactsCache::add(Contacts&& contacts, bool& personal) +{ + personal = seq_find_if(contacts,[&](auto&& c){ + return is_personal(c.email); }) != contacts.cend(); + + for (auto&& contact: contacts) { + contact.personal = personal; + add(std::move(contact)); + } +} + +const Contact* +ContactsCache::_find(const std::string& email) const +{ + std::lock_guard<std::mutex> l_{priv_->mtx_}; + + const auto it = priv_->contacts_.find(email); + if (it == priv_->contacts_.end()) + return {}; + else + return &it->second; +} + +void +ContactsCache::clear() +{ + std::lock_guard<std::mutex> l_{priv_->mtx_}; + + ++priv_->dirty_; + + priv_->contacts_.clear(); +} + +std::size_t +ContactsCache::size() const +{ + std::lock_guard<std::mutex> l_{priv_->mtx_}; + + return priv_->contacts_.size(); +} + + +/** + * This is used for sorting the Contacts in order of relevance. A highly + * specific algorithm, but the details don't matter _too_ much. + * + * This is currently used for the ordering in mu-cfind and auto-completion in + * mu4e, if the various completion methods don't override it... + */ +constexpr auto RecentOffset{15 * 24 * 3600}; +struct ContactLessThan { + ContactLessThan() + : recently_{::time({}) - RecentOffset} {} + + bool operator()(const Mu::Contact& ci1, const Mu::Contact& ci2) const { + // non-personal is less relevant. + if (ci1.personal != ci2.personal) + return ci1.personal < ci2.personal; + + // older is less relevant for recent messages + if (std::max(ci1.message_date, ci2.message_date) > recently_ && + ci1.message_date != ci2.message_date) + return ci1.message_date < ci2.message_date; + + // less frequent is less relevant + if (ci1.frequency != ci2.frequency) + return ci1.frequency < ci2.frequency; + + // if all else fails, alphabetically + return ci1.email < ci2.email; + } + // only sort recently seen contacts by recency; approx 15 days. + // this changes during the lifetime, but that's all fine. + const time_t recently_; +}; + +using ContactSet = std::set<std::reference_wrapper<const Contact>, + ContactLessThan>; + +void +ContactsCache::for_each(const EachContactFunc& each_contact) const +{ + std::lock_guard<std::mutex> l_{priv_->mtx_}; + + // first sort them for 'rank' + ContactSet sorted; + for (const auto& item : priv_->contacts_) + sorted.emplace(item.second); + + // return in _reverse_ order, so we get the most relevant ones first. + for (auto it = sorted.rbegin(); it != sorted.rend(); ++it) { + if (!each_contact(*it)) + break; + } +} + +static bool +address_matches(const std::string& addr, const StringVec& plain, const std::vector<Regex>& regexes) +{ + for (auto&& p : plain) + if (g_ascii_strcasecmp(addr.c_str(), p.c_str()) == 0) + return true; + + for (auto&& rx : regexes) { + if (rx.matches(addr)) + return true; + } + + return false; +} + +bool +ContactsCache::is_personal(const std::string& addr) const +{ + return address_matches(addr, priv_->personal_plain_, priv_->personal_rx_); +} + +bool +ContactsCache::is_ignored(const std::string& addr) const +{ + return address_matches(addr, priv_->ignored_plain_, priv_->ignored_rx_); +} + +bool +ContactsCache::is_valid(const std::string& addr) const +{ + return priv_->is_valid_email(addr); +} + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +static void +test_mu_contacts_cache_base() +{ + MemDb xdb{}; + Config cdb{xdb}; + ContactsCache contacts(cdb); + + g_assert_true(contacts.empty()); + g_assert_cmpuint(contacts.size(), ==, 0); + + contacts.add(Mu::Contact("foo.bar@example.com", + "Foo", {}, 12345)); + g_assert_false(contacts.empty()); + g_assert_cmpuint(contacts.size(), ==, 1); + + contacts.add(Mu::Contact("cuux@example.com", "Cuux", {}, + 54321)); + + g_assert_cmpuint(contacts.size(), ==, 2); + + contacts.add( + Mu::Contact("foo.bar@example.com", "Foo", {}, 77777)); + g_assert_cmpuint(contacts.size(), ==, 2); + + contacts.add( + Mu::Contact("Foo.Bar@Example.Com", "Foo", {}, 88888)); + g_assert_cmpuint(contacts.size(), ==, 2); + // note: replaces first. + + { + const auto info = contacts._find("bla@example.com"); + g_assert_false(info); + } + + { + const auto info = contacts._find("foo.BAR@example.com"); + g_assert_true(info); + + g_assert_cmpstr(info->email.c_str(), ==, "Foo.Bar@Example.Com"); + } + + contacts.clear(); + g_assert_true(contacts.empty()); + g_assert_cmpuint(contacts.size(), ==, 0); +} + +static void +test_mu_contacts_cache_personal() +{ + MemDb xdb{}; + Config cdb{xdb}; + cdb.set<Config::Id::PersonalAddresses> + (StringVec{{"foo@example.com", "bar@cuux.org", "/bar-.*@fnorb.f./"}}); + ContactsCache contacts{cdb}; + + g_assert_true(contacts.is_personal("foo@example.com")); + g_assert_true(contacts.is_personal("Bar@CuuX.orG")); + g_assert_true(contacts.is_personal("bar-123abc@fnorb.fi")); + g_assert_true(contacts.is_personal("bar-zzz@fnorb.fr")); + + g_assert_false(contacts.is_personal("foo@bar.com")); + g_assert_false(contacts.is_personal("BÂr@CuuX.orG")); + g_assert_false(contacts.is_personal("bar@fnorb.fi")); + g_assert_false(contacts.is_personal("bar-zzz@fnorb.xr")); +} + +static void +test_mu_contacts_cache_ignored() +{ + MemDb xdb{}; + Config cdb{xdb}; + cdb.set<Config::Id::IgnoredAddresses> + (StringVec{{"foo@example.com", "bar@cuux.org", "/bar-.*@fnorb.f./"}}); + ContactsCache contacts{cdb}; + + g_assert_true(contacts.is_ignored("foo@example.com")); + g_assert_true(contacts.is_ignored("Bar@CuuX.orG")); + g_assert_true(contacts.is_ignored("bar-123abc@fnorb.fi")); + g_assert_true(contacts.is_ignored("bar-zzz@fnorb.fr")); + + g_assert_false(contacts.is_ignored("foo@bar.com")); + g_assert_false(contacts.is_ignored("BÂr@CuuX.orG")); + g_assert_false(contacts.is_ignored("bar@fnorb.fi")); + g_assert_false(contacts.is_ignored("bar-zzz@fnorb.xr")); + + g_assert_cmpuint(contacts.size(),==,0); + contacts.add(Mu::Contact{"a@example.com", "a", 123, true, 1000, 0}); + g_assert_cmpuint(contacts.size(),==,1); + contacts.add(Mu::Contact{"foo@example.com", "b", 123, true, 1000, 0}); // ignored + contacts.add(Mu::Contact{"bar-123abc@fnorb.fi", "c", 123, true, 1000, 0}); // ignored + g_assert_cmpuint(contacts.size(),==,1); + contacts.add(Mu::Contact{"b@example.com", "d", 123, true, 1000, 0}); + g_assert_cmpuint(contacts.size(),==,2); +} + + + +static void +test_mu_contacts_cache_foreach() +{ + MemDb xdb{}; + Config cdb{xdb}; + ContactsCache ccache(cdb); + + ccache.add(Mu::Contact{"a@example.com", "a", 123, true, 1000, 0}); + ccache.add(Mu::Contact{"b@example.com", "b", 456, true, 1000, 0}); + + { + size_t n{}; + g_assert_false(ccache.empty()); + g_assert_cmpuint(ccache.size(),==,2); + ccache.for_each([&](auto&& contact) { ++n; return false; }); + g_assert_cmpuint(n,==,1); + } + + { + size_t n{}; + g_assert_false(ccache.empty()); + g_assert_cmpuint(ccache.size(),==,2); + ccache.for_each([&](auto&& contact) { ++n; return true; }); + g_assert_cmpuint(n,==,2); + } + + { + size_t n{}; + ccache.clear(); + g_assert_true(ccache.empty()); + g_assert_cmpuint(ccache.size(),==,0); + ccache.for_each([&](auto&& contact) { ++n; return true; }); + g_assert_cmpuint(n,==,0); + } +} + + + +static void +test_mu_contacts_cache_sort() +{ + auto result_chars = [](const Mu::ContactsCache& ccache)->std::string { + std::string str; + if (g_test_verbose()) + fmt::print("contacts-cache:\n"); + + ccache.for_each([&](auto&& contact) { + if (g_test_verbose()) + fmt::print("\t- {}\n", contact.display_name()); + str += contact.name; + return true; + }); + return str; + }; + + const auto now{std::time({})}; + + // "first" means more relevant + + { /* recent messages, newer comes first */ + + MemDb xdb{}; + Config cdb{xdb}; + ContactsCache ccache(cdb); + + ccache.add(Mu::Contact{"a@example.com", "a", now, true, 1000, 0}); + ccache.add(Mu::Contact{"b@example.com", "b", now-1, true, 1000, 0}); + assert_equal(result_chars(ccache), "ab"); + } + + { /* non-recent messages, more frequent comes first */ + + MemDb xdb{}; + Config cdb{xdb}; + ContactsCache ccache(cdb); + + ccache.add(Mu::Contact{"a@example.com", "a", now-2*RecentOffset, true, 1000, 0}); + ccache.add(Mu::Contact{"b@example.com", "b", now-3*RecentOffset, true, 2000, 0}); + assert_equal(result_chars(ccache), "ba"); + } + + { /* personal comes first */ + MemDb xdb{}; + Config cdb{xdb}; + ContactsCache ccache(cdb); + + ccache.add(Mu::Contact{"a@example.com", "a", now-5*RecentOffset, true, 1000, 0}); + ccache.add(Mu::Contact{"b@example.com", "b", now, false, 8000, 0}); + assert_equal(result_chars(ccache), "ab"); + } + + { /* if all else fails, reverse-alphabetically */ + MemDb xdb{}; + Config cdb{xdb}; + ContactsCache ccache(cdb); + + ccache.add(Mu::Contact{"a@example.com", "a", now, false, 1000, 0}); + ccache.add(Mu::Contact{"b@example.com", "b", now, false, 1000, 0}); + g_assert_cmpuint(ccache.size(),==,2); + assert_equal(result_chars(ccache), "ba"); + } +} + +static void +test_mu_contacts_valid_address() +{ + MemDb xdb{}; + Config cdb{xdb}; + ContactsCache ccache(cdb); + + g_assert_true(ccache.is_valid("a@example.com")); + g_assert_false(ccache.is_valid("a***@@booa@example..com")); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/contacts-cache/base", test_mu_contacts_cache_base); + g_test_add_func("/contacts-cache/personal", test_mu_contacts_cache_personal); + g_test_add_func("/contacts-cache/ignored", test_mu_contacts_cache_ignored); + g_test_add_func("/contacts-cache/for-each", test_mu_contacts_cache_foreach); + g_test_add_func("/contacts-cache/sort", test_mu_contacts_cache_sort); + g_test_add_func("/contacts-cache/valid-address", test_mu_contacts_valid_address); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/mu-contacts-cache.hh b/lib/mu-contacts-cache.hh new file mode 100644 index 0000000..d31c9dc --- /dev/null +++ b/lib/mu-contacts-cache.hh @@ -0,0 +1,171 @@ +/* +** Copyright (C) 2020-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef __MU_CONTACTS_CACHE_HH__ +#define __MU_CONTACTS_CACHE_HH__ + +#include <glib.h> +#include <time.h> +#include <memory> +#include <functional> +#include <chrono> +#include <string> +#include <time.h> +#include <inttypes.h> +#include <utils/mu-utils.hh> + +#include "mu-config.hh" +#include <message/mu-message.hh> + +namespace Mu { + +class ContactsCache { +public: + /** + * Construct a new ContactsCache object + * + * @param config db configuration database object + */ + ContactsCache(Config& config); + + /** + * DTOR + * + */ + ~ContactsCache(); + + /** + * Add a contact + * + * Invalid email address are not cached (but we log a warning); neither + * are "ignored" addresses (see --ignored-address in mu-init(1)) + * + * @param contact a Contact object + */ + void add(Contact&& contact); + + + /** + * Add a contacts sequence; this should be used for the contacts of a + * specific message, and determines if it is a "personal" message: + * if any of the contacts matches one of the personal addresses, + * any of the senders/recipients are considered "personal" + * + * Invalid email address are not cached (but we log a warning); neither + * are "ignored" addresses (see --ignored-address in mu-init(1)) + * + * @param contacts a Contact object sequence + * @param is_personal receives true if any of the contacts was personal; + * false otherwise + */ + void add(Contacts&& contacts, bool& is_personal); + void add(Contacts&& contacts) { + bool _ignore; + add(std::move(contacts), _ignore); + } + + /** + * Clear all contacts + * + */ + void clear(); + + /** + * Get the number of contacts + * + + * @return number of contacts + */ + std::size_t size() const; + + /** + * Are there no contacts? + * + * @return true or false + */ + bool empty() const { return size() == 0; } + + /** + * Serialize contact information. This all marks the data as + * non-dirty + */ + void serialize() const; + + /** + * Does this look like a 'personal' address? + * + * @param addr some e-mail address + * + * @return true or false + */ + bool is_personal(const std::string& addr) const; + + /** + * Does this look like an email-address that should be ignored? + * + * @param addr some e-mail address + * + * @return true or false + */ + bool is_ignored(const std::string& addr) const; + + /** + * Does this look like a valid email-address? + * + * @param addr some e-mail address + * + * @return true or false + */ + bool is_valid(const std::string& addr) const; + + /** + * Find a contact based on the email address. This is not safe, since + * the returned ptr can be invalidated at any time; only for unit-tests. + * + * @param email email address + * + * @return contact info, or {} if not found + */ + const Contact* _find(const std::string& email) const; + + /** + * Prototype for a callable that receives a contact + * + * @param contact some contact + * + * @return to get more contacts; false otherwise + */ + using EachContactFunc = std::function<bool(const Contact& contact_info)>; + + /** + * Invoke some callable for each contact, in _descending_ order of rank (i.e., the + * highest ranked contacts come first). + * + * @param each_contact function invoked for each contact + */ + void for_each(const EachContactFunc& each_contact) const; + +private: + struct Private; + std::unique_ptr<Private> priv_; +}; + +} // namespace Mu + +#endif /* __MU_CONTACTS_CACHE_HH__ */ diff --git a/lib/mu-indexer.cc b/lib/mu-indexer.cc new file mode 100644 index 0000000..e764933 --- /dev/null +++ b/lib/mu-indexer.cc @@ -0,0 +1,663 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-indexer.hh" + +#include <config.h> + +#include <atomic> +#include <algorithm> +#include <mutex> +#include <vector> +#include <thread> +#include <condition_variable> +#include <iostream> +#include <atomic> +#include <chrono> +using namespace std::chrono_literals; + +#include "mu-store.hh" + +#include "mu-scanner.hh" +#include "utils/mu-async-queue.hh" +#include "utils/mu-error.hh" + +#include "utils/mu-utils-file.hh" + +using namespace Mu; + +struct IndexState { + enum State { Idle, + Scanning, + Finishing, + Cleaning + }; + static const char* name(State s) { + switch (s) { + case Idle: + return "idle"; + case Scanning: + return "scanning"; + case Finishing: + return "finishing"; + case Cleaning: + return "cleaning"; + default: + return "<error>"; + } + } + + bool operator==(State rhs) const { + return state_.load() == rhs; + } + bool operator!=(State rhs) const { + return state_.load() != rhs; + } + void change_to(State new_state) { + mu_debug("changing indexer state {}->{}", name((State)state_), + name((State)new_state)); + state_.store(new_state); + } + +private: + std::atomic<State> state_{Idle}; +}; + +struct Indexer::Private { + Private(Mu::Store& store) + : store_{store}, scanner_{store_.root_maildir(), + [this](auto&& path, + auto&& statbuf, auto&& info) { + return handler(path, statbuf, info); + }}, + max_message_size_{store_.config().get<Mu::Config::Id::MaxMessageSize>()}, + was_empty_{store.empty()} { + + mu_message("created indexer for {} -> " + "{} (batch-size: {}; was-empty: {}; ngrams: {})", + store.root_maildir(), store.path(), + store.config().get<Mu::Config::Id::BatchSize>(), + was_empty_, + store.config().get<Mu::Config::Id::SupportNgrams>()); + } + + ~Private() { + stop(); + } + + bool dir_predicate(const std::string& path, const struct dirent* dirent) const; + bool handler(const std::string& fullpath, struct stat* statbuf, Scanner::HandleType htype); + + void maybe_start_worker(); + void item_worker(); + void scan_worker(); + + bool add_message(const std::string& path); + + bool cleanup(); + bool start(const Indexer::Config& conf, bool block); + bool stop(); + + bool is_running() const { return state_ != IndexState::Idle; } + + Indexer::Config conf_; + Store& store_; + Scanner scanner_; + const size_t max_message_size_; + + ::time_t dirstamp_{}; + std::size_t max_workers_; + std::vector<std::thread> workers_; + std::thread scanner_worker_; + + struct WorkItem { + std::string full_path; + enum Type { + Dir, + File + }; + Type type; + }; + + AsyncQueue<WorkItem> todos_; + + Progress progress_{}; + IndexState state_{}; + std::mutex lock_, w_lock_; + std::atomic<time_t> completed_{}; + bool was_empty_{}; +}; + +bool +Indexer::Private::handler(const std::string& fullpath, struct stat* statbuf, + Scanner::HandleType htype) +{ + switch (htype) { + case Scanner::HandleType::EnterDir: + case Scanner::HandleType::EnterNewCur: { + if (fullpath.length() > MaxTermLength) { + // currently the db uses the path as a key, and + // therefore it cannot be too long. We'd get an error + // later anyway but for now it's useful for surviving + // circular symlinks + mu_warning("'{}' is too long; ignore", fullpath); + return false; + } + + // in lazy-mode, we ignore this dir if its dirstamp suggest it + // is up-to-date (this is _not_ always true; hence we call it + // lazy-mode); only for actual message dirs, since the dir + // tstamps may not bubble up.U + dirstamp_ = store_.dirstamp(fullpath); + if (conf_.lazy_check && dirstamp_ >= statbuf->st_ctime && + htype == Scanner::HandleType::EnterNewCur) { + mu_debug("skip {} (seems up-to-date: {:%FT%T} >= {:%FT%T})", + fullpath, mu_time(dirstamp_), mu_time(statbuf->st_ctime)); + return false; + } + + // don't index dirs with '.noindex' + auto noindex = ::access((fullpath + "/.noindex").c_str(), F_OK) == 0; + if (noindex) { + mu_debug("skip {} (has .noindex)", fullpath); + return false; // don't descend into this dir. + } + + // don't index dirs with '.noupdate', unless we do a full + // (re)index. + if (!conf_.ignore_noupdate) { + auto noupdate = ::access((fullpath + "/.noupdate").c_str(), F_OK) == 0; + if (noupdate) { + mu_debug("skip {} (has .noupdate)", fullpath); + return false; + } + } + + mu_debug("checked {}", fullpath); + return true; + } + case Scanner::HandleType::LeaveDir: { + todos_.push({fullpath, WorkItem::Type::Dir}); + return true; + } + + case Scanner::HandleType::File: { + ++progress_.checked; + + if ((size_t)statbuf->st_size > max_message_size_) { + mu_debug("skip {} (too big: {} bytes)", fullpath, statbuf->st_size); + return false; + } + + // if the message is not in the db yet, or not up-to-date, queue + // it for updating/inserting. + if (statbuf->st_ctime <= dirstamp_ && store_.contains_message(fullpath)) + return false; + + // push the remaining messages to our "todo" queue for + // (re)parsing and adding/updating to the database. + todos_.push({fullpath, WorkItem::Type::File}); + return true; + } + default: + g_return_val_if_reached(false); + return false; + } +} + +void +Indexer::Private::maybe_start_worker() +{ + std::lock_guard lock{w_lock_}; + + if (todos_.size() > workers_.size() && workers_.size() < max_workers_) { + workers_.emplace_back(std::thread([this] { item_worker(); })); + mu_debug("added worker {}", workers_.size()); + } +} + +bool +Indexer::Private::add_message(const std::string& path) +{ + /* + * Having the lock here makes things a _lot_ slower. + * + * The reason for having the lock is some helgrind warnings; + * but it believed those are _false alarms_ + * https://gitlab.gnome.org/GNOME/glib/-/issues/2662 + * + * For now, set the lock as we were seeing some db corruption. + */ + std::unique_lock lock{w_lock_}; + auto msg{Message::make_from_path(path, store_.message_options())}; + if (!msg) { + mu_warning("failed to create message from {}: {}", path, msg.error().what()); + return false; + } + // if the store was empty, we know that the message is completely new + // and can use the fast path (Xapians 'add_document' rather tahn + // 'replace_document) + auto res = store_.add_message(msg.value(), was_empty_); + if (!res) { + mu_warning("failed to add message @ {}: {}", path, res.error().what()); + return false; + } + + return true; +} + +void +Indexer::Private::item_worker() +{ + WorkItem item; + + mu_debug("started worker"); + + while (state_ == IndexState::Scanning) { + if (!todos_.pop(item, 250ms)) + continue; + try { + switch (item.type) { + case WorkItem::Type::File: { + if (G_LIKELY(add_message(item.full_path))) + ++progress_.updated; + } break; + case WorkItem::Type::Dir: + store_.set_dirstamp(item.full_path, ::time(NULL)); + break; + default: + g_warn_if_reached(); + break; + } + } catch (const Mu::Error& er) { + mu_warning("error adding message @ {}: {}", item.full_path, er.what()); + } + + maybe_start_worker(); + std::this_thread::yield(); + } +} + +bool +Indexer::Private::cleanup() +{ + mu_debug("starting cleanup"); + + size_t n{}; + std::vector<Store::Id> orphans; // store messages without files. + store_.for_each_message_path([&](Store::Id id, const std::string& path) { + ++n; + if (::access(path.c_str(), R_OK) != 0) { + mu_debug("cannot read {} (id={}); queuing for removal from store", + path, id); + orphans.emplace_back(id); + } + + return state_ == IndexState::Cleaning; + }); + + if (orphans.empty()) + mu_debug("nothing to clean up"); + else { + mu_debug("removing {} stale message(s) from store", orphans.size()); + store_.remove_messages(orphans); + progress_.removed += orphans.size(); + } + + return true; +} + +void +Indexer::Private::scan_worker() +{ + XapianDb::Transaction tx{store_.xapian_db()}; // RAII + + progress_.reset(); + if (conf_.scan) { + mu_debug("starting scanner"); + if (!scanner_.start()) { // blocks. + mu_warning("failed to start scanner"); + state_.change_to(IndexState::Idle); + return; + } + mu_debug("scanner finished with {} file(s) in queue", todos_.size()); + } + + // now there may still be messages in the work queue... + // finish those; this is a bit ugly; perhaps we should + // handle SIGTERM etc. + + if (!todos_.empty()) { + const auto workers_size = std::invoke([this] { + std::lock_guard lock{w_lock_}; + return workers_.size(); + }); + mu_debug("process {} remaining message(s) with {} worker(s)", + todos_.size(), workers_size); + while (!todos_.empty()) + std::this_thread::sleep_for(100ms); + } + // and let the worker finish their work. + state_.change_to(IndexState::Finishing); + for (auto&& w : workers_) + if (w.joinable()) + w.join(); + + if (conf_.cleanup) { + mu_debug("starting cleanup"); + state_.change_to(IndexState::Cleaning); + cleanup(); + mu_debug("cleanup finished"); + } + + completed_ = ::time({}); + store_.config().set<Mu::Config::Id::LastIndex>(completed_); + state_.change_to(IndexState::Idle); +} + +bool +Indexer::Private::start(const Indexer::Config& conf, bool block) +{ + stop(); + + conf_ = conf; + if (conf_.max_threads == 0) { + /* benchmarking suggests that ~4 threads is the fastest (the + * real bottleneck is the database, so adding more threads just + * slows things down) + */ + max_workers_ = std::min(4U, std::thread::hardware_concurrency()); + } else + max_workers_ = conf.max_threads; + + if (store_.empty() && conf_.lazy_check) { + mu_debug("turn off lazy check since we have an empty store"); + conf_.lazy_check = false; + } + + mu_debug("starting indexer with <= {} worker thread(s)", max_workers_); + mu_debug("indexing: {}; clean-up: {}", conf_.scan ? "yes" : "no", + conf_.cleanup ? "yes" : "no"); + + state_.change_to(IndexState::Scanning); + /* kick off the first worker, which will spawn more if needed. */ + workers_.emplace_back(std::thread([this] { item_worker(); })); + /* kick the disk-scanner thread */ + scanner_worker_ = std::thread([this] { scan_worker(); }); + + mu_debug("started indexer in {}-mode", block ? "blocking" : "non-blocking"); + if (block) { + while(is_running()) { + using namespace std::chrono_literals; + std::this_thread::sleep_for(100ms); + } + } + + return true; +} + +bool +Indexer::Private::stop() +{ + scanner_.stop(); + + todos_.clear(); + if (scanner_worker_.joinable()) + scanner_worker_.join(); + + state_.change_to(IndexState::Idle); + for (auto&& w : workers_) + if (w.joinable()) + w.join(); + workers_.clear(); + + return true; +} + +Indexer::Indexer(Store& store) + : priv_{std::make_unique<Private>(store)} +{} + +Indexer::~Indexer() = default; + +bool +Indexer::start(const Indexer::Config& conf, bool block) +{ + const auto mdir{priv_->store_.root_maildir()}; + if (G_UNLIKELY(access(mdir.c_str(), R_OK) != 0)) { + mu_critical("'{}' is not readable: {}", mdir, g_strerror(errno)); + return false; + } + + std::lock_guard lock(priv_->lock_); + if (is_running()) + return true; + + return priv_->start(conf, block); +} + +bool +Indexer::stop() +{ + std::lock_guard lock{priv_->lock_}; + + if (!is_running()) + return true; + + mu_debug("stopping indexer"); + return priv_->stop(); +} + +bool +Indexer::is_running() const +{ + return priv_->is_running(); +} + +const Indexer::Progress& +Indexer::progress() const +{ + priv_->progress_.running = priv_->state_ == IndexState::Idle ? false : true; + + return priv_->progress_; +} + +::time_t +Indexer::completed() const +{ + return priv_->completed_; +} + + +#if BUILD_TESTS +#include "mu-test-utils.hh" + +static void +test_index_basic() +{ + allow_warnings(); + + TempDir tdir; + auto store = Store::make_new(tdir.path(), MU_TESTMAILDIR2); + assert_valid_result(store); + g_assert_true(store->empty()); + + Indexer& idx{store->indexer()}; + + g_assert_false(idx.is_running()); + g_assert_true(idx.stop()); + g_assert_cmpuint(idx.completed(),==, 0); + + const auto& prog{idx.progress()}; + g_assert_false(prog.running); + g_assert_cmpuint(prog.checked,==, 0); + g_assert_cmpuint(prog.updated,==, 0); + g_assert_cmpuint(prog.removed,==, 0); + + Indexer::Config conf{}; + conf.ignore_noupdate = true; + + { + const auto start{time({})}; + g_assert_true(idx.start(conf)); + while (idx.is_running()) + g_usleep(10000); + + g_assert_false(idx.is_running()); + g_assert_true(idx.stop()); + + g_assert_cmpuint(idx.completed() - start, <, 5); + + g_assert_false(prog.running); + g_assert_cmpuint(prog.checked,==, 14); + g_assert_cmpuint(prog.updated,==, 14); + g_assert_cmpuint(prog.removed,==, 0); + + g_assert_cmpuint(store->size(),==,14); + } + + conf.lazy_check = true; + conf.max_threads = 1; + conf.ignore_noupdate = false; + + { + const auto start{time({})}; + g_assert_true(idx.start(conf)); + while (idx.is_running()) + g_usleep(10000); + + g_assert_false(idx.is_running()); + g_assert_true(idx.stop()); + + g_assert_cmpuint(idx.completed() - start, <, 3); + + g_assert_false(prog.running); + g_assert_cmpuint(prog.checked,==, 0); + g_assert_cmpuint(prog.updated,==, 0); + g_assert_cmpuint(prog.removed,==, 0); + + g_assert_cmpuint(store->size(),==, 14); + } +} + + +static void +test_index_lazy() +{ + allow_warnings(); + + TempDir tdir; + auto store = Store::make_new(tdir.path(), MU_TESTMAILDIR2); + assert_valid_result(store); + g_assert_true(store->empty()); + Indexer& idx{store->indexer()}; + + Indexer::Config conf{}; + conf.lazy_check = true; + conf.ignore_noupdate = false; + + const auto start{time({})}; + g_assert_true(idx.start(conf)); + while (idx.is_running()) + g_usleep(10000); + + g_assert_false(idx.is_running()); + g_assert_true(idx.stop()); + + g_assert_cmpuint(idx.completed() - start, <, 3); + + const auto& prog{idx.progress()}; + g_assert_false(prog.running); + g_assert_cmpuint(prog.checked,==, 6); + g_assert_cmpuint(prog.updated,==, 6); + g_assert_cmpuint(prog.removed,==, 0); + + g_assert_cmpuint(store->size(),==, 6); +} + +static void +test_index_cleanup() +{ + allow_warnings(); + + TempDir tdir; + auto mdir = join_paths(tdir.path(), "Test"); + { + auto res = run_command({"cp", "-r", MU_TESTMAILDIR2, mdir}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code,==, 0); + } + + auto store = Store::make_new(tdir.path(), mdir); + assert_valid_result(store); + g_assert_true(store->empty()); + Indexer& idx{store->indexer()}; + + Indexer::Config conf{}; + conf.ignore_noupdate = true; + + g_assert_true(idx.start(conf)); + while (idx.is_running()) + g_usleep(10000); + + g_assert_false(idx.is_running()); + g_assert_true(idx.stop()); + g_assert_cmpuint(store->size(),==, 14); + + // remove a message + { + auto mpath = join_paths(mdir, "bar", "cur", "mail6"); + auto res = run_command({"rm", mpath}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code,==, 0); + } + + // no cleanup, # stays the same + conf.cleanup = false; + g_assert_true(idx.start(conf)); + while (idx.is_running()) + g_usleep(10000); + g_assert_false(idx.is_running()); + g_assert_true(idx.stop()); + g_assert_cmpuint(store->size(),==, 14); + + // cleanup, message is gone from store. + conf.cleanup = true; + g_assert_true(idx.start(conf)); + while (idx.is_running()) + g_usleep(10000); + g_assert_false(idx.is_running()); + g_assert_true(idx.stop()); + g_assert_cmpuint(store->size(),==, 13); +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/index/basic", test_index_basic); + g_test_add_func("/index/lazy", test_index_lazy); + g_test_add_func("/index/cleanup", test_index_cleanup); + + return g_test_run(); + +} +#endif /*BUILD_TESTS*/ diff --git a/lib/mu-indexer.hh b/lib/mu-indexer.hh new file mode 100644 index 0000000..3ea1fb6 --- /dev/null +++ b/lib/mu-indexer.hh @@ -0,0 +1,122 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_INDEXER_HH__ +#define MU_INDEXER_HH__ + +#include <atomic> +#include <memory> +#include <chrono> + +namespace Mu { + +class Store; + +/// An object abstracting the index process. +class Indexer { +public: + /** + * Construct an indexer object + * + * @param store the message store to use + */ + Indexer(Store& store); + + /** + * DTOR + */ + ~Indexer(); + + /// A configuration object for the indexer + struct Config { + bool scan{true}; + /**< scan for new messages */ + bool cleanup{true}; + /**< clean messages no longer in the file system */ + size_t max_threads{}; + /**< maximum # of threads to use */ + bool ignore_noupdate{}; + /**< ignore .noupdate files */ + bool lazy_check{}; + /**< whether to skip directories that don't have a changed + * mtime */ + }; + + /** + * Start indexing. If already underway, do nothing. This returns + * immediately after starting, with the work being done in the + * background, unless blocking = true + * + * @param conf a configuration object + * + * @return true if starting worked or an indexing process was already + * underway; false otherwise. + * + */ + bool start(const Config& conf, bool block=false); + + /** + * Stop indexing. If not indexing, do nothing. + * + * @return true if we stopped indexing, or indexing was not underway; false otherwise. + */ + bool stop(); + + /** + * Is an indexing process running? + * + * @return true or false. + */ + bool is_running() const; + + // Object describing current progress + struct Progress { + void reset() { + running = false; + checked = updated = removed = 0; + } + std::atomic<bool> running{}; /**< Is an index operation in progress? */ + std::atomic<size_t> checked{}; /**< Number of messages checked for changes */ + std::atomic<size_t> updated{}; /**< Number of messages (re)parsed/added/updated */ + std::atomic<size_t> removed{}; /**< Number of message removed from store */ + }; + + /** + * Get an object describing the current progress. The progress object + * describes the most recent indexing job, and is reset upon a fresh + * start(). + * + * @return a progress object. + */ + const Progress& progress() const; + + /** + * Last time indexing was completed. + * + * @return the time or 0 + */ + ::time_t completed() const; + +private: + struct Private; + std::unique_ptr<Private> priv_; +}; + +} // namespace Mu +#endif /* MU_INDEXER_HH__ */ diff --git a/lib/mu-maildir.cc b/lib/mu-maildir.cc new file mode 100644 index 0000000..5166b17 --- /dev/null +++ b/lib/mu-maildir.cc @@ -0,0 +1,455 @@ +/* +** Copyright (C) 2008-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to 59the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include <string> +#include <unistd.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <sys/wait.h> +#include <fcntl.h> +#include <stdlib.h> + +#include <string.h> +#include <errno.h> +#include <glib/gprintf.h> +#include <gio/gio.h> + +#include "glibconfig.h" +#include "mu-maildir.hh" +#include "utils/mu-utils.hh" +#include "utils/mu-utils-file.hh" + +using namespace Mu; + +#define MU_MAILDIR_NOINDEX_FILE ".noindex" +#define MU_MAILDIR_NOUPDATE_FILE ".noupdate" + +/* On Linux (and some BSD), we have entry->d_type, but some file + * systems (XFS, ReiserFS) do not support it, and set it DT_UNKNOWN. + * On other OSs, notably Solaris, entry->d_type is not present at all. + * For these cases, we use lstat (in get_dtype) as a slower fallback, + * and return it in the d_type parameter + */ +static unsigned char +get_dtype(struct dirent* dentry, const std::string& path, bool use_lstat) +{ +#ifdef HAVE_STRUCT_DIRENT_D_TYPE + + if (dentry->d_type == DT_UNKNOWN) + goto slowpath; + if (dentry->d_type == DT_LNK && !use_lstat) + goto slowpath; + + return dentry->d_type; /* fastpath */ + +slowpath: +#endif /*HAVE_STRUCT_DIRENT_D_TYPE*/ + + return determine_dtype(path, use_lstat); +} + +static Mu::Result<void> +create_maildir(const std::string& path, mode_t mode) +{ + if (path.empty()) + return Err(Error{Error::Code::File, "path must not be empty"}); + + std::array<std::string,3> subdirs = {"new", "cur", "tmp"}; + for (auto&& subdir: subdirs) { + + const auto fullpath{join_paths(path, subdir)}; + + /* if subdir already exists, don't try to re-create + * it */ + if (check_dir(fullpath, true/*readable*/, true/*writable*/)) + continue; + + int rv{g_mkdir_with_parents(fullpath.c_str(), static_cast<int>(mode))}; + + /* note, g_mkdir_with_parents won't detect an error if + * there's already such a dir, but with the wrong + * permissions; so we need to check */ + if (rv != 0 || !check_dir(fullpath, true/*readable*/, true/*writable*/)) + return Err(Error{Error::Code::File, + "creating dir failed for {}: {}", + fullpath, g_strerror(errno)}); + } + + return Ok(); +} + +static Mu::Result<void> /* create a noindex file if requested */ +create_noindex(const std::string& path) +{ + const auto noindexpath{join_paths(path, MU_MAILDIR_NOINDEX_FILE)}; + + /* note, if the 'close' failed, creation may still have succeeded...*/ + int fd = ::creat(noindexpath.c_str(), 0644); + if (fd < 0 || ::close(fd) != 0) + return Err(Error{Error::Code::File, + "error creating .noindex: {}", g_strerror(errno)}); + else + return Ok(); +} + +Mu::Result<void> +Mu::maildir_mkdir(const std::string& path, mode_t mode, bool noindex) +{ + if (auto&& created{create_maildir(path, mode)}; !created) + return created; // fail. + else if (!noindex) + return Ok(); + + if (auto&& created{create_noindex(path)}; !created) + return created; //fail + + return Ok(); +} + +/* determine whether the source message is in 'new' or in 'cur'; + * we ignore messages in 'tmp' for obvious reasons */ +static Mu::Result<void> +check_subdir(const std::string& src, bool& in_cur) +{ + char *srcpath{g_path_get_dirname(src.c_str())}; + + bool invalid{}; + if (g_str_has_suffix(srcpath, "cur")) + in_cur = true; + else if (g_str_has_suffix(srcpath, "new")) + in_cur = false; + else + invalid = true; + + g_free(srcpath); + + if (invalid) + return Err(Error{Error::Code::File, "invalid source message '{}'", src}); + else + return Ok(); +} + +static Mu::Result<std::string> +get_target_fullpath(const std::string& src, const std::string& targetpath, + bool unique_names) +{ + bool in_cur{}; + if (auto&& res = check_subdir(src, in_cur); !res) + return Err(std::move(res.error())); + + const auto srcfile{basename(src)}; + + /* create target-path; note: make the filename *cough* unique by + * including a hash of the srcname in the targetname. This helps if + * there are copies of a message (which all have the same basename) + */ + if (unique_names) + return join_paths(targetpath, in_cur ? "cur" : "new", + mu_format("{:08x}-{}", g_str_hash(src.c_str()), srcfile)); + else + return join_paths(targetpath, in_cur ? "cur" : "new", srcfile.c_str()); +} + +Result<void> +Mu::maildir_link(const std::string& src, const std::string& targetpath, + bool unique_names) +{ + auto path_res{get_target_fullpath(src, targetpath, unique_names)}; + if (!path_res) + return Err(std::move(path_res.error())); + + auto rv{::symlink(src.c_str(), path_res->c_str())}; + if (rv != 0) + return Err(Error{Error::Code::File, + "error creating link {} => {}: {}", + *path_res, src, g_strerror(errno)}); + + return Ok(); +} + +static bool +clear_links(const std::string& path, DIR* dir) +{ + bool res; + struct dirent* dentry; + + res = true; + errno = 0; + + while ((dentry = ::readdir(dir))) { + + if (dentry->d_name[0] == '.') + continue; /* ignore .,.. other dotdirs */ + + const auto fullpath{join_paths(path, dentry->d_name)}; + const auto d_type = get_dtype(dentry, fullpath.c_str(), + true/*lstat*/); + switch(d_type) { + case DT_LNK: + if (::unlink(fullpath.c_str()) != 0) { + /* LCOV_EXCL_START*/ + mu_warning("error unlinking {}: {}", fullpath, g_strerror(errno)); + res = false; + /* LCOV_EXCL_STOP*/ + } else + mu_debug("unlinked linksdir {}", fullpath); + break; + case DT_DIR: { + DIR* subdir{::opendir(fullpath.c_str())}; + /* LCOV_EXCL_START*/ + if (!subdir) { + mu_warning("error opening dir {}: {}", fullpath, g_strerror(errno)); + res = false; + } + if (!clear_links(fullpath, subdir)) + res = false; + /* LCOV_EXCL_STOP*/ + ::closedir(subdir); + } break; + default: + break; + } + } + + return res; +} + +Mu::Result<void> +Mu::maildir_clear_links(const std::string& path) +{ + const auto dir{::opendir(path.c_str())}; + if (!dir) + return Err(Error{Error::Code::File, "failed to open {}: {}", + path, g_strerror(errno)}); + + clear_links(path, dir); + ::closedir(dir); + + return Ok(); +} + +/* LCOV_EXCL_START*/ +static Mu::Result<void> +msg_move_verify(const std::string& src, const std::string& dst) +{ + /* double check -- is the target really there? */ + if (::access(dst.c_str(), F_OK) != 0) + return Err(Error{Error::Code::File, + "can't find target ({}->{})", src, dst}); + + if (::access(src.c_str(), F_OK) == 0) { + if (src == dst) { + mu_warning("moved {} to itself", src); + } + /* this could happen if some other tool (for mail syncing) is + * interfering */ + mu_debug("source is still there ({}->{})", src, dst); + } + + mu_debug("moved {} -> {}", src, dst); + + return Ok(); +} +/* LCOV_EXCL_STOP*/ + +/* LCOV_EXCL_START*/ +// don't use this right now, since it gives as (false alarm) +// valgrind warning in tests +/* use GIO to move files; this is slower than rename() so only use + * this when needed: when moving across filesystems */ +G_GNUC_UNUSED static Mu::Result<void> +msg_move_g_file(const std::string& src, const std::string& dst) +{ + GFile *srcfile{g_file_new_for_path(src.c_str())}; + GFile *dstfile{g_file_new_for_path(dst.c_str())}; + + GError* err{}; + auto res = g_file_move(srcfile, dstfile, + G_FILE_COPY_OVERWRITE, + NULL, NULL, NULL, &err); + g_clear_object(&srcfile); + g_clear_object(&dstfile); + + if (res) + return Ok(); + else + return Err(Error::Code::File, &err, "error moving {} -> {}", src, dst); +} +/* LCOV_EXCL_STOP*/ + +/* use mv to move files; this is slower than rename() so only use this when + * needed: when moving across filesystems */ +G_GNUC_UNUSED static Mu::Result<void> +msg_move_mv_file(const std::string& src, const std::string& dst) +{ + if (auto res{run_command0({"/bin/mv", src, dst})}; !res) + return Err(Error::Code::File, "error moving {}->{}; err={}", src, dst, res.error()); + else + return Ok(); +} + +Mu::Result<void> +Mu::maildir_move_message(const std::string& oldpath, + const std::string& newpath, + bool assume_remote) +{ + mu_debug("moving {} --> {} (assume-remote:{})", oldpath, newpath, assume_remote); + + if (::access(oldpath.c_str(), R_OK) != 0) + return Err(Error{Error::Code::File, "cannot read {}", oldpath}); + + if (oldpath == newpath) + return Ok(); // nothing to do. + + if (!assume_remote) { /* for testing */ + + if (::rename(oldpath.c_str(), newpath.c_str()) == 0) /* seems it worked; double-check */ + return msg_move_verify(oldpath, newpath); + /* LCOV_EXCL_START*/ + if (errno != EXDEV) /* some unrecoverable error occurred */ + return Err(Error{Error::Code::File, "error moving {} -> {}: {}", + oldpath, newpath, strerror(errno)}); + /* LCOV_EXCL_STOP*/ + } + + /* the EXDEV / assume-remote case -- source and target live on different + * file systems + * + * we can choose either msg_move_gio_file or msg_move_mv_file; + * we use the latter for now, since the former gives some (false) + * valgrind alarms. + * */ + if (auto&& res{msg_move_mv_file(oldpath, newpath)}; !res) + return res; + else + return msg_move_verify(oldpath, newpath); +} + +static std::string +reinvent_filename_base() +{ + return mu_format("{}.{:08x}{:08x}.{}", ::time({}), + g_random_int(), g_get_monotonic_time(), g_get_host_name()); +} + +/** + * Determine the destination filename + * + * @param file a filename + * @param flags flags for the destination + * @param new_name whether to change the basename + * + * @return the destion filename. + */ +static std::string +determine_dst_filename(const std::string& file, Flags flags, + bool new_name) +{ + /* Recalculate a unique new base file name */ + auto&& parts{message_file_parts(file)}; + if (new_name) + parts.base = reinvent_filename_base(); + + /* for a New message, there are no flags etc.; so we only return the + * name sans suffix */ + if (any_of(flags & Flags::New)) + return std::move(parts.base); + + const auto flagstr{ + to_string( + flags_filter( + flags, MessageFlagCategory::Mailfile))}; + + return parts.base + parts.separator + "2," + flagstr; +} + + +/* + * sanity checks + */ +static Mu::Result<void> +check_determine_target_params (const std::string& old_path, + const std::string& root_maildir_path, + const std::string& target_maildir, + Flags newflags) +{ + if (!g_path_is_absolute(old_path.c_str())) + return Err(Error{Error::Code::File, + "old_path is not absolute ({})", old_path}); + + if (!g_path_is_absolute(root_maildir_path.c_str())) + return Err(Error{Error::Code::File, + "root maildir path is not absolute ({})", + root_maildir_path}); + + if (!target_maildir.empty() && target_maildir[0] != '/') + return Err(Error{Error::Code::File, + "target maildir must be empty or start with / ({})", + target_maildir}); + + if (old_path.find(root_maildir_path) != 0) + return Err(Error{Error::Code::File, + "old-path must be below root-maildir ({}) ({})", + old_path, root_maildir_path}); + + if (any_of(newflags & Flags::New) && newflags != Flags::New) + return Err(Error{Error::Code::File, + "if the New flag is specified, it must be the only flag"}); + return Ok(); +} + + +Mu::Result<std::string> +Mu::maildir_determine_target(const std::string& old_path, + const std::string& root_maildir_path, + const std::string& target_maildir, + Flags newflags, + bool new_name) +{ + newflags = flags_maildir_file(newflags); // filter out irrelevant flags. + + /* sanity checks */ + if (const auto checked{check_determine_target_params( + old_path, root_maildir_path, target_maildir, newflags)}; !checked) + return Err(Error{std::move(checked.error())}); + + /* + * this gets us the source maildir filesystem path, the directory + * in which new/ & cur/ lives, and the source file + */ + const auto src{base_message_dir_file(old_path)}; + if (!src) + return Err(src.error()); + const auto& [src_mdir, src_file, is_new] = *src; + + /* if target_mdir is empty, the src_dir does not change (though cur/ + * maybe become new or vice-versa) */ + const auto dst_mdir = target_maildir.empty() ? src_mdir : + join_paths(root_maildir_path, target_maildir); + + /* now calculate the message name (incl. its immediate parent dir) */ + const auto dst_file{determine_dst_filename(src_file, newflags, new_name)}; + + /* and the complete path name. */ + const std::string subdir{(none_of(newflags & Flags::New)) ? "cur" : "new"}; + + return join_paths(dst_mdir, subdir,dst_file); +} diff --git a/lib/mu-maildir.hh b/lib/mu-maildir.hh new file mode 100644 index 0000000..e9e4c75 --- /dev/null +++ b/lib/mu-maildir.hh @@ -0,0 +1,120 @@ +/* +** Copyright (C) 2008-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_MAILDIR_HH__ +#define MU_MAILDIR_HH__ + +#include <string> +#include <utils/mu-result.hh> + +#include <glib.h> +#include <time.h> +#include <sys/types.h> /* for mode_t */ +#include <message/mu-message.hh> + +namespace Mu { + +/** + * Create a new maildir. if parts of the maildir already exists, those will + * simply be ignored. + * + * IOW, if you try to create the same maildir twice, the second will simply be a + * no-op (without any errors). Note, if the function fails 'halfway', it will + * *not* try to remove the parts the were created. it *will* create any parent + * dirs that are not yet existent. + * + * @param path the path (missing components will be created, as in 'mkdir -p'). + * must be non-empty + * @param mode the file mode (e.g., 0755) + * @param noindex add a .noindex file to the maildir, so it will be excluded + * from indexing by 'mu index' + * + * @return a valid result (!!result) or an Error + */ +Result<void> maildir_mkdir(const std::string& path, mode_t mode=0700, + bool noindex=false); + +/** + * Create a symbolic link to a mail message + * + * @param src the full path to the source message + * @param targetpath the path to the target maildir; ie., *not* + * MyMaildir/cur, but just MyMaildir/. The function will figure out + * the correct subdir then. + * @param unique_names whether to create unique names; should be true unless + * for tests. + * + * @return a valid result (!!result) or an Error + */ +Result<void> maildir_link(const std::string& src, const std::string& targetpath, + bool unique_names=true); +/** + * Recursively delete all the symbolic links in a directory tree + * + * @param dir top dir + * + * @return a valid result (!!result) or an Error + */ +Result<void> maildir_clear_links(const std::string& dir); + +/** + * Move a message file to another maildir. If the target exists, it is overwritten. + * + * @param oldpath an absolute file system path to an existing message in an + * actual maildir + * @param newpath the absolute full path to the target file + * @param assume_remote assume the target is on a different file-system, + * and hence rename() won't work and we need another method + * + * @return a valid result or an Error + */ +Result<void> maildir_move_message(const std::string& oldpath, + const std::string& newpath, + bool assume_remote = false); + +/** + * Determine the target path for a to-be-moved message; i.e. this does not + * actually move the message, only calculate the path. + * + * @param old_path an absolute file system path to an existing message in an + * actual maildir + * @param root_maildir_path the absolute file system path under which + * all maildirs live. + * @param target_maildir the target maildir; note that this the base-level + * Maildir, ie. /home/user/Maildir/archive, and must _not_ include the + * 'cur' or 'new' part. Note that the target maildir must be on the + * same filesystem. Can be empty if the message should not be moved to + * a different maildir; note that this may still involve a + * move to another directory (say, from new/ to cur/) + * @param flags to set for the target (influences the filename, path). + * Any non-Maildir/File flags are ignored. + * @param new_name whether to change the basename of the file + * + * @return Full path name of the target file or an Error + */ +Result<std::string> +maildir_determine_target(const std::string& old_path, + const std::string& root_maildir_path, + const std::string& target_maildir, + Flags newflags, + bool new_name); + +} // namespace Mu + +#endif /*MU_MAILDIR_HH__*/ diff --git a/lib/mu-query-macros.cc b/lib/mu-query-macros.cc new file mode 100644 index 0000000..1fb682b --- /dev/null +++ b/lib/mu-query-macros.cc @@ -0,0 +1,160 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-query-macros.hh" + +#include <glib.h> +#include <unordered_map> + +#include "utils/mu-utils.hh" + +using namespace Mu; + +constexpr auto MU_BOOKMARK_GROUP = "mu"; + +struct QueryMacros::Private { + Private(const Config& conf): conf_{conf} {} + + + Result<void> import_key_file(GKeyFile *kfile); + + const Config& conf_; + std::unordered_map<std::string, std::string> macros_{}; +}; + +Result<void> +QueryMacros::Private::import_key_file(GKeyFile *kfile) +{ + if (!kfile) + return Err(Error::Code::InvalidArgument, "invalid key-file"); + + GError *err{}; + size_t num{}; + gchar **keys{g_key_file_get_keys(kfile, MU_BOOKMARK_GROUP, &num, &err)}; + if (!keys) + return Err(Error::Code::File, &err/*cons*/,"failed to read keys"); + + for (auto key = keys; key && *key; ++key) { + + auto rawval{g_key_file_get_string(kfile, MU_BOOKMARK_GROUP, *key, &err)}; + if (!rawval) { + g_strfreev(keys); + return Err(Error::Code::File, &err/*cons*/,"failed to read key '{}'", *key); + } + + auto val{to_string_gchar(std::move(rawval))}; + macros_.erase(val); // we want to replace + macros_.emplace(std::string(*key), std::move(val)); + ++num; + } + + g_strfreev(keys); + mu_debug("imported {} query macro(s); total {}", num, macros_.size()); + return Ok(); +} + +QueryMacros::QueryMacros(const Config& conf): + priv_{std::make_unique<Private>(conf)} {} + +QueryMacros::~QueryMacros() = default; + +Result<void> +QueryMacros::load_bookmarks(const std::string& path) +{ + GError *err{}; + GKeyFile *kfile{g_key_file_new()}; + if (!g_key_file_load_from_file(kfile, path.c_str(), G_KEY_FILE_NONE, &err)) { + g_key_file_unref(kfile); + return Err(Error::Code::File, &err/*cons*/, + "failed to read bookmarks from {}", path); + } + + auto&& res = priv_->import_key_file(kfile); + g_key_file_unref(kfile); + + return res; +} + +Option<std::string> +QueryMacros::find_macro(const std::string& name) const +{ + if (const auto it{priv_->macros_.find(name)}; it != priv_->macros_.end()) + return it->second; + else + return Nothing; +} + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" +#include "utils/mu-utils-file.hh" + +static void +test_bookmarks() +{ + MemDb db; + Config conf_db{db}; + QueryMacros qm{conf_db}; + + TempDir tdir{}; + const auto bmfile{join_paths(tdir.path(), "bookmarks.ini")}; + std::ofstream os{bmfile}; + + mu_println(os, "# test\n" + "[mu]\n" + "foo=subject:bar"); + os.close(); + + auto res = qm.load_bookmarks(bmfile); + assert_valid_result(res); + + assert_equal(qm.find_macro("foo").value_or(""), "subject:bar"); + assert_equal(qm.find_macro("bar").value_or("nope"), "nope"); +} + + +static void +test_bookmarks_fail() +{ + + MemDb db; + Config conf_db{db}; + QueryMacros qm{conf_db}; + + auto res = qm.load_bookmarks("/foo/bar/non-existent"); + g_assert_false(!!res); +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/query/macros/bookmarks", test_bookmarks); + g_test_add_func("/query/macros/bookmarks-fail", test_bookmarks_fail); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/mu-query-macros.hh b/lib/mu-query-macros.hh new file mode 100644 index 0000000..1b62615 --- /dev/null +++ b/lib/mu-query-macros.hh @@ -0,0 +1,75 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + + +#ifndef MU_QUERY_MACROS_HH__ +#define MU_QUERY_MACROS_HH__ + +#include <string> +#include <memory> + +#include <utils/mu-result.hh> +#include <utils/mu-option.hh> + +#include "mu-config.hh" + +namespace Mu { + +class QueryMacros{ +public: + /** + * Construct QueryMacros object + * + * @param conf config object ref + */ + QueryMacros(const Config& conf); + + /** + * DTOR + */ + ~QueryMacros(); + + /** + * Read bookmarks (ie. macros) from a bookmark-file + * + * @param bookmarks_file path to the bookmarks file + * + * @return Ok or some error + */ + Result<void> load_bookmarks(const std::string& bookmarks_file); + + + /** + * Find a macro (aka 'bookmark') by its name + * + * @param name the name of the bookmark + * + * @return the macro value or Nothing if not found + */ + Option<std::string> find_macro(const std::string& name) const; + +private: + struct Private; + std::unique_ptr<Private> priv_; +}; + + +} // namespace Mu + +#endif /* MU_QUERY_MACROS_HH__ */ diff --git a/lib/mu-query-match-deciders.cc b/lib/mu-query-match-deciders.cc new file mode 100644 index 0000000..999d609 --- /dev/null +++ b/lib/mu-query-match-deciders.cc @@ -0,0 +1,223 @@ +/* +** Copyright (C) 2020-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-query-match-deciders.hh" + +#include "mu-query-results.hh" +#include "utils/mu-option.hh" + +using namespace Mu; + + +// We use a MatchDecider to gather information about the matches, and decide +// whether to include them in the results. +// +// Note that to include the "related" messages, we need _two_ queries; the first +// one to get the initial matches (called the Leader-Query) and a Related-Query, +// to get the Leader matches + all messages that have a thread-id seen in the +// Leader matches. +// +// We use the MatchDecider to gather information and use it for both queries. + +struct MatchDecider : public Xapian::MatchDecider { + MatchDecider(QueryFlags qflags, DeciderInfo& info) : qflags_{qflags}, decider_info_{info} {} + /** + * Update the match structure with unreadable/duplicate flags + * + * @param doc a Xapian document. + * + * @return a new QueryMatch object + */ + QueryMatch make_query_match(const Xapian::Document& doc) const + { + QueryMatch qm{}; + + auto msgid{opt_string(doc, Field::Id::MessageId) + .value_or(*opt_string(doc, Field::Id::Path))}; + if (!decider_info_.message_ids.emplace(std::move(msgid)).second) + qm.flags |= QueryMatch::Flags::Duplicate; + + const auto path{opt_string(doc, Field::Id::Path)}; + if (!path || ::access(path->c_str(), R_OK) != 0) + qm.flags |= QueryMatch::Flags::Unreadable; + + return qm; + } + + /** + * Should this message be included in the results? + * + * @param qm a query match + * + * @return true or false + */ + bool should_include(const QueryMatch& qm) const + { + if (any_of(qflags_ & QueryFlags::SkipDuplicates) && + any_of(qm.flags & QueryMatch::Flags::Duplicate)) + return false; + + if (any_of(qflags_ & QueryFlags::SkipUnreadable) && + any_of(qm.flags & QueryMatch::Flags::Unreadable)) + return false; + + return true; + } + /** + * Gather thread ids from this match. + * + * @param doc the document (message) + * + */ + void gather_thread_ids(const Xapian::Document& doc) const + { + auto thread_id{opt_string(doc, Field::Id::ThreadId)}; + if (thread_id) + decider_info_.thread_ids.emplace(std::move(*thread_id)); + } + +protected: + const QueryFlags qflags_; + DeciderInfo& decider_info_; + +private: + Option<std::string> opt_string(const Xapian::Document& doc, Field::Id id) const noexcept { + const auto value_no{field_from_id(id).value_no()}; + std::string val = xapian_try([&] { return doc.get_value(value_no); }, std::string{""}); + if (val.empty()) + return Nothing; + else + return Some(std::move(val)); + } +}; + +struct MatchDeciderLeader final : public MatchDecider { + MatchDeciderLeader(QueryFlags qflags, DeciderInfo& info) : MatchDecider(qflags, info) {} + /** + * operator() + * + * This receives the documents considered during a Xapian query, and + * is to return either true (keep) or false (ignore) + * + * We use this to potentiallly avoid certain messages (documents): + * - with QueryFlags::SkipUnreadable this will return false for message + * that are not readable in the file-system + * - with QueryFlags::SkipDuplicates this will return false for messages + * whose message-id was seen before. + * + * Even if we do not skip these messages entirely, we remember whether + * they were unreadable/duplicate (in the QueryMatch::Flags), so we can + * quickly find that info when doing the second 'related' query. + * + * The "leader" query. Matches here get the Leader flag unless they are + * duplicates / unreadable. We check the duplicate/readable status + * regardless of whether SkipDuplicates/SkipUnreadable was passed + * (to gather that information); however those flags + * affect our true/false verdict. + * + * @param doc xapian document + * + * @return true or false + */ + bool operator()(const Xapian::Document& doc) const override { + // by definition, we haven't seen the docid before, + // so no need to search + auto it = decider_info_.matches.emplace(doc.get_docid(), make_query_match(doc)); + it.first->second.flags |= QueryMatch::Flags::Leader; + + return should_include(it.first->second); + } +}; + +std::unique_ptr<Xapian::MatchDecider> +Mu::make_leader_decider(QueryFlags qflags, DeciderInfo& info) +{ + return std::make_unique<MatchDeciderLeader>(qflags, info); +} + +struct MatchDeciderRelated final : public MatchDecider { + MatchDeciderRelated(QueryFlags qflags, DeciderInfo& info) : MatchDecider(qflags, info) {} + /** + * operator() + * + * This receives the documents considered during a Xapian query, and + * is to return either true (keep) or false (ignore) + * + * We use this to potentially avoid certain messages (documents): + * - with QueryFlags::SkipUnreadable this will return false for message + * that are not readable in the file-system + * - with QueryFlags::SkipDuplicates this will return false for messages + * whose message-id was seen before. + * + * Unlike in the "leader" decider (scroll up), we don't need to remember + * messages we won't include. + * + * @param doc xapian document + * + * @return true or false + */ + bool operator()(const Xapian::Document& doc) const override { + // we may have seen this match in the "Leader" query. + const auto it = decider_info_.matches.find(doc.get_docid()); + if (it != decider_info_.matches.end()) + return should_include(it->second); + + auto qm{make_query_match(doc)}; + if (should_include(qm)) { + qm.flags |= QueryMatch::Flags::Related; + decider_info_.matches.emplace(doc.get_docid(), std::move(qm)); + return true; + } else + return false; // nope. + } +}; + +std::unique_ptr<Xapian::MatchDecider> +Mu::make_related_decider(QueryFlags qflags, DeciderInfo& info) +{ + return std::make_unique<MatchDeciderRelated>(qflags, info); +} + +struct MatchDeciderThread final : public MatchDecider { + MatchDeciderThread(QueryFlags qflags, DeciderInfo& info) : MatchDecider{qflags, info} {} + /** + * operator() + * + * This receives the documents considered during a Xapian query, and + * is to return either true (keep) or false (ignore) + * + * Only include documents that earlier checks have decided to include. + * + * @param doc xapian document + * + * @return true or false + */ + bool operator()(const Xapian::Document& doc) const override { + // we may have seen this match in the "Leader" query, + // or in the second (unbuounded) related query; + const auto it{decider_info_.matches.find(doc.get_docid())}; + return it != decider_info_.matches.end() && !it->second.thread_path.empty(); + } +}; + +std::unique_ptr<Xapian::MatchDecider> +Mu::make_thread_decider(QueryFlags qflags, DeciderInfo& info) +{ + return std::make_unique<MatchDeciderThread>(qflags, info); +} diff --git a/lib/mu-query-match-deciders.hh b/lib/mu-query-match-deciders.hh new file mode 100644 index 0000000..bd19605 --- /dev/null +++ b/lib/mu-query-match-deciders.hh @@ -0,0 +1,76 @@ +/* +** Copyright (C) 2021-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_QUERY_MATCH_DECIDERS_HH__ +#define MU_QUERY_MATCH_DECIDERS_HH__ + +#include <unordered_set> +#include <unordered_map> +#include <memory> + +#include "mu-xapian-db.hh" + +#include "mu-query-results.hh" + +namespace Mu { +using StringSet = std::unordered_set<std::string>; + +struct DeciderInfo { + QueryMatches matches; + StringSet thread_ids; + StringSet message_ids; +}; + +/** + * Make a "leader" decider, that is, a MatchDecider for either a singular or the + * first query in the leader/related pair of queries. Gather information for + * threading, and the subsequent "related" query. + * + * @param qflags query flags + * @param match_info receives information about the matches. + * + * @return a unique_ptr to a match decider. + */ +std::unique_ptr<Xapian::MatchDecider> make_leader_decider(QueryFlags qflags, DeciderInfo& info); + +/** + * Make a "related" decider, that is, a MatchDecider for the second query + * in the leader/related pair of queries. + * + * @param qflags query flags + * @param match_info receives information about the matches. + * + * @return a unique_ptr to a match decider. + */ +std::unique_ptr<Xapian::MatchDecider> make_related_decider(QueryFlags qflags, DeciderInfo& info); + +/** + * Make a "thread" decider, that is, a MatchDecider that removes all but the + * document excepts for the ones found during initial/related searches. + * + * @param qflags query flags + * @param match_info receives information about the matches. + * + * @return a unique_ptr to a match decider. + */ +std::unique_ptr<Xapian::MatchDecider> make_thread_decider(QueryFlags qflags, DeciderInfo& info); + +} // namespace Mu + +#endif /* MU_QUERY_MATCH_DECIDERS_HH__ */ diff --git a/lib/mu-query-parser.cc b/lib/mu-query-parser.cc new file mode 100644 index 0000000..87242dd --- /dev/null +++ b/lib/mu-query-parser.cc @@ -0,0 +1,485 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-query-parser.hh" + +#include <string_view> +#include <variant> +#include <type_traits> +#include <iostream> + +#include "utils/mu-utils.hh" +#include "utils/mu-sexp.hh" +#include "utils/mu-option.hh" +#include <glib.h> +#include "utils/mu-utils-file.hh" + +using namespace Mu; + +// Sexp extensions... +static Sexp& +prepend(Sexp& s, Sexp&& e) +{ + s.list().insert(s.list().begin(), std::move(e)); + return s; +} + +static Option<Sexp&> +second(Sexp& s) +{ + if (s.listp() && !s.empty() && s.cbegin() + 1 != s.cend()) + return *(s.begin()+1); + else + return Nothing; +} + + +static bool +looks_like_matcher(const Sexp& sexp) +{ + // all the "terminal values" (from the Mu parser's pov) + const std::array<Sexp::Symbol, 5> value_syms = { + placeholder_sym, phrase_sym, regex_sym, range_sym, wildcard_sym + }; + + if (!sexp.listp() || sexp.empty() || !sexp.front().symbolp()) + return false; + + const auto symbol{sexp.front().symbol()}; + if (seq_some(value_syms, [&](auto &&sym) { return symbol == sym; })) + return true; + else if (!!field_from_name(symbol.name) || field_is_combi(symbol.name)) + return true; + else + return false; +} + +struct ParseContext { + bool expand; + std::vector<std::string> warnings; +}; + + + + +/** + * Indexable fields become _phrase_ fields if they contain + * wordbreakable data; + * + * @param field + * @param val + * + * @return + */ +static Option<Sexp> +phrasify(const Field& field, const Sexp& val) +{ + if (!field.is_phrasable_term() || !val.stringp()) + return Nothing; // nothing to phrasify + + auto words{utf8_wordbreak(val.string())}; + if (words.find(' ') == std::string::npos) + return Nothing; // nothing to phrasify + + auto phrase = Sexp { + Sexp::Symbol{field.name}, + Sexp{phrase_sym, Sexp{std::move(words)}}}; + + // if the field both a normal term & phrasable, match both + // if they are different + if (val.string() != words) + return Sexp{or_sym, + Sexp {Sexp::Symbol{field.name}, Sexp(val.string())}, + std::move(phrase)}; + else + return phrase; +} + + +/* + * Grammar + * + * query -> factor { (<OR> | <XOR>) factor } + * factor -> unit { [<AND>] unit } + * unit -> matcher | <NOT> query | <(> query <)> + * matcher + */ + +static Sexp query(Sexp& tokens, ParseContext& ctx); + + +static Sexp +matcher(Sexp& tokens, ParseContext& ctx) +{ + if (tokens.empty()) + return {}; + + auto val{*tokens.head()}; + tokens.pop_front(); + /* special case: if we find some non-matcher type here, we need to second-guess the token */ + if (!looks_like_matcher(val)) + val = Sexp{placeholder_sym, val.symbol().name}; + + const auto fieldsym{val.front().symbol()}; + + // Note the _expand_ case is what we use when processing the query 'for real'; + // the non-expand case is only to have a bit more human-readable Sexp for use + // mu find's '--analyze' + // + // Re: phrase-fields We map something like 'subject:hello-world' + // to + // (or (subject "hello-world" (subject (phrase "hello world")))) + + if (ctx.expand) { /* should we expand meta-fields? */ + auto fields = fields_from_name(fieldsym == placeholder_sym ? "" : fieldsym.name); + if (!fields.empty()) { + Sexp vals{}; + vals.add(or_sym); + for (auto&& field: fields) + if (auto&& phrase{phrasify(field, *second(val))}; phrase) + vals.add(std::move(*phrase)); + else + vals.add(Sexp{Sexp::Symbol{field.name}, Sexp{*second(val)}}); + val = std::move(vals); + } + + } + + if (auto&& field{field_from_name(fieldsym.name)}; field) { + if (auto&& phrase(phrasify(*field, *second(val))); phrase) + val = std::move(*phrase); + } + + return val; +} + +static Sexp +unit(Sexp& tokens, ParseContext& ctx) +{ + if (tokens.head_symbolp(not_sym)) { /* NOT */ + tokens.pop_front(); + Sexp sub{unit(tokens, ctx)}; + + /* special case: interpret "not" as a matcher instead; */ + if (sub.empty()) + return matcher(prepend(tokens, Sexp{placeholder_sym, not_sym.name}), ctx); + + /* we try to optimize: double negations are removed */ + if (sub.head_symbolp(not_sym)) + return *second(sub); + else + return Sexp(not_sym, std::move(sub)); + + } else if (tokens.head_symbolp(open_sym)) { /* ( sub) */ + tokens.pop_front(); + Sexp sub{query(tokens, ctx)}; + if (tokens.head_symbolp(close_sym)) + tokens.pop_front(); + else { + //g_warning("expected <)>"); + } + return sub; + } + + /* matcher */ + return matcher(tokens, ctx); +} + + +static Sexp +factor(Sexp& tokens, ParseContext& ctx) +{ + Sexp un = unit(tokens, ctx); + + /* query 'a b' is to be interpreted as 'a AND b'; + * + * we need an implicit AND if the head symbol is either + * a matcher (value) or the start of a sub-expression */ + auto implicit_and = [&]() { + if (tokens.head_symbolp(open_sym)) + return true; + else if (tokens.head_symbolp(not_sym)) // turn a lone 'not' -> 'and not' + return true; + else if (auto&& head{tokens.head()}; head) + return looks_like_matcher(*head); + else + return false; + }; + + Sexp uns; + while (true) { + if (tokens.head_symbolp(and_sym)) + tokens.pop_front(); + else if (!implicit_and()) + break; + + if (auto&& un2 = unit(tokens, ctx); !un2.empty()) + uns.add(std::move(un2)); + else + break; + } + + if (!uns.empty()) { + un = Sexp{and_sym, std::move(un)}; + un.add_list(std::move(uns)); + } + + return un; +} + +static Sexp +query(Sexp& tokens, ParseContext& ctx) +{ + /* note: we flatten (or (or ( or ...)) etc. here; + * for optimization (since Xapian likes flat trees) */ + + Sexp fact = factor(tokens, ctx); + Sexp or_factors, xor_factors; + while (true) { + auto factors = std::invoke([&]()->Option<Sexp&> { + + if (tokens.head_symbolp(or_sym)) + return or_factors; + else if (tokens.head_symbolp(xor_sym)) + return xor_factors; + else + return Nothing; + }); + + if (!factors) + break; + + tokens.pop_front(); + factors->add(factor(tokens, ctx)); + } + + // a bit clumsy... + + if (!or_factors.empty() && xor_factors.empty()) { + fact = Sexp{or_sym, std::move(fact)}; + fact.add_list(std::move(or_factors)); + } else if (or_factors.empty() && !xor_factors.empty()) { + fact = Sexp{xor_sym, std::move(fact)}; + fact.add_list(std::move(xor_factors)); + } else if (!or_factors.empty() && !xor_factors.empty()) { + fact = Sexp{or_sym, std::move(fact)}; + fact.add_list(std::move(or_factors)); + prepend(xor_factors, xor_sym); + fact.add(std::move(xor_factors)); + } + + return fact; +} + +Sexp +Mu::parse_query(const std::string& expr, bool expand) +{ + ParseContext context; + context.expand = expand; + + if (auto&& items = process_query(expr); !items.listp()) + throw std::runtime_error("tokens must be a list-sexp"); + else + return query(items, context); +} + + +#if defined(BUILD_PARSE_QUERY)||defined(BUILD_PARSE_QUERY_EXPAND) +int +main (int argc, char *argv[]) +{ + if (argc < 2) { + mu_printerrln("expected: {} <query>", argv[0]); + return 1; + } + + std::string expr; + for (auto i = 1; i < argc; ++i) { + expr += argv[i]; + expr += " "; + } + + auto&& sexp = parse_query(expr, +#ifdef BUILD_PARSE_QUERY_EXPAND + true/*expand*/ +#else + false/*don't expand*/ +#endif + ); + mu_println("{}", sexp.to_string()); + return 0; +} +#endif // BUILD_PARSE_QUERY || BUILD_PARSE_QUERY_EXPAND + + + +#if BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +using TestCase = std::pair<std::string, std::string>; + +static void +test_parser_basic() +{ + std::vector<TestCase> cases = { + // single term + TestCase{R"(a)", R"((_ "a"))"}, + // a and b + TestCase{R"(a and b)", R"((and (_ "a") (_ "b")))"}, + // a and b and c + TestCase{R"(a and b and c)", R"((and (_ "a") (_ "b") (_ "c")))"}, + // a or b + TestCase{R"(a or b)", R"((or (_ "a") (_ "b")))"}, + // a or b and c + TestCase{R"(a or b and c)", R"((or (_ "a") (and (_ "b") (_ "c"))))"}, + // a and b or c + TestCase{R"(a and b or c)", R"((or (and (_ "a") (_ "b")) (_ "c")))"}, + // not a + TestCase{R"(not a)", R"((not (_ "a")))"}, + // lone not + TestCase{R"(not)", R"((_ "not"))"}, + // a and (b or c) + TestCase{R"(a and (b or c))", R"((and (_ "a") (or (_ "b") (_ "c"))))"}, + // not a and not b + TestCase{R"(not a and b)", R"((and (not (_ "a")) (_ "b")))"}, + // a not b + TestCase{R"(a not b)", R"((and (_ "a") (not (_ "b"))))"}, + }; + + for (auto&& test: cases) { + auto&& sexp{parse_query(test.first)}; + //mu_message ("'{}' <=> '{}'", sexp.to_string(), test.second); + assert_equal(sexp.to_string(), test.second); + } +} + +static void +test_parser_recover() +{ + std::vector<TestCase> cases = { + // implicit AND + TestCase{R"(a b)", R"((and (_ "a") (_ "b")))"}, + // a or or (second to be used as value) + TestCase{R"(a or and)", R"((or (_ "a") (_ "and")))"}, + // missing end ) + TestCase{R"(a and ()", R"((_ "a"))"}, + // missing end ) + TestCase{R"(a and (b)", R"((and (_ "a") (_ "b")))"}, + }; + + for (auto&& test: cases) { + auto&& sexp{parse_query(test.first)}; + assert_equal(sexp.to_string(), test.second); + } +} + + +static void +test_parser_fields() +{ + std::vector<TestCase> cases = { + // simple field + TestCase{R"(s:hello)", R"((subject "hello"))"}, + // field, wildcard, regexp + TestCase{R"(subject:a* recip:/b/)", + R"((and (subject (wildcard "a")) (recip (regex "b"))))"}, + TestCase{R"(from:hello or subject:world)", + R"((or (from "hello") (subject "world")))"}, + }; + + for (auto&& test: cases) { + auto&& sexp{parse_query(test.first)}; + assert_equal(sexp.to_string(), test.second); + } +} + +static void +test_parser_expand() +{ + std::vector<TestCase> cases = { + // simple field + TestCase{R"(recip:a)", R"((or (to "a") (cc "a") (bcc "a")))"}, + // field, wildcard, regexp + TestCase{R"(a*)", + R"((or (to (wildcard "a")) (cc (wildcard "a")) (bcc (wildcard "a")) (from (wildcard "a")) (subject (wildcard "a")) (body (wildcard "a")) (embed (wildcard "a"))))"}, + TestCase{R"(a xor contact:b)", + R"((xor (or (to "a") (cc "a") (bcc "a") (from "a") (subject "a") (body "a") (embed "a")) (or (to "b") (cc "b") (bcc "b") (from "b"))))"} + }; + + for (auto&& test: cases) { + auto&& sexp{parse_query(test.first, true/*expand*/)}; + assert_equal(sexp.to_string(), test.second); + } +} + + +static void +test_parser_range() +{ + std::vector<TestCase> cases = { + TestCase{R"(size:1)", R"((size (range "1" "1")))"}, + TestCase{R"(size:2..)", R"((size (range "2" "")))"}, + TestCase{R"(size:..1k)", R"((size (range "" "1024")))"}, + TestCase{R"(size:..)", R"((size (range "" "")))"}, + }; + + for (auto&& test: cases) { + auto&& sexp{parse_query(test.first, true/*expand*/)}; + assert_equal(sexp.to_string(), test.second); + } +} + +static void +test_parser_optimize() +{ + std::vector<TestCase> cases = { + TestCase{R"(not a)", R"((not (_ "a")))"}, + TestCase{R"(not not a)", R"((_ "a"))"}, + TestCase{R"(not not not a)", R"((not (_ "a")))"}, + TestCase{R"(not not not not a)", R"((_ "a"))"}, + }; + + + for (auto&& test: cases) { + auto&& sexp{parse_query(test.first)}; + assert_equal(sexp.to_string(), test.second); + } +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/query-parser/basic", test_parser_basic); + g_test_add_func("/query-parser/recover", test_parser_recover); + g_test_add_func("/query-parser/fields", test_parser_fields); + g_test_add_func("/query-parser/range", test_parser_range); + g_test_add_func("/query-parser/expand", test_parser_expand); + g_test_add_func("/query-parser/optimize", test_parser_optimize); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/mu-query-parser.hh b/lib/mu-query-parser.hh new file mode 100644 index 0000000..72b23a7 --- /dev/null +++ b/lib/mu-query-parser.hh @@ -0,0 +1,114 @@ +/* +** Copyright (C) 2023-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include <string> + +#include "mu-xapian-db.hh" + +#include "utils/mu-sexp.hh" +#include "utils/mu-result.hh" +#include "mu-store.hh" + +namespace Mu { +/* + * Some useful symbol-sexps + */ +static inline const auto placeholder_sym = "_"_sym; +static inline const auto phrase_sym = "phrase"_sym; +static inline const auto regex_sym = "regex"_sym; +static inline const auto range_sym = "range"_sym; +static inline const auto wildcard_sym = "wildcard"_sym; + +static inline const auto open_sym = "("_sym; +static inline const auto close_sym = ")"_sym; + +static inline const auto and_sym = "and"_sym; +static inline const auto or_sym = "or"_sym; +static inline const auto xor_sym = "xor"_sym; +static inline const auto not_sym = "not"_sym; +static inline const auto and_not_sym = "and-not"_sym; + + +/* + * We take a query, then parse it into a human-readable s-expression and then + * turn that s-expression into a Xapian query + * + * some query: + * "from:hello or subject:world" + * + * 1. tokenize-query + * => ((from "hello") or (subject "world")) + * + * 2. parse-query + * => (or (from "hello") (subject "world")) + * + * 3. xapian-query + * => Query((Fhello OR Sworld)) + * * + */ + +/** + * Analyze the query expression and express it as a Sexp-list with the sequence + * of elements. + * + * @param expr a search expression + * + * @return Sexp with the sequence of elements + */ +Sexp process_query(const std::string& expr); + +/** + * Parse the query expression and create a parse-tree expressed as an Sexp + * object (tree). + * + * Internally, this processes the stream into element (see process_query()) and + * processes the tokens into a Sexp. This sexp is meant to be human-readable. + * + * @param expr a search expression + * @param expand whether to expand meta-fields (such as '_', 'recip', 'contacts') + * + * @return Sexp with the parse tree + */ +Sexp parse_query(const std::string& expr, bool expand=false); + +/** + * Make a Xapian Query for the given string expression. + * + * This uses parse_query() and turns the S-expression into a Xapian::Query. + * Unlike mere parsing, this uses the information in the store to resolve + * wildcard / regex queries. + * + * @param store the message store + * @param expr a string expression + * @param flavor type of parser to use + * + * @return a Xapian query result or an error. + */ +enum struct ParserFlags { + None = 0 << 0, + SupportNgrams = 1 << 0, /**< Support Xapian's Ngrams for CJK etc. handling */ + XapianParser = 1 << 1, /**< For testing only, use Xapian's + * built-in QueryParser; this is not + * fully compatible with mu, only useful + * for debugging. */ +}; +Result<Xapian::Query> make_xapian_query(const Store& store, const std::string& expr, + ParserFlags flag=ParserFlags::None) noexcept; + +MU_ENABLE_BITOPS(ParserFlags); +} // namespace Mu diff --git a/lib/mu-query-processor.cc b/lib/mu-query-processor.cc new file mode 100644 index 0000000..592beb4 --- /dev/null +++ b/lib/mu-query-processor.cc @@ -0,0 +1,527 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-query-parser.hh" + +#include <string_view> +#include <variant> +#include <type_traits> +#include <iostream> + +#include "utils/mu-option.hh" +#include <glib.h> +#include "utils/mu-utils-file.hh" + +using namespace Mu; + +/** + * An 'Element' here is a rather rich version of what is traditionally + * considered a (lexical) token. + * + * We try to determine as much as possible during the analysis phase; which is + * quite a bit (given the fairly simple query language), and the parsing phase + * only has to deal with the putting these elements in a tree. + * + * During analysis: + * 1) separate the query into a sequence strings + * 2) for each of these strings + * - Does it look like an Op? ('or', 'and' etc.) --> Op + * - Otherwise: treat as a Basic field ([field]:value) + * - Whitespace in value? -> promote to Phrase + * - otherwise: + * - Is value a regex (in /<regex>/) -> promote to Regex + * - Is value a wildcard (ends in '*') -> promote to Wildcard + * - is value a range (a..b) -> promote to Range + * + * After analysis, we have the sequence of element as a Sexp, which can then be + * fed to the parser. We attempt to make the Sexp as human-readable as possible. + */ +struct Element { + enum struct Bracket { Open, Close} ; + enum struct Op { And, Or, Xor, Not, AndNot }; + + template<typename ValueType> + struct FieldValue { + FieldValue(const ValueType& v): field{}, value{v}{} + + template<typename StringType> + FieldValue(const StringType& fname, const ValueType& v): + field{std::string{fname}}, value{v}{} + template<typename StringType> + FieldValue(const Option<StringType>& fname, const ValueType& v) { + if (fname) + field = std::string{*fname}; + value = v; + } + + Option<std::string> field{}; + ValueType value{}; + }; + struct Basic: public FieldValue<std::string> {using FieldValue::FieldValue;}; + struct Regex: public FieldValue<std::string> {using FieldValue::FieldValue;}; + struct Wildcard: public FieldValue<std::string> {using FieldValue::FieldValue;}; + struct Range: public FieldValue<std::pair<std::string, std::string>> { + using FieldValue::FieldValue; }; + + using ValueType = std::variant< + /* */ + Bracket, + /* op */ + Op, + /* string values */ + std::string, + /* value types */ + Basic, + Regex, + Wildcard, + Range + >; + + // helper + template <typename T, typename U> + struct decay_equiv: + std::is_same<typename std::decay<T>::type, U>::type {}; + + Element(Bracket b): value{b} {} + Element(Op op): value{op} {} + + template<typename T, + typename std::enable_if<std::is_base_of<class FieldValue<T>, T>::value>::type = 0> + Element(const std::string& field, const T& val): value{T{field, val}} {} + + Element(const std::string& val): value{val} {} + + template<typename T> + Option<T&> get_opt() { + if (std::holds_alternative<T>(value)) + return std::get<T>(value); + else + return Nothing; + } + + Sexp sexp() const { + return std::visit([](auto&& arg)->Sexp { + + auto field_sym = [](const Option<std::string>& field) { + return field ? Sexp::Symbol{*field} : placeholder_sym; + }; + + using T = std::decay_t<decltype(arg)>; + + if constexpr (std::is_same_v<T, Bracket>) { + switch(arg) { + case Bracket::Open: + return open_sym; + case Bracket::Close: + return close_sym; + default: + throw std::logic_error("invalid bracket type"); + } + } else if constexpr (std::is_same_v<T, Op>) { + switch(arg) { + case Op::And: + return and_sym; + case Op::Or: + return or_sym; + case Op::Xor: + return xor_sym; + case Op::Not: + return not_sym; + case Op::AndNot: + return and_not_sym; + default: + throw std::logic_error("invalid op type"); + } + } else if constexpr (std::is_same_v<T, Basic>) { + return Sexp { field_sym(arg.field), arg.value }; + } else if constexpr (std::is_same_v<T, Regex>) { + return Sexp { field_sym(arg.field), Sexp{ regex_sym, arg.value}}; + } else if constexpr (std::is_same_v<T, Wildcard>) { + return Sexp { field_sym(arg.field), Sexp{ wildcard_sym, arg.value}}; + } else if constexpr (std::is_same_v<T, Range>) { + return Sexp {field_sym(arg.field), + Sexp{ range_sym, arg.value.first, arg.value.second }}; + } else if constexpr (std::is_same_v<T, std::string>) { + throw std::logic_error("no bare strings should be here"); + } else + throw std::logic_error("uninvited visitor"); + }, value); + } + + ValueType value; +}; + +using Elements = std::vector<Element>; + + + +/** + * Remove first character from string and return it. + * + * @param[in,out] str a string + * @param[in,out] pos position in _original_ string + * + * @return a char or 0 if there is none. + */ +static char +read_char(std::string& str, size_t& pos) +{ + if (str.empty()) + return {}; + + auto kar{str.at(0)}; + str.erase(0, 1); + ++pos; + + return kar; +} + +/** + * Restore kar at the beginning of the string + * + * @param[in,out] str a string + * @param[in,out] pos position in _original_ string + * @param kar a character + */ +static void +unread_char(std::string& str, size_t& pos, char kar) +{ + str = kar + str; + --pos; +} + + +/** + * Remove the the next element from the string and return it + * + * @param[in,out] str a string + * @param[in,out] pos position in _original_ string * + * + * @return an Element or Nothing + */ +static Option<Element> +next_element(std::string& str, size_t& pos) +{ + bool quoted{}, escaped{}; + std::string value{}; + + auto is_separator = [](char c) { return c == ' '|| c == '(' || c == ')'; }; + + while (!str.empty()) { + + auto kar = read_char(str, pos); + + if (kar == '\\') { + escaped = !escaped; + if (escaped) + continue; + } + + if (kar == '"' && !escaped) { + if (!escaped && quoted) + return Element{value}; + else { + quoted = true; + continue; + } + } + + if (!quoted && !escaped && is_separator(kar)) { + if (!value.empty()) { + unread_char(str, pos, kar); + return Element{value}; + } + + if (quoted || kar == ' ') + continue; + + switch (kar) { + case '(': + return Element{Element::Bracket::Open}; + case ')': + return Element{Element::Bracket::Close}; + default: + break; + } + } + + value += kar; + escaped = false; + } + + if (value.empty()) + return Nothing; + else + return Element{value}; +} + + +static Option<Element> +opify(Element&& element) +{ + auto&& str{element.get_opt<std::string>()}; + if (!str) + return element; + + static const std::unordered_map<std::string, Element::Op> ops = { + { "and", Element::Op::And }, + { "or", Element::Op::Or}, + { "xor", Element::Op::Xor }, + { "not", Element::Op::Not }, + // AndNot only appears during parsing. + }; + + if (auto&& it = ops.find(utf8_flatten(*str)); it != ops.end()) + element.value = it->second; + + return element; +} + +static Option<Element> +basify(Element&& element) +{ + auto&& str{element.get_opt<std::string>()}; + if (!str) + return element; + + const auto pos = str->find(':'); + if (pos == std::string::npos) { + element.value = Element::Basic{*str}; + return element; + } + + const auto fname{str->substr(0, pos)}; + if (auto&& field{field_from_name(fname)}; field) { + auto val{str->substr(pos + 1)}; + if (field == Field::Id::Flags) { + if (auto&& finfo{flag_info(val)}; finfo) + element.value = Element::Basic{field->name, + std::string{finfo->name}}; + else + element.value = Element::Basic{*str}; + } else if (field == Field::Id::Priority) { + if (auto&& prio{priority_from_name(val)}; prio) + element.value = Element::Basic{field->name, + std::string{priority_name(*prio)}}; + else + element.value = Element::Basic{*str}; + } else + element.value = Element::Basic{std::string{field->name}, + str->substr(pos + 1)}; + } else if (field_is_combi(fname)) + element.value = Element::Basic{fname, str->substr(pos +1)}; + else + element.value = Element::Basic{*str}; + + return element; +} + +static Option<Element> +wildcardify(Element&& element) +{ + auto&& basic{element.get_opt<Element::Basic>()}; + if (!basic) + return element; + + auto&& val{basic->value}; + if (val.size() < 2 || val[val.size()-1] != '*') + return element; + + val.erase(val.size() - 1); + element.value = Element::Wildcard{basic->field, val}; + + return element; +} + +static Option<Element> +regexpify(Element&& element) +{ + auto&& str{element.get_opt<Element::Basic>()}; + if (!str) + return element; + + auto&& val{str->value}; + if (val.size() < 3 || val[0] != '/' || val[val.size()-1] != '/') + return element; + + val.erase(val.size() - 1); + val.erase(0, 1); + element.value = Element::Regex{str->field, std::move(val)}; + + return element; +} + +// handle range-fields: Size, Date, Changed +static Option<Element> +rangify(Element&& element) +{ + auto&& str{element.get_opt<Element::Basic>()}; + if (!str) + return element; + + if (!str->field) + return element; + + auto&& field = field_from_name(*str->field); + if (!field || !field->is_range()) + return element; + + /* yes: get the range */ + auto&& range = std::invoke([&]()->std::pair<std::string, std::string> { + const auto val{str->value}; + const auto pos{val.find("..")}; + + if (pos == std::string::npos) + return { val, val }; + else + return {val.substr(0, pos), val.substr(pos + 2)}; + }); + + if (field->id == Field::Id::Size) { + int64_t s1{range.first.empty() ? -1 : + parse_size(range.first, false/*first*/).value_or(-1)}; + int64_t s2{range.second.empty() ? -1 : + parse_size(range.second, true/*last*/).value_or(-1)}; + if (s2 >= 0 && s1 > s2) + std::swap(s1, s2); + element.value = Element::Range{str->field, + {s1 < 0 ? "" : std::to_string(s1), + s2 < 0 ? "" : std::to_string(s2)}}; + + } else if (field->id == Field::Id::Date || field->id == Field::Id::Changed) { + auto tstamp=[](auto&& str, auto&& first)->int64_t { + return str.empty() ? -1 : + parse_date_time(str, first ,false/*local*/).value_or(-1); + }; + int64_t lower{tstamp(range.first, true/*lower*/)}; + int64_t upper{tstamp(range.second, false/*upper*/)}; + if (lower >= 0 && upper >= 0 && lower > upper) { + // can't simply swap due to rounding up/down + lower = tstamp(range.second, true/*lower*/); + upper = tstamp(range.first, false/*upper*/); + } + // use "Zulu" time. + element.value = Element::Range{ + str->field, + {lower < 0 ? "" : + mu_format("{:%FT%TZ}",mu_time(lower, true/*utc*/)), + upper < 0 ? "" : + mu_format("{:%FT%TZ}", mu_time(upper, true/*utc*/))}}; + } + + return element; +} + +static Elements +process(const std::string& expr) +{ + Elements elements{}; + size_t offset{0}; + + /* all control chars become SPC */ + std::string str{expr}; + for (auto& c: str) + c = ::iscntrl(c) ? ' ' : c; + + while(!str.empty()) { + auto&& element = next_element(str, offset) + .and_then(opify) + .and_then(basify) + .and_then(regexpify) + .and_then(wildcardify) + .and_then(rangify); + if (element) + elements.emplace_back(std::move(element.value())); + } + + return elements; +} + +Sexp +Mu::process_query(const std::string& expr) +{ + const auto& elements{::process(expr)}; + + Sexp sexp{}; + for (auto&& elm: elements) + sexp.add(elm.sexp()); + + return sexp; +} + +#ifdef BUILD_PROCESS_QUERY +int +main (int argc, char *argv[]) +{ + if (argc < 2) { + mu_printerrln("expected: process-query <query>"); + return 1; + } + + std::string expr; + for (auto i = 1; i < argc; ++i) { + expr += argv[i]; + expr += " "; + } + + auto sexp = process_query(expr); + mu_println("{}", sexp.to_string()); + + return 0; +} +#endif /*BUILD_ANALYZE_QUERY*/ + +#if BUILD_TESTS +/* + * + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +using TestCase = std::pair<std::string, std::string>; + +static void +test_processor() +{ + std::vector<TestCase> cases = { + // basics + TestCase{R"(hello world)", R"(((_ "hello") (_ "world")))"}, + TestCase{R"(maildir:/"hello world")", R"(((maildir "/hello world")))"}, + TestCase{R"(flag:deleted)", R"(((_ "flag:deleted")))"} // non-existing flags + }; + + for (auto&& test: cases) { + auto&& sexp{process_query(test.first)}; + assert_equal(sexp.to_string(), test.second); + } +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/query-parser/processor", test_processor); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/mu-query-results.hh b/lib/mu-query-results.hh new file mode 100644 index 0000000..0123ab4 --- /dev/null +++ b/lib/mu-query-results.hh @@ -0,0 +1,422 @@ +/* +** Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_QUERY_RESULTS_HH__ +#define MU_QUERY_RESULTS_HH__ + +#include <algorithm> +#include <limits> +#include <stdexcept> +#include <string> +#include <unordered_map> +#include <unordered_set> +#include <limits> +#include <ostream> +#include <cmath> +#include <memory> + +#include <unistd.h> +#include <fcntl.h> +#include <glib.h> + +#include <mu-xapian-db.hh> +#include <utils/mu-utils.hh> +#include <utils/mu-option.hh> + +#include <message/mu-message.hh> + +namespace Mu { + +/** + * This implements a QueryResults structure, which capture the results of a + * Xapian query, and a QueryResultsIterator, which gives C++-compliant iterator + * to go over the results. and finally QueryThreader (in query-threader.cc) which + * calculates the threads, using the JWZ algorithm. + */ + +/// Flags that influence now matches are presented (or skipped) +enum struct QueryFlags { + None = 0, /**< no flags */ + Descending = 1 << 0, /**< sort z->a */ + SkipUnreadable = 1 << 1, /**< skip unreadable msgs */ + SkipDuplicates = 1 << 2, /**< skip duplicate msgs */ + IncludeRelated = 1 << 3, /**< include related msgs */ + Threading = 1 << 4, /**< calculate threading info */ + // internal + Leader = 1 << 5, /**< This is the leader query (for internal use + * only)*/ +}; +MU_ENABLE_BITOPS(QueryFlags); + +/// Stores all the essential information for sorting the results. +struct QueryMatch { + /// Flags for a match (message) found + enum struct Flags { + None = 0, /**< No Flags */ + Leader = 1 << 0, /**< Mark direct matches as leader */ + Related = 1 << 1, /**< A related message */ + Unreadable = 1 << 2, /**< No readable file */ + Duplicate = 1 << 3, /**< Message-id seen before */ + + Root = 1 << 10, /**< Is this the thread-root? */ + First = 1 << 11, /**< Is this the first message in a thread? */ + Last = 1 << 12, /**< Is this the last message in a thread? */ + Orphan = 1 << 13, /**< Is this message without a parent? */ + HasChild = 1 << 14, /**< Does this message have a child? */ + + ThreadSubject = 1 << 20, /**< Message holds subject for (sub)thread */ + }; + + Flags flags{Flags::None}; /**< Flags */ + std::string date_key; /**< The date-key (for sorting all sub-root levels) */ + // the thread subject is the subject of the first message in a thread, + // and any message that has a different subject compared to its predecessor + // (ignoring prefixes such as Re:) + // + // otherwise, it is empty. + std::string subject; /**< subject for this message */ + size_t thread_level{}; /**< The thread level */ + std::string thread_path; /**< The hex-numerial path in the thread, ie. '00:01:0a' */ + std::string thread_date; /**< date of newest message in thread */ + + bool operator<(const QueryMatch& rhs) const { return date_key < rhs.date_key; } + + bool has_flag(Flags flag) const; +}; + +MU_ENABLE_BITOPS(QueryMatch::Flags); + +inline bool +QueryMatch::has_flag(QueryMatch::Flags flag) const +{ + return any_of(flags & flag); +} + +/* LCOV_EXCL_START */ +static inline std::ostream& +operator<<(std::ostream& os, QueryMatch::Flags mflags) +{ + if (mflags == QueryMatch::Flags::None) { + os << "<none>"; + return os; + } + + if (any_of(mflags & QueryMatch::Flags::Leader)) + os << "leader "; + if (any_of(mflags & QueryMatch::Flags::Unreadable)) + os << "unreadable "; + if (any_of(mflags & QueryMatch::Flags::Duplicate)) + os << "dup "; + + if (any_of(mflags & QueryMatch::Flags::Root)) + os << "root "; + if (any_of(mflags & QueryMatch::Flags::Related)) + os << "related "; + if (any_of(mflags & QueryMatch::Flags::First)) + os << "first "; + if (any_of(mflags & QueryMatch::Flags::Last)) + os << "last "; + if (any_of(mflags & QueryMatch::Flags::Orphan)) + os << "orphan "; + if (any_of(mflags & QueryMatch::Flags::HasChild)) + os << "has-child "; + + return os; +} + +inline std::ostream& +operator<<(std::ostream& os, const QueryMatch& qmatch) +{ + os << "qm:[" << qmatch.thread_path << "]: " // " (" << qmatch.thread_level << "): " + << "> date:<" << qmatch.date_key << "> " + << "flags:{" << qmatch.flags << "}"; + + return os; +} +/* LCOV_EXCL_STOP*/ + +using QueryMatches = std::unordered_map<Xapian::docid, QueryMatch>; + + +/// +/// This is a view over the Xapian::MSet, which can optionally filter unreadable +/// / duplicate messages. +/// +/// Note, we internally skip unreadable/duplicate messages (when asked too); those +/// skipped ones do _not_ count towards the max_size +/// +class QueryResultsIterator { +public: + using iterator_category = std::output_iterator_tag; + using value_type = Message; + using difference_type = void; + using pointer = void; + using reference = void; + + QueryResultsIterator(Xapian::MSetIterator mset_it, QueryMatches& query_matches) + : mset_it_{mset_it}, query_matches_{query_matches} { + } + + /** + * Increment the iterator (we don't support post-increment) + * + * @return an updated iterator, or end() if we were already at end() + */ + QueryResultsIterator& operator++() { + ++mset_it_; + mdoc_ = Nothing; + return *this; + } + + /** + * (Non)Equivalence operators + * + * @param rhs some other iterator + * + * @return true or false + */ + bool operator==(const QueryResultsIterator& rhs) const { return mset_it_ == rhs.mset_it_; } + bool operator!=(const QueryResultsIterator& rhs) const { return mset_it_ != rhs.mset_it_; } + + QueryResultsIterator& operator*() { return *this; } + const QueryResultsIterator& operator*() const { return *this; } + + + /** + * Get the Xapian::Document this iterator is pointing at, + * or an empty document when looking at end(). + * + * @return a document + */ + Option<Xapian::Document> document() const { + return xapian_try([this]()->Option<Xapian::Document> { + auto doc{mset_it_.get_document()}; + if (doc.get_docid() == 0) + return Nothing; + else + return Some(std::move(doc)); + }, Nothing); + } + + + /** + * get the corresponding Message for this iter, if any + * + * @return a Message or Nothing + */ + Option<Message> message() const { + if (auto&& xdoc{document()}; !xdoc) + return Nothing; + else if (auto&& doc{Message::make_from_document(std::move(xdoc.value()))}; + !doc) + return Nothing; + else + return Some(std::move(doc.value())); + } + + /** + * Get the doc-id for the document this iterator is pointing at, or 0 + * when looking at end. + * + * @return a doc-id. + */ + Xapian::docid doc_id() const { return *mset_it_; } + + /** + * Get the message-id for the document (message) this iterator is + * pointing at, or not when not available + * + * @return a message-id + */ + Option<std::string> message_id() const noexcept { + return opt_string(Field::Id::MessageId); + } + + /** + * Get the thread-id for the document (message) this iterator is + * pointing at, or Nothing. + * + * @return a message-id + */ + Option<std::string> thread_id() const noexcept { + return opt_string(Field::Id::ThreadId); + } + + /** + * Get the file-system path for the document (message) this iterator is + * pointing at, or Nothing. + * + * @return a filesystem path + */ + Option<std::string> path() const noexcept { + return opt_string(Field::Id::Path); + } + + /** + * Get the a sortable date str for the document (message) the iterator + * is pointing at. pointing at, or Nothing. This (encoded) string + * has the same sort-order as the corresponding date. + * + * @return a filesystem path + */ + Option<std::string> date_str() const noexcept { + return opt_string(Field::Id::Date); + } + + /** + * Get the subject for the document (message) this iterator is pointing + * at. + * + * @return the subject + */ + Option<std::string> subject() const noexcept { + return opt_string(Field::Id::Subject); + } + + /** + * Get the references for the document (messages) this is iterator is + * pointing at, or empty if pointing at end of if no references are + * available. + * + * @return references + */ + std::vector<std::string> references() const noexcept { + return mu_document().string_vec_value(Field::Id::References); + } + + /** + * Get some value from the document, or Nothing if empty. + * + * @param id a message field id + * + * @return the value + */ + Option<std::string> opt_string(Field::Id id) const noexcept { + if (auto&& val{mu_document().string_value(id)}; val.empty()) + return Nothing; + else + return Some(std::move(val)); + } + + /** + * Get the Query match info for this message. + * + * @return the match info. + */ + QueryMatch& query_match() { + g_assert(query_matches_.find(doc_id()) != query_matches_.end()); + return query_matches_.find(doc_id())->second; + } + const QueryMatch& query_match() const { + g_assert(query_matches_.find(doc_id()) != query_matches_.end()); + return query_matches_.find(doc_id())->second; + } + +private: + /** + * Get a (cached) reference for the Mu::Document corresponding + * to the current iter. + * + * @return cached mu document, + */ + const Mu::Document& mu_document() const { + if (!mdoc_) { + if (auto xdoc = document(); !xdoc) + std::runtime_error("iter without document"); + else + mdoc_ = Mu::Document{xdoc.value()}; + } + return mdoc_.value(); + } + + mutable Option<Mu::Document> mdoc_; // cache. + Xapian::MSetIterator mset_it_; + QueryMatches& query_matches_; +}; + + +static inline auto +format_as(const QueryResultsIterator& it) +{ + return it.path().value_or("<no path>"); +} + +constexpr auto MaxQueryResultsSize = std::numeric_limits<size_t>::max(); + +class QueryResults { +public: + /// Helper types + using iterator = QueryResultsIterator; + using const_iterator = const iterator; + + /** + * Construct a QueryResults object + * + * @param mset an Xapian::MSet with matches + */ + QueryResults(const Xapian::MSet& mset, QueryMatches&& query_matches) + : mset_{mset}, query_matches_{std::move(query_matches)} + { + } + /** + * Is this QueryResults object empty (ie., no matches)? + * + * @return true are false + */ + bool empty() const { return mset_.empty(); } + + /** + * Get the number of matches in this QueryResult + * + * @return number of matches + */ + size_t size() const { return mset_.size(); } + + /** + * Get the begin iterator to the results. + * + * @return iterator + */ + const iterator begin() const { return QueryResultsIterator(mset_.begin(), query_matches_); } + + /** + * Get the end iterator to the results. + * + * @return iterator + */ + const_iterator end() const { return QueryResultsIterator(mset_.end(), query_matches_); } + + /** + * Get the query-matches for these QueryResults. The non-const + * version can be use to _steal_ the query results, by moving + * them. + * + * @return query-matches + */ + const QueryMatches& query_matches() const { return query_matches_; } + QueryMatches& query_matches() { return query_matches_; } + +private: + const Xapian::MSet mset_; + mutable QueryMatches query_matches_; +}; + +} // namespace Mu + +#endif /* MU_QUERY_RESULTS_HH__ */ diff --git a/lib/mu-query-threads.cc b/lib/mu-query-threads.cc new file mode 100644 index 0000000..6d99281 --- /dev/null +++ b/lib/mu-query-threads.cc @@ -0,0 +1,957 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-query-threads.hh" +#include <message/mu-message.hh> + +#include <set> +#include <unordered_set> +#include <list> +#include <cassert> +#include <cstring> +#include <iostream> +#include <iomanip> + +#include <utils/mu-option.hh> + +using namespace Mu; + +struct Container { + using Containers = std::vector<Container*>; + + Container() = default; + Container(Option<QueryMatch&> msg) : query_match{msg} {} + Container(const Container&) = delete; + Container(Container&&) = default; + + void add_child(Container& new_child) + { + new_child.parent = this; + children.emplace_back(&new_child); + } + void remove_child(Container& child) + { + children.erase(find_child(child)); + assert(!has_child(child)); + } + + Containers::iterator find_child(Container& child) + { + return std::find_if(children.begin(), children.end(), [&](auto&& c) { + return c == &child; + }); + } + Containers::const_iterator find_child(Container& child) const + { + return std::find_if(children.begin(), children.end(), [&](auto&& c) { + return c == &child; + }); + } + bool has_child(Container& child) const { return find_child(child) != children.cend(); } + + bool is_reachable(Container* other) const + { + auto up{ur_parent()}; + return up && up == other->ur_parent(); + } + template <typename Func> void for_each_child(Func&& func) + { + auto it{children.rbegin()}; + while (it != children.rend()) { + auto next = std::next(it); + func(*it); + it = next; + } + } + // During sorting, this is the cached value for the (recursive) date-key + // of this container -- ie.. either the one from the first of its + // children, or from its query-match, if it has no children. + // + // Note that the sub-root-levels of threads are always sorted by date, + // in ascending order, regardless of whatever sorting was specified for + // the root-level. + + std::string thread_date_key; + + Option<QueryMatch&> query_match; + bool is_nuked{}; + Container* parent{}; + Containers children; + + using ContainerVec = std::vector<Container*>; + + private: + const Container* ur_parent() const + { + assert(this->parent != this); + return parent ? parent->ur_parent() : this; + } +}; + +using Containers = Container::Containers; +using ContainerVec = Container::ContainerVec; + +/* LCOV_EXCL_START */ +static std::ostream& +operator<<(std::ostream& os, const Container& container) +{ + os << "container: " << std::right << std::setw(10) << &container + << ": parent: " << std::right << std::setw(10) << container.parent << " [" + << container.thread_date_key << "]" + << "\n children: "; + + for (auto&& c : container.children) + os << std::right << std::setw(10) << c << " "; + + os << (container.is_nuked ? " nuked" : ""); + + if (container.query_match) + os << "\n " << container.query_match.value(); + + return os; +} +/* LCOV_EXCL_STOP */ + +using IdTable = std::unordered_map<std::string, Container>; +using DupTable = std::multimap<std::string, Container>; + +static void +handle_duplicates(IdTable& id_table, DupTable& dup_table) +{ + size_t n{}; + + for (auto&& dup : dup_table) { + const auto msgid{dup.first}; + auto it = id_table.find(msgid); + if (it == id_table.end()) + continue; + + // add duplicates as fake children + char buf[32]; + ::snprintf(buf, sizeof(buf), "dup-%zu", ++n); + it->second.add_child(id_table.emplace(buf, std::move(dup.second)).first->second); + } +} + +template <typename QueryResultsType> +static IdTable +determine_id_table(QueryResultsType& qres) +{ + // 1. For each query_match + IdTable id_table; + DupTable dups; + for (auto&& mi : qres) { + const auto msgid{mi.message_id().value_or(*mi.path())}; + // Step 0 (non-JWZ): filter out dups, handle those at the end + if (mi.query_match().has_flag(QueryMatch::Flags::Duplicate)) { + dups.emplace(msgid, mi.query_match()); + continue; + } + // 1.A If id_table contains an empty Container for this ID: + // Store this query_match (query_match) in the Container's query_match (value) slot. + // Else: + // Create a new Container object holding this query_match (query-match); + // Index the Container by Query_Match-ID + auto c_it = id_table.find(msgid); + auto& container = [&]() -> Container& { + if (c_it != id_table.end()) { + if (!c_it->second.query_match) // hmm, dup? + c_it->second.query_match = mi.query_match(); + return c_it->second; + } else { + // Else: + // Create a new Container object holding this query_match + // (query-match); Index the Container by Query_Match-ID + return id_table.emplace(msgid, mi.query_match()).first->second; + } + }(); + + // We sort by date (ascending), *except* for the root; we don't + // know what query_matchs will be at the root level yet, so remember + // both. Moreover, even when sorting the top-level in descending + // order, still sort the thread levels below that in ascending + // order. + container.thread_date_key = container.query_match->date_key = + mi.date_str().value_or(""); + // initial guess for the thread-date; might be updated + // later. + + // remember the subject, we use it to determine the (sub)thread subject + container.query_match->subject = mi.subject().value_or(""); + + // 1.B + // For each element in the query_match's References field: + Container* parent_ref_container{}; + for (const auto& ref : mi.references()) { + // grand_<n>-parent -> grand_<n-1>-parent -> ... -> parent. + + // Find a Container object for the given Query_Match-ID; If it exists, use + // it; otherwise make one with a null Query_Match. + auto ref_container = [&]() -> Container* { + auto ref_it = id_table.find(ref); + if (ref_it == id_table.end()) + ref_it = id_table.emplace(ref, Nothing).first; + return &ref_it->second; + }(); + + // Link the References field's Containers together in the order implied + // by the References header. + // * If they are already linked, don't change the existing links. + // + // * Do not add a link if adding that link would introduce a loop: that is, + // before asserting A->B, search down the children of B to see if A is + // reachable, and also search down the children of A to see if B is + // reachable. If either is already reachable as a child of the other, + // don't add the link. + if (parent_ref_container && !ref_container->parent) { + if (!parent_ref_container->is_reachable(ref_container)) + parent_ref_container->add_child(*ref_container); + // else + // g_message ("%u: reachable %s -> %s", __LINE__, + // msgid.c_str(), ref.c_str()); + } + + parent_ref_container = ref_container; + } + + // Add the query_match to the chain. + if (parent_ref_container && !container.parent) { + if (!parent_ref_container->is_reachable(&container)) + parent_ref_container->add_child(container); + // else + // g_message ("%u: reachable %s -> parent", __LINE__, + // msgid.c_str()); + } + } + + // non-JWZ: add duplicate messages. + handle_duplicates(id_table, dups); + + return id_table; +} + +/// Recursively walk all containers under the root set. +/// For each container: +/// +/// If it is an empty container with no children, nuke it. +/// +/// Note: Normally such containers won't occur, but they can show up when two +/// query_matchs have References lines that disagree. For example, assuming A and +/// B are query_matchs, and 1, 2, and 3 are references for query_matchs we haven't +/// seen: +/// +/// A has references: 1, 2, 3 +/// B has references: 1, 3 +/// +/// There is ambiguity as to whether 3 is a child of 1 or of 2. So, +/// depending on the processing order, we might end up with either +/// +/// -- 1 +/// |-- 2 +/// \-- 3 +/// |-- A +/// \-- B +/// +/// or +/// +/// -- 1 +/// |-- 2 <--- non root childless container! +/// \-- 3 +/// |-- A +/// \-- B +/// +/// If the Container has no Query_Match, but does have children, remove this +/// container but promote its children to this level (that is, splice them in +/// to the current child list.) +/// +/// Do not promote the children if doing so would promote them to the root +/// set -- unless there is only one child, in which case, do. + +static void +prune(Container* child) +{ + Container* container{child->parent}; + + for (auto& grandchild : child->children) { + grandchild->parent = container; + if (container) + container->children.emplace_back(grandchild); + } + + child->children.clear(); + child->is_nuked = true; + + if (container) + container->remove_child(*child); +} + +static bool +prune_empty_containers(Container& container) +{ + Containers to_prune; + + container.for_each_child([&](auto& child) { + if (prune_empty_containers(*child)) + to_prune.emplace_back(child); + }); + + for (auto& child : to_prune) + prune(child); + + // Never nuke these. + if (container.query_match) + return false; + + // If it is an empty container with no children, nuke it. + // + // If the Container is empty, but does have children, remove this + // container but promote its children to this level (that is, splice them in + // to the current child list.) + // + // Do not promote the children if doing so would promote them to the root + // set -- unless there is only one child, in which case, do. + // const auto rootset_child{!container.parent->parent}; + if (container.parent || container.children.size() <= 1) + return true; // splice/nuke it. + + return false; +} + +static void +prune_empty_containers(IdTable& id_table) +{ + for (auto&& item : id_table) { + auto& child(item.second); + if (child.parent) + continue; // not a root child. + + if (prune_empty_containers(item.second)) + prune(&child); + } +} + +// +// Sorting. +// + +/// Register some information about a match (i.e., message) that we can use for +/// subsequent queries. +using ThreadPath = std::vector<unsigned>; +inline std::string +to_string(const ThreadPath& tpath, size_t digits) +{ + std::string str; + str.reserve(tpath.size() * digits); + + bool first{true}; + for (auto&& segm : tpath) { + str += mu_format("{}{:0{}x}", first ? "" : ":", segm, digits); + first = false; + } + + return str; +} + +static bool // compare subjects, ignore anything before the last ':<space>*' +subject_matches(const std::string& sub1, const std::string& sub2) +{ + auto search_str = [](const std::string& s) -> const char* { + const auto pos = s.find_last_of(':'); + if (pos == std::string::npos) + return s.c_str(); + else { + const auto pos2 = s.find_first_not_of(' ', pos + 1); + return s.c_str() + (pos2 == std::string::npos ? pos : pos2); + } + }; + + return g_strcmp0(search_str(sub1), search_str(sub2)) == 0; +} + +static bool +update_container(Container& container, + bool descending, + ThreadPath& tpath, + size_t seg_size, + const std::string& prev_subject = "") +{ + if (!container.children.empty()) { + Container* first = container.children.front(); + if (first->query_match) + first->query_match->flags |= QueryMatch::Flags::First; + Container* last = container.children.back(); + if (last->query_match) + last->query_match->flags |= QueryMatch::Flags::Last; + } + + if (!container.query_match) + return false; // nothing else to do. + + auto& qmatch(*container.query_match); + if (!container.parent) + qmatch.flags |= QueryMatch::Flags::Root; + else if (!container.parent->query_match) + qmatch.flags |= QueryMatch::Flags::Orphan; + + if (!container.children.empty()) + qmatch.flags |= QueryMatch::Flags::HasChild; + + if (qmatch.has_flag(QueryMatch::Flags::Root) || prev_subject.empty() || + !subject_matches(prev_subject, qmatch.subject)) + qmatch.flags |= QueryMatch::Flags::ThreadSubject; + + if (descending && container.parent) { + // trick xapian by giving it "inverse" sorting key so our + // ascending-date sorted threads stay in that order + tpath.back() = ((1U << (4 * seg_size)) - 1) - tpath.back(); + } + + qmatch.thread_path = to_string(tpath, seg_size); + qmatch.thread_level = tpath.size() - 1; + + // ensure thread root comes before its children + if (descending) + qmatch.thread_path += ":z"; + + return true; +} + +static void +update_containers(Containers& children, + bool descending, + ThreadPath& tpath, + size_t seg_size, + std::string& prev_subject) +{ + size_t idx{0}; + + for (auto&& c : children) { + tpath.emplace_back(idx++); + if (c->query_match) { + update_container(*c, descending, tpath, seg_size, prev_subject); + prev_subject = c->query_match->subject; + } + update_containers(c->children, descending, tpath, seg_size, prev_subject); + tpath.pop_back(); + } +} + +static void +update_containers(ContainerVec& root_vec, bool descending, size_t n) +{ + ThreadPath tpath; + tpath.reserve(n); + + const auto seg_size = static_cast<size_t>(std::ceil(std::log2(n) / 4.0)); + /*note: 4 == std::log2(16)*/ + + size_t idx{0}; + for (auto&& c : root_vec) { + tpath.emplace_back(idx++); + std::string prev_subject; + if (update_container(*c, descending, tpath, seg_size)) + prev_subject = c->query_match->subject; + update_containers(c->children, descending, tpath, seg_size, prev_subject); + tpath.pop_back(); + } +} + +static void +sort_container(Container& container) +{ + // 1. childless container. + if (container.children.empty()) + return; // no children; nothing to sort. + + // 2. container with children. + // recurse, depth-first: sort the children + for (auto& child : container.children) + sort_container(*child); + + // now sort this level. + std::sort(container.children.begin(), container.children.end(), [&](auto&& c1, auto&& c2) { + return c1->thread_date_key < c2->thread_date_key; + }); + + // and 'bubble up' the date of the *newest* message with a date. We + // reasonably assume that it's later than its parent. + const auto& newest_date = container.children.back()->thread_date_key; + if (!newest_date.empty()) + container.thread_date_key = newest_date; +} + +static void +sort_siblings(IdTable& id_table, bool descending) +{ + if (id_table.empty()) + return; + + // unsorted vec of root containers. We can + // only sort these _after_ sorting the children. + ContainerVec root_vec; + for (auto&& item : id_table) { + if (!item.second.parent && !item.second.is_nuked) + root_vec.emplace_back(&item.second); + } + + // now sort all threads _under_ the root set (by date/ascending) + for (auto&& c : root_vec) + sort_container(*c); + + // and then sort the root set. + // + // The difference with the sub-root containers is that at the top-level, + // we can sort either in ascending or descending order, while on the + // subroot level it's always in ascending order. + // + // Note that unless we're testing, _xapian_ will handle + // the ascending/descending of the top level. + std::sort(root_vec.begin(), root_vec.end(), [&](auto&& c1, auto&& c2) { +#ifdef BUILD_TESTS + if (descending) + return c2->thread_date_key < c1->thread_date_key; + else +#endif /*BUILD_TESTS*/ + return c1->thread_date_key < c2->thread_date_key; + }); + + // now all is sorted... final step is to determine thread paths and + // other flags. + update_containers(root_vec, descending, id_table.size()); +} + +/* LCOV_EXCL_START */ +static std::ostream& +operator<<(std::ostream& os, const IdTable& id_table) +{ + os << "------------------------------------------------\n"; + for (auto&& item : id_table) { + os << item.first << " => " << item.second << "\n"; + } + os << "------------------------------------------------\n"; + + std::set<std::string> ids; + for (auto&& item : id_table) { + if (item.second.query_match) + ids.emplace(item.second.query_match->thread_path); + } + + for (auto&& id : ids) { + auto it = std::find_if(id_table.begin(), id_table.end(), [&](auto&& item) { + return item.second.query_match && + item.second.query_match->thread_path == id; + }); + assert(it != id_table.end()); + os << it->first << ": " << it->second << '\n'; + } + return os; +} +/* LCOV_EXCL_STOP */ + +template <typename Results> +static void +calculate_threads_real(Results& qres, bool descending) +{ + // Step 1: build the id_table + auto id_table{determine_id_table(qres)}; + + if (g_test_verbose()) + std::cout << "*** id-table(1):\n" << id_table << "\n"; + + // // Step 2: get the root set + // // Step 3: discard id_table + // Nope: id-table owns the containers. + // Step 4: prune empty containers + prune_empty_containers(id_table); + + // Step 5: group root-set by subject. + // Not implemented. + + // Step 6: we're done threading + + // Step 7: sort siblings. The segment-size is the number of hex-digits + // in the thread-path string (so we can lexically compare them.) + sort_siblings(id_table, descending); + + // Step 7a:. update querymatches + for (auto&& item : id_table) { + Container& c{item.second}; + if (c.query_match) + c.query_match->thread_date = c.thread_date_key; + } + // if (g_test_verbose()) + // std::cout << "*** id-table(2):\n" << id_table << "\n"; +} + +void +Mu::calculate_threads(Mu::QueryResults& qres, bool descending) +{ + calculate_threads_real(qres, descending); +} + +#ifdef BUILD_TESTS + +struct MockQueryResult { + MockQueryResult(const std::string& message_id_arg, + const std::string& date_arg, + const std::vector<std::string>& refs_arg = {}) + : message_id_{message_id_arg}, date_{date_arg}, refs_{refs_arg} + { + } + MockQueryResult(const std::string& message_id_arg, + const std::vector<std::string>& refs_arg = {}) + : MockQueryResult(message_id_arg, "", refs_arg) + { + } + Option<std::string> message_id() const { return message_id_; } + Option<std::string> path() const { return path_; } + Option<std::string> date_str() const { return date_; } + Option<std::string> subject() const { return subject_; } + QueryMatch& query_match() { return query_match_; } + const QueryMatch& query_match() const { return query_match_; } + const std::vector<std::string>& references() const { return refs_; } + + std::string path_; + std::string message_id_; + QueryMatch query_match_{}; + std::string date_; + std::string subject_; + std::vector<std::string> refs_; +}; + +using MockQueryResults = std::vector<MockQueryResult>; + +G_GNUC_UNUSED static std::ostream& +operator<<(std::ostream& os, const MockQueryResults& qrs) +{ + for (auto&& mi : qrs) + os << mi.query_match().thread_path << " :: " << mi.message_id().value_or("<none>") + << std::endl; + + return os; +} + +static void +calculate_threads(MockQueryResults& qres, bool descending) +{ + calculate_threads_real(qres, descending); +} + +using Expected = std::vector<std::pair<std::string, std::string>>; + +static void +assert_thread_paths(const MockQueryResults& qrs, const Expected& expected) +{ + for (auto&& exp : expected) { + auto it = std::find_if(qrs.begin(), qrs.end(), [&](auto&& qr) { + return qr.message_id().value_or("") == exp.first || + qr.path().value_or("") == exp.first; + }); + g_assert_true(it != qrs.end()); + mu_debug("thread-path ({}@{}): expected: '{}'; got '{}'", + it->message_id().value_or("<none>"), + it->path().value_or("<none>"), + exp.second, it->query_match().thread_path); + g_assert_cmpstr(exp.second.c_str(), ==, it->query_match().thread_path.c_str()); + } +} + +static void +test_sort_ascending() +{ + auto results = MockQueryResults{MockQueryResult{"m1", "1", {"m2"}}, + MockQueryResult{"m2", "2", {"m3"}}, + MockQueryResult{"m3", "3", {}}, + MockQueryResult{"m4", "4", {}}}; + + calculate_threads(results, false); + + assert_thread_paths(results, {{"m1", "0:0:0"}, {"m2", "0:0"}, {"m3", "0"}, {"m4", "1"}}); +} + +static void +test_sort_descending() +{ + auto results = MockQueryResults{MockQueryResult{"m1", "1", {"m2"}}, + MockQueryResult{"m2", "2", {"m3"}}, + MockQueryResult{"m3", "3", {}}, + MockQueryResult{"m4", "4", {}}}; + + calculate_threads(results, true); + + assert_thread_paths(results, + {{"m1", "1:f:f:z"}, {"m2", "1:f:z"}, {"m3", "1:z"}, {"m4", "0:z"}}); +} + +static void +test_id_table_inconsistent() +{ + auto results = MockQueryResults{ + MockQueryResult{"m1", "1", {"m2"}}, // 1->2 + MockQueryResult{"m2", "2", {"m1"}}, // 2->1 + MockQueryResult{"m3", "3", {"m3"}}, // self ref + MockQueryResult{"m4", "4", {"m3", "m5"}}, + MockQueryResult{"m5", "5", {"m4", "m4"}}, // dup parent + }; + + calculate_threads(results, false); + assert_thread_paths(results, + { + {"m2", "0"}, + {"m1", "0:0"}, + {"m3", "1"}, + {"m5", "1:0"}, + {"m4", "1:0:0"}, + }); +} + +static void +test_dups_dup_last() +{ + MockQueryResult r1{"m1", "1", {}}; + r1.query_match().flags |= QueryMatch::Flags::Leader; + r1.path_ = "/path1"; + + MockQueryResult r1_dup{"m1", "1", {}}; + r1_dup.query_match().flags |= QueryMatch::Flags::Duplicate; + r1_dup.path_ = "/path2"; + + auto results = MockQueryResults{r1, r1_dup}; + + calculate_threads(results, false); + + assert_thread_paths(results, + { + {"/path1", "0"}, + {"/path2", "0:0"}, + }); +} + +static void +test_dups_dup_first() +{ + // now dup becomes the leader; this will _demote_ + // r1. + + MockQueryResult r1_dup{"m1", "1", {}}; + r1_dup.query_match().flags |= QueryMatch::Flags::Duplicate; + r1_dup.path_ = "/path1"; + + MockQueryResult r1{"m1", "1", {}}; + r1.query_match().flags |= QueryMatch::Flags::Leader; + r1.path_ = "/path2"; + + auto results = MockQueryResults{r1_dup, r1}; + + calculate_threads(results, false); + + assert_thread_paths(results, { + {"/path2", "0"}, + {"/path1", "0:0"}, + }); +} + +static void +test_dups_dup_multi() +{ + // now dup becomes the leader; this will _demote_ + // r1. + + MockQueryResult r1_dup1{"m1", "1", {}}; + r1_dup1.query_match().flags |= QueryMatch::Flags::Duplicate; + r1_dup1.path_ = "/path1"; + + MockQueryResult r1_dup2{"m1", "1", {}}; + r1_dup2.query_match().flags |= QueryMatch::Flags::Duplicate; + r1_dup2.path_ = "/path2"; + + MockQueryResult r1{"m1", "1", {}}; + r1.query_match().flags |= QueryMatch::Flags::Leader; + r1.path_ = "/path3"; + + auto results = MockQueryResults{r1_dup1, r1_dup2, r1}; + calculate_threads(results, false); + + assert_thread_paths(results, { + {"/path3", "0"}, + {"/path1", "0:0"}, + {"/path2", "0:1"}, + }); +} + + + + +static void +test_do_not_prune_root_empty_with_children() +{ + // m7 should not be nuked + auto results = MockQueryResults{ + MockQueryResult{"x1", "1", {"m7"}}, + MockQueryResult{"x2", "2", {"m7"}}, + }; + + calculate_threads(results, false); + + assert_thread_paths(results, + { + {"x1", "0:0"}, + {"x2", "0:1"}, + }); +} + +static void +test_prune_root_empty_with_child() +{ + // m7 should be nuked + auto results = MockQueryResults{ + MockQueryResult{"m1", "1", {"m7"}}, + }; + + calculate_threads(results, false); + + assert_thread_paths(results, + { + {"m1", "0"}, + }); +} + +static void +test_prune_empty_with_children() +{ + // m6 should be nuked + auto results = MockQueryResults{ + MockQueryResult{"m1", "1", {"m7", "m6"}}, + MockQueryResult{"m2", "2", {"m7", "m6"}}, + }; + + calculate_threads(results, false); + + assert_thread_paths(results, + { + {"m1", "0:0"}, + {"m2", "0:1"}, + }); +} + +static void +test_thread_info_ascending() +{ + auto results = MockQueryResults{ + MockQueryResult{"m1", "5", {}}, + MockQueryResult{"m2", "1", {}}, + MockQueryResult{"m3", "3", {"m2"}}, + MockQueryResult{"m4", "2", {"m2"}}, + // orphan siblings + MockQueryResult{"m10", "6", {"m9"}}, + MockQueryResult{"m11", "7", {"m9"}}, + }; + calculate_threads(results, false); + + assert_thread_paths(results, + { + {"m2", "0"}, // 2 + {"m4", "0:0"}, // 2 + {"m3", "0:1"}, // 3 + {"m1", "1"}, // 5 + + {"m10", "2:0"}, // 6 + {"m11", "2:1"}, // 7 + }); + + g_assert_true(results[0].query_match().has_flag(QueryMatch::Flags::Root)); + g_assert_true(results[1].query_match().has_flag(QueryMatch::Flags::Root | + QueryMatch::Flags::HasChild)); + g_assert_true(results[2].query_match().has_flag(QueryMatch::Flags::Last)); + g_assert_true(results[3].query_match().has_flag(QueryMatch::Flags::First)); + g_assert_true(results[4].query_match().has_flag(QueryMatch::Flags::Orphan | + QueryMatch::Flags::First)); + g_assert_true( + results[5].query_match().has_flag(QueryMatch::Flags::Orphan | QueryMatch::Flags::Last)); +} + +static void +test_thread_info_descending() +{ + auto results = MockQueryResults{ + MockQueryResult{"m1", "5", {}}, + MockQueryResult{"m2", "1", {}}, + MockQueryResult{"m3", "3", {"m2"}}, + MockQueryResult{"m4", "2", {"m2"}}, + // orphan siblings + MockQueryResult{"m10", "6", {"m9"}}, + MockQueryResult{"m11", "7", {"m9"}}, + }; + calculate_threads(results, true /*descending*/); + + assert_thread_paths(results, + { + {"m1", "1:z"}, // 5 + {"m2", "2:z"}, // 2 + {"m4", "2:f:z"}, // 2 + {"m3", "2:e:z"}, // 3 + + {"m10", "0:f:z"}, // 6 + {"m11", "0:e:z"}, // 7 + }); + g_assert_true(results[0].query_match().has_flag(QueryMatch::Flags::Root)); + g_assert_true(results[1].query_match().has_flag(QueryMatch::Flags::Root | + QueryMatch::Flags::HasChild)); + g_assert_true(results[2].query_match().has_flag(QueryMatch::Flags::Last)); + g_assert_true(results[3].query_match().has_flag(QueryMatch::Flags::First)); + + g_assert_true( + results[4].query_match().has_flag(QueryMatch::Flags::Orphan | QueryMatch::Flags::Last)); + g_assert_true(results[5].query_match().has_flag(QueryMatch::Flags::Orphan | + QueryMatch::Flags::First)); +} + +int +main(int argc, char* argv[]) +try { + g_test_init(&argc, &argv, NULL); + + g_test_add_func("/threader/sort/ascending", test_sort_ascending); + g_test_add_func("/threader/sort/decending", test_sort_descending); + + g_test_add_func("/threader/id-table-inconsistent", test_id_table_inconsistent); + g_test_add_func("/threader/dups/dup-last", test_dups_dup_last); + g_test_add_func("/threader/dups/dup-first", test_dups_dup_first); + g_test_add_func("/threader/dups/dup-multi", test_dups_dup_multi); + + g_test_add_func("/threader/prune/do-not-prune-root-empty-with-children", + test_do_not_prune_root_empty_with_children); + g_test_add_func("/threader/prune/prune-root-empty-with-child", + test_prune_root_empty_with_child); + g_test_add_func("/threader/prune/prune-empty-with-children", + test_prune_empty_with_children); + + g_test_add_func("/threader/thread-info/ascending", test_thread_info_ascending); + g_test_add_func("/threader/thread-info/descending", test_thread_info_descending); + + return g_test_run(); +} catch (const std::runtime_error& re) { + std::cerr << re.what() << "\n"; + return 1; +} catch (...) { + std::cerr << "caught exception\n"; + return 1; +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/mu-query-threads.hh b/lib/mu-query-threads.hh new file mode 100644 index 0000000..5aab888 --- /dev/null +++ b/lib/mu-query-threads.hh @@ -0,0 +1,41 @@ +/* +** Copyright (C) 2021 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_QUERY_THREADS__ +#define MU_QUERY_THREADS__ + +#include "mu-query-results.hh" + +namespace Mu { +/** + * Calculate the threads for these query results; that is, determine the + * thread-paths for each message, so we can let Xapian order them in the correct + * order. + * + * Note - threads are sorted chronologically, and the messages below the top + * level are always sorted in ascending orde + * + * @param qres query results + * @param descending whether to sort the top-level in descending order + */ +void calculate_threads(QueryResults& qres, bool descending); + +} // namespace Mu + +#endif /*MU_QUERY_THREADS__*/ diff --git a/lib/mu-query-xapianizer.cc b/lib/mu-query-xapianizer.cc new file mode 100644 index 0000000..11aeee0 --- /dev/null +++ b/lib/mu-query-xapianizer.cc @@ -0,0 +1,521 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-query-parser.hh" + +#include <string_view> +#include <variant> +#include <array> +#include <type_traits> + +#include "utils/mu-option.hh" +#include <glib.h> +#include "utils/mu-utils-file.hh" + +using namespace Mu; + +// backward compat +#ifndef HAVE_XAPIAN_FLAG_NGRAMS +#define FLAG_NGRAMS FLAG_CJK_NGRAM +#endif /*HAVE_XAPIAN_FLAG_NGRAMS*/ + +/** + * Expand terms for scripts without explicit word-breaks (e.g. + * Chinese/Japanese/Korean) in the way that Xapian expects it - + * use Xapian's built-in QueryParser just for that. + */ +static Result<Xapian::Query> +ngram_expand(const Field& field, const std::string& str) +{ + Xapian::QueryParser qp; + const auto pfx{std::string(1U, field.xapian_prefix())}; + + qp.set_default_op(Xapian::Query::OP_OR); + + return qp.parse_query(str, Xapian::QueryParser::FLAG_NGRAMS, pfx); +} + + +static Option<Sexp> +tail(Sexp&& s) +{ + if (!s.listp() || s.empty()) + return Nothing; + + s.list().erase(s.list().begin(), s.list().begin() + 1); + + return s; +} + +Option<std::string> +head_symbol(const Sexp& s) +{ + if (!s.listp() || s.empty() || !s.head() || !s.head()->symbolp()) + return Nothing; + + return s.head()->symbol().name; +} + + +Option<std::string> +string_nth(const Sexp& args, size_t n) +{ + if (!args.listp() || args.size() < n + 1) + return Nothing; + + if (auto&& item{args.list().at(n)}; !item.stringp()) + return Nothing; + else + return item.string(); +} + +static Result<Xapian::Query> +phrase(const Field& field, Sexp&& s) +{ + if (!field.is_phrasable_term()) + return Err(Error::Code::InvalidArgument, + "field {} does not support phrases", field.name); + + if (s.size() == 1 && s.front().stringp()) { + auto&& words{split(s.front().string(), " ")}; + std::vector<Xapian::Query> phvec; + phvec.reserve(words.size()); + for(auto&& w: words) + phvec.emplace_back(Xapian::Query{field.xapian_term(std::move(w))}); + return Xapian::Query{Xapian::Query::OP_PHRASE, + phvec.begin(), phvec.end()}; + } else + return Err(Error::Code::InvalidArgument, + "invalid phrase for field {}: '{}'", field.name, s.to_string()); +} + +static Result<Xapian::Query> +regex(const Store& store, const Field& field, const std::string& rx_str) +{ + auto&& str{utf8_flatten(rx_str)}; + auto&& rx{Regex::make(str, G_REGEX_OPTIMIZE)}; + if (!rx) { + mu_warning("invalid regexp: '{}': {}", str, rx.error().what()); + return Xapian::Query::MatchNothing; + } + + std::vector<Xapian::Query> rxvec; + store.for_each_term(field.id, [&](auto&& str) { + if (auto&& val{str.data() + 1}; rx->matches(val)) + rxvec.emplace_back(field.xapian_term(std::string_view{val})); + return true; + }); + + return Xapian::Query(Xapian::Query::OP_OR, rxvec.begin(), rxvec.end()); +} + + + +static Result<Xapian::Query> +range(const Field& field, Sexp&& s) +{ + auto&& r0{string_nth(s, 0)}; + auto&& r1{string_nth(s, 1)}; + if (!r0 || !r1) + return Err(Error::Code::InvalidArgument, "expected 2 range values"); + + // in the sexp, we use iso date/time for human readability; now convert to + // time_t + auto iso_to_lexnum=[](const std::string& s)->Option<std::string> { + if (s.empty()) + return s; + if (auto&& t{parse_date_time(s, true, true/*utc*/)}; !t) + return Nothing; + else + return to_lexnum(*t); + }; + + if (field == Field::Id::Date || field == Field::Id::Changed) { + // iso -> time_t + r0 = iso_to_lexnum(*r0); + r1 = iso_to_lexnum(*r1); + } else if (field == Field::Id::Size) { + if (!r0->empty()) + r0 = to_lexnum(::atoll(r0->c_str())); + if (!r1->empty()) + r1 = to_lexnum(::atoll(r1->c_str())); + } else + return Err(Error::Code::InvalidArgument, + "unsupported range field {}", field.name); + + if (r0->empty() && r1->empty()) + return Xapian::Query::MatchNothing; // empty range matches nothing. + else if (r0->empty() && !r1->empty()) + return Xapian::Query(Xapian::Query::OP_VALUE_LE, + field.value_no(), *r1); + else if (!r0->empty() && r1->empty()) + return Xapian::Query(Xapian::Query::OP_VALUE_GE, + field.value_no(), *r0); + else + return Xapian::Query(Xapian::Query::OP_VALUE_RANGE, + field.value_no(), *r0, *r1); +} + + + +using OpPair = std::pair<const std::string_view, Xapian::Query::op>; +static constexpr std::array<OpPair, 4> LogOpPairs = {{ + { "and", Xapian::Query::OP_AND }, + { "or", Xapian::Query::OP_OR }, + { "xor", Xapian::Query::OP_XOR }, + { "not", Xapian::Query::OP_AND_NOT } + }}; + +static Option<Xapian::Query::op> +find_log_op(const std::string& opname) +{ + for (auto&& p: LogOpPairs) + if (p.first == opname) + return p.second; + + return Nothing; +} + +static Result<Xapian::Query> parse(const Store& store, Sexp&& s, Mu::ParserFlags flags); + +static Result<Xapian::Query> +parse_logop(const Store& store, Xapian::Query::op op, Sexp&& args, Mu::ParserFlags flags) +{ + if (!args.listp() || args.empty()) + return Err(Error::Code::InvalidArgument, + "expected non-empty list but got", args.to_string()); + + std::vector<Xapian::Query> qs; + for (auto&& elm: args.list()) { + if (auto&& q{parse(store, std::move(elm), flags)}; !q) + return Err(std::move(q.error())); + else + qs.emplace_back(std::move(*q)); + } + + switch(op) { + case Xapian::Query::OP_AND_NOT: + // TODO: optimize AND_NOT + if (qs.size() != 1) + return Err(Error::Code::InvalidArgument, + "expected single argument for NOT"); + else + return Xapian::Query{op, Xapian::Query::MatchAll, qs.at(0)}; + + case Xapian::Query::OP_AND: + case Xapian::Query::OP_OR: + case Xapian::Query::OP_XOR: + return Xapian::Query(op, qs.begin(), qs.end()); + + default: + return Err(Error::Code::InvalidArgument, "unexpected xapian op"); + } +} + + +static Result<Xapian::Query> +parse_field_matcher(const Store& store, const Field& field, + const std::string& match_sym, Sexp&& args) +{ + auto&& str0{string_nth(args, 0)}; + + if (match_sym == wildcard_sym.name && str0) + return Xapian::Query{Xapian::Query::OP_WILDCARD, + field.xapian_term(*str0)}; + else if (match_sym == range_sym.name && !!str0) + return range(field, std::move(args)); + else if (match_sym == regex_sym.name && !!str0) + return regex(store, field, *str0); + else if (match_sym == phrase_sym.name) + return phrase(field, std::move(args)); + + return Err(Error::Code::InvalidArgument, + "invalid field '{}'/'{}' matcher: {}", + field.name, match_sym, args.to_string()); +} + +static Result<Xapian::Query> +parse_basic(const Field &field, Sexp &&vals, Mu::ParserFlags flags) +{ + auto ngrams = any_of(flags & ParserFlags::SupportNgrams); + if (!vals.stringp()) + return Err(Error::Code::InvalidArgument, "expected string"); + + auto&& val{vals.string()}; + + switch (field.id) { + case Field::Id::Flags: + if (auto&& finfo{flag_info(val)}; finfo) + return Xapian::Query{field.xapian_term(finfo->shortcut_lower())}; + else + return Err(Error::Code::InvalidArgument, "invalid flag '{}'", val); + case Field::Id::Priority: + if (auto&& prio{priority_from_name(val)}; prio) + return Xapian::Query{field.xapian_term(to_char(*prio))}; + else + return Err(Error::Code::InvalidArgument, "invalid priority '{}'", val); + default: { + auto q{Xapian::Query{field.xapian_term(val)}}; + if (ngrams) { // special case: cjk; see if we can create an expanded query. + if (field.is_phrasable_term() && contains_unbroken_script(val)) + if (auto&& ng{ngram_expand(field, val)}; ng) + return ng; + } + return q; + }} +} + +static Result<Xapian::Query> +parse(const Store& store, Sexp&& s, Mu::ParserFlags flags) +{ + auto&& headsym{head_symbol(s)}; + if (!headsym) + return Err(Error::Code::InvalidArgument, + "expected (symbol ...) but got {}", s.to_string()); + + // ie., something like (or|and| ... ....) + if (auto&& logop{find_log_op(*headsym)}; logop) { + if (auto&& args{tail(std::move(s))}; !args) + return Err(Error::Code::InvalidArgument, + "expected (logop ...) but got {}", + s.to_string()); + else + return parse_logop(store, *logop, std::move(*args), flags); + + } + // something like (field ...) + else if (auto&& field{field_from_name(*headsym)}; field) { + + auto&& rest{tail(std::move(s))}; + if (!rest || rest->empty()) + return Err(Error::Code::InvalidArgument, + "expected field-value or field-matcher"); + + auto&& matcher{rest->front()}; + // field-value: (field "value"); ensure "value" is there + if (matcher.stringp()) + return parse_basic(*field, std::move(matcher), flags); + + // otherwise, we expect a field-matcher, e.g. (field (phrase "a b c")) + // ensure the matcher is a list starting with a symbol + auto&& match_sym{head_symbol(matcher)}; + if (!match_sym) + return Err(Error::Code::InvalidArgument, + "expected field-matcher"); + + if (auto&& args{tail(std::move(matcher))}; !args) + return Err(Error::Code::InvalidArgument, "expected matcher arguments"); + else + return parse_field_matcher(store, *field, + *match_sym, std::move(*args)); + } + return Err(Error::Code::InvalidArgument, "unexpected sexp {}", s.to_string()); +} + +/* LCOV_EXCL_START*/ +// parse the way Xapian's internal parser does it; for testing. +static Xapian::Query +xapian_query_classic(const std::string& expr, Mu::ParserFlags flags) +{ + Xapian::QueryParser xqp; + + // add prefixes + field_for_each([&](auto&& field){ + + if (!field.is_searchable()) + return; + + const auto prefix{std::string(1U, field.xapian_prefix())}; + std::vector<std::string> names = { + std::string{field.name}, + std::string(1U, field.shortcut) + }; + if (!field.alias.empty()) + names.emplace_back(std::string{field.alias}); + + for (auto&& name: names) + xqp.add_prefix(name, prefix); + }); + + auto xflags = Xapian::QueryParser::FLAG_PHRASE | + Xapian::QueryParser::FLAG_BOOLEAN | + Xapian::QueryParser::FLAG_WILDCARD; + + if (any_of(flags & ParserFlags::SupportNgrams)) + xflags |= Xapian::QueryParser::FLAG_NGRAMS; + + xqp.set_default_op(Xapian::Query::OP_AND); + return xqp.parse_query(expr, xflags); +} +/* LCOV_EXCL_STOP*/ + +Result<Xapian::Query> +Mu::make_xapian_query(const Store& store, const std::string& expr, Mu::ParserFlags flags) noexcept +{ + if (any_of(flags & Mu::ParserFlags::XapianParser)) + return xapian_query_classic(expr, flags); + + return parse(store, Mu::parse_query(expr, true/*expand*/), flags); +} + + +#ifdef BUILD_XAPIANIZE_QUERY +int +main (int argc, char *argv[]) +{ + if (argc < 2) { + mu_printerrln("expected: parse-query <query>"); + return 1; + } + + auto store = Store::make(runtime_path(Mu::RuntimePath::XapianDb)); + if (!store) { + mu_printerrln("error: {}", store.error()); + return 2; + } + + std::string expr; + for (auto i = 1; i < argc; ++i) { + expr += argv[i]; + expr += " "; + } + + if (auto&& query{make_xapian_query(*store, expr)}; !query) { + mu_printerrln("error: {}", query.error()); + return 1; + } else + mu_println("mu: {}", query->get_description()); + + if (auto&& query{make_xapian_query(*store, expr, ParserFlags::XapianParser)}; !query) { + mu_printerrln("error: {}", query.error()); + return 2; + } else + mu_println("xp: {}", query->get_description()); + + return 0; + + +} +#endif /*BUILD_XAPIANIZE_QUERY*/ + +#if BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +using TestCase = std::pair<std::string, std::string>; + +static void +test_sexp() +{ + /* tail */ + g_assert_false(!!tail(Sexp{})); + auto t = tail(Sexp{1,2,3}); + g_assert_true(!!t && t->listp() && t->size() == 2); + + /* head_symbol */ + g_assert_false(!!head_symbol(Sexp{})); + assert_equal(head_symbol(Sexp{"foo"_sym, 1, 2}).value_or("bar"), "foo"); + + /* string_nth */ + g_assert_false(!!string_nth(Sexp{}, 123)); + g_assert_false(!!string_nth(Sexp{1, 2, 3}, 1)); + assert_equal(string_nth(Sexp{"aap", "noot", "mies"}, 2).value_or("wim"), "mies"); +} + + +static void +test_xapian() +{ + allow_warnings(); + + auto&& testhome{unwrap(make_temp_dir())}; + auto&& dbpath{runtime_path(RuntimePath::XapianDb, testhome)}; + auto&& store{unwrap(Store::make_new(dbpath, join_paths(testhome, "test-maildir")))}; + + // Xapian internal format (get_description()) is _not_ guaranteed + // to be the same between versions + auto&& zz{make_xapian_query(store, R"(subject:"hello world")")}; + assert_valid_result(zz); + /* LCOV_EXCL_START*/ + if (zz->get_description() != R"(Query((Shello world OR (Shello PHRASE 2 Sworld))))") { + mu_println("{}", zz->get_description()); + if (mu_test_mu_hacker()) { + // in the mu hacker case, we want to be warned if Xapian changed. + g_critical("xapian version mismatch"); + g_assert_true(false); + } else { + g_test_skip("incompatible xapian descriptions"); + return; + } + } + /* LCOV_EXCL_STOP*/ + + std::vector<TestCase> cases = { + + TestCase{R"(i:87h766tzzz.fsf@gnus.org)", R"(Query(I87h766tzzz.fsf@gnus.org))"}, + TestCase{R"(subject:foo to:bar)", R"(Query((Sfoo AND Tbar)))"}, + TestCase{R"(subject:"cuux*")", R"(Query(WILDCARD SYNONYM Scuux))"}, + TestCase{R"(subject:"hello world")", + R"(Query((Shello world OR (Shello PHRASE 2 Sworld))))"}, + TestCase{R"(subject:/boo/")", R"(Query())"}, + + // logic + TestCase{R"(not)", R"(Query((Tnot OR Cnot OR Hnot OR Fnot OR Snot OR Bnot OR Enot)))"}, + TestCase{R"(from:a and (from:b or from:c))", R"(Query((Fa AND (Fb OR Fc))))"}, + // optimize? + TestCase{R"(not from:a and to:b)", R"(Query(((<alldocuments> AND_NOT Fa) AND Tb)))"}, + TestCase{R"(cc:a not bcc:b)", R"(Query((Ca AND (<alldocuments> AND_NOT Hb))))"}, + + // ranges. + TestCase{R"(size:1..10")", R"(Query(VALUE_RANGE 17 g1 ga))"}, + TestCase{R"(size:10..1")", R"(Query(VALUE_RANGE 17 g1 ga))"}, + TestCase{R"(size:10..")", R"(Query(VALUE_GE 17 ga))"}, + TestCase{R"(size:..10")", R"(Query(VALUE_LE 17 ga))"}, + TestCase{R"(size:10")", R"(Query(VALUE_RANGE 17 ga ga))"}, // change? + TestCase{R"(size:..")", R"(Query())"}, + }; + + for (auto&& test: cases) { + auto&& xq{make_xapian_query(store, test.first)}; + assert_valid_result(xq); + assert_equal(xq->get_description(), test.second); + } + + remove_directory(testhome); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + Xapian::QueryParser qp; + + g_test_add_func("/query-parser/sexp", test_sexp); + g_test_add_func("/query-parser/xapianizer", test_xapian); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/mu-query.cc b/lib/mu-query.cc new file mode 100644 index 0000000..5b76005 --- /dev/null +++ b/lib/mu-query.cc @@ -0,0 +1,303 @@ +/* +** Copyright (C) 2008-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation; either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include <mu-query.hh> + +#include <stdexcept> +#include <string> +#include <cctype> +#include <cstring> +#include <sstream> +#include <cmath> + +#include <stdlib.h> +#include <glib/gstdio.h> + +#include "mu-xapian-db.hh" +#include "mu-query-results.hh" +#include "mu-query-match-deciders.hh" +#include "mu-query-threads.hh" + +#include "mu-query-parser.hh" + +using namespace Mu; + +struct Query::Private { + Private(const Store& store) : + store_{store}, + parser_flags_{any_of(store_.message_options() & Message::Options::SupportNgrams) ? + ParserFlags::SupportNgrams : ParserFlags::None} {} + + Xapian::Enquire make_enquire(const std::string& expr, Field::Id sortfield_id, + QueryFlags qflags) const; + Xapian::Enquire make_related_enquire(const StringSet& thread_ids, + Field::Id sortfield_id, + QueryFlags qflags) const; + + Option<QueryResults> run_threaded(QueryResults&& qres, Xapian::Enquire& enq, + QueryFlags qflags, size_t max_size) const; + Option<QueryResults> run_singular(const std::string& expr, + Field::Id sortfield_id, + QueryFlags qflags, size_t maxnum) const; + Option<QueryResults> run_related(const std::string& expr, + Field::Id sortfield_id, + QueryFlags qflags, size_t maxnum) const; + + Option<QueryResults> run(const std::string& expr, + Field::Id sortfield_id, QueryFlags qflags, + size_t maxnum) const; + const Store& store_; + const ParserFlags parser_flags_; +}; + +Query::Query(const Store& store) : priv_{std::make_unique<Private>(store)} {} + +Query::~Query() = default; + +static Xapian::Enquire& +sort_enquire(Xapian::Enquire& enq, Field::Id sortfield_id, QueryFlags qflags) +{ + const auto value_no{field_from_id(sortfield_id).value_no()}; + enq.set_sort_by_value(value_no, any_of(qflags & QueryFlags::Descending)); + + return enq; +} + +static Xapian::Query +make_query(const Store& store, const std::string& expr, ParserFlags parser_flags) +{ + if (expr.empty() || expr == R"("")") + return Xapian::Query::MatchAll; + else { + if (auto&& q{make_xapian_query(store, expr, parser_flags)}; !q) { + mu_warning("error in query '{}': {}", expr, q.error().what()); + return Xapian::Query::MatchNothing; + } else + return q.value(); + } +} + +Xapian::Enquire +Query::Private::make_enquire(const std::string& expr, + Field::Id sortfield_id, + QueryFlags qflags) const +{ + auto enq{store_.xapian_db().enquire()}; + enq.set_query(make_query(store_, expr, parser_flags_)); + sort_enquire(enq, sortfield_id, qflags); + + return enq; +} + +Xapian::Enquire +Query::Private::make_related_enquire(const StringSet& thread_ids, + Field::Id sortfield_id, + QueryFlags qflags) const +{ + auto enq{store_.xapian_db().enquire()}; + std::vector<Xapian::Query> qvec; + qvec.reserve(thread_ids.size()); + + for (auto&& t : thread_ids) + qvec.emplace_back(field_from_id(Field::Id::ThreadId).xapian_term(t)); + + Xapian::Query qr{Xapian::Query::OP_OR, qvec.begin(), qvec.end()}; + enq.set_query(qr); + + sort_enquire(enq, sortfield_id, qflags); + + return enq; +} + +struct ThreadKeyMaker : public Xapian::KeyMaker { + ThreadKeyMaker(const QueryMatches& matches) : match_info_(matches) {} + std::string operator()(const Xapian::Document& doc) const override { + const auto it{match_info_.find(doc.get_docid())}; + return (it == match_info_.end()) ? "" : it->second.thread_path; + } + const QueryMatches& match_info_; +}; + +Option<QueryResults> +Query::Private::run_threaded(QueryResults&& qres, Xapian::Enquire& enq, QueryFlags qflags, + size_t maxnum) const +{ + const auto descending{any_of(qflags & QueryFlags::Descending)}; + + calculate_threads(qres, descending); + + ThreadKeyMaker key_maker{qres.query_matches()}; + enq.set_sort_by_key(&key_maker, descending); + + DeciderInfo minfo; + minfo.matches = qres.query_matches(); + auto mset{enq.get_mset(0, maxnum, {}, make_thread_decider(qflags, minfo).get())}; + mset.fetch(); + + return QueryResults{mset, std::move(qres.query_matches())}; +} + +Option<QueryResults> +Query::Private::run_singular(const std::string& expr, + Field::Id sortfield_id, + QueryFlags qflags, size_t maxnum) const +{ + // i.e. a query _without_ related messages, but still possibly + // with threading. + // + // In the threading case, the sortfield-id is ignored, we always sort by + // date (since threading the threading results are always by date.) + + const auto singular_qflags{qflags | QueryFlags::Leader}; + const auto threading{any_of(qflags & QueryFlags::Threading)}; + + DeciderInfo minfo{}; +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wextra" + auto enq{make_enquire(expr, threading ? Field::Id::Date : sortfield_id, qflags)}; +#pragma GCC diagnostic ignored "-Wswitch-default" +#pragma GCC diagnostic pop + auto mset{enq.get_mset(0, maxnum, {}, + make_leader_decider(singular_qflags, minfo).get())}; + mset.fetch(); + + auto qres{QueryResults{mset, std::move(minfo.matches)}}; + + return threading ? run_threaded(std::move(qres), enq, qflags, maxnum) : qres; +} + +static Option<std::string> +opt_string(const Xapian::Document& doc, Field::Id id) noexcept +{ + const auto value_no{field_from_id(id).value_no()}; + std::string val = + xapian_try([&] { return doc.get_value(value_no); }, std::string{""}); + if (val.empty()) + return Nothing; + else + return Some(std::move(val)); +} + +Option<QueryResults> +Query::Private::run_related(const std::string& expr, + Field::Id sortfield_id, + QueryFlags qflags, size_t maxnum) const +{ + // i.e. a query _with_ related messages and possibly with threading. + // + // In the threading case, the sortfield-id is ignored, we always sort by + // date (since threading the threading results are always by date.); + // moreover, in either threaded or non-threaded case, we sort the first + // ("leader") query by date, i.e, we prefer the newest or oldest + // (descending) messages. + const auto leader_qflags{QueryFlags::Leader | qflags}; + const auto threading{any_of(qflags & QueryFlags::Threading)}; + + // Run our first, "leader" query + DeciderInfo minfo{}; + auto enq{make_enquire(expr, Field::Id::Date, leader_qflags)}; + const auto mset{ + enq.get_mset(0, maxnum, {}, make_leader_decider(leader_qflags, minfo).get())}; + + // Gather the thread-ids we found + mset.fetch(); + minfo.thread_ids.reserve(mset.size()); + for (auto it = mset.begin(); it != mset.end(); ++it) + if (auto thread_id{opt_string(it.get_document(), Field::Id::ThreadId)}; thread_id) + minfo.thread_ids.emplace(std::move(*thread_id)); + + // Now, determine the "related query". + // + // In the threaded-case, we search among _all_ messages, since complete + // threads are preferred; no need to sort in that case since the search + // is unlimited and the sorting happens during threading. + auto r_enq = std::invoke([&]{ + if (threading) + return make_related_enquire(minfo.thread_ids, Field::Id::Date, + qflags); + else + return make_related_enquire(minfo.thread_ids, sortfield_id, qflags); + }); + + const auto r_mset{r_enq.get_mset(0, threading ? store_.size() : maxnum, {}, + make_related_decider(qflags, minfo).get())}; + auto qres{QueryResults{r_mset, std::move(minfo.matches)}}; + return threading ? run_threaded(std::move(qres), r_enq, qflags, maxnum) : qres; +} + +Option<QueryResults> +Query::Private::run(const std::string& expr, Field::Id sortfield_id, QueryFlags qflags, + size_t maxnum) const +{ + const auto eff_maxnum{maxnum == 0 ? store_.size() : maxnum}; + + if (any_of(qflags & QueryFlags::IncludeRelated)) + return run_related(expr, sortfield_id, qflags, eff_maxnum); + else + return run_singular(expr, sortfield_id, qflags, eff_maxnum); +} + +Result<QueryResults> +Query::run(const std::string& expr, Field::Id sortfield_id, + QueryFlags qflags, size_t maxnum) const +{ + // some flags are for internal use only. + g_return_val_if_fail(none_of(qflags & QueryFlags::Leader), + Err(Error::Code::InvalidArgument, "cannot pass Leader flag")); + + StopWatch sw{ + mu_format("query: '{}'; (related:{}; threads:{}; ngrams:{}; max-size:{})", + expr, + any_of(qflags & QueryFlags::IncludeRelated) ? "yes" : "no", + any_of(qflags & QueryFlags::Threading) ? "yes" : "no", + any_of(priv_->parser_flags_ & ParserFlags::SupportNgrams) ? "yes" : "no", + maxnum == 0 ? std::string{"∞"} : std::to_string(maxnum))}; + + return xapian_try_result([&]{ + if (auto&& res = priv_->run(expr, sortfield_id, qflags, maxnum); res) + return Result<QueryResults>(Ok(std::move(res.value()))); + else + return Result<QueryResults>(Err(Error::Code::Query, + "failed to run query")); + }); +} + +size_t +Query::count(const std::string& expr) const +{ + return xapian_try( + [&] { + const auto enq{priv_->make_enquire(expr, {}, {})}; + auto mset{enq.get_mset(0, priv_->store_.size())}; + mset.fetch(); + return mset.size(); + }, + 0); +} + +/* LCOV_EXCL_START*/ +std::string +Query::parse(const std::string& expr, bool xapian) const +{ + if (xapian) + return make_query(priv_->store_, expr, + priv_->parser_flags_).get_description(); + else + return parse_query(expr).to_string(); +} +/* LCOV_EXCL_STOP*/ diff --git a/lib/mu-query.hh b/lib/mu-query.hh new file mode 100644 index 0000000..7ca1275 --- /dev/null +++ b/lib/mu-query.hh @@ -0,0 +1,100 @@ +/* +** Copyright (C) 2008-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify +** it under the terms of the GNU General Public License as published by +** the Free Software Foundation; either version 3 of the License, or +** (at your option) any later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef __MU_QUERY_HH__ +#define __MU_QUERY_HH__ + +#include <memory> + +#include <glib.h> +#include <mu-store.hh> +#include <mu-query-results.hh> +#include <utils/mu-utils.hh> +#include <utils/mu-option.hh> +#include <utils/mu-result.hh> +#include <message/mu-message.hh> + +namespace Mu { + +class Query { +public: + /** + * Run a query on the store + * + * @param expr the search expression + * @param sortfield_id the sortfield-id. Default to Date + * @param flags query flags + * @param maxnum maximum number of results to return. 0 for 'no limit' + * + * @return the query-results or an error + */ + Result<QueryResults> run(const std::string& expr, + Field::Id sortfield_id = Field::Id::Date, + QueryFlags flags = QueryFlags::None, + size_t maxnum = 0) const; + + /** + * run a Xapian query to count the number of matches; for the syntax, please + * refer to the mu-query manpage + * + * @param expr the search expression; use "" to match all messages + * + * @return the number of matches + */ + size_t count(const std::string& expr = "") const; + + /** + * For debugging, get the internal string representation of the parsed + * query + * + * @param expr a xapian search expression + * @param xapian if true, show Xapian's internal representation, + * otherwise, mu's. + + * @return the string representation of the query + */ + std::string parse(const std::string& expr, bool xapian) const; + +private: + friend class Store; + + /** + * Construct a new Query instance. + * + * @param store a MuStore object + */ + Query(const Store& store); + /** + * DTOR + * + */ + ~Query(); + + /** + * Move CTOR + * + * @param other + */ + + struct Private; + std::unique_ptr<Private> priv_; +}; +} // namespace Mu + +#endif /*__MU_QUERY_HH__*/ diff --git a/lib/mu-scanner.cc b/lib/mu-scanner.cc new file mode 100644 index 0000000..bbc8d7e --- /dev/null +++ b/lib/mu-scanner.cc @@ -0,0 +1,425 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include "mu-scanner.hh" + +#include "config.h" + +#include <chrono> +#include <mutex> +#include <atomic> +#include <thread> +#include <cstring> + +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> + +#include <glib.h> + +#include "utils/mu-utils.hh" +#include "utils/mu-utils-file.hh" +#include "utils/mu-error.hh" + +using namespace Mu; + +using Mode = Scanner::Mode; + +/* + * dentry->d_ino, dentry->d_type may not be available + */ +struct dentry_t { + dentry_t(const struct dirent *dentry): +#if HAVE_DIRENT_D_INO + d_ino{dentry->d_ino}, +#endif /*HAVE_DIRENT_D_INO*/ + +#if HAVE_DIRENT_D_TYPE + d_type(dentry->d_type), +#endif /*HAVE_DIRENT_D_TYPE*/ + d_name{static_cast<const char*>(dentry->d_name)} {} +#if HAVE_DIRENT_D_INO + ino_t d_ino; +#endif /*HAVE_DIRENT_D_INO*/ + +#if HAVE_DIRENT_D_TYPE + unsigned char d_type; +#endif /*HAVE_DIRENT_D_TYPE*/ + + std::string d_name; +}; + +struct Scanner::Private { + Private(const std::string& root_dir, Scanner::Handler handler, Mode mode): + root_dir_{root_dir}, handler_{handler}, mode_{mode} { + if (root_dir_.length() > PATH_MAX) + throw Mu::Error{Error::Code::InvalidArgument, "path is too long"}; + if (!handler_) + throw Mu::Error{Error::Code::InvalidArgument, "missing handler"}; + } + ~Private() { stop(); } + + Result<void> start(); + void stop(); + + bool process_dentry(const std::string& path, const dentry_t& dentry, + bool is_maildir); + bool process_dir(const std::string& path, bool is_maildir); + + int lazy_stat(const char *fullpath, struct stat *stat_buf, + const dentry_t& dentry); + + bool maildirs_only_mode() const { return mode_ == Mode::MaildirsOnly; } + + const std::string root_dir_; + const Scanner::Handler handler_; + Mode mode_; + std::atomic<bool> running_{}; + std::mutex lock_; +}; + +static bool +ignore_dentry(const dentry_t& dentry) +{ + const auto d_name{dentry.d_name.c_str()}; + + /* dotdir? */ + if (d_name[0] == '\0' || (d_name[1] == '\0' && d_name[0] == '.') || + (d_name[2] == '\0' && d_name[0] == '.' && d_name[1] == '.')) + return true; + + if (d_name[0] != 't' && d_name[0] != 'h' && d_name[0] != '.') + return false; /* don't ignore */ + + if (::strcmp(d_name, "tmp") == 0 || ::strcmp(d_name, "hcache.db") == 0) + return true; // ignore + + if (d_name[0] == '.') + for (auto dname : { "nnmaildir", "notmuch", "noindex", "noupdate"}) + if (::strcmp(d_name + 1, dname) == 0) + return true; + + return false; /* don't ignore */ +} + + +/* + * stat() if necessary (we'd like to avoid it), which we can if we only need the + * file-type and we already have that from the dentry. + */ +int +Scanner::Private::lazy_stat(const char *path, struct stat *stat_buf, const dentry_t& dentry) +{ +#if HAVE_DIRENT_D_TYPE + if (maildirs_only_mode()) { + switch (dentry.d_type) { + case DT_REG: + stat_buf->st_mode = S_IFREG; + return 0; + case DT_DIR: + stat_buf->st_mode = S_IFDIR; + return 0; + default: + /* LNK is inconclusive; we need a stat. */ + break; + } + } +#endif /*HAVE_DIRENT_D_TYPE*/ + + int res = ::stat(path, stat_buf); + if (res != 0) + mu_warning("failed to stat {}: {}", path, g_strerror(errno)); + + return res; +} + + +bool +Scanner::Private::process_dentry(const std::string& path, const dentry_t& dentry, + bool is_maildir) +{ + if (ignore_dentry(dentry)) + return true; + + auto call_handler=[&](auto&& path, auto&& statbuf, auto&& htype)->bool { + return maildirs_only_mode() ? true : handler_(path, statbuf, htype); + }; + + const auto fullpath{join_paths(path, dentry.d_name)}; + struct stat statbuf{}; + if (lazy_stat(fullpath.c_str(), &statbuf, dentry) != 0) + return false; + + if (maildirs_only_mode() && S_ISDIR(statbuf.st_mode) && dentry.d_name == "cur") { + handler_(path/*without cur*/, {}, Scanner::HandleType::Maildir); + return true; // found maildir; no need to recurse further. + } + + if (S_ISDIR(statbuf.st_mode)) { + const auto new_cur = dentry.d_name == "cur" || dentry.d_name == "new"; + const auto htype = + new_cur ? + Scanner::HandleType::EnterNewCur : + Scanner::HandleType::EnterDir; + + const auto res = call_handler(fullpath, &statbuf, htype); + if (!res) + return true; // skip + + process_dir(fullpath, new_cur); + return call_handler(fullpath, &statbuf, Scanner::HandleType::LeaveDir); + + } else if (S_ISREG(statbuf.st_mode) && is_maildir) + return call_handler(fullpath, &statbuf, Scanner::HandleType::File); + + mu_debug("skip {} (neither maildir-file nor directory)", fullpath); + + return true; +} + +bool +Scanner::Private::process_dir(const std::string& path, bool is_maildir) +{ + if (!running_) + return true; /* we're done */ + + if (G_UNLIKELY(path.length() > PATH_MAX)) { + // note: unlikely to hit this, one case would be a self-referential + // symlink; that should be caught earlier, so this is just a backstop. + mu_warning("path is too long: {}", path); + return false; + } + + const auto dir{::opendir(path.c_str())}; + if (G_UNLIKELY(!dir)) { + mu_warning("failed to scan dir {}: {}", path, g_strerror(errno)); + return false; + } + + std::vector<dentry_t> dir_entries; + while (running_) { + errno = 0; + if (const auto& dentry{::readdir(dir)}; dentry) { +#if HAVE_DIRENT_D_TYPE /* optimization: filter out non-dirs early. NB not all file-systems support + * returning the file-type in `d_type`, so don't skip `DT_UNKNOWN`. + */ + if (maildirs_only_mode() && + dentry->d_type != DT_DIR && + dentry->d_type != DT_LNK && + dentry->d_type != DT_UNKNOWN) + continue; +#endif /*HAVE_DIRENT_D_TYPE*/ + dir_entries.emplace_back(dentry); + continue; + } else if (errno != 0) { + mu_warning("failed to read {}: {}", path, g_strerror(errno)); + continue; + } + + break; + } + ::closedir(dir); + +#if HAVE_DIRENT_D_INO + // sort by i-node; much faster on rotational (HDDs) devices and on SSDs + // sort is quick enough to not matter much + std::sort(dir_entries.begin(), dir_entries.end(), + [](auto&& d1, auto&& d2){ return d1.d_ino < d2.d_ino; }); +#endif /*HAVEN_DIRENT_D_INO*/ + + // now process... + for (auto&& dentry: dir_entries) + process_dentry(path, dentry, is_maildir); + + return true; +} + +Result<void> +Scanner::Private::start() +{ + const auto mode{F_OK | R_OK}; + if (G_UNLIKELY(::access(root_dir_.c_str(), mode) != 0)) + return Err(Error::Code::File, "'{}' is not readable: {}", root_dir_, + g_strerror(errno)); + + struct stat statbuf {}; + if (G_UNLIKELY(::stat(root_dir_.c_str(), &statbuf) != 0)) + return Err(Error::Code::File, "'{}' is not stat'able: {}", + root_dir_, g_strerror(errno)); + + if (G_UNLIKELY(!S_ISDIR(statbuf.st_mode))) + return Err(Error::Code::File, + "'{}' is not a directory", root_dir_); + + running_ = true; + mu_debug("starting scan @ {}", root_dir_); + + const auto bname{basename(root_dir_)}; + const auto is_maildir = bname == "cur" || bname == "new"; + + const auto start{std::chrono::steady_clock::now()}; + process_dir(root_dir_, is_maildir); + const auto elapsed = std::chrono::steady_clock::now() - start; + mu_debug("finished scan of {} in {} ms", root_dir_, to_ms(elapsed)); + running_ = false; + + return Ok(); +} + +void +Scanner::Private::stop() +{ + if (running_) { + mu_debug("stopping scan"); + running_ = false; + } +} + +Scanner::Scanner(const std::string& root_dir, Scanner::Handler handler, Mode flavor) + : priv_{std::make_unique<Private>(root_dir, handler, flavor)} +{} + +Scanner::~Scanner() = default; + +Result<void> +Scanner::start() +{ + if (priv_->running_) + return Ok(); // nothing to do + + auto res = priv_->start(); /* blocks */ + priv_->running_ = false; + + return res; +} + +void +Scanner::stop() +{ + std::lock_guard l(priv_->lock_); + priv_->stop(); +} + +bool +Scanner::is_running() const +{ + return priv_->running_; +} + + +#if BUILD_TESTS +/* LCOV_EXCL_START*/ +#include "mu-test-utils.hh" + +static void +test_scan_maildirs() +{ + allow_warnings(); + + size_t count{}; + Scanner scanner{ + MU_TESTMAILDIR, + [&](const std::string& fullpath, const struct stat* statbuf, auto&& htype) -> bool { + ++count; + g_usleep(10000); + return true; + }}; + assert_valid_result(scanner.start()); + scanner.stop(); + count = 0; + assert_valid_result(scanner.start()); + + while (scanner.is_running()) { g_usleep(100000); } + + // very rudimentary test... + g_assert_cmpuint(count,==,23); +} + +static void +test_count_maildirs() +{ + allow_warnings(); + + std::vector<std::string> dirs; + Scanner scanner{ + MU_TESTMAILDIR2, + [&](const std::string& fullpath, const struct stat* statbuf, auto&& htype) -> bool { + dirs.emplace_back(basename(fullpath)); + return true; + }, Scanner::Mode::MaildirsOnly}; + assert_valid_result(scanner.start()); + + while (scanner.is_running()) { g_usleep(1000); } + + g_assert_cmpuint(dirs.size(),==,3); + g_assert_true(seq_find_if(dirs, [](auto& p){return p == "bar";}) != dirs.end()); + g_assert_true(seq_find_if(dirs, [](auto& p){return p == "Foo";}) != dirs.end()); + g_assert_true(seq_find_if(dirs, [](auto& p){return p == "wom_bat";}) != dirs.end()); +} + +static void +test_fail_nonexistent() +{ + allow_warnings(); + + Scanner scanner{"/foo/bar/non-existent", + [&](auto&& a1, auto&& a2, auto&& a3){ return false; }}; + g_assert_false(scanner.is_running()); + g_assert_false(!!scanner.start()); + g_assert_false(scanner.is_running()); +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/scanner/scan-maildirs", test_scan_maildirs); + g_test_add_func("/scanner/count-maildirs", test_count_maildirs); + g_test_add_func("/scanner/fail-nonexistent", test_fail_nonexistent); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ + +#if BUILD_LIST_MAILDIRS + +static bool +on_path(const std::string& path, struct stat* statbuf, Scanner::HandleType htype) +{ + mu_println("{}", path); + return true; +} + +int +main (int argc, char *argv[]) +{ + if (argc < 2) { + mu_printerrln("expected: path to maildir"); + return 1; + } + + Scanner scanner{argv[1], on_path, Mode::MaildirsOnly}; + + scanner.start(); + + return 0; +} +/* LCOV_EXCL_STOP*/ +#endif /*BUILD_LIST_MAILDIRS*/ diff --git a/lib/mu-scanner.hh b/lib/mu-scanner.hh new file mode 100644 index 0000000..e124c52 --- /dev/null +++ b/lib/mu-scanner.hh @@ -0,0 +1,122 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_SCANNER_HH__ +#define MU_SCANNER_HH__ + +#include <functional> +#include <memory> +#include <utils/mu-result.hh> + +#include <dirent.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> + +namespace Mu { + +/** + * @brief Maildir scanner + * + * Scans maildir (trees) recursively, and calls the Handler callback for + * directories & files. + * + * It filters out (i.e., does *not* call the handler for): + * - files starting with '.' + * - files that do not live in a cur / new leaf maildir + * - directories '.' and '..' and 'tmp' +*/ +class Scanner { + public: + enum struct HandleType { + /* + * Mode: All + */ + File, + EnterNewCur, /* cur/ or new/ */ + EnterDir, /* some other directory */ + LeaveDir, + /* + * Mode: Maildir + */ + Maildir, + }; + + /** + * Callback handler function + * + * path: full file-system path + * statbuf: stat result or nullptr (for Mode::MaildirsOnly) + * htype: HandleType. For Mode::MaildirsOnly only Maildir + */ + using Handler = std::function< + bool(const std::string& path, struct stat* statbuf, HandleType htype)>; + + /** + * Running mode for this Scanner + */ + enum struct Mode { + All, /**< Vanilla */ + MaildirsOnly /**< Only return maildir to handler */ + }; + + /** + * Construct a scanner object for scanning a directory, recursively. + * + * If handler is a directory + * + * @param root_dir root dir to start scanning + * @param handler handler function for some direntry + * @param options options to influence behavior + */ + Scanner(const std::string& root_dir, Handler handler, Mode mode = Mode::All); + + /** + * DTOR + */ + ~Scanner(); + + /**# + * Start the scan; this is a blocking call than runs until + * finished or (from another thread) stop() is called. + * + * @return Ok if starting worked; an Error otherwise + */ + Result<void> start(); + + /** + * Request stopping the scan if it's running; otherwise do nothing + */ + void stop(); + + /** + * Is a scan currently running? + * + * @return true or false + */ + bool is_running() const; + +private: + struct Private; + std::unique_ptr<Private> priv_; +}; + +} // namespace Mu + +#endif /* MU_SCANNER_HH__ */ diff --git a/lib/mu-script.cc b/lib/mu-script.cc new file mode 100644 index 0000000..81d481b --- /dev/null +++ b/lib/mu-script.cc @@ -0,0 +1,162 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include "config.h" + +#include "mu-script.hh" +#include "mu/mu-options.hh" +#include "utils/mu-utils.hh" +#include "utils/mu-option.hh" + +#include <fstream> +#include <iostream> + +#ifdef BUILD_GUILE +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wredundant-decls" +#include <libguile.h> +#pragma GCC diagnostic pop +#endif /*BUILD_GUILE*/ + +using namespace Mu; + +static std::string +get_name(const std::string& path) +{ + auto pos = path.find_last_of("/"); + if (pos == std::string::npos) + return path; + + auto name = path.substr(pos + 1); + + pos = name.find_last_of("."); + if (pos == std::string::npos) + return name; + + return name.substr(0, pos); +} + + +static Mu::Option<Mu::ScriptInfo> +get_info(std::string&& path, const std::string& prefix) +{ + std::ifstream file{path}; + if (!file.is_open()) { + mu_warning ("failed to open {}", path); + return Nothing; + } + + Mu::ScriptInfo info{}; + info.path = path; + info.name = get_name(path); + + std::string line; + while (std::getline(file, line)) { + + if (line.find(prefix) != 0) + continue; + + line = line.substr(prefix.length()); + + if (info.oneline.empty()) + info.oneline = line; + else + info.description += line; + } + + // std::cerr << "ONELINE: " << info.oneline << '\n'; + // std::cerr << "DESCR : " << info.description << '\n'; + + return info; +} + + + +static void +script_infos_in_dir(const std::string& scriptdir, Mu::ScriptInfos& infos) +{ + DIR *dir = opendir(scriptdir.c_str()); + if (!dir) { + mu_debug("failed to open '{}': {}", scriptdir, + g_strerror(errno)); + return; + } + + const std::string ext{".scm"}; + + struct dirent *dentry; + while ((dentry = readdir(dir))) { + + if (!g_str_has_suffix(dentry->d_name, ext.c_str())) + continue; + + auto&& info = get_info(scriptdir + "/" + dentry->d_name, ";; INFO: "); + if (!info) + continue; + + infos.emplace_back(std::move(*info)); + } + + closedir(dir); /* ignore error checking... */ +} + + +Mu::ScriptInfos +Mu::script_infos(const Mu::ScriptPaths& paths) +{ + /* create a list of names, paths */ + ScriptInfos infos; + for (auto&& dir: paths) { + script_infos_in_dir(dir, infos); + } + + std::sort(infos.begin(), infos.end(), [](auto&& i1, auto&& i2) { + return i1.name < i2.name; + }); + + return infos; +} + +Result<void> +Mu::run_script(const std::string& path, + const std::vector<std::string>& args) +{ +#ifndef BUILD_GUILE + return Err(Error::Code::Script, + "guile script support is not available"); +#else + std::string mainargs; + for (auto&& arg: args) + mainargs += mu_format("{}\"{}\"", mainargs.empty() ? "" : " ", arg); + auto expr = mu_format("(main '(\"{}\" {}))", get_name(path), mainargs); + + std::vector<const char*> argv = { + GUILE_BINARY, + "-l", path.c_str(), + "-c", expr.c_str(), + }; + + /* does not return */ + scm_boot_guile(argv.size(), const_cast<char**>(argv.data()), + [](void *closure, int argc, char **argv) { + scm_shell(argc, argv); + }, NULL); + + return Ok(); +#endif /*BUILD_GUILE*/ +} diff --git a/lib/mu-script.hh b/lib/mu-script.hh new file mode 100644 index 0000000..48ff45a --- /dev/null +++ b/lib/mu-script.hh @@ -0,0 +1,65 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#ifndef MU_SCRIPT_HH__ +#define MU_SCRIPT_HH__ + +#include <string> +#include <vector> + +#include <utils/mu-result.hh> + +namespace Mu { + +/** + * Information about a script. + * + */ +struct ScriptInfo { + std::string name; /**< Name of script */ + std::string path; /**< Full path to script */ + std::string oneline; /**< One-line description */ + std::string description; /**< More help */ +}; + +/// Sequence of script infos. +using ScriptInfos = std::vector<ScriptInfo>; + +/** + * Get information about the available scripts + * + * @return infos + */ +using ScriptPaths = std::vector<std::string>; +ScriptInfos script_infos(const ScriptPaths& paths); + + +/** + * Run some specific script + * + * @param path full path to the scripts + * @param args argument vector to pass to the script + * + * @return Ok() or some error; however, note that this does not return after succesfully + * starting a script. + */ +Result<void> run_script(const std::string& path, const std::vector<std::string>& args); + +} // namepace Mu + +#endif /* MU_SCRIPT_HH__ */ diff --git a/lib/mu-server.cc b/lib/mu-server.cc new file mode 100644 index 0000000..62c9ca0 --- /dev/null +++ b/lib/mu-server.cc @@ -0,0 +1,1087 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-server.hh" + +#include "message/mu-message.hh" + +#include <fstream> +#include <sstream> +#include <string> +#include <algorithm> +#include <atomic> +#include <thread> +#include <mutex> +#include <variant> +#include <functional> + +#include <cstring> +#include <glib.h> +#include <glib/gprintf.h> +#include <unistd.h> + +#include "mu-maildir.hh" +#include "mu-query.hh" +#include "mu-store.hh" + +#include "utils/mu-utils.hh" +#include "utils/mu-utils-file.hh" + +#include "utils/mu-option.hh" +#include "utils/mu-command-handler.hh" +#include "utils/mu-readline.hh" + +using namespace Mu; + +/* LCOV_EXCL_START */ + +/// output stream to _either_ a file or to a stringstream +struct OutputStream { + /** + * Construct an OutputStream for a tempfile + * + * @param tmp_dir dir for temp files + */ + OutputStream(const std::string& tmp_dir): + fname_{join_paths(tmp_dir, + mu_format("mu-{}.eld", g_get_monotonic_time()))}, + out_{std::ofstream{fname_}} { + if (!out().good()) + throw Mu::Error{Error::Code::File, "failed to create temp-file"}; + } + /** + * Construct an OutputStream for a stringstream + * + * @param cdr name of the output (e.g., "contacts") + * + * @return + */ + OutputStream(): out_{std::ostringstream{}} {} + + /** + * Get a writable ostream + * + * @return an ostream + */ + std::ostream& out() { + if (std::holds_alternative<std::ofstream>(out_)) + return std::get<std::ofstream>(out_); + else + return std::get<std::ostringstream>(out_); + } + + /// conversion + operator std::ostream&() { return out(); } + + /** + * Get the output as a string, either something like, either a lisp form + * or a the full path to a temp file containing the same. + * + * @return lisp form or path + */ + std::string to_string() const { + return std::holds_alternative<std::ostringstream>(out_) ? + std::get<std::ostringstream>(out_).str() : + quote(fname_); + } + + /** + * Delete file, if any. Only do this when the OutputStream is no + * longer needed. + */ + void unlink () { + if (fname_.empty()) + return; + if (auto&&res{::unlink(fname_.c_str())}; res != 0) + mu_warning("failed to unlink '{}'", ::strerror(res)); + else + mu_debug("unlinked output-stream {}", fname_); + } + +private: + std::string fname_; + using OutType = std::variant<std::ofstream, std::ostringstream>; + OutType out_; +}; + + + +/// @brief object to manage the server-context for all commands. +struct Server::Private { + Private(Store& store, const Server::Options& opts, Output output) + : store_{store}, options_{opts}, output_{output}, + command_handler_{make_command_map()}, + keep_going_{true}, + tmp_dir_{unwrap(make_temp_dir())} + {} + + ~Private() { + indexer().stop(); + if (index_thread_.joinable()) + index_thread_.join(); + if (!tmp_dir_.empty()) + remove_directory(tmp_dir_); + } + // + // construction helpers + // + CommandHandler::CommandInfoMap make_command_map(); + + // + // acccessors + Store& store() { return store_; } + const Store& store() const { return store_; } + Indexer& indexer() { return store().indexer(); } + //CommandMap& command_map() const { return command_map_; } + + // + // invoke + // + bool invoke(const std::string& expr) noexcept; + + // + // output + void output(const std::string& str, Server::OutputFlags flags = {}) const { + if (output_) + output_(str, flags); + } + void output_sexp(const Sexp& sexp, Server::OutputFlags flags = {}) const { + output(sexp.to_string(), flags); + } + + size_t output_results(const QueryResults& qres, size_t batch_size) const; + + // + // handlers for various commands. + // + void add_handler(const Command& cmd); + void compose_handler(const Command& cmd); + void contacts_handler(const Command& cmd); + void data_handler(const Command& cmd); + void find_handler(const Command& cmd); + void help_handler(const Command& cmd); + void index_handler(const Command& cmd); + void move_handler(const Command& cmd); + void mkdir_handler(const Command& cmd); + void ping_handler(const Command& cmd); + void queries_handler(const Command& cmd); + void quit_handler(const Command& cmd); + void remove_handler(const Command& cmd); + void view_handler(const Command& cmd); + +private: + void move_docid(Store::Id docid, Option<std::string> flagstr, + bool new_name, bool no_view); + + void perform_move(Store::Id docid, + const Message& msg, + const std::string& maildirarg, + Flags flags, + bool new_name, + bool no_view); + + void view_mark_as_read(Store::Id docid, Message&& msg, bool rename); + + OutputStream make_output_stream() const { + if (options_.allow_temp_file) + return OutputStream{tmp_dir_}; + else + return OutputStream{}; + } + + std::ofstream make_temp_file_stream(std::string& fname) const; + + Store& store_; + Server::Options options_; + Server::Output output_; + const CommandHandler command_handler_; + std::atomic<bool> keep_going_{}; + std::thread index_thread_; + std::string tmp_dir_; +}; + +static void +append_metadata(std::string& str, const QueryMatch& qmatch) +{ + const auto td{::atoi(qmatch.thread_date.c_str())}; + + str += mu_format(" :meta (:path \"{}\" :level {} :date \"{}\" " + ":data-tstamp ({} {} 0)", + qmatch.thread_path, + qmatch.thread_level, + qmatch.thread_date, + static_cast<unsigned>(td >> 16), + static_cast<unsigned>(td & 0xffff)); + + if (qmatch.has_flag(QueryMatch::Flags::Root)) + str += " :root t"; + if (qmatch.has_flag(QueryMatch::Flags::Related)) + str += " :related t"; + if (qmatch.has_flag(QueryMatch::Flags::First)) + str += " :first-child t"; + if (qmatch.has_flag(QueryMatch::Flags::Last)) + str += " :last-child t"; + if (qmatch.has_flag(QueryMatch::Flags::Orphan)) + str += " :orphan t"; + if (qmatch.has_flag(QueryMatch::Flags::Duplicate)) + str += " :duplicate t"; + if (qmatch.has_flag(QueryMatch::Flags::HasChild)) + str += " :has-child t"; + if (qmatch.has_flag(QueryMatch::Flags::ThreadSubject)) + str += " :thread-subject t"; + + str += ')'; +} + +/* + * A message here consists of a message s-expression with optionally a :docid + * and/or :meta expression added. + * + * We could parse the sexp and use the Sexp APIs to add some things... but... + * it's _much_ faster to directly work on the string representation: remove the + * final ')', add a few items, and add the ')' again. + */ +static std::string +msg_sexp_str(const Message& msg, Store::Id docid, const Option<QueryMatch&> qm) +{ + auto&& sexpstr{msg.document().sexp_str()}; + + if (docid != 0 || qm) { + sexpstr.reserve(sexpstr.size () + (docid == 0 ? 0 : 16) + (qm ? 64 : 0)); + + // remove the closing ( ... ) + sexpstr.erase(sexpstr.end() - 1); + + if (docid != 0) + sexpstr += " :docid " + to_string(docid); + if (qm) + append_metadata(sexpstr, *qm); + + sexpstr += ')'; // ... end close it again. + } + + return sexpstr; +} + + +CommandHandler::CommandInfoMap +Server::Private::make_command_map() +{ + CommandHandler::CommandInfoMap cmap; + + using CommandInfo = CommandHandler::CommandInfo; + using ArgMap = CommandHandler::ArgMap; + using ArgInfo = CommandHandler::ArgInfo; + using Type = Sexp::Type; + using Type = Sexp::Type; + + cmap.emplace( + "add", + CommandInfo{ + ArgMap{{":path", ArgInfo{Type::String, true, "file system path to the message"}}}, + "add a message to the store", + [&](const auto& params) { add_handler(params); }}); + + cmap.emplace( + "contacts", + CommandInfo{ + ArgMap{{":personal", ArgInfo{Type::Symbol, false, "only personal contacts"}}, + {":after", + ArgInfo{Type::String, false, "only contacts seen after time_t string"}}, + {":tstamp", ArgInfo{Type::String, false, "return changes since tstamp"}}, + {":maxnum", ArgInfo{Type::Number, false, "max number of contacts to return"}} + }, + "get contact information", + [&](const auto& params) { contacts_handler(params); }}); + + cmap.emplace( + "data", + CommandInfo{ + ArgMap{{":kind", ArgInfo{Type::Symbol, true, "kind of data (maildirs)"}}}, + "request data of some kind", + [&](const auto& params) { data_handler(params); }}); + + cmap.emplace( + "find", + CommandInfo{ + ArgMap{{":query", ArgInfo{Type::String, true, "search expression"}}, + {":threads", + ArgInfo{Type::Symbol, false, "whether to include threading information"}}, + {":sortfield", ArgInfo{Type::Symbol, false, "the field to sort results by"}}, + {":descending", + ArgInfo{Type::Symbol, false, "whether to sort in descending order"}}, + {":batch-size", ArgInfo{Type::Number, false, "batch size for result"}}, + {":maxnum", ArgInfo{Type::Number, false, "maximum number of result (hint)"}}, + {":skip-dups", + ArgInfo{Type::Symbol, + false, + "whether to skip messages with duplicate message-ids"}}, + {":include-related", + ArgInfo{Type::Symbol, + false, + "whether to include other message related to matching ones"}}}, + "query the database for messages", + [&](const auto& params) { find_handler(params); }}); + + cmap.emplace( + "help", + CommandInfo{ + ArgMap{{":command", ArgInfo{Type::Symbol, false, "command to get information for"}}, + {":full", ArgInfo{Type::Symbol, false, "show full descriptions"}}}, + "get information about one or all commands", + [&](const auto& params) { help_handler(params); }}); + cmap.emplace( + "index", + CommandInfo{ + ArgMap{{":my-addresses", ArgInfo{Type::List, false, "list of 'my' addresses"}}, + {":cleanup", + ArgInfo{Type::Symbol, + false, + "whether to remove stale messages from the store"}}, + {":lazy-check", + ArgInfo{Type::Symbol, + false, + "whether to avoid indexing up-to-date directories"}}}, + "scan maildir for new/updated/removed messages", + [&](const auto& params) { index_handler(params); }}); + cmap.emplace( + "mkdir", + CommandInfo{ + ArgMap{ + {":path", ArgInfo{Type::String, true, "location for the new maildir"}}, + {":update", ArgInfo{Type::Symbol, false, + "whether to send an update after creating"}} + }, "create a new maildir", + [&](const auto& params) { mkdir_handler(params); }}); + cmap.emplace( + "move", + CommandInfo{ + ArgMap{ + {":docid", ArgInfo{Type::Number, false, "document-id"}}, + {":msgid", ArgInfo{Type::String, false, "message-id"}}, + {":flags", ArgInfo{Type::String, false, "new flags for the message"}}, + {":maildir", ArgInfo{Type::String, false, "the target maildir"}}, + {":rename", ArgInfo{Type::Symbol, false, "change filename when moving"}}, + {":no-view", + ArgInfo{Type::Symbol, false, "if set, do not hint at updating the view"}}, + }, + "move messages and/or change their flags", + [&](const auto& params) { move_handler(params); }}); + cmap.emplace( + "ping", + CommandInfo{ + ArgMap{}, + "ping the mu-server and get server information in the response", + [&](const auto& params) { ping_handler(params); }}); + + cmap.emplace( + "queries", + CommandInfo{ + ArgMap{ + {":queries", + ArgInfo{Type::List, false, "queries for which to get read/unread numbers"}}, + }, + "get unread/totals information for a list of queries", + [&](const auto& params) { queries_handler(params); }}); + + cmap.emplace("quit", CommandInfo{{}, "quit the mu server", [&](const auto& params) { + quit_handler(params); + }}); + + cmap.emplace( + "remove", + CommandInfo{ + ArgMap{{":docid", + ArgInfo{Type::Number, false, "document-id for the message to remove"}}, + {":path", + ArgInfo{Type::String, false, "document-id for the message to remove"}} + }, + "remove a message from filesystem and database, using either :docid or :path", + [&](const auto& params) { remove_handler(params); }}); + + cmap.emplace( + "view", + CommandInfo{ArgMap{ + {":docid", ArgInfo{Type::Number, false, "document-id"}}, + {":msgid", ArgInfo{Type::String, false, "message-id"}}, + {":path", ArgInfo{Type::String, false, "message filesystem path"}}, + {":mark-as-read", + ArgInfo{Type::Symbol, false, "mark message as read (if not already)"}}, + {":rename", ArgInfo{Type::Symbol, false, "change filename when moving"}}, + }, + "view a message. exactly one of docid/msgid/path must be specified", + [&](const auto& params) { view_handler(params); }}); + return cmap; +} + +bool +Server::Private::invoke(const std::string& expr) noexcept +{ + auto make_error=[](auto&& code, auto&& msg) { + return Sexp().put_props( + ":error", Error::error_number(code), + ":message", msg); + }; + + if (!keep_going_) + return false; + try { + auto cmd{Command::make_parse(std::string{expr})}; + if (!cmd) + throw cmd.error(); + + auto res = command_handler_.invoke(*cmd); + if (!res) + throw res.error(); + + } catch (const Mu::Error& me) { + output_sexp(make_error(me.code(), mu_format("{}", + me.what()))); + keep_going_ = true; + } catch (const Xapian::Error& xerr) { + output_sexp(make_error(Error::Code::Internal, + mu_format("xapian error: {}: {}", + xerr.get_type(), xerr.get_description()))); + keep_going_ = false; + } catch (const std::runtime_error& re) { + output_sexp(make_error(Error::Code::Internal, + mu_format("caught runtime exception: {}", + re.what()))); + keep_going_ = false; + } catch (const std::out_of_range& oore) { + output_sexp(make_error(Error::Code::Internal, + mu_format("caught out-of-range exception: {}", + oore.what()))); + keep_going_ = false; + } catch (const std::exception& e) { + output_sexp(make_error(Error::Code::Internal, + mu_format(" exception: {}", e.what()))); + keep_going_ = false; + } catch (...) { + output_sexp(make_error(Error::Code::Internal, + mu_format("something went wrong: quitting"))); + keep_going_ = false; + } + + return keep_going_; +} + +/* 'add' adds a message to the database, and takes two parameters: 'path', which + * is the full path to the message, and 'maildir', which is the maildir this + * message lives in (e.g. "/inbox"). + * + * responds with an (added . <message sexp>) forr the new message + */ +void +Server::Private::add_handler(const Command& cmd) +{ + auto path{cmd.string_arg(":path")}; + const auto docid_res{store().add_message(*path)}; + + if (!docid_res) + throw docid_res.error(); + + const auto docid{docid_res.value()}; + output_sexp(Sexp().put_props(":info", "add"_sym, + ":path", *path, + ":docid", docid)); + + auto msg_res{store().find_message(docid)}; + if (!msg_res) + throw Error(Error::Code::Store, + "failed to get message at {} (docid={})", *path, docid); + + output(mu_format("(:update {})", + msg_sexp_str(msg_res.value(), docid, {}))); +} + +void +Server::Private::contacts_handler(const Command& cmd) +{ + const auto personal = cmd.boolean_arg(":personal"); + const auto afterstr = cmd.string_arg(":after").value_or(""); + const auto tstampstr = cmd.string_arg(":tstamp").value_or(""); + const auto maxnum = cmd.number_arg(":maxnum").value_or(0 /*unlimited*/); + + const auto after{afterstr.empty() ? 0 : parse_date_time(afterstr, true).value_or(0)}; + const auto tstamp = g_ascii_strtoll(tstampstr.c_str(), NULL, 10); + + mu_debug("find {} contacts last seen >= {:%c} (tstamp: {})", + personal ? "personal" : "any", mu_time(after), tstamp); + + auto match_contact = [&](const Contact& ci)->bool { + if (ci.tstamp < tstamp) + return false; /* already seen? */ + else if (personal && !ci.personal) + return false; /* not personal? */ + else if (ci.message_date < after) + return false; /* too old? */ + else + return true; + }; + + auto n{0}; + auto&& out{make_output_stream()}; + mu_print(out, "("); + store().contacts_cache().for_each([&](const Contact& ci) { + if (!match_contact(ci)) + return true; // continue + mu_println(out.out(), "{}", quote(ci.display_name())); + ++n; + return maxnum == 0 || n < maxnum; + }); + mu_print(out, ")"); + output(mu_format("(:contacts {}\n:tstamp \"{}\")", + out.to_string(), g_get_monotonic_time())); + + mu_debug("sent {} of {} contact(s)", n, store().contacts_cache().size()); +} + +void +Server::Private::data_handler(const Command& cmd) +{ + const auto request_type{unwrap(cmd.symbol_arg(":kind"))}; + + if (request_type == "maildirs") { + auto&& out{make_output_stream()}; + mu_print(out, "("); + for (auto&& mdir: store().maildirs()) + mu_println(out, "{}", quote(std::move(mdir))); + mu_print(out, ")"); + output(mu_format("(:maildirs {})", out.to_string())); + } else + throw Error(Error::Code::InvalidArgument, + "invalid request type '{}'", request_type); +} + + +/* + * creating a message object just to get a path seems a bit excessive maybe + * mu_store_get_path could be added if this turns out to be a problem + */ +static std::string +path_from_docid(const Store& store, Store::Id docid) +{ + auto msg{store.find_message(docid)}; + if (!msg) + throw Error(Error::Code::Store, "could not get message from store"); + + if (auto path{msg->path()}; path.empty()) + throw Error(Error::Code::Store, "could not get path for message {}", + docid); + else + return path; +} + +static std::vector<Store::Id> +determine_docids(const Store& store, const Command& cmd) +{ + auto docid{cmd.number_arg(":docid").value_or(0)}; + const auto msgid{cmd.string_arg(":msgid").value_or("")}; + + if ((docid == 0) == msgid.empty()) + throw Error(Error::Code::InvalidArgument, + "precisely one of docid and msgid must be specified"); + + if (docid != 0) + return {static_cast<Store::Id>(docid)}; + else + return store.find_duplicates(msgid); +} + +size_t +Server::Private::output_results(const QueryResults& qres, size_t batch_size) const +{ + // create an output stream with a file name + size_t n{}, batch_n{}; + auto&& out{make_output_stream()}; + // structured bindings / lambda don't work with some clang. + + mu_print(out, "("); + for (auto&& mi: qres) { + + auto msg{mi.message()}; + if (!msg) + continue; + + auto qm{mi.query_match()}; // construct sexp for a single header. + mu_println(out, "{}", msg_sexp_str(*msg, mi.doc_id(), qm)); + ++n; + ++batch_n; + + if (n % batch_size == 0) { + // batch complete + mu_print(out, ")"); + batch_size = 5000; + output(mu_format("(:headers {})", out.to_string())); + batch_n = 0; + // start a new batch + out = make_output_stream(); + mu_print(out, "("); + } + } + + mu_print(out, ")"); + if (batch_n > 0) + output(mu_format("(:headers {})", out.to_string())); + else + out.unlink(); + + return n; +} + + +void +Server::Private::find_handler(const Command& cmd) +{ + const auto q{cmd.string_arg(":query").value_or("")}; + const auto threads{cmd.boolean_arg(":threads")}; + // perhaps let mu4e set this as frame-lines of the appropriate frame. + const auto batch_size{cmd.number_arg(":batch-size").value_or(200)}; + const auto descending{cmd.boolean_arg(":descending")}; + const auto maxnum{cmd.number_arg(":maxnum").value_or(-1) /*unlimited*/}; + const auto skip_dups{cmd.boolean_arg(":skip-dups")}; + const auto include_related{cmd.boolean_arg(":include-related")}; + + // complicated! + auto sort_field_id = std::invoke([&]()->Field::Id { + if (const auto arg = cmd.symbol_arg(":sortfield"); !arg) + return Field::Id::Date; + else if (arg->length() < 2) + throw Error{Error::Code::InvalidArgument, "invalid sort field '{}'", + *arg}; + else if (const auto field{field_from_name(arg->substr(1))}; !field) + throw Error{Error::Code::InvalidArgument, "invalid sort field '{}'", + *arg}; + else + return field->id; + }); + + if (batch_size < 1) + throw Error{Error::Code::InvalidArgument, "invalid batch-size {}", batch_size}; + + auto qflags{QueryFlags::SkipUnreadable}; // don't show unreadables. + if (descending) + qflags |= QueryFlags::Descending; + if (skip_dups) + qflags |= QueryFlags::SkipDuplicates; + if (include_related) + qflags |= QueryFlags::IncludeRelated; + if (threads) + qflags |= QueryFlags::Threading; + + StopWatch sw{mu_format("{} (indexing: {})", __func__, + indexer().is_running() ? "yes" : "no")}; + + // we need to _lock_ the store while querying (which likely consists of + // multiple actual queries) + grabbing the results. + std::lock_guard l{store_.lock()}; + auto qres{store_.run_query(q, sort_field_id, qflags, maxnum)}; + if (!qres) + throw Error(Error::Code::Query, "failed to run query: {}", qres.error().what()); + + /* before sending new results, send an 'erase' message, so the frontend + * knows it should erase the headers buffer. this will ensure that the + * output of two finds will not be mixed. */ + output_sexp(Sexp().put_props(":erase", Sexp::t_sym)); + const auto bsize{static_cast<size_t>(batch_size)}; + const auto foundnum = output_results(*qres, bsize); + output_sexp(Sexp().put_props(":found", foundnum)); +} + +void +Server::Private::help_handler(const Command& cmd) +{ + const auto command{cmd.symbol_arg(":command").value_or("")}; + const auto full{cmd.bool_arg(":full").value_or(!command.empty())}; + auto&& info_map{command_handler_.info_map()}; + + if (command.empty()) { + mu_println(";; Commands are single-line s-expressions of the form\n" + ";; (<command-name> :param1 val1 :param2 val2 ...)\n" + ";; For instance:\n;; (help :command mkdir)\n" + ";; to get more information about the 'mkdir' command\n;;\n" + ";; The following commands are available:"); + } + + std::vector<std::string> names; + for (auto&& name_cmd: info_map) + names.emplace_back(name_cmd.first); + + std::sort(names.begin(), names.end()); + + for (auto&& name : names) { + const auto& info{info_map.find(name)->second}; + + if (!command.empty() && name != command) + continue; + + mu_println(";; {:<12} -- {}", name, info.docstring); + + if (!full) + continue; + + for (auto&& argname : info.sorted_argnames()) { + const auto& arg{info.args.find(argname)}; + mu_println(";; {:<17} :: {:<24} -- {}", + arg->first, to_string(arg->second), + arg->second.docstring); + } + mu_println(";;"); + } +} + +static Sexp +get_stats(const Indexer::Progress& stats, const std::string& state) +{ + Sexp sexp; + sexp.put_props( + ":info", "index"_sym, + ":status", Sexp::Symbol(state), + ":checked", static_cast<int>(stats.checked), + ":updated", static_cast<int>(stats.updated), + ":cleaned-up", static_cast<int>(stats.removed)); + + return sexp; +} + +void +Server::Private::index_handler(const Command& cmd) +{ + Mu::Indexer::Config conf{}; + conf.cleanup = cmd.boolean_arg(":cleanup"); + conf.lazy_check = cmd.boolean_arg(":lazy-check"); + // ignore .noupdate with an empty store. + conf.ignore_noupdate = store().empty(); + + indexer().stop(); + if (index_thread_.joinable()) + index_thread_.join(); + + // start a background track. + index_thread_ = std::thread([this, conf = std::move(conf)] { + StopWatch sw{"indexing"}; + indexer().start(conf); + while (indexer().is_running()) { + std::this_thread::sleep_for(std::chrono::milliseconds(2000)); + output_sexp(get_stats(indexer().progress(), "running"), + Server::OutputFlags::Flush); + } + output_sexp(get_stats(indexer().progress(), "complete"), + Server::OutputFlags::Flush); + }); +} + +void +Server::Private::mkdir_handler(const Command& cmd) +{ + const auto path{cmd.string_arg(":path").value_or("<error>")}; + const auto update{cmd.boolean_arg(":update")}; + + if (path.find(store().root_maildir()) != 0) + throw Error{Error::Code::File, "maildir is not below root-maildir"}; + + if (auto&& res = maildir_mkdir(path, 0755, false); !res) + throw res.error(); + + /* mu4e does a lot of opportunistic 'mkdir', only send it updates when + * requested */ + if (!update) + return; + + output_sexp(Sexp().put_props(":info", "mkdir", + ":message", + mu_format("{} has been created", path))); +} + +void +Server::Private::perform_move(Store::Id docid, + const Message& msg, + const std::string& maildirarg, + Flags flags, + bool new_name, + bool no_view) +{ + bool different_mdir{}; + auto maildir{maildirarg}; + if (maildir.empty()) { + maildir = msg.maildir(); + different_mdir = false; + } else /* are we moving to a different mdir, or is it just flags? */ + different_mdir = maildir != msg.maildir(); + + Store::MoveOptions move_opts{Store::MoveOptions::DupFlags}; + if (new_name) + move_opts |= Store::MoveOptions::ChangeName; + + /* note: we get back _all_ the messages that changed; the first is the + * primary mover; the rest (if present) are any dups affected */ + const auto id_paths{unwrap(store().move_message(docid, maildir, flags, move_opts))}; + for (auto& [id,path]: id_paths) { + auto idmsg{store().find_message(id)}; + if (!idmsg) + throw Error{Error::Code::Xapian, "cannot find message for id {}", id}; + + auto sexpstr = "(:update " + msg_sexp_str(*idmsg, id, {}); + /* note, the :move t thing is a hint to the frontend that it + * could remove the particular header */ + if (different_mdir) + sexpstr += " :move t"; + if (!no_view && id == docid) + sexpstr += " :maybe-view t"; + sexpstr += ')'; + output(std::move(sexpstr)); + } +} + +static Flags +calculate_message_flags(const Message& msg, Option<std::string> flagopt) +{ + const auto flags = std::invoke([&]()->Option<Flags>{ + if (!flagopt) + return msg.flags(); + else + return flags_from_expr(*flagopt, msg.flags()); + }); + + if (!flags) + throw Error{Error::Code::InvalidArgument, + "invalid flags '{}'", flagopt.value_or("")}; + else + return flags.value(); +} + +void +Server::Private::move_docid(Store::Id docid, + Option<std::string> flagopt, + bool new_name, + bool no_view) +{ + if (docid == Store::InvalidId) + throw Error{Error::Code::InvalidArgument, "invalid docid"}; + + auto msg{store_.find_message(docid)}; + if (!msg) + throw Error{Error::Code::Store, "failed to get message from store"}; + + const auto flags = calculate_message_flags(msg.value(), flagopt); + perform_move(docid, *msg, "", flags, new_name, no_view); +} + +/* + * 'move' moves a message to a different maildir and/or changes its flags. + * parameters are *either* a 'docid:' or 'msgid:' pointing to the message, a + * 'maildir:' for the target maildir, and a 'flags:' parameter for the new + * flags. + * + * With :msgid, this is "opportunistic": it's not an error when the given + * message-id does not exist. This is e.g. for the case when tagging possible + * related messages. + */ +void +Server::Private::move_handler(const Command& cmd) +{ + auto maildir{cmd.string_arg(":maildir").value_or("")}; + const auto flagopt{cmd.string_arg(":flags")}; + const auto rename{cmd.boolean_arg(":rename")}; + const auto no_view{cmd.boolean_arg(":noupdate")}; + const auto docids{determine_docids(store_, cmd)}; + + if (docids.empty()) { + if (!!cmd.string_arg(":msgid")) { + // msgid not found: no problem. + mu_debug("no move: '{}' not found", + *cmd.string_arg(":msgid")); + return; + } + // however, if we wanted to be move by msgid, it's worth raising + // an error. + throw Mu::Error{Error::Code::Store, + "message not found in store (docid={})", + cmd.number_arg(":docid").value_or(0)}; + } else if (docids.size() > 1) { + if (!maildir.empty()) // ie. duplicate message-ids. + throw Mu::Error{Error::Code::Store, + "cannot move multiple messages at the same time"}; + // multi. + for (auto&& docid : docids) + move_docid(docid, flagopt, rename, no_view); + return; + } else { + const auto docid{docids.at(0)}; + auto msg = store().find_message(docid) + .or_else([&]{throw Error{Error::Code::InvalidArgument, + "cannot find message {}", docid};}).value(); + + /* if maildir was not specified, take the current one */ + if (maildir.empty()) + maildir = msg.maildir(); + + /* determine the real target flags, which come from the flags-parameter + * we received (ie., flagstr), if any, plus the existing message + * flags. */ + const auto flags = calculate_message_flags(msg, flagopt); + perform_move(docid, msg, maildir, flags, rename, no_view); + } +} + +void +Server::Private::ping_handler(const Command& cmd) +{ + const auto storecount{store().size()}; + if (storecount == (unsigned)-1) + throw Error{Error::Code::Store, "failed to read store"}; + Sexp addrs; + for (auto&& addr : store().config().get<Config::Id::PersonalAddresses>()) + addrs.add(addr); + + output_sexp(Sexp() + .put_props(":pong", "mu") + .put_props(":props", + Sexp().put_props( + ":version", VERSION, + ":personal-addresses", std::move(addrs), + ":database-path", store().path(), + ":root-maildir", store().root_maildir(), + ":doccount", storecount))); +} + +void +Server::Private::queries_handler(const Command& cmd) +{ + const auto queries{cmd.string_vec_arg(":queries") + .value_or(std::vector<std::string>{})}; + + Sexp qresults; + for (auto&& q : queries) { + const auto count{store_.count_query(q)}; + const auto unreadq{mu_format("flag:unread AND ({})", q)}; + const auto unread{store_.count_query(unreadq)}; + qresults.add(Sexp().put_props(":query", q, + ":count", count, + ":unread", unread)); + } + + output_sexp(Sexp(":queries"_sym, std::move(qresults))); +} + + +void +Server::Private::quit_handler(const Command& cmd) +{ + keep_going_ = false; +} + +void +Server::Private::remove_handler(const Command& cmd) +{ + auto docid_opt{cmd.number_arg(":docid")}; + auto path_opt{cmd.string_arg(":path")}; + + if (!!docid_opt == !!path_opt) + throw Error(Error::Code::InvalidArgument, + "must pass precisely one of :docid and :path"); + std::string path; + Store::Id docid{}; + if (docid = docid_opt.value_or(0); docid != 0) + path = path_from_docid(store(), docid); + else + path = path_opt.value(); + + if (::unlink(path.c_str()) != 0 && errno != ENOENT) + throw Error(Error::Code::File, + "could not delete {}: {}", path, g_strerror(errno)); + + if (!store().remove_message(path)) + mu_warning("failed to remove message @ {} ({}) from store", path, docid); + else + mu_debug("removed message @ {} @ ({})", path, docid); + + output_sexp(Sexp().put_props(":remove", docid)); // act as if it worked. +} + +void +Server::Private::view_mark_as_read(Store::Id docid, Message&& msg, bool rename) +{ + auto new_flags = [](const Message& m)->Option<Flags> { + auto nflags = flags_from_delta_expr("+S-u-N", m.flags()); + if (!nflags || nflags == m.flags()) + return Nothing; // nothing to do + else + return nflags; + }; + + auto&& nflags = new_flags(msg); + if (!nflags) { // nothing to move, just send the message for viewing. + output(mu_format("(:view {})", msg_sexp_str(msg, docid, {}))); + return; + } + + // move message + dups, present results. + Store::MoveOptions move_opts{Store::MoveOptions::DupFlags}; + if (rename) + move_opts |= Store::MoveOptions::ChangeName; + + const auto ids{Store::id_vec(unwrap(store().move_message(docid, {}, nflags, move_opts)))}; + for (auto&& [id, moved_msg]: store().find_messages(ids)) + output(mu_format("({} {})", id == docid ? ":view" : ":update", + msg_sexp_str(moved_msg, id, {}))); +} + +void +Server::Private::view_handler(const Command& cmd) +{ + StopWatch sw{mu_format("{} (indexing: {})", __func__, indexer().is_running())}; + + const auto mark_as_read{cmd.boolean_arg(":mark-as-read")}; + /* for now, do _not_ rename, as it seems to confuse mbsync */ + const auto rename{false}; + //const auto rename{get_bool_or(params, ":rename")}; + + const auto docids{determine_docids(store(), cmd)}; + + if (docids.empty()) + throw Error{Error::Code::Store, "failed to find message for view"}; + const auto docid{docids.at(0)}; + auto msg = store().find_message(docid) + .or_else([]{throw Error{Error::Code::Store, + "failed to find message for view"};}).value(); + + /* if the message should not be marked-as-read, we're done. */ + if (!mark_as_read) + output(mu_format("(:view {})", msg_sexp_str(msg, docid, {}))); + else + view_mark_as_read(docid, std::move(msg), rename); + /* otherwise, mark message and and possible dups as read */ +} + +Server::Server(Store& store, const Server::Options& opts, Server::Output output) + : priv_{std::make_unique<Private>(store, opts, output)} +{} + +Server::~Server() = default; + +bool +Server::invoke(const std::string& expr) noexcept +{ + return priv_->invoke(expr); +} + +/* LCOV_EXCL_STOP */ diff --git a/lib/mu-server.hh b/lib/mu-server.hh new file mode 100644 index 0000000..0ceaa68 --- /dev/null +++ b/lib/mu-server.hh @@ -0,0 +1,89 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_SERVER_HH__ +#define MU_SERVER_HH__ + +#include <memory> +#include <functional> + +#include <utils/mu-sexp.hh> +#include <utils/mu-utils.hh> +#include <mu-store.hh> + +/* LCOV_EXCL_START */ + +namespace Mu { + +/** + * @brief Implements the mu server, as used by mu4e. + */ +class Server { +public: + enum struct OutputFlags { + None = 0, + Flush = 1 << 0, /**< flush output buffer after */ + }; + + /** + * Prototype for output function + * + * @param str a string + * @param flags flags that influence the behavior + */ + using Output = std::function<void(const std::string& str, OutputFlags flags)>; + + struct Options { + bool allow_temp_file; /**< temp file optimization allowed? */ + }; + + /** + * Construct a new server + * + * @param store a message store object + * @param output callable for the server responses. + */ + Server(Store& store, const Options& opts, Output output); + + /** + * DTOR + */ + ~Server(); + + /** + * Invoke a call on the server. + * + * @param expr the s-expression to call + * + * @return true if we the server is still ready for more + * calls, false when it should quit. + */ + bool invoke(const std::string& expr) noexcept; + +private: + struct Private; + std::unique_ptr<Private> priv_; +}; +MU_ENABLE_BITOPS(Server::OutputFlags); + +} // namespace Mu + +/* LCOV_EXCL_STOP */ + +#endif /* MU_SERVER_HH__ */ diff --git a/lib/mu-store.cc b/lib/mu-store.cc new file mode 100644 index 0000000..eb08eac --- /dev/null +++ b/lib/mu-store.cc @@ -0,0 +1,673 @@ +/* +** Copyright (C) 2021-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-store.hh" + +#include <chrono> +#include <mutex> +#include <array> +#include <cstdlib> +#include <stdexcept> +#include <unordered_map> +#include <atomic> +#include <type_traits> +#include <iostream> +#include <cstring> + +#include "mu-maildir.hh" +#include "mu-query.hh" +#include "mu-xapian-db.hh" +#include "mu-scanner.hh" + +#include "utils/mu-error.hh" + +#include "utils/mu-utils.hh" +#include <utils/mu-utils-file.hh> + +using namespace Mu; + +static_assert(std::is_same<Store::Id, Xapian::docid>::value, "wrong type for Store::Id"); + +// Properties +constexpr auto ExpectedSchemaVersion = MU_STORE_SCHEMA_VERSION; + +static std::string +remove_slash(const std::string& str) +{ + auto clean{str}; + while (!clean.empty() && clean[clean.length() - 1] == '/') + clean.pop_back(); + + return clean; +} + +struct Store::Private { + + Private(const std::string& path, bool readonly): + xapian_db_{XapianDb(path, readonly ? XapianDb::Flavor::ReadOnly + : XapianDb::Flavor::Open)}, + config_{xapian_db_}, + contacts_cache_{config_}, + root_maildir_{remove_slash(config_.get<Config::Id::RootMaildir>())}, + message_opts_{make_message_options(config_)} + {} + + Private(const std::string& path, const std::string& root_maildir, + Option<const Config&> conf): + xapian_db_{XapianDb(path, XapianDb::Flavor::CreateOverwrite)}, + config_{make_config(xapian_db_, root_maildir, conf)}, + contacts_cache_{config_}, + root_maildir_{remove_slash(config_.get<Config::Id::RootMaildir>())}, + message_opts_{make_message_options(config_)} + {} + + ~Private() try { + mu_debug("closing store @ {}", xapian_db_.path()); + if (!xapian_db_.read_only()) + contacts_cache_.serialize(); + } catch (...) { + mu_critical("caught exception in store dtor"); + } + + Config make_config(XapianDb& xapian_db, const std::string& root_maildir, + Option<const Config&> conf) { + + if (!g_path_is_absolute(root_maildir.c_str())) + throw Error{Error::Code::File, + "root maildir path is not absolute ({})", + root_maildir}; + + Config config{xapian_db}; + if (conf) + config.import_configurable(*conf); + + config.set<Config::Id::RootMaildir>(remove_slash(root_maildir)); + config.set<Config::Id::SchemaVersion>(ExpectedSchemaVersion); + + return config; + } + + Message::Options make_message_options(const Config& conf) { + if (conf.get<Config::Id::SupportNgrams>()) + return Message::Options::SupportNgrams; + else + return Message::Options::None; + } + + Option<Message> find_message_unlocked(Store::Id docid) const; + Store::IdVec find_duplicates_unlocked(const Store& store, + const std::string& message_id) const; + + Result<Store::Id> add_message_unlocked(Message& msg); + Result<Store::Id> update_message_unlocked(Message& msg, Store::Id docid); + Result<Store::Id> update_message_unlocked(Message& msg, const std::string& old_path); + + + using PathMessage = std::pair<std::string, Message>; + Result<PathMessage> move_message_unlocked(Message&& msg, + Option<const std::string&> target_mdir, + Option<Flags> new_flags, + MoveOptions opts); + XapianDb xapian_db_; + Config config_; + ContactsCache contacts_cache_; + std::unique_ptr<Indexer> indexer_; + + const std::string root_maildir_; + const Message::Options message_opts_; + + std::mutex lock_; +}; + + +Result<Store::Id> +Store::Private::add_message_unlocked(Message& msg) +{ + auto&& docid{xapian_db_.add_document(msg.document().xapian_document())}; + if (docid) + mu_debug("added message @ {}; docid = {}", msg.path(), *docid); + + return docid; +} + + +Result<Store::Id> +Store::Private::update_message_unlocked(Message& msg, Store::Id docid) +{ + auto&& res{xapian_db_.replace_document(docid, msg.document().xapian_document())}; + if (res) + mu_debug("updated message @ {}; docid = {}", msg.path(), *res); + + return res; +} + +Result<Store::Id> +Store::Private::update_message_unlocked(Message& msg, const std::string& path_to_replace) +{ + return xapian_db_.replace_document( + field_from_id(Field::Id::Path).xapian_term(path_to_replace), + msg.document().xapian_document()); +} + +Option<Message> +Store::Private::find_message_unlocked(Store::Id docid) const +{ + if (auto&& doc{xapian_db_.document(docid)}; !doc) + return Nothing; + else if (auto&& msg{Message::make_from_document(std::move(*doc))}; !msg) + return Nothing; + else + return Some(std::move(*msg)); +} + +Store::IdVec +Store::Private::find_duplicates_unlocked(const Store& store, + const std::string& message_id) const +{ + if (message_id.empty() || message_id.size() > MaxTermLength) { + mu_warning("invalid message-id '{}'", message_id); + return {}; + } + + auto expr{mu_format("{}:{}", + field_from_id(Field::Id::MessageId).shortcut, + message_id)}; + if (auto&& res{store.run_query(expr)}; !res) { + mu_warning("error finding message-ids: {}", res.error().what()); + return {}; + + } else { + Store::IdVec ids; + ids.reserve(res->size()); + for (auto&& mi: *res) + ids.emplace_back(mi.doc_id()); + return ids; + } +} + + +Store::Store(const std::string& path, Store::Options opts) + : priv_{std::make_unique<Private>(path, none_of(opts & Store::Options::Writable))} +{ + if (none_of(opts & Store::Options::Writable) && + any_of(opts & Store::Options::ReInit)) + throw Mu::Error(Error::Code::InvalidArgument, + "Options::ReInit requires Options::Writable"); + + const auto s_version{config().get<Config::Id::SchemaVersion>()}; + if (any_of(opts & Store::Options::ReInit)) { + /* don't try to recover from version with an incompatible scheme */ + if (s_version < 500) + throw Mu::Error(Error::Code::CannotReinit, + "old schema ({}) is too old to re-initialize from", + s_version).add_hint("Invoke 'mu init' without '--reinit'; " + "see mu-init(1) for details"); + const auto old_root_maildir{root_maildir()}; + + MemDb mem_db; + Config old_config(mem_db); + old_config.import_configurable(config()); + + this->priv_.reset(); + /* and create a new one "in place" */ + Store new_store(path, old_root_maildir, old_config); + this->priv_ = std::move(new_store.priv_); + } + + /* otherwise, the schema version should match. */ + if (s_version != ExpectedSchemaVersion) + throw Mu::Error(Error::Code::SchemaMismatch, + "expected schema-version {}, but got {}", + ExpectedSchemaVersion, s_version). + add_hint("Please (re)initialize with 'mu init'; see mu-init(1) for details"); +} + +Store::Store(const std::string& path, + const std::string& root_maildir, + Option<const Config&> conf): + priv_{std::make_unique<Private>(path, root_maildir, conf)} +{} + +Store::Store(Store&& other) +{ + priv_ = std::move(other.priv_); + priv_->indexer_.reset(); +} + +Store::~Store() = default; + +Store::Statistics +Store::statistics() const +{ + Statistics stats{}; + + stats.size = size(); + stats.last_change = config().get<Config::Id::LastChange>(); + stats.last_index = config().get<Config::Id::LastIndex>(); + + return stats; +} + +const XapianDb& +Store::xapian_db() const +{ + return priv_->xapian_db_; +} + +XapianDb& +Store::xapian_db() +{ + return priv_->xapian_db_; +} + +const Config& +Store::config() const +{ + return priv_->config_; +} + +Config& +Store::config() +{ + return priv_->config_; +} + +const std::string& +Store::root_maildir() const { + return priv_->root_maildir_; +} + +const ContactsCache& +Store::contacts_cache() const +{ + return priv_->contacts_cache_; +} + +Indexer& +Store::indexer() +{ + std::lock_guard guard{priv_->lock_}; + + if (xapian_db().read_only()) + throw Error{Error::Code::Store, "no indexer for read-only store"}; + else if (!priv_->indexer_) + priv_->indexer_ = std::make_unique<Indexer>(*this); + + return *priv_->indexer_.get(); +} + +Result<Store::Id> +Store::add_message(Message& msg, bool is_new) +{ + const auto mdir{maildir_from_path(msg.path(), root_maildir())}; + if (!mdir) + return Err(mdir.error()); + if (auto&& res = msg.set_maildir(mdir.value()); !res) + return Err(res.error()); + + // we shouldn't mix ngrams/non-ngrams messages. + if (any_of(msg.options() & Message::Options::SupportNgrams) != + any_of(message_options() & Message::Options::SupportNgrams)) + return Err(Error::Code::InvalidArgument, "incompatible message options"); + + /* add contacts from this message to cache; this cache + * also determines whether those contacts are _personal_, i.e. match + * our personal addresses. + * + * if a message has any personal contacts, mark it as personal; do + * this by updating the message flags. + */ + bool is_personal{}; + priv_->contacts_cache_.add(msg.all_contacts(), is_personal); + if (is_personal) + msg.set_flags(msg.flags() | Flags::Personal); + + std::lock_guard guard{priv_->lock_}; + auto&& res = is_new ? + priv_->add_message_unlocked(msg) : + priv_->update_message_unlocked(msg, msg.path()); + if (!res) + return Err(res.error()); + + mu_debug("added {}{}message @ {}; docid = {}", + is_new ? "new " : "", is_personal ? "personal " : "", msg.path(), *res); + + return res; +} + +Result<Store::Id> +Store::add_message(const std::string& path, bool is_new) +{ + if (auto msg{Message::make_from_path(path, priv_->message_opts_)}; !msg) + return Err(msg.error()); + else + return add_message(msg.value(), is_new); +} + + +bool +Store::remove_message(const std::string& path) +{ + const auto term{field_from_id(Field::Id::Path).xapian_term(path)}; + + std::lock_guard guard{priv_->lock_}; + + xapian_db().delete_document(term); + mu_debug("deleted message @ {} from store", path); + return true; +} + +void +Store::remove_messages(const std::vector<Store::Id>& ids) +{ + std::lock_guard guard{priv_->lock_}; + + XapianDb::Transaction tx (xapian_db()); // RAII + + for (auto&& id : ids) + xapian_db().delete_document(id); +} + + +Option<Message> +Store::find_message(Store::Id docid) const +{ + std::lock_guard guard{priv_->lock_}; + + return priv_->find_message_unlocked(docid); +} + +Option<Store::Id> +Store::find_message_id(const std::string& path) const +{ + constexpr auto path_field{field_from_id(Field::Id::Path)}; + + std::lock_guard guard{priv_->lock_}; + + auto enq{xapian_db().enquire()}; + enq.set_query(Xapian::Query{path_field.xapian_term(path)}); + + if (auto mset{enq.get_mset(0, 1)}; mset.empty()) + return Nothing; // message not found + else + return Some(*mset.begin()); +} + + +Store::IdMessageVec +Store::find_messages(IdVec ids) const +{ + std::lock_guard guard{priv_->lock_}; + + IdMessageVec id_msgs; + for (auto&& id: ids) { + if (auto&& msg{priv_->find_message_unlocked(id)}; msg) + id_msgs.emplace_back(std::make_pair(id, std::move(*msg))); + } + + return id_msgs; +} + +/** + * Move a message in store and filesystem; with DryRun, only calculate the target name. + * + * Lock is assumed taken already + * + * @param id message id + * @param target_mdir target_mdir (or Nothing for current) + * @param new_flags new flags (or Nothing) + * @param opts move_options + * + * @return the Message after the moving, or an Error + */ +Result<Store::Private::PathMessage> +Store::Private::move_message_unlocked(Message&& msg, + Option<const std::string&> target_mdir, + Option<Flags> new_flags, + MoveOptions opts) +{ + const auto old_path = msg.path(); + const auto target_flags = new_flags.value_or(msg.flags()); + const auto target_maildir = target_mdir.value_or(msg.maildir()); + + /* 1. first determine the file system path of the target */ + const auto target_path = + maildir_determine_target(msg.path(), root_maildir_, + target_maildir, target_flags, + any_of(opts & MoveOptions::ChangeName)); + if (!target_path) + return Err(target_path.error()); + + // in dry-run mode, we only determine the target-path + if (none_of(opts & MoveOptions::DryRun)) { + + /* 2. let's move it */ + if (const auto res = maildir_move_message(msg.path(), target_path.value()); !res) + return Err(res.error()); + + /* 3. file move worked, now update the message with the new info.*/ + if (auto&& res = msg.update_after_move( + target_path.value(), target_maildir, target_flags); !res) + return Err(res.error()); + + /* 4. update message worked; re-store it */ + if (auto&& res = update_message_unlocked(msg, old_path); !res) + return Err(res.error()); + } + + /* 6. Profit! */ + return Ok(PathMessage{std::move(*target_path), std::move(msg)}); +} + +Store::IdVec +Store::find_duplicates(const std::string& message_id) const +{ + std::lock_guard guard{priv_->lock_}; + + return priv_->find_duplicates_unlocked(*this, message_id); +} + + +Result<Store::IdPathVec> +Store::move_message(Store::Id id, + Option<const std::string&> target_mdir, + Option<Flags> new_flags, + MoveOptions opts) +{ + auto filter_dup_flags=[](Flags old_flags, Flags new_flags) -> Flags { + new_flags = flags_keep_unmutable(old_flags, new_flags, Flags::Draft); + new_flags = flags_keep_unmutable(old_flags, new_flags, Flags::Flagged); + new_flags = flags_keep_unmutable(old_flags, new_flags, Flags::Trashed); + return new_flags; + }; + + std::lock_guard guard{priv_->lock_}; + + auto msg{priv_->find_message_unlocked(id)}; + if (!msg) + return Err(Error::Code::Store, "cannot find message <{}>", id); + + const auto message_id{msg->message_id()}; + auto res{priv_->move_message_unlocked(std::move(*msg), target_mdir, new_flags, opts)}; + if (!res) + return Err(res.error()); + + IdPathVec id_paths{{id, res->first}}; + if (none_of(opts & Store::MoveOptions::DupFlags) || message_id.empty() || !new_flags) + return Ok(std::move(id_paths)); + + /* handle the dup-flags case; i.e. apply (a subset of) the flags to + * all messages with the same message-id as well */ + auto dups{priv_->find_duplicates_unlocked(*this, message_id)}; + for (auto&& dupid: dups) { + + if (dupid == id) + continue; // already + + auto dup_msg{priv_->find_message_unlocked(dupid)}; + if (!dup_msg) + continue; // no such message + + /* For now, don't change Draft/Flagged/Trashed */ + const auto dup_flags{filter_dup_flags(dup_msg->flags(), *new_flags)}; + /* use the updated new_flags and MoveOptions without DupFlags (so we don't + * recurse) */ + opts = opts & ~MoveOptions::DupFlags; + if (auto dup_res = priv_->move_message_unlocked( + std::move(*dup_msg), Nothing, dup_flags, opts); !dup_res) + mu_warning("failed to move dup: {}", dup_res.error().what()); + else + id_paths.emplace_back(dupid, dup_res->first); + } + + // sort the dup paths by name; + std::sort(id_paths.begin() + 1, id_paths.end(), + [](const auto& idp1, const auto& idp2) { return idp1.second < idp2.second; }); + + return Ok(std::move(id_paths)); +} + +Store::IdVec +Store::id_vec(const IdPathVec& ips) +{ + IdVec idv; + for (auto&& ip: ips) + idv.emplace_back(ip.first); + + return idv; +} + + +time_t +Store::dirstamp(const std::string& path) const +{ + std::string ts; + + { + std::unique_lock lock{priv_->lock_}; + ts = xapian_db().metadata(path); + } + + return ts.empty() ? 0 /*epoch*/ : ::strtoll(ts.c_str(), {}, 16); +} + +void +Store::set_dirstamp(const std::string& path, time_t tstamp) +{ + std::unique_lock lock{priv_->lock_}; + + xapian_db().set_metadata(path, mu_format("{:x}", tstamp)); +} + +bool +Store::contains_message(const std::string& path) const +{ + std::unique_lock lock{priv_->lock_}; + + return xapian_db().term_exists(field_from_id(Field::Id::Path).xapian_term(path)); +} + +std::size_t +Store::for_each_message_path(Store::ForEachMessageFunc msg_func) const +{ + size_t n{}; + + xapian_try([&] { + std::lock_guard guard{priv_->lock_}; + auto enq{xapian_db().enquire()}; + + enq.set_query(Xapian::Query::MatchAll); + enq.set_cutoff(0, 0); + + Xapian::MSet matches(enq.get_mset(0, xapian_db().size())); + constexpr auto path_no{field_from_id(Field::Id::Path).value_no()}; + for (auto&& it = matches.begin(); it != matches.end(); ++it, ++n) + if (!msg_func(*it, it.get_document().get_value(path_no))) + break; + }); + + return n; +} + +std::size_t +Store::for_each_term(Field::Id field_id, Store::ForEachTermFunc func) const +{ + return xapian_db().all_terms(field_from_id(field_id).xapian_term(), func); +} + +std::mutex& +Store::lock() const +{ + return priv_->lock_; +} + +Result<QueryResults> +Store::run_query(const std::string& expr, + Field::Id sortfield_id, + QueryFlags flags, size_t maxnum) const +{ + return Query{*this}.run(expr, sortfield_id, flags, maxnum); +} + +size_t +Store::count_query(const std::string& expr) const +{ + return xapian_try([&] { + std::lock_guard guard{priv_->lock_}; + Query q{*this}; + return q.count(expr); }, 0); +} + +std::string +Store::parse_query(const std::string& expr, bool xapian) const +{ + return xapian_try([&] { + std::lock_guard guard{priv_->lock_}; + Query q{*this}; + + return q.parse(expr, xapian); + }, std::string{}); +} + + +std::vector<std::string> +Store::maildirs() const +{ + std::vector<std::string> mdirs; + const auto prefix_size{root_maildir().size()}; + + Scanner::Handler handler = [&](const std::string& path, auto&& _1, auto&& _2) { + auto md{path.substr(prefix_size)}; + mdirs.emplace_back(md.empty() ? "/" : std::move(md)); + return true; + }; + + Scanner scanner{root_maildir(), handler, Scanner::Mode::MaildirsOnly}; + scanner.start(); + std::sort(mdirs.begin(), mdirs.end()); + + return mdirs; +} + +Message::Options +Store::message_options() const +{ + return priv_->message_opts_; +} diff --git a/lib/mu-store.hh b/lib/mu-store.hh new file mode 100644 index 0000000..dd49045 --- /dev/null +++ b/lib/mu-store.hh @@ -0,0 +1,490 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_STORE_HH__ +#define MU_STORE_HH__ + +#include <string> +#include <vector> +#include <mutex> +#include <ctime> +#include <memory> + +#include "mu-contacts-cache.hh" +#include "mu-xapian-db.hh" +#include "mu-config.hh" +#include "mu-indexer.hh" +#include "mu-query-results.hh" + +#include <utils/mu-utils.hh> +#include <utils/mu-utils.hh> +#include <utils/mu-option.hh> + +#include <message/mu-message.hh> + +namespace Mu { + +class Store { +public: + using Id = Xapian::docid; /**< Id for a message in the store */ + static constexpr Id InvalidId = 0; /**< Invalid store id */ + using IdVec = std::vector<Id>; /**< Vector of document ids */ + using IdPathVec = std::vector<std::pair<Id, std::string>>; + /**< vector of id, path pairs */ + + /** + * Configuration options. + */ + enum struct Options { + None = 0, /**< No specific options */ + Writable = 1 << 0, /**< Open in writable mode */ + ReInit = 1 << 1, /**< Re-initialize based on existing */ + }; + + /** + * Make a store for an existing document database + * + * @param path path to the database + * @param options startup options + * + * @return A store or an error. + */ + static Result<Store> make(const std::string& path, + Options opts=Options::None) noexcept { + return xapian_try_result( + [&]{return Ok(Store{path, opts});}); + } + + /** + * Construct a store for a not-yet-existing document database + * + * @param path path to the database + * @param root_maildir absolute path to maildir to use for this store + * @param conf a configuration object + * + * @return a store or an error + */ + static Result<Store> make_new(const std::string& path, + const std::string& root_maildir, + Option<const Config&> conf={}) noexcept { + return xapian_try_result( + [&]{return Ok(Store(path, root_maildir, conf));}); + } + + /** + * Move CTOR + * + */ + Store(Store&&); + + /** + * DTOR + */ + ~Store(); + + /** + * Store statistics. Unlike the properties, these can change + * during the lifetime of a store. + * + */ + struct Statistics { + size_t size; /**< number of messages in store */ + ::time_t last_change; /**< last time any update happened */ + ::time_t last_index; /**< last time an indexing op was performed */ + }; + + /** + * Get store statistics + * + * @return statistics + */ + Statistics statistics() const; + + /** + * Get the underlying xapian db object + * + * @return the XapianDb for this store + */ + const XapianDb& xapian_db() const; + XapianDb& xapian_db(); + + /** + * Get the Config for this store + * + * @return the Config + */ + const Config& config() const; + Config& config(); + + /** + * Get the ContactsCache object for this store + * + * @return the Contacts object + */ + const ContactsCache& contacts_cache() const; + + /** + * Get the Indexer associated with this store. It is an error to call + * this on a read-only store. + * + * @return the indexer. + */ + Indexer& indexer(); + + /** + * Run a query; see the `mu-query` man page for the syntax. + * + * Multi-threaded callers must acquire the lock and keep it + * at least as long as the return value. + * + * @param expr the search expression + * @param sortfieldid the sortfield-id. If the field is NONE, sort by DATE + * @param flags query flags + * @param maxnum maximum number of results to return. 0 for 'no limit' + * + * @return the query-results or an error. + */ + std::mutex& lock() const; + Result<QueryResults> run_query(const std::string& expr, + Field::Id sortfield_id = Field::Id::Date, + QueryFlags flags = QueryFlags::None, + size_t maxnum = 0) const; + + /** + * run a Xapian query merely to count the number of matches; for the + * syntax, please refer to the mu-query manpage + * + * @param expr the search expression; use "" to match all messages + * + * @return the number of matches + */ + size_t count_query(const std::string& expr = "") const; + + /** + * For debugging, get the internal string representation of the parsed + * query + * + * @param expr a xapian search expression + * @param xapian if true, show Xapian's internal representation, + * otherwise, mu's. + * + * @return the string representation of the query + */ + std::string parse_query(const std::string& expr, bool xapian) const; + + /** + * Add or update a message to the store. When planning to write many + * messages, it's much faster to do so in a transaction. If so, set + * @param in_transaction to true. When done with adding messages, call + * commit(). + * + * Optimization: If you are sure the message (i.e., a message with the + * given file-system path) does not yet exist in the database, ie., when + * doing the initial indexing, set @p is_new to true since we then don't + * have to check for the existing message. + * + * @param msg a message + * @param is_new whether this is a completely new message + * + * @return the doc id of the added message or an error. + */ + Result<Id> add_message(Message& msg, bool is_new = false); + Result<Id> add_message(const std::string& path, bool is_new = false); + + /** + * Remove a message from the store. It will _not_ remove the message + * from the file system. + * + * @param path the message path. + * + * @return true if removing happened; false otherwise. + */ + bool remove_message(const std::string& path); + + /** + * Remove a number if messages from the store. It will _not_ remove the + * message from the file system. + * + * @param ids vector with store ids for the message + */ + void remove_messages(const std::vector<Id>& ids); + + /** + * Remove a message from the store. It will _not_ remove the message + * from the file system. + * + * @param id the store id for the message + */ + void remove_message(Id id) { remove_messages({id}); } + + /** + * Find message in the store. + * + * @param id doc id for the message to find + * + * @return a message (if found) or Nothing + */ + Option<Message> find_message(Id id) const; + + /** + * Find a message's docid based on its path + * + * @param path path to the message + * + * @return the docid or Nothing if not found + */ + Option<Id> find_message_id(const std::string& path) const; + + /** + * Find the messages for the given ids + * + * @param ids document ids for the message + * + * @return id, message pairs for the messages found + * (which not necessarily _all_ of the ids) + */ + using IdMessageVec = std::vector<std::pair<Id, Message>>; + IdMessageVec find_messages(IdVec ids) const; + + /** + * Find the ids for all messages with a give message-id + * + * @param message_id a message id + * + * @return the ids of all messages with the given message-id + */ + IdVec find_duplicates(const std::string& message_id) const; + + /** + * does a certain message exist in the store already? + * + * @param path the message path + * + * @return true if the message exists in the store, false otherwise + */ + bool contains_message(const std::string& path) const; + + /** + * Options for moving + * + */ + enum struct MoveOptions { + None = 0, /**< Defaults */ + ChangeName = 1 << 0, /**< Change the name when moving */ + DupFlags = 1 << 1, /**< Update flags for duplicate messages too */ + DryRun = 1 << 2, /**< Don't really move, just determine target paths */ + }; + + /** + * Move a message both in the filesystem and in the store. After a successful move, the + * message is updated. + * + * @param id the id for some message + * @param target_mdir the target maildir (if any) + * @param new_flags new flags (if any) + * @param opts move options + * + * @return Result, either an IdPathVec with ids and paths for the moved message(s) or some + * error. Note that in case of success at least one message is returned, and only with + * MoveOptions::DupFlags can it be more than one. + * + * The first element of the IdPathVec, is the main message that got move; any subsequent + * (if any) are the duplicate paths, sorted by path-name. + */ + Result<IdPathVec> move_message(Store::Id id, + Option<const std::string&> target_mdir = Nothing, + Option<Flags> new_flags = Nothing, + MoveOptions opts = MoveOptions::None); + /** + * Convert IdPathVec -> IdVec + * + * @param ips idpath vector + * + * @return vector of ids + */ + static IdVec id_vec(const IdPathVec& ips); + + /** + * Prototype for the ForEachMessageFunc + * + * @param id :t store Id for the message + * @param path: the absolute path to the message + * + * @return true if for_each should continue; false to quit + */ + using ForEachMessageFunc = std::function<bool(Id, const std::string&)>; + + /** + * Call @param func for each document in the store. This takes a lock on + * the store, so the func should _not_ call any other Store:: methods. + * + * @param func a Callable invoked for each message. + * + * @return the number of times func was invoked + */ + size_t for_each_message_path(ForEachMessageFunc func) const; + + /** + * Prototype for the ForEachTermFunc + * + * @param term: + * + * @return true if for_each should continue; false to quit + */ + using ForEachTermFunc = std::function<bool(const std::string&)>; + + /** + * Call @param func for each term for the given field in the store. This + * takes a lock on the store, so the func should _not_ call any other + * Store:: methods. + * + * @param id the field id + * @param func a Callable invoked for each message. + * + * @return the number of times func was invoked + */ + size_t for_each_term(Field::Id id, ForEachTermFunc func) const; + + /** + * Get the timestamp for some message, or 0 if not found + * + * @param path the path + * + * @return the timestamp, or 0 if not found + */ + time_t message_tstamp(const std::string& path) const; + + /** + * Get the timestamp for some directory + * + * @param path the path + * + * @return the timestamp, or 0 if not found + */ + time_t dirstamp(const std::string& path) const; + + /** + * Set the timestamp for some directory + * + * @param path a filesystem path + * @param tstamp the timestamp for that path + */ + void set_dirstamp(const std::string& path, time_t tstamp); + + /* + * + * Some convenience + * + */ + + /** + * Get the Xapian database-path for this store + * + * @return the path + */ + const std::string& path() const { return xapian_db().path(); } + + /** + * Get the root-maildir for this store + * + * @return the root-maildir + */ + const std::string& root_maildir() const; + + /** + * Get the number of messages in the store + * + * @return the number + */ + size_t size() const { return xapian_db().size(); } + + /** + * Is the store empty? + * + * @return true or false + */ + bool empty() const { return xapian_db().empty(); } + + + /** + * Get the list of maildirs, that is, the list of maildirs + * under root_maildir, without file-system prefix. + * + * This does a file-system scan. + * + * @return list of maildirs + */ + std::vector<std::string> maildirs() const; + + + /** + * Compatible message-options for this store + * + * @return message-options. + */ + Message::Options message_options() const; + + + /* + * _almost_ private + */ + + /** + * Get a reference to the private data. For internal use. + * + * @return private reference. + */ + struct Private; + std::unique_ptr<Private>& priv() { return priv_; } + const std::unique_ptr<Private>& priv() const { return priv_; } + +private: + /** + * Construct a store for an existing document database + * + * @param path path to the database + * @param options startup options + */ + Store(const std::string& path, Options opts=Options::None); + + /** + * Construct a store for a not-yet-existing document database + * + * @param path path to the database + * @param config a configuration object + */ + Store(const std::string& path, const std::string& root_maildir, + Option<const Config&> conf); + + std::unique_ptr<Private> priv_; +}; + +MU_ENABLE_BITOPS(Store::Options); +MU_ENABLE_BITOPS(Store::MoveOptions); + +static inline std::string +format_as(const Store& store) +{ + return mu_format("store ({}/{})", format_as(store.xapian_db()), + store.root_maildir()); +} + +} // namespace Mu + +#endif /* MU_STORE_HH__ */ diff --git a/lib/mu-xapian-db.cc b/lib/mu-xapian-db.cc new file mode 100644 index 0000000..a8c897a --- /dev/null +++ b/lib/mu-xapian-db.cc @@ -0,0 +1,148 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + + +#include "mu-xapian-db.hh" +#include "utils/mu-utils.hh" +#include <inttypes.h> +#include <mu-config.hh> + +#include <mutex> + +using namespace Mu; + +const Xapian::Database& +XapianDb::db() const +{ + if (std::holds_alternative<Xapian::WritableDatabase>(db_)) + return std::get<Xapian::WritableDatabase>(db_); + else + return std::get<Xapian::Database>(db_); +} + +Xapian::WritableDatabase& +XapianDb::wdb() +{ + if (read_only()) + throw std::runtime_error("database is read-only"); + return std::get<Xapian::WritableDatabase>(db_); +} + +bool +XapianDb::read_only() const +{ + return !std::holds_alternative<Xapian::WritableDatabase>(db_); +} + +const std::string& +XapianDb::path() const +{ + return path_; +} + +void +XapianDb::set_timestamp(const std::string_view key) +{ + wdb().set_metadata(std::string{key}, mu_format("{}", ::time({}))); +} + +using Flavor = XapianDb::Flavor; + +static std::string +make_path(const std::string& db_path, Flavor flavor) +{ + if (flavor != Flavor::ReadOnly) { + /* we do our own flushing, set Xapian's internal one as + * the backstop*/ + g_setenv("XAPIAN_FLUSH_THRESHOLD", "500000", 1); + /* create path if needed */ + if (g_mkdir_with_parents(db_path.c_str(), 0700) != 0) + throw Error(Error::Code::File, "failed to create database dir {}: {}", + db_path, ::strerror(errno)); + } + + return db_path; +} + +static XapianDb::DbType +make_db(const std::string& db_path, Flavor flavor) +{ + switch (flavor) { + + case Flavor::ReadOnly: + return Xapian::Database(db_path); + case Flavor::Open: + return Xapian::WritableDatabase(db_path, Xapian::DB_OPEN); + case Flavor::CreateOverwrite: + return Xapian::WritableDatabase(db_path, Xapian::DB_CREATE_OR_OVERWRITE); + /* LCOV_EXCL_START*/ + default: + throw std::logic_error("unknown flavor"); + /* LCOV_EXCL_STOP*/ + } +} + +XapianDb::XapianDb(const std::string& db_path, Flavor flavor): + path_(make_path(db_path, flavor)), + db_(make_db(path_, flavor)), + batch_size_{Config(*this).get<Config::Id::BatchSize>()} +{ + if (flavor == Flavor::CreateOverwrite) + set_timestamp(MetadataIface::created_key); + + mu_debug("created {} / {} (batch-size: {})", flavor, *this, batch_size_); +} + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" +#include "config.h" +#include "mu-store.hh" + +static void +test_errors() +{ + allow_warnings(); + + TempDir tdir; + auto store = Store::make_new(tdir.path(), MU_TESTMAILDIR2); + assert_valid_result(store); + g_assert_true(store->empty()); + + XapianDb xdb(tdir.path(), Flavor::ReadOnly); + g_assert_true(xdb.read_only()); + + g_assert_false(!!xdb.delete_document("Boo")); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/xapian-db/errors", test_errors); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/lib/mu-xapian-db.hh b/lib/mu-xapian-db.hh new file mode 100644 index 0000000..f9753c6 --- /dev/null +++ b/lib/mu-xapian-db.hh @@ -0,0 +1,575 @@ +/* +** Copyright (C) 2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_XAPIAN_DB_HH__ +#define MU_XAPIAN_DB_HH__ + +#include <variant> +#include <memory> +#include <string> +#include <mutex> +#include <functional> +#include <unordered_map> + +#include <glib.h> + +#include <utils/mu-result.hh> +#include <utils/mu-utils.hh> + +/* starting with 1.4.6, Xapian supports C++ move semantics, + * but only with XAPIAN_MOVE_SEMANTICS defined + */ +#ifndef XAPIAN_MOVE_SEMANTICS +#define XAPIAN_MOVE_SEMANTICS +#endif /*XAPIAN_MOVE_SEMANTICS*/ +#include <xapian.h> + +namespace Mu { + +// LCOV_EXCL_START + +// avoid exception-handling boilerplate. +template <typename Func> void +xapian_try(Func&& func) noexcept +try { + func(); +} catch (const Xapian::Error& xerr) { + mu_critical("{}: xapian error '{}'", __func__, xerr.get_msg()); +} catch (const std::runtime_error& re) { + mu_critical("{}: runtime error: {}", __func__, re.what()); +} catch (const std::exception& e) { + mu_critical("{}: caught std::exception: {}", __func__, e.what()); +} catch (...) { + mu_critical("{}: caught exception", __func__); +} + +template <typename Func, typename Default = std::invoke_result<Func>> auto +xapian_try(Func&& func, Default&& def) noexcept -> std::decay_t<decltype(func())> +try { + return func(); +} catch (const Xapian::DocNotFoundError& xerr) { + return static_cast<Default>(def); +} catch (const Xapian::Error& xerr) { + mu_warning("{}: xapian error '{}'", __func__, xerr.get_msg()); + return static_cast<Default>(def); +} catch (const std::runtime_error& re) { + mu_critical("{}: runtime error: {}", __func__, re.what()); + return static_cast<Default>(def); +} catch (const std::exception& e) { + mu_critical("{}: caught std::exception: {}", __func__, e.what()); + return static_cast<Default>(def); +} catch (...) { + mu_critical("{}: caught exception", __func__); + return static_cast<Default>(def); +} + +template <typename Func> auto +xapian_try_result(Func&& func) noexcept -> std::decay_t<decltype(func())> +try { + return func(); +} catch (const Xapian::DatabaseNotFoundError& nferr) { + return Err(Error{Error::Code::Xapian, "failed to open database"}. + add_hint("Try (re)creating using `mu init'")); +} catch (const Xapian::DatabaseLockError& dlerr) { + return Err(Error{Error::Code::StoreLock, "database locked"}. + add_hint("Perhaps mu is already running?")); +} catch (const Xapian::DatabaseCorruptError& dcerr) { + return Err(Error{Error::Code::Xapian, "failed to read database"}. + add_hint("Try (re)creating using `mu init'")); +} catch (const Xapian::DocNotFoundError& dnferr) { + return Err(Error{Error::Code::Xapian, "message not found in database"}. + add_hint("Try reopening the database")); +} catch (const Xapian::Error& xerr) { + return Err(Error::Code::Xapian, "{}", xerr.get_msg()); +} catch (const std::runtime_error& re) { + return Err(Error::Code::Internal, "runtime error: {}", re.what()); +} catch (const std::exception& e) { + return Err(Error::Code::Internal, "caught std::exception: {}", e.what()); +} catch (...) { + return Err(Error::Code::Internal, "caught exception"); +} + +// LCOV_EXCL_STOP + +/// abstract base +struct MetadataIface { + virtual ~MetadataIface(){} + virtual void set_metadata(const std::string& name, const std::string& val) = 0; + virtual std::string metadata(const std::string& name) const = 0; + virtual bool read_only() const = 0; + + using each_func = std::function<void(const std::string&, const std::string&)>; + virtual void for_each(each_func&& func) const =0; + + /* + * These are special: handled on the Xapian db level + * rather than Config + */ + static inline constexpr std::string_view created_key = "created"; + static inline constexpr std::string_view last_change_key = "last-change"; +}; + + +/// In-memory db +struct MemDb: public MetadataIface { + /** + * Create a new memdb + * + * @param readonly read-only? (for testing) + */ + MemDb(bool readonly=false):read_only_{readonly} {} + + /** + * Set some metadata + * + * @param name key name + * @param val value + */ + void set_metadata(const std::string& name, const std::string& val) override { + map_.erase(name); + map_[name] = val; + } + + /** + * Get metadata for given key, empty if not found + * + * @param name key name + * + * @return string + */ + std::string metadata(const std::string& name) const override { + if (auto&& it = map_.find(name); it != map_.end()) + return it->second; + else + return {}; + } + + /** + * Is this db read-only? + * + * @return true or false + */ + bool read_only() const override { return read_only_; } + + + /** + * Invoke function for each key/value pair. Do not call + * @this from each_func(). + * + * @param func a function + */ + void for_each(MetadataIface::each_func&& func) const override { + for (const auto& [key, value] : map_) + func(key, value); + } + +private: + std::unordered_map<std::string, std::string> map_; + const bool read_only_; +}; + +/** + * Fairly thin wrapper around Xapian::Database and Xapian::WritableDatabase + * with just the things we need + locking + exception handling + */ +class XapianDb: public MetadataIface { +#define DB_LOCKED std::unique_lock lock__{lock_}; +public: + /** + * Type of database to create. + * + */ + enum struct Flavor { + ReadOnly, /**< Read-only database */ + Open, /**< Open existing read-write */ + CreateOverwrite, /**< Create new or overwrite existing */ + }; + + /** + * XapianDb CTOR. This may throw some Xapian exception. + * + * @param db_path path to the database + * @param flavor kind of database + */ + XapianDb(const std::string& db_path, Flavor flavor); + + /** + * DTOR + */ + ~XapianDb() { + if (tx_level_ > 0) + mu_warning("inconsistent transaction level ({})", tx_level_); + if (tx_level_ > 0) { + mu_debug("closing db after committing {} change(s)", changes_); + xapian_try([this]{ DB_LOCKED; wdb().commit_transaction(); }); + } else + mu_debug("closing db"); + } + + /** + * Is the database read-only? + * + * @return true or false + */ + bool read_only() const override; + + /** + * Path to the database; empty for in-memory databases + * + * @return path to database + */ + const std::string& path() const; + + /** + * Get a description of the Xapian database + * + * @return description + */ + const std::string description() const { + return db().get_description(); + } + + /** + * Get the number of documents (messages) in the database + * + * @return number + */ + size_t size() const noexcept { + return xapian_try([this]{ + DB_LOCKED; return db().get_doccount(); }, 0); + } + + /** + * Is the the base empty? + * + * @return true or false + */ + size_t empty() const noexcept { return size() == 0; } + + /** + * Get a database enquire object for queries. + * + * @return an enquire object + */ + Xapian::Enquire enquire() const { + DB_LOCKED; return Xapian::Enquire(db()); + } + + /** + * Get a document from the database if there is one + * + * @param id id of the document + * + * @return the document or an error + */ + Result<Xapian::Document> document(Xapian::docid id) const { + return xapian_try_result([&]{ + DB_LOCKED; return Ok(db().get_document(id)); }); + } + + /** + * Get metadata for the given key + * + * @param key key (non-empty) + * + * @return the value or empty + */ + std::string metadata(const std::string& key) const override { + return xapian_try([&]{ + DB_LOCKED; return db().get_metadata(key);}, ""); + } + + /** + * Set metadata for the given key + * + * @param key key (non-empty) + * @param val new value for key + */ + void set_metadata(const std::string& key, const std::string& val) override { + xapian_try([&] { DB_LOCKED; wdb().set_metadata(key, val); + maybe_commit(); }); + } + + /** + * Invoke function for each key/value pair. This is called with the lock + * held, so do not call functions on @this is each_func(). + * + * @param each_func a function + */ + //using each_func = MetadataIface::each_func; + void for_each(MetadataIface::each_func&& func) const override { + xapian_try([&]{ + DB_LOCKED; + for (auto&& it = db().metadata_keys_begin(); + it != db().metadata_keys_end(); ++it) + func(*it, db().get_metadata(*it)); + }); + } + + /** + * Does the given term exist in the database? + * + * @param term some term + * + * @return true or false + */ + bool term_exists(const std::string& term) const { + return xapian_try([&]{ + DB_LOCKED; return db().term_exists(term);}, false); + } + + /** + * Add a new document to the database + * + * @param doc a document (message) + * + * @return new docid or 0 + */ + Result<Xapian::docid> add_document(const Xapian::Document& doc) { + return xapian_try_result([&]{ + DB_LOCKED; + auto&& id{wdb().add_document(doc)}; + set_timestamp(MetadataIface::last_change_key); + maybe_commit(); + return Ok(std::move(id)); + }); + } + + /** + * Replace document in database + * + * @param term unique term + * @param id docid + * @param doc replacement document + * + * @return new docid or an error + */ + Result<Xapian::docid> + replace_document(const std::string& term, const Xapian::Document& doc) { + return xapian_try_result([&]{ + DB_LOCKED; + auto&& id{wdb().replace_document(term, doc)}; + set_timestamp(MetadataIface::last_change_key); + maybe_commit(); + return Ok(std::move(id)); + }); + } + Result<Xapian::docid> + replace_document(Xapian::docid id, const Xapian::Document& doc) { + return xapian_try_result([&]{ + DB_LOCKED; + wdb().replace_document(id, doc); + set_timestamp(MetadataIface::last_change_key); + maybe_commit(); + return Ok(std::move(id)); + }); + } + + /** + * Delete document(s) for the given term or id + * + * @param term a term + * + * @return Ok or Error + */ + Result<void> delete_document(const std::string& term) { + return xapian_try_result([&]{ + DB_LOCKED; + wdb().delete_document(term); + set_timestamp(MetadataIface::last_change_key); + maybe_commit(); + return Ok(); + }); + } + Result<void> delete_document(Xapian::docid id) { + return xapian_try_result([&]{ + DB_LOCKED; + wdb().delete_document(id); + set_timestamp(MetadataIface::last_change_key); + maybe_commit(); + return Ok(); + }); + } + + template<typename Func> + size_t all_terms(const std::string& prefix, Func&& func) const { + DB_LOCKED; + size_t n{}; + for (auto it = db().allterms_begin(prefix); it != db().allterms_end(prefix); ++it) { + if (!func(*it)) + break; + ++n; + } + return n; + } + + /* + * If the "transaction ref count" > 0 (with inc_transactions());, we run + * in "transaction mode". That means that the subsequent Xapian mutation + * are part of a transactions, which is flushed when the number of + * changes reaches the batch size, _or_ the transaction ref count is + * decreased to 0 (dec_transactions()). * + */ + + /** + * Increase the transaction level; needs to be balance by dec_transactions() + */ + void inc_transaction_level() { + xapian_try([this]{ + DB_LOCKED; + if (tx_level_ == 0) {// need to start the Xapian transaction? + mu_debug("begin transaction"); + wdb().begin_transaction(); + } + ++tx_level_; + mu_debug("ind'd tx level to {}", tx_level_); + }); + } + + /** + * Decrease the transaction level (to balance inc_transactions()) + * + * If the level reach 0, perform a Xapian commit. + */ + void dec_transaction_level() { + xapian_try([this]{ + DB_LOCKED; + if (tx_level_ == 0) { + mu_critical("cannot dec transaction-level)"); + throw std::runtime_error("cannot dec transactions"); + } + + --tx_level_; + if (tx_level_ == 0) {// need to commit the Xapian transaction? + mu_debug("committing {} changes", changes_); + changes_ = 0; + wdb().commit_transaction(); + } + + mu_debug("dec'd tx level to {}", tx_level_); + }); + } + + /** + * Are we inside a transaction? + * + * @return true or false + */ + bool in_transaction() const { DB_LOCKED; return tx_level_ > 0; } + + + /** + * RAII Transaction object + * + */ + struct Transaction { + Transaction(XapianDb& db): db_{db} { + db_.inc_transaction_level(); + } + ~Transaction() { + db_.dec_transaction_level(); + } + private: + XapianDb& db_; + }; + + + /** + * Manually request the Xapian DB to be committed to disk. This won't + * do anything while in a transaction. + */ + void commit() { + xapian_try([this]{ + DB_LOCKED; + if (tx_level_ == 0) { + mu_info("committing xapian-db @ {}", path_); + wdb().commit(); + } else + mu_debug("not committing while in transaction"); + }); + } + + using DbType = std::variant<Xapian::Database, Xapian::WritableDatabase>; +private: + + /** + * To be called after all changes, with DB_LOCKED held. + */ + void maybe_commit() { + // in transaction-mode and enough changes, commit them + // and start a new transaction + if (tx_level_ > 0 && ++changes_ >= batch_size_) { + mu_debug("batch full ({}/{}); committing change", changes_, batch_size_); + wdb().commit_transaction(); + wdb().commit(); + --tx_level_; + changes_ = 0; + wdb().begin_transaction(); + ++tx_level_; + } + } + + void set_timestamp(const std::string_view key); + + /** + * Get a reference to the underlying database + * + * @return db database reference + */ + const Xapian::Database& db() const; + /** + * Get a reference to the underlying writable database. It is + * an error to call this on a read-only database. + * + * @return db writable database reference + */ + Xapian::WritableDatabase& wdb(); + + mutable std::mutex lock_; + std::string path_; + DbType db_; + size_t tx_level_{}; + const size_t batch_size_; + size_t changes_{}; +}; + +constexpr std::string_view +format_as(XapianDb::Flavor flavor) +{ + switch(flavor) { + case XapianDb::Flavor::CreateOverwrite: + return "create-overwrite"; + case XapianDb::Flavor::Open: + return "open"; + case XapianDb::Flavor::ReadOnly: + return "read-only"; + default: + return "??"; + } +} + +static inline std::string +format_as(const XapianDb& db) +{ + return mu_format("{} @ {}", db.description(), db.path()); +} + +} // namespace Mu + +#endif /* MU_XAPIAN_DB_HH__ */ diff --git a/lib/tests/bench-indexer.cc b/lib/tests/bench-indexer.cc new file mode 100644 index 0000000..e76f9f8 --- /dev/null +++ b/lib/tests/bench-indexer.cc @@ -0,0 +1,553 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include <glib.h> +#include <string> +#include <thread> +#include <vector> +#include <iostream> +#include <fstream> + +#include <utils/mu-utils.hh> +#include <utils/mu-regex.hh> +#include <mu-store.hh> +#include "mu-maildir.hh" + +#include "utils/mu-test-utils.hh" + +using namespace Mu; + +constexpr auto test_msg = +R"(Return-Path: <htcondor-users-bounces@cs.wisc.edu> +Received: from pop3.web.de [212.227.17.177] + by localhost with POP3 (fetchmail-6.4.6) + for <arne@localhost> (single-drop); Fri, 26 Jun 2020 12:56:08 +0200 (CEST) +Received: from jeeves.cs.wisc.edu ([128.105.6.16]) by mx-ha.web.de (mxweb112 + [212.227.17.8]) with ESMTPS (Nemesis) id 1MdMYE-1jFXaM2gnA-00ZKvt for + <@ID@@web.de>; Fri, 26 Jun 2020 01:28:11 +0200 +Received: from jeeves.cs.wisc.edu (localhost [127.0.0.1]) + by jeeves.cs.wisc.edu (8.14.4/8.14.4) with ESMTP id 05PNLgek013419; + Thu, 25 Jun 2020 18:22:23 -0500 +Received: from shale.cs.wisc.edu (shale.cs.wisc.edu [128.105.6.25]) + by jeeves.cs.wisc.edu (8.14.4/8.14.4) with ESMTP id 05PNLaf0013414 + (version=TLSv1/SSLv3 cipher=AES256-GCM-SHA384 bits=256 verify=OK) + for <htcondor-users@jeeves.cs.wisc.edu>; Thu, 25 Jun 2020 18:21:36 -0500 +Received: from smtp7.wiscmail.wisc.edu (wmmta4.doit.wisc.edu [144.92.197.245]) + by shale.cs.wisc.edu (8.14.4/8.14.4) with ESMTP id 05PNLaMK013694 + (version=TLSv1/SSLv3 cipher=DHE-RSA-AES128-GCM-SHA256 bits=128 + verify=NO) + for <htcondor-users@cs.wisc.edu>; Thu, 25 Jun 2020 18:21:36 -0500 +Received: from USG02-CY1-obe.outbound.protection.office365.us + ([23.103.209.108]) by smtp7.wiscmail.wisc.edu + (Oracle Communications Messaging Server 8.0.2.4.20190812 64bit (built + Aug 12 + 2019)) with ESMTPS id <0QCI042LC8VUXFC0@smtp7.wiscmail.wisc.edu> for + htcondor-users@cs.wisc.edu (ORCPT htcondor-users@cs.wisc.edu); Thu, + 25 Jun 2020 18:21:31 -0500 (CDT) +X-Spam-Report: IsSpam=no, Probability=11%, Hits= RETURN_RECEIPT 0.5, + FROM_US_TLD 0.1, HTML_00_01 0.05, HTML_00_10 0.05, SUPERLONG_LINE 0.05, + BODYTEXTP_SIZE_3000_LESS 0, BODY_SIZE_10000_PLUS 0, DKIM_SIGNATURE 0, + KNOWN_MTA_TFX 0, NO_URI_HTTPS 0, SPF_PASS 0, SXL_IP_TFX_WM 0, + WEBMAIL_SOURCE 0, WEBMAIL_XOIP 0, WEBMAIL_X_IP_HDR 0, __ANY_URI 0, + __ARCAUTH_DKIM_PASSED 0, __ARCAUTH_DMARC_PASSED 0, __ARCAUTH_PASSED 0, + __ATTACHMENT_SIZE_0_10K 0, __ATTACHMENT_SIZE_10_25K 0, + __BODY_NO_MAILTO 0, + __CT 0, __CTYPE_HAS_BOUNDARY 0, __CTYPE_MULTIPART 0, __HAS_ATTACHMENT 0, + __HAS_ATTACHMENT1 0, __HAS_ATTACHMENT2 0, __HAS_FROM 0, __HAS_MSGID 0, + __HAS_XOIP 0, __HIGHBITS 0, __MIME_TEXT_P 0, __MIME_TEXT_P1 0, + __MIME_TEXT_P2 0, __MIME_VERSION 0, __MULTIPLE_RCPTS_TO_X2 0, + __NO_HTML_TAG_RAW 0, __RETURN_RECEIPT_TO 0, __SANE_MSGID 0, + __TO_MALFORMED_2 0, __TO_NAME 0, __TO_NAME_DIFF_FROM_ACC 0, + __TO_NO_NAME 0, + __TO_REAL_NAMES 0, __URI_IN_BODY 0, __URI_MAILTO 0, __URI_NOT_IMG 0, + __URI_NO_PATH 0, __URI_NS , __URI_WITHOUT_PATH 0 +X-Wisc-Doma: @ID@X@numerica.us,numerica.us +X-Wisc-Env-From-B64: d2VzbGV5LnRheWxvckBudW1lcmljYS51cw== +X-Spam-PmxInfo: Server=avs-13, Version=6.4.7.2805085, + Antispam-Engine: 2.7.2.2107409, Antispam-Data: 2020.6.25.231519, + AntiVirus-Engine: 5.74.0, AntiVirus-Data: 2020.6.25.5740002, + SenderIP=[23.103.209.108] +X-Wisc-DKIM-Verify: @ID@XXXXXXX@numerica.us,numericaus.onmicrosoft.com!pass +X-Spam-Score: * +ARC-Seal: i=1; a=rsa-sha256; s=arcselector5401; d=microsoft.com; cv=none; + b=KyXoddJsnsHsBwhdlO5rcljgMRaylJAUAxWTjG4jQL1C8XJAMgeERtH2sRffdjibYUFfSuDUNJmrTrvrbjKGUt2I8J2M2MgUB/upMoroVPNBrP1Fy9wMeZJQuSS4r4KjZZktsl2i8eq667pzOZO6+wX2IA5M7YtxDqglcWOE6btWzbABVjx+9eCXMt0eMd1+UI6ABK8Frd33EFQLKT0h/cxidWR9l+0gCMAcRxsLrQ82+ckU606AIV/DA1E4Tq7ADe/+CRv4QszDN93pWL/1N2/OOh9vFTs9g9ZG6uXjN+Km/IAdylPbfHgKW60ev3/Bvv6N3pA7DjpuiKj6BnW7mQ== +ARC-Message-Signature: i=1; a=rsa-sha256; c=relaxed/relaxed; d=microsoft.com; + s=arcselector5401; + h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; + bh=OZrj1we1ZUH0xBMhJ5/F6EQnB0cmitFs2xZW1fLMRNs=; + b=Pq07a3u26s2UdpucJuVQ0h68272wx46Wp61x/30TelPPFLCRxVjmlH1U3IBmIsZ1jOEtGXFJRv65L3HmwGxRUdLlMOdPRB64BBfHQ9NGWUBykKQmOrJNGJs635nEdpugpzngzIdcg1PS5vHxPJAnOeqoo71OVPI3JqPrPEn2TJJgb9J6PApexkqIbVl35prGPsyS/t2IlYw3/ihWzORG6wvqJeqedgpJTBXeGaDoMa+MQ1BeUsdvybh8+hau4ASpM5lwyeXlGmJ5mUTZi39jp+dFdDrmCj/VM4ezeuXeH9+HFtDjKLZJaTDWUID0IBcr91BaoQE/4r6y+lpkah6LLQ== +ARC-Authentication-Results: i=1; mx.microsoft.com 1; spf=pass + smtp.mailfrom=numerica.us; + dmarc=pass action=none header.from=numerica.us; + dkim=pass header.d=numerica.us; arc=none +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; + d=numericaus.onmicrosoft.com; s=selector1-numericaus-onmicrosoft-com; + h=From:Date:Subject:Message-ID:Content-Type:MIME-Version:X-MS-Exchange-SenderADCheck; + bh=OZrj1we1ZUH0xBMhJ5/F6EQnB0cmitFs2xZW1fLMRNs=; + b=cFn0eL5k2IKry9U8qa8mbVaxRiyicUAWzRc3NUtj+VEbgShfrz8SO6FPX20WTQQJg/Fu/3isqsSEUt+9NSEEbgd5eQ1EVz5E/JVeNjPe9GXR0JEF/g3f6yM7CO+kKTvXSRvQjce683U0j7Aj1pSDEktoVNP4xvOS2Gx9VjdWTmc= +Received: from DM3P110MB0474.NAMP110.PROD.OUTLOOK.COM (2001:489a:200:413::10) + by DM3P110MB0490.NAMP110.PROD.OUTLOOK.COM (2001:489a:200:413::14) + with Microsoft SMTP Server + (version=TLS1_2, cipher=TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384) + id 15.20.3109.25; Thu, 25 Jun 2020 23:21:07 +0000 +Received: from DM3P110MB0474.NAMP110.PROD.OUTLOOK.COM + ([fe80::f548:f084:9867:9375]) by DM3P110MB0474.NAMP110.PROD.OUTLOOK.COM + ([fe80::f548:f084:9867:9375%11]) with mapi id 15.20.3131.024; Thu, + 25 Jun 2020 23:21:07 +0000 +From: Raul Endymion <XXXXXXXXXXXXXXXXXXXX@numerica.us> +To: "'htcondor-users@cs.wisc.edu'" <htcondor-users@cs.wisc.edu> +Thread-topic: OPINIONS WANTED: Are there any blatent downsides I am missing to + the following Condor configuration +Thread-index: AdZLRbEvYoEDBZChS62aOHgPzKD8kw== +Date: Thu, 25 Jun 2020 23:21:06 +0000 +Message-id: <DM3P110MB04746CDBA55B3E597EFD1877FA920@DM3P110MB0474.NAMP110.PROD.OUTLOOK.COM> +Accept-Language: en-US +Content-language: en-US +X-MS-Has-Attach: yes +X-MS-TNEF-Correlator: +X-Originating-IP: [50.233.29.54] +x-ms-publictraffictype: Email +x-ms-office365-filtering-correlation-id: f4edecdf-5582-4b2d-226a-08d8195e7007 +x-ms-traffictypediagnostic: DM3P110MB0490: +x-microsoft-antispam-prvs: <DM3P110MB04907C313C4243B4FE42A20CFA920@DM3P110MB0490.NAMP110.PROD.OUTLOOK.COM> +x-ms-oob-tlc-oobclassifiers: OLM:10000; +x-forefront-prvs: 0445A82F82 +x-ms-exchange-senderadcheck: 1 +x-microsoft-antispam: BCL:0; +X-Forefront-Antispam-Report: CIP:255.255.255.255; CTRY:; LANG:en; SCL:1; SRV:; + IPV:NLI; SFV:NSPM; H:DM3P110MB0474.NAMP110.PROD.OUTLOOK.COM; + PTR:; CAT:NONE; SFTY:; + SFS:(346002)(366004)(6916009)(83380400001)(71200400001)(8936002)(55016002)(5660300002)(8676002)(9686003)(66616009)(64756008)(33656002)(52536014)(66446008)(66476007)(66556008)(66946007)(99936003)(76116006)(186003)(86362001)(26005)(7696005)(6506007)(44832011)(508600001)(2906002)(80162005)(80862006)(491001)(554374003); + DIR:OUT; SFP:1102; +x-ms-exchange-transport-forked: True +MIME-version: 1.0 +X-OriginatorOrg: numerica.us +X-MS-Exchange-CrossTenant-Network-Message-Id: f4edecdf-5582-4b2d-226a-08d8195e7007 +X-MS-Exchange-CrossTenant-OriginalArrivalTime: 25 Jun 2020 23:21:06.8341 (UTC) +X-MS-Exchange-CrossTenant-fromentityheader: Hosted +X-MS-Exchange-CrossTenant-id: fae7a2ae-df1d-444e-91be-babb0900b9c2 +X-MS-Exchange-CrossTenant-mailboxtype: HOSTED +X-MS-Exchange-Transport-CrossTenantHeadersStamped: DM3P110MB0490 +Subject: [HTCondor-users] OPINIONS WANTED: Are there any blatent downsides I + am missing to the following Condor configuration +X-BeenThere: htcondor-users@cs.wisc.edu +X-Mailman-Version: 2.1.19 +Precedence: list +List-Id: HTCondor-Users Mail List <htcondor-users.cs.wisc.edu> +List-Unsubscribe: <https://lists.cs.wisc.edu/mailman/options/htcondor-users>, + <mailto:htcondor-users-request@cs.wisc.edu?subject=unsubscribe> +List-Archive: <https://www-auth.cs.wisc.edu/lists/htcondor-users/> +List-Post: <mailto:htcondor-users@cs.wisc.edu> +List-Help: <mailto:htcondor-users-request@cs.wisc.edu?subject=help> +List-Subscribe: <https://lists.cs.wisc.edu/mailman/listinfo/htcondor-users>, + <mailto:htcondor-users-request@cs.wisc.edu?subject=subscribe> +Reply-To: HTCondor-Users Mail List <htcondor-users@cs.wisc.edu> +Content-Type: multipart/mixed; boundary="===============0678627779074767862==" +Errors-To: htcondor-users-bounces@cs.wisc.edu +Sender: "HTCondor-users" <htcondor-users-bounces@cs.wisc.edu> +Envelope-To: <@ID@XXXXX@web.de> +X-UI-Filterresults: unknown:2;V03:K0:cdojl5YHfkg=:jhTbQXp38SL2za/LB4M7aUwpyw + 5rDHoN1+/ScH/O9/G1fKWbGryQ203thF+1ZrHUOOwq8MVOc5SsoqzSTsaNbEAdthFcDDz3Oui + SHxX1hdpV3UOjZEHzWlpjEjRe7t74g2RI/ESELmkPuLg/LZC7SjAsg70cTJBIfDPYxJkJAcUl + 9W6OEBsmtTDO0va/EQRYjfkpoF9tjfmfMNw9KSKHuDdqZu2Xfak8mQKnWsoxWeUkD31r60iPC + yikbj7KP5AlHaWMzyTTdlvtjYRLfSuUSe1uqjI5NWCnZDDjz7zODoaWPp7p2U/MQenXEjN6+M + WnZL6ZC8AGtze/hYgOCXcLf4ydQ7m9YueJiY5nDn7g+cwnhxypVNFTL5NjSpKKXbkzbyu9Tdl + ez+92g/9pGW17iOo5NrFtfctLlmCEH0RxjouKI7FBmv3bIvFC4FvfghiNf7OZmRg2/nT5i+1o + AICYNAx2y5CezKsKM2f1tm60dkydQIR8pK45dDKZPz3i7NeJm9dknZ2OYFTnucUvdPaT8nR43 + cK3kk2QUE48Ngo/0NwepSGrV9TkOt+hY3PUYkXWp/mwP2QPSjy4cALyvLyKwG24qZ9CiiRLMV + KqPFlCRnoDG5MHJ4d0krFlqmg8rNsWzV3oWfMNKFZmD24lVUmWGb+ExxbCFc0xzIt12o/EqBw + nVkXLu+E0apM+cmG6ubYfOymRoUpiKsZI9ivc+mAaEE+v2RBzcAURzlhzQHIn81onvzbQwCge + tMtBkSEfqyoa1HjalX4B9WQ90M42K+7xW039ydakQ7JOeYVpkPXYoBF7mbrXRckhMXjQatLQ8 + MWA8+U031Xfa1ueOIfCCkzJ49wyx1LoLPyqdjCvnzaRd72yNEMJ5zM/itMIPE9reIHBtpom0i + RhIYdJFDrL+SKqE68lcJakCcF3R+VLApwLKOr0HChGQjdEk7c/rm5E0dF5f3oYlHf591QoXIJ + h16yfcJYe6fMo1YYunkvbEDFPpzttIq7aIk0FzxrOdRvj3yQajDbwOpYI/5T/DaabPn3M8lK7 + 8pn7LrbmyCaLHhkYMS4h3SDkYWsifza6vkldizrK7IPf6KhS7AhTkbnEonWS6454GLUg1nYGX + W5Qp/G9LzvjtEGQMcwnCN5jb5zq7o3f+9FrROKjpwFxL+mL85CEXY/KMOVpf6hDJJfSyu6/X6 + FpbwlJVLFdGeA0/+xcKcmutpkJACgK2kHqvZ8MZxt+5jBJWVlIDLZKa8/IoGWC+ikLX2/hPNB + 4TU89QYG5ygPmwwDXruFG7N7jVURZceHqWNKtqegS6YQ5nirsPJWJR7jzgr+HbntUaQETXNpn + QrxpsVHXfqRu2GlP5h28RaIpvBVUcwqrs+eLJELStvBzyAmaVPVoKFjEWFfwrmE89W6Bmz2W3 + kHExOq3hI3gDsGXKjTjT/kjHkaHmtnVUXr4vqovf8Ht4Vwmtf0S4xsgpYjnYjUIzG9eiwIFAZ + hL2gvjwW51qtMvybf01C50xTiS9GSfO0SR7meBPA67skcA+wFo11wmwXsUk1irpKnC+Y9hVZX + 2vPkfZ1T2VXNo997cQC59lBpi/TU5gnuM7H/Vcl7tF3Lqtmqut7s6HkPWCegDZ3O2W7shH7aZ + 1bOXbO+W/SNC+WcMnj+fhuP+dHcrt0Vw4RD9knJOOzdZTH3OCli/vpjqgTbCKEaWMhCIeM2g0 + RiLFxTeTEEBCa49bwa8n2r4T/vA3duZd8F/DNKvWTfhRr1Mxtz3n15EOar13fFijtnieEiv4/ + vO/5uRF+H86Fcoua7B8AswThbiG1vou6M48g0Zo6iGEcrueKEaHMI4XM7wQF77KazMdn5f1BP + +KyQX83aHJN/qGniXgF8yu+h0M7Nf0YrTteYQd2C/HZrIA8IaLqqvLoGRl7dRBnbZiP7jRdQm + 1YEYtjX4XBoShrXPfIxPnJBUBnnOaePYxOJkS2FaBv19jPkMnyc9xuJYD7JOTFnXKzAnoaBqT + OR+dGrLLGZ1MM/0gqclKTv7Hcce+6CJyTWkx5mq42w49HFI/kdHBRxU8xIRv4B9l0ePf9EbWr + cDcrssee//6KXiRmF4fm7jq828/uhj8MIJet9sIU5ncKwHEse3I4YmVT5+dB+ZGZh0gbJPFj6 + xcICpshhYct+euMCdNfy3lkxiRr76RwfBzLAOP5+1U3GAx/hcsL2AgyBHMwWo+Kkeq8pPy4YI + pQMxJyylI6JMa/DbBggnDk+xNZpRKo/XA4lAJY57DCOPL2ZcL8kU2aCd5LjtYHK0ZWSFtOjxs + oIEr/f2vvg+zibxzaANBzylZn3yPe9pI/IBefu9fL4MVaYY3aboxuncX4fyi0VH0WbFkSYXRi + a7LIu3LI2LTU13C/LE7j9hmxP6TApyiXi14f0GSa2sbF6HWp2v2rhYM7h67AAn3SQgvcJLpgb + Hz5ABb/OAk6ABVEl+a483zexJ6iT2P0gYc08zmewy8Jf8AD9r846k9pGZuhBaOHREx3bA16Bj + uWYh3QzSI6MQoJM3XbBGLVkX36Lfj54T9kk97lLaxfbGPuNoyOV9iTBKxts3m2KD+52iH3EEi + glbH6HNIUHyCHdEXsXyGVFwfM9V7OQcVO/g266KIQ74wU16x/Zdsq4p/1PcRXHRnoMxP/pUrj + EOLWzFU71qzC/OSkYWRil9HXUyucTFGQ0N08jZNXctI9lElWtgq3iI+Cz2F20rz+LJGhSHSkZ + 0G5JgXrtspeJN5yoH6TOE0hblr5sZcAM0wiSP7x/hPBeYHswzTA5/laWMn++9aTPVgpPaJ9/x + wyLm55OZr4Jl+StWd3MqLCgiRB3cNGrDX7f8Eqnj4wfCHiGIUHewD4qrfXraZQhIk17W+9JyD + osmUiVD9ZRdNCY2eNnu8ZkJ4uzKl44lwLL43sInKBjdAHlnoxrR2FOrYXbnU31ujwxdeUr6Hs + xPFy0Git0CpWCWYmaz37KA8GW7PE4ffWzcfCmz6AKBrbHcCreeUnyqnSEDy9ubnz7mcLRnu3W + RAWi6diI8gcS9g0+r4z5PtZX9rveXRekHJ4k08VuYVmdiz3gjXmHPlm9IKPEAbygP2EYgjwGE + RbReLc8xHJlfLbwdXyGw0HU= + +--===============0678627779074767862== +Content-language: en-US +Content-type: multipart/signed; protocol="application/x-pkcs7-signature"; + micalg=2.16.840.1.101.3.4.2.3; + boundary="----=_NextPart_000_0018_01D64B14.F58791A0" + +------=_NextPart_000_0018_01D64B14.F58791A0 +Content-Type: text/plain; + charset="utf-8" +Content-Transfer-Encoding: quoted-printable + +Hey! + +I am architecting our final HTCondor configuration over here and I have = +an idea I am unsure about and I would like to ask some experienced users = +for their opinion. + +Background, we have a small, relatively homogenous cluster (with no = +special universes) and less than 10 users. Since each user has their own = +workstation separate from our cluster I thought the following = +configuration would suit our needs, but I want to make sure there isn't = +a huge disadvantage I am missing: + +1. Set the Central Manager to be highly available to the point of = +tolerating N cluster machine failures +2. Put a Submit on each of the users' workstations (I am a little = +worried about the resource usage of the condor_shadow and condor_schedd, = +my users are already running into RAM consumption issues over time as it = +is) +3. Place an Execute on each of the cluster machines, which would lead to = +the central manager being on a machine that is also executing jobs + +Fortunately both my users' and cluster machines all have access to the = +same network storage, and we have centralized authentication so we can = +just use our users' credentials to authenticate everywhere.=20 + +Before I set this in dry mud, does anyone have any retrospective = +recommendations I could benefit hearing from, since I am still pretty = +new to the project? + +Thank you! +-Raul + +Raul Endymion =E2=80=93 Cluster Manager +Numerica Corporation (www.numerica.us) +5042 Technology Parkway #100 +Fort Collins, Colorado 80528 +=E2=98=8E=EF=B8=8F (970) 207 2233 +=F0=9F=93=A7 @ID@XXXXXXXXXXXXXXX@numerica.us + + + +------=_NextPart_000_0018_01D64B14.F58791A0 +Content-Type: application/pkcs7-signature; + name="smime.p7s" +Content-Transfer-Encoding: base64 +Content-Disposition: attachment; + filename="smime.p7s" + +MIAGCSqGSIb3DQEHAqCAMIACAQExDzANBglghkgBZQMEAgMFADCABgkqhkiG9w0BBwEAAKCCEv4w +ggWpMIIDkaADAgECAhAV2Tfkh0+gtEu0gskeSMTdMA0GCSqGSIb3DQEBCwUAMFsxEjAQBgoJkiaJ +k/IsZAEZFgJ1czEYMBYGCgmSJomT8ixkARkWCG51bWVyaWNhMRIwEAYKCZImiZPyLGQBGRYCYWQx +FzAVBgNVBAMTDmFkLUdJTEdBTEFELUNBMB4XDTE2MDcyNDE5NTcxM1oXDTM2MDcyNDIwMDcxMlow +WzESMBAGCgmSJomT8ixkARkWAnVzMRgwFgYKCZImiZPyLGQBGRYIbnVtZXJpY2ExEjAQBgoJkiaJ +k/IsZAEZFgJhZDEXMBUGA1UEAxMOYWQtR0lMR0FMQUQtQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4IC +DwAwggIKAoICAQCq+/935KPrc8clxrq76k7GrrUHRbsM4FCfyrWicGPZsOKbJfcoloF2EAfj6AYR +QyU/l9um/8NqW+cu6/TY6YcY622L+UtT1QWC/Kt0kVL7cTtZN+VK/BkjcDVbUOqdeFY1q0tMzdco +WFxqjayGRYnX6oEZ7krDsGtJBBET/504Z3vDq/0ZD3lNG2dCWp1y+3VzUcb+OKkOPwMGHpw3gZM5 +lZN/znB7d7qwxFSRoLzZZB3nZKKJHcp2ZuyJR+pCT5VdHGGV4gpVQKuL49/UoJBA0o8Kv0DGPByD ++LVwhlyFMi2jlnCd5lqiWRw9JAE3fqS/Di/cGbMjXMI2CplBj+GmZH8fgy4BQRwmsOUELTaYkJyJ +otcHGENO1+xYrR/lFEQLhh+8V2IJvBM2G1dgJ3EuEslL4q0xGeYLZJd7Z9xvXkAJaX/eWjHWICFI +zbsH/6fBqXYow/V8hfZhb20dGGnPESXPqMv/1mLgUIqr++Fjl6zKM5mYZuHlmrtd+eLgg7VsjDvh +cMxdQnju+jzJflxlmY2KSwt5lsu7viqmQyqVUnHFaEsV116B0uCROc5o1pBdRMdeeLrRoj6xPVlc +IzmIZz3wZERxCAWeJqBx5d1kXe+cDL4pMNQ/hmah4mshjtyOGv+oEgcdxzUQ72W7JNLhSv8C6gpU +eQwPq8usFAvUOwIDAQABo2kwZzATBgkrBgEEAYI3FAIEBh4EAEMAQTAOBgNVHQ8BAf8EBAMCAYYw +DwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUF+CLMX/eZk96ElRSeiEHqnsujqEwEAYJKwYBBAGC +NxUBBAMCAQAwDQYJKoZIhvcNAQELBQADggIBACcwALtn+SFUx+YTrLCFY+Ghh4yubQt3YdEI6hOQ +JnmNPKsUEzCvoRE5L2ZLkG2VhJNX3KAJmXgkZMCGBPbiA/65r/cbYqZATQEG/g9aVicz/IBHXvg4 +7+YDDN9VpRy8c93AZNNTRf83Pw+CDsdIGG7mg8rc0tiCgt0V3gN0wF8oRSsb/trqd+ujk41bvaPw +Rl+8JUeRN0Pq9lH4VGGk9GEIQv8JXhr2VKFmJcGKLB+qvMRvWQZ5oPTGDE3pUYI5q8f7/fMiJKU6 +hb9l+tXP7uDLWIawg/MoUc2BwAThyXFk9LZhkYWYpzbaf2Ez2JYieD4ey8RjEKvis9mF6Z/p6+69 +GbYvuf2bRikYenrmboXCUO820totjP2UyHczexZsMP/XznmyDJuN+BDLzLjm7ks8lXDwpF/Kqnjm +1EyiQI0OB4cn889yM039U7raJeHpuiwju2/YO6krE+plLQhkM7pl6v6Ly/ZKICwDfbcU8k8LE4+K +3VaXmVYRYbSXx8l2Ke0CWKNfehBGQ024gKjNt8t7gCgInG5s+roumqeKyfCWlhYll1FAxEQmwP/6 +966y7uJrGLra0VUjdppbZpAENSF0pdX08VfsasSZ20hnCaLWO1b3i0ZOBLBAoNzeCm+BdS6DAOhy +JnHHZ+OBoiaYwCSjSvTDmHyQkNK3wmu+/wyNMIIGnDCCBISgAwIBAgITbwAAAEFhCq43is5OqAAA +AAAAQTANBgkqhkiG9w0BAQsFADBbMRIwEAYKCZImiZPyLGQBGRYCdXMxGDAWBgoJkiaJk/IsZAEZ +FghudW1lcmljYTESMBAGCgmSJomT8ixkARkWAmFkMRcwFQYDVQQDEw5hZC1HSUxHQUxBRC1DQTAe +Fw0xOTA3MjIxNDE4MDFaFw0yMTA3MjIxNDI4MDFaMFwxEjAQBgoJkiaJk/IsZAEZFgJ1czEYMBYG +CgmSJomT8ixkARkWCG51bWVyaWNhMRIwEAYKCZImiZPyLGQBGRYCYWQxGDAWBgNVBAMTD2FkLUNF +TEVCUklBTi1DQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAKRLgjg0yC0P2jLwTCIA +V/zEGk/PEc3pZxNAo7m0I/SXdNulUEkjxai5Wq53i0EhWVLpUU8XY3joXax46yCMqh0PUn90QmMD +BybLyFDX6av8tVS5cQs0HbTZdIuj7A/dsKzKKIrSHd3SQ9MLNPRkSRdhagmf5LCF1Y4xEEiuAA/H +XdYAxGIcl8n6b2CcLlZzq4W13Ipv8FIZoDsG1u0b9NGfeSOOHidi5kdD6r8lM5PaSPmZsl5PdKK6 ++E1Y6rBCvITu0MBo5Tjuwt5cok3Ve0BK5Fg89aIL2/rMicm20qG6nbqxLhHeR0mhPO98KIIzDoeL +rLpAlWS7GoPvJqbRzxsCAwEAAaOCAlYwggJSMBAGCSsGAQQBgjcVAQQDAgEBMCMGCSsGAQQBgjcV +AgQWBBSv5TU1Bjnw5n3u1iO2y+BHQXk7MTAdBgNVHQ4EFgQUoeMyqBhiyBcgwJN8zbr7pRbgs+sw +GQYJKwYBBAGCNxQCBAweCgBTAHUAYgBDAEEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQFMAMB +Af8wHwYDVR0jBBgwFoAUF+CLMX/eZk96ElRSeiEHqnsujqEwgdMGA1UdHwSByzCByDCBxaCBwqCB +v4aBvGxkYXA6Ly8vQ049YWQtR0lMR0FMQUQtQ0EsQ049R2lsZ2FsYWQsQ049Q0RQLENOPVB1Ymxp +YyUyMEtleSUyMFNlcnZpY2VzLENOPVNlcnZpY2VzLENOPUNvbmZpZ3VyYXRpb24sREM9YWQsREM9 +bnVtZXJpY2EsREM9dXM/Y2VydGlmaWNhdGVSZXZvY2F0aW9uTGlzdD9iYXNlP29iamVjdENsYXNz +PWNSTERpc3RyaWJ1dGlvblBvaW50MIHGBggrBgEFBQcBAQSBuTCBtjCBswYIKwYBBQUHMAKGgaZs +ZGFwOi8vL0NOPWFkLUdJTEdBTEFELUNBLENOPUFJQSxDTj1QdWJsaWMlMjBLZXklMjBTZXJ2aWNl +cyxDTj1TZXJ2aWNlcyxDTj1Db25maWd1cmF0aW9uLERDPWFkLERDPW51bWVyaWNhLERDPXVzP2NB +Q2VydGlmaWNhdGU/YmFzZT9vYmplY3RDbGFzcz1jZXJ0aWZpY2F0aW9uQXV0aG9yaXR5MA0GCSqG +SIb3DQEBCwUAA4ICAQBmRoSlPe++k7tsAJOvq0+0dNI6yk6gOBmY4g5jL9NTEjSxPWkeYegIwLr2 +UqpiIIZmAh9e9v3z0T2egVyRqNezLPXLkg/2gUfV6D0kRyKtG5mL0yAn/0hkkVyf6jWJpCKmH77x +0w3UpnfKs79jv5YpQDhC2eRFivN50HhIkigLWScPq4zd81ghmN8VFTHVQmsGua/mm1Oj5/pBFuQF +B4ljon1N//wX5ZJZaUlJR9eR9tM9m+Gyds2flr5+mZT6Zgm26fKiC5zs91aGnzqGx6s30jfXELP2 +FjFrrR46ooV7ehhnyBlCACxIWqXe5sSZsSh9oEYZ7Ux5Vq0thkfArBWsF7HA+LovKCUyHLcXbVBB +6/VAwZ3GLYi/bqbVIEFlVRu4nv/JyKWwoGbAhGyzZNWoeHszFrEIQbQMoMsEumVkMZreE6AxP+zb +6JPPOjlhpymtMo54z1MDYJPyo4HmcpL4xUjHZgqgOxMrbHC4oIVLvKZ/scbVBhPnd0tHHSZqj3ZS +gfTvG/ut/tLNTXXe48PkLBw4KguhbLm61Elu3wJALT0UL+ENgUWwb7csUGQBqOyPAHXGYnf/ACOc +UBqQckcrK8Jq3u8rnCloW3uDw86hw7MFM+YjmhVRdYRxpJmhKVPT6Amufp2WsSVId8q3CSqTH33L +fcxbV1n7hLWHA67MhTCCBq0wggWVoAMCAQICEycAAAsJMaw2RjtHZFUAAQAACwkwDQYJKoZIhvcN +AQELBQAwXDESMBAGCgmSJomT8ixkARkWAnVzMRgwFgYKCZImiZPyLGQBGRYIbnVtZXJpY2ExEjAQ +BgoJkiaJk/IsZAEZFgJhZDEYMBYGA1UEAxMPYWQtQ0VMRUJSSUFOLUNBMB4XDTIwMDUxMjE1MDk0 +MloXDTIxMDcyMjE0MjgwMVowgcExEjAQBgoJkiaJk/IsZAEZFgJ1czEYMBYGCgmSJomT8ixkARkW +CG51bWVyaWNhMRIwEAYKCZImiZPyLGQBGRYCYWQxETAPBgNVBAsTCE51bWVyaWNhMQ4wDAYDVQQL +EwVVc2VyczEYMBYGA1UECxMPUHJlc2VudCBJbnRlcm5zMRYwFAYDVQQDEw1XZXNsZXkgVGF5bG9y +MSgwJgYJKoZIhvcNAQkBFhl3ZXNsZXkudGF5bG9yQG51bWVyaWNhLnVzMIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEA5clDLapXkiLVXhAFP9GJv+JJkt+cacyvWaX9xEvqMQXOXb7MqO5E +DJE8XPMfxaX84WhuMMePOc9SNUKpDtTa2SHz+AOom+JH38ce2gfrdOPwez/e6RrUb3o8ZvMr3hJl +Yy+6vEFEADIICfHSlIjkLJbGNFTRDccvkOPjD2W+fmzFAtWyNb/eqM+mwdTuXjOxTvP6V34zJsvc +YKJUzhhD8jI7GdqOoNoirTlaMVTH5udK0P2KvzD6F0LfwcOlc3bTvY9uI585xhdniK4yAIka8OMq +5zmyEQLYOadcVSscjAlkC1sQ0gbwL3AdwS+bntryq+2Ds380OJ+Z1Uy7TRkeBQIDAQABo4IDADCC +AvwwPAYJKwYBBAGCNxUHBC8wLQYlKwYBBAGCNxUI9/Bss4wDhbmBGISeqheH4YBfgSWC6qJEgcjE +IgIBZQIBKDATBgNVHSUEDDAKBggrBgEFBQcDBDAOBgNVHQ8BAf8EBAMCBaAwGwYJKwYBBAGCNxUK +BA4wDDAKBggrBgEFBQcDBDBEBgkqhkiG9w0BCQ8ENzA1MA4GCCqGSIb3DQMCAgIAgDAOBggqhkiG +9w0DBAICAIAwBwYFKw4DAgcwCgYIKoZIhvcNAwcwHQYDVR0OBBYEFDZHoDwoOKD5uzpF/2CcZSeg +XWLmMB8GA1UdIwQYMBaAFKHjMqgYYsgXIMCTfM26+6UW4LPrMIHVBgNVHR8Egc0wgcowgceggcSg +gcGGgb5sZGFwOi8vL0NOPWFkLUNFTEVCUklBTi1DQSxDTj1DZWxlYnJpYW4sQ049Q0RQLENOPVB1 +YmxpYyUyMEtleSUyMFNlcnZpY2VzLENOPVNlcnZpY2VzLENOPUNvbmZpZ3VyYXRpb24sREM9YWQs +REM9bnVtZXJpY2EsREM9dXM/Y2VydGlmaWNhdGVSZXZvY2F0aW9uTGlzdD9iYXNlP29iamVjdENs +YXNzPWNSTERpc3RyaWJ1dGlvblBvaW50MIHHBggrBgEFBQcBAQSBujCBtzCBtAYIKwYBBQUHMAKG +gadsZGFwOi8vL0NOPWFkLUNFTEVCUklBTi1DQSxDTj1BSUEsQ049UHVibGljJTIwS2V5JTIwU2Vy +dmljZXMsQ049U2VydmljZXMsQ049Q29uZmlndXJhdGlvbixEQz1hZCxEQz1udW1lcmljYSxEQz11 +cz9jQUNlcnRpZmljYXRlP2Jhc2U/b2JqZWN0Q2xhc3M9Y2VydGlmaWNhdGlvbkF1dGhvcml0eTBS +BgNVHREESzBJoCwGCisGAQQBgjcUAgOgHgwcd2VzbGV5LnRheWxvckBhZC5udW1lcmljYS51c4EZ +d2VzbGV5LnRheWxvckBudW1lcmljYS51czANBgkqhkiG9w0BAQsFAAOCAQEAX3zFhiDYU+vQap2J +hiysyC9L7nkL7VI2OQWg4Z/JnNJTFiA6BwtoDYAT4qq1Jix4hZc+g78Gj99OnkhlBQDe9Hq12yI9 +muboQSDAYO6iDK76wQv3Rt8Fl4SUD4Ygwy52QrkTDrj/HZxTNask5p/2ilGBJnG9KT2VbEgGJkP9 +kXn1vAgOl3BCxgjdWekWCvxpmffr+Z3UtmQIiZAB3OsKcgdsSy9pveTMjxtKJemaH3kpXQiTgCev +CMuWZb3YnqXI8Fd+uUw6HwA4c+ZH62G9Q8KGkwXyhOPizmm3UeSlMo27yUCE+cF5EIHBxpGJ6z83 +7MbxMVKnS1Wz1n8MtW2ezDGCBCEwggQdAgEBMHMwXDESMBAGCgmSJomT8ixkARkWAnVzMRgwFgYK +CZImiZPyLGQBGRYIbnVtZXJpY2ExEjAQBgoJkiaJk/IsZAEZFgJhZDEYMBYGA1UEAxMPYWQtQ0VM +RUJSSUFOLUNBAhMnAAALCTGsNkY7R2RVAAEAAAsJMA0GCWCGSAFlAwQCAwUAoIICfzAYBgkqhkiG +9w0BCQMxCwYJKoZIhvcNAQcBMBwGCSqGSIb3DQEJBTEPFw0yMDA2MjUyMzIwNDRaME8GCSqGSIb3 +DQEJBDFCBEBaj66vdgjAhEO0p7lO6X44h+LpUlAcROa5Hi4Jp5aWS4hU8CuqOrH12y2GRNmNhKLa +0YieL4fCL3YqDRfop79NMFIGCyqGSIb3DQEJEAIBMUMwQQQdAAAAABAAAACgLzslsB99TKIYKeHy +Wh5cAQAAAACAAQAwHTAbgRl3ZXNsZXkudGF5bG9yQG51bWVyaWNhLnVzMIGCBgkrBgEEAYI3EAQx +dTBzMFwxEjAQBgoJkiaJk/IsZAEZFgJ1czEYMBYGCgmSJomT8ixkARkWCG51bWVyaWNhMRIwEAYK +CZImiZPyLGQBGRYCYWQxGDAWBgNVBAMTD2FkLUNFTEVCUklBTi1DQQITJwAACwkxrDZGO0dkVQAB +AAALCTCBhAYLKoZIhvcNAQkQAgsxdaBzMFwxEjAQBgoJkiaJk/IsZAEZFgJ1czEYMBYGCgmSJomT +8ixkARkWCG51bWVyaWNhMRIwEAYKCZImiZPyLGQBGRYCYWQxGDAWBgNVBAMTD2FkLUNFTEVCUklB +Ti1DQQITJwAACwkxrDZGO0dkVQABAAALCTCBkwYJKoZIhvcNAQkPMYGFMIGCMAsGCWCGSAFlAwQB +KjALBglghkgBZQMEARYwCgYIKoZIhvcNAwcwCwYJYIZIAWUDBAECMA4GCCqGSIb3DQMCAgIAgDAN +BggqhkiG9w0DAgIBQDALBglghkgBZQMEAgMwCwYJYIZIAWUDBAICMAsGCWCGSAFlAwQCATAHBgUr +DgMCGjANBgkqhkiG9w0BAQEFAASCAQBNFxhcbK6Rmw0Xyu+79cH5kUsXENcdUaJPKlegcY/gl2BZ +0CPpGcRnwz6z8OPYjvw3jrkiAE8nBbuCKu1CPtuk1h4Cybk7exyMybYvK5xge+N+dz2mFipRfGSY +rl/ztX1jyvcDruxaSJwb8WMhAGs505yfaCJfwgFOI3QGi+wUunbOIKy3QQZTXDv89yslZqi0wmeI +8sVRqSAYZRIPEylwS9CU2ReK9BJlfVLZnNP1At4gHE6S2hk8T0eVeLT8uhQiUXXJe4644UoPhoA4 +Fxgm7Q62KT6yP9O7c4eZzmQ4A9hdlWM6CtZ5pgMAzLOrVFdypzSc+S1j8DqcFkALCw83AAAAAAAA + +------=_NextPart_000_0018_01D64B14.F58791A0-- + +--===============0678627779074767862== +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +_______________________________________________ +HTCondor-users mailing list +To unsubscribe, send a message to htcondor-users-request@cs.wisc.edu with a +subject: Unsubscribe +You can also unsubscribe by visiting +https://lists.cs.wisc.edu/mailman/listinfo/htcondor-users + +The archives can be found at: +https://lists.cs.wisc.edu/archive/htcondor-users/ +--===============0678627779074767862==--)"; + +static std::string +message(const Regex& rx, size_t id) +{ + char buf[16]; + ::snprintf(buf, sizeof(buf), "%zu", id); + + return to_string_gchar( + g_regex_replace(rx, test_msg, -1, 0, buf, + G_REGEX_MATCH_DEFAULT, {})); +} + +struct TestData { + size_t num_maildirs; + size_t num_messages; + size_t num_threads; +}; + + +static void +setup(const TestData& tdata) +{ + /* create toplevel */ + auto top_maildir = std::string{BENCH_MAILDIRS}; + int res = g_mkdir_with_parents(top_maildir.c_str(), 0700); + g_assert_cmpuint(res,==, 0); + + /* create maildirs */ + for (size_t i = 0; i != tdata.num_maildirs; ++i) { + const auto mdir = mu_format("{}/maildir-{}", top_maildir, i); + auto res = maildir_mkdir(mdir); + g_assert(!!res); + } + const auto rx = Regex::make("@ID@"); + /* create messages */ + for (size_t n = 0; n != tdata.num_messages; ++n) { + auto mpath = mu_format("{}/maildir-{}/cur/msg-{}:2,S", + top_maildir, n % tdata.num_maildirs, + n); + std::ofstream stream(mpath); + auto msg = message(*rx, n); + stream.write(msg.c_str(), msg.size()); + g_assert_true(stream.good()); + } +} + +static void +tear_down() +{ + /* ugly */ + GError *err{}; + const auto cmd{mu_format("/bin/rm -rf '{}' '{}'", BENCH_MAILDIRS, BENCH_STORE)}; + if (!g_spawn_command_line_sync(cmd.c_str(), NULL, NULL, NULL, &err)) { + mu_warning("error: {}", err ? err->message : "?"); + g_clear_error(&err); + } +} + +void +black_hole(void) +{ + return; /* do nothing */ +} + +static void +benchmark_indexer(gconstpointer testdata) +{ + using namespace std::chrono_literals; + using Clock = std::chrono::steady_clock; + const auto tdata = reinterpret_cast<const TestData*>(testdata); + + setup(*tdata); + auto start = Clock::now(); + + { + auto store{Store::make_new(BENCH_STORE, BENCH_MAILDIRS)}; + g_assert_true(!!store); + Indexer::Config conf{}; + conf.max_threads = tdata->num_threads; + + auto res = store->indexer().start(conf); + g_assert_true(res); + while(store->indexer().is_running()) { + std::this_thread::sleep_for(100ms); + } + g_assert_cmpuint(store->size(),==, tdata->num_messages); + } + + const auto elapsed = Clock::now() - start; + std::cout << "indexed " << tdata->num_messages << " messages in " + << tdata->num_maildirs << " maildirs in " + << to_ms(elapsed) << "ms; " + << to_us(elapsed) / tdata->num_messages << " μs/message; " + << static_cast<size_t>(1000*tdata->num_messages / to_ms(elapsed)) + << " messages/s" + << " (" << tdata->num_threads << " thread(s))\n"; + + tear_down(); +} + +int +main(int argc, char *argv[]) +{ + size_t num_maildirs{}, num_messages{}; + + mu_test_init(&argc, &argv); + + if (g_test_perf()) { + num_maildirs = 20; + num_messages = 5000; + } else { + num_maildirs = 10; + num_messages = 1000; + } + + g_log_set_handler( + NULL, + (GLogLevelFlags)(G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL | G_LOG_FLAG_RECURSION), + (GLogFunc)black_hole, + NULL); + + + size_t thread_num{}; + const auto tnum = g_getenv("THREAD_NUM"); + if (tnum) + thread_num = ::strtol(tnum, NULL, 10); + + if (thread_num != 0) { + /* THREAD_NUM specified */ + static TestData tdata{num_maildirs, num_messages, thread_num}; + char *name = g_strdup_printf("/bench/indexer/%zu-cores", thread_num); + g_test_add_data_func(name, &tdata, benchmark_indexer); + g_free(name); + } else { + /* no THREAD_NUM specified */ + + const size_t hw_threads = std::thread::hardware_concurrency(); + + { + static TestData tdata{num_maildirs, num_messages, 1}; + g_test_add_data_func("/bench/indexer/1-core", &tdata, benchmark_indexer); + } + + if (hw_threads > 2) { + static TestData tdata{num_maildirs, num_messages, hw_threads/2}; + char *name = g_strdup_printf("/bench/indexer/%zu-cores", hw_threads/2); + g_test_add_data_func(name, &tdata, benchmark_indexer); + g_free(name); + } + + if (hw_threads > 1) { + static TestData tdata{num_maildirs, num_messages, hw_threads}; + char *name = g_strdup_printf("/bench/indexer/%zu-cores", hw_threads); + g_test_add_data_func(name, &tdata, benchmark_indexer); + g_free(name); + } + } + + tear_down(); + + return g_test_run(); +} diff --git a/lib/tests/meson.build b/lib/tests/meson.build new file mode 100644 index 0000000..39b5b38 --- /dev/null +++ b/lib/tests/meson.build @@ -0,0 +1,147 @@ +## Copyright (C) 2021-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# +# tests +# + + +# +# unit tests +# + +test('test-threads', + executable('test-threads', + '../mu-query-threads.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) +test('test-contacts-cache', + executable('test-contacts-cache', + '../mu-contacts-cache.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-config', + executable('test-config', + '../mu-config.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-query-macros', + executable('test-query-macros', + '../mu-query-macros.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [lib_mu_dep])) + +test('test-query-processor', + executable('test-query-processor', + '../mu-query-processor.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [lib_mu_dep])) + +test('test-query-parser', + executable('test-query-parser', + '../mu-query-parser.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [lib_mu_dep])) + +test('test-query-xapianizer', + executable('test-query-xapianizer', + '../mu-query-xapianizer.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [lib_mu_dep])) + + +test('test-indexer', + executable('test-indexer', + '../mu-indexer.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, config_h_dep, + lib_mu_dep])) + +test('test-scanner', + executable('test-scanner', + '../mu-scanner.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, config_h_dep, + lib_mu_utils_dep])) + +test('test-xapian-db', + executable('test-xapian-db', + '../mu-xapian-db.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [lib_mu_dep, config_h_dep])) + +test('test-maildir', + executable('test-maildir', + 'test-mu-maildir.cc', + install: false, + dependencies: [glib_dep, lib_mu_dep])) +test('test-msg', + executable('test-msg', + 'test-mu-msg.cc', + install: false, + dependencies: [glib_dep, lib_mu_dep])) +test('test-store', + executable('test-store', + 'test-mu-store.cc', + install: false, + dependencies: [glib_dep, lib_mu_dep])) +test('test-query', + executable('test-query', + 'test-query.cc', + install: false, + dependencies: [glib_dep, gmime_dep, lib_mu_dep])) + +test('test-store-query', + executable('test-store-query', + 'test-mu-store-query.cc', + install: false, + dependencies: [glib_dep, gmime_dep, lib_mu_dep])) +# +# benchmarks +# +bench_maildirs=join_paths(meson.current_build_dir(), 'maildirs') +bench_store=join_paths(meson.current_build_dir(), 'store') +bench_indexer_exe = executable( + 'bench-indexer', + 'bench-indexer.cc', + install:false, + cpp_args:['-DBENCH_MAILDIRS="' + bench_maildirs + '"', + '-DBENCH_STORE="' + bench_store + '"', + ], + dependencies: [lib_mu_dep, glib_dep]) + +benchmark('bench-indexer', bench_indexer_exe, args: ['-m', 'perf']) + +# +# below does _not_ pass; it is believed that it's a false alarm. +# https://gitlab.gnome.org/GNOME/glib/-/issues/2662 + +# also register benchmark as a normal test so it gets included for +# valgrind/helgrind etc. +# test('test-bench-indexer', bench_indexer_exe, +# args : ['-m', 'quick'], env: ['THREADNUM=16']) diff --git a/lib/tests/test-mu-container.cc b/lib/tests/test-mu-container.cc new file mode 100644 index 0000000..4fb1939 --- /dev/null +++ b/lib/tests/test-mu-container.cc @@ -0,0 +1,80 @@ +/* +** Copyright (C) 2014 Jakub Sitnicki <jsitnicki@gmail.com> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include <glib.h> + +#include "utils/mu-test-utils.hh" +#include "mu-container.hh" + +static gboolean +container_has_children(const MuContainer* c) +{ + return c && c->child; +} + +static gboolean +container_is_sibling_of(const MuContainer* c, const MuContainer* sibling) +{ + const MuContainer* cur; + + for (cur = c; cur; cur = cur->next) { + if (cur == sibling) + return TRUE; + } + + return container_is_sibling_of(sibling, c); +} + +static void +test_mu_container_splice_children_when_parent_has_no_siblings(void) +{ + MuContainer *child, *parent, *root_set; + + child = mu_container_new(NULL, 0, "child"); + parent = mu_container_new(NULL, 0, "parent"); + parent = mu_container_append_children(parent, child); + + root_set = parent; + root_set = mu_container_splice_children(root_set, parent); + + g_assert(root_set != NULL); + g_assert(!container_has_children(parent)); + g_assert(container_is_sibling_of(root_set, child)); + + mu_container_destroy(parent); + mu_container_destroy(child); +} + +int +main(int argc, char* argv[]) +{ + g_test_init(&argc, &argv, NULL); + + g_test_add_func("/mu-container/mu-container-splice-children-when-parent-has-no-siblings", + test_mu_container_splice_children_when_parent_has_no_siblings); + + g_log_set_handler( + NULL, + (GLogLevelFlags)(G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL | G_LOG_FLAG_RECURSION), + (GLogFunc)black_hole, + NULL); + + return g_test_run(); +} diff --git a/lib/tests/test-mu-maildir.cc b/lib/tests/test-mu-maildir.cc new file mode 100644 index 0000000..aee8189 --- /dev/null +++ b/lib/tests/test-mu-maildir.cc @@ -0,0 +1,557 @@ +/* +** Copyright (C) 2008-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include <glib.h> +#include <glib/gstdio.h> + +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <vector> +#include <fstream> + +#include "utils/mu-test-utils.hh" +#include "mu-maildir.hh" +#include "utils/mu-utils.hh" +#include "utils/mu-utils-file.hh" +#include "utils/mu-result.hh" + +using namespace Mu; + +static void +test_maildir_mkdir_01() +{ + TempDir temp_dir; + auto mdir = join_paths(temp_dir.path(), "cuux"); + auto res{maildir_mkdir(mdir, 0755, false/*!noindex*/)}; + assert_valid_result(res); + + for (auto sub : {"tmp", "cur", "new"}) { + auto subpath = join_paths(mdir, sub); + g_assert_cmpuint(g_access(subpath.c_str(), R_OK), ==, 0); + g_assert_cmpuint(g_access(subpath.c_str(), W_OK), ==, 0); + } + + auto noindex = join_paths(mdir, ".noindex"); + g_assert_cmpuint(g_access(noindex.c_str(), F_OK), !=, 0); +} + +static void +test_maildir_mkdir_02() +{ + TempDir temp_dir; + auto mdir = join_paths(temp_dir.path(), "cuux"); + auto res{maildir_mkdir(mdir, 0755, true/*noindex*/)}; + assert_valid_result(res); + + for (auto sub : {"tmp", "cur", "new"}) { + auto subpath = join_paths(mdir, sub); + g_assert_cmpuint(g_access(subpath.c_str(), R_OK), ==, 0); + g_assert_cmpuint(g_access(subpath.c_str(), W_OK), ==, 0); + } + + auto noindex = join_paths(mdir, ".noindex"); + g_assert_cmpuint(g_access(noindex.c_str(), F_OK), ==, 0); +} + +static void +test_maildir_mkdir_03() +{ + TempDir temp_dir; + auto mdir = join_paths(temp_dir.path(), "cuux"); + + // create part already + auto curdir = join_paths(mdir, "cur"); + g_assert_cmpuint(g_mkdir_with_parents(curdir.c_str(), 0755), ==, 0); + + auto res{maildir_mkdir(mdir, 0755, false/*!noindex*/)}; + assert_valid_result(res); + + // should still work. + for (auto sub : {"tmp", "cur", "new"}) { + auto subpath = join_paths(mdir, sub); + g_assert_cmpuint(g_access(subpath.c_str(), R_OK), ==, 0); + g_assert_cmpuint(g_access(subpath.c_str(), W_OK), ==, 0); + } + + auto noindex = join_paths(mdir, ".noindex"); + g_assert_cmpuint(g_access(noindex.c_str(), F_OK), !=, 0); +} + + +static void +test_maildir_mkdir_04() +{ + allow_warnings(); + + if (geteuid() == 0) { + g_test_skip("not useful when run as root"); + return; + } + + TempDir temp_dir; + auto mdir = join_paths(temp_dir.path(), "cuux"); + g_assert_cmpuint(g_mkdir_with_parents(mdir.c_str(), 0755), ==, 0); + + auto curdir = join_paths(mdir, "cur"); + g_assert_cmpuint(g_mkdir_with_parents(curdir.c_str(), 0000), ==, 0); + + /* this should fail now, because cur is not read/writable */ + auto res = maildir_mkdir(mdir, 0755, false); + g_assert_false(!!res); +} + +static gboolean +ignore_error(const char* log_domain, GLogLevelFlags log_level, const gchar* msg, gpointer user_data) +{ + return FALSE; /* don't abort */ +} + +static void +test_maildir_mkdir_05(void) +{ + /* this must fail */ + g_test_log_set_fatal_handler((GTestLogFatalFunc)ignore_error, NULL); + + g_assert_false(!!maildir_mkdir({}, 0755, true)); +} + +[[maybe_unused]] static void +assert_matches_regexp(const char* str, const char* rx) +{ + if (!g_regex_match_simple(rx, str, (GRegexCompileFlags)0, (GRegexMatchFlags)0)) { + if (g_test_verbose()) + g_print("%s does not match %s", str, rx); + g_assert(0); + } +} + + +static void +test_determine_target_ok(void) +{ + struct TestCase { + std::string old_path; + std::string root_maildir; + std::string target_maildir; + Flags new_flags; + bool new_name; + std::string expected; + }; + const std::vector<TestCase> testcases = { + TestCase{ /* change some flags */ + "/home/foo/Maildir/test/cur/123456:2,FR", + "/home/foo/Maildir", + {}, + Flags::Seen | Flags::Passed, + false, + "/home/foo/Maildir/test/cur/123456:2,PS" + }, + + TestCase{ /* from cur -> new */ + "/home/foo/Maildir/test/cur/123456:2,FR", + "/home/foo/Maildir", + {}, + Flags::New, + false, + "/home/foo/Maildir/test/new/123456" + }, + + TestCase{ /* from new->cur */ + "/home/foo/Maildir/test/cur/123456", + "/home/foo/Maildir", + {}, + Flags::Seen | Flags::Flagged, + false, + "/home/foo/Maildir/test/cur/123456:2,FS" + }, + + TestCase{ /* change maildir */ + "/home/foo/Maildir/test/cur/123456:2,FR", + "/home/foo/Maildir", + "/test2", + Flags::Flagged | Flags::Replied, + false, + "/home/foo/Maildir/test2/cur/123456:2,FR" + }, + TestCase{ /* remove all flags */ + "/home/foo/Maildir/test/new/123456", + "/home/foo/Maildir", + {}, + Flags::None, + false, + "/home/foo/Maildir/test/cur/123456:2," + }, + }; + + for (auto&& testcase: testcases) { + const auto res = maildir_determine_target( + testcase.old_path, + testcase.root_maildir, + testcase.target_maildir, + testcase.new_flags, + testcase.new_name); + g_assert_true(!!res); + g_assert_cmpstr(testcase.expected.c_str(), ==, + res.value().c_str()); + } +} + + + +static void +test_determine_target_fail(void) +{ + struct TestCase { + std::string old_path; + std::string root_maildir; + std::string target_maildir; + Flags new_flags; + bool new_name; + std::string expected; + }; + const std::vector<TestCase> testcases = { + TestCase{ /* fail: no absolute path */ + "../foo/Maildir/test/cur/123456:2,FR-not-absolute", + "/home/foo/Maildir", + {}, + Flags::Seen | Flags::Passed, + false, + "/home/foo/Maildir/test/cur/123456:2,PS" + }, + + TestCase{ /* fail: no absolute root */ + "/home/foo/Maildir/test/cur/123456:2,FR", + "../foo/Maildir-not-absolute", + {}, + Flags::New, + false, + "/home/foo/Maildir/test/new/123456" + }, + + TestCase{ /* fail: maildir must start with '/' */ + "/home/foo/Maildir/test/cur/123456", + "/home/foo/Maildir", + "mymaildirwithoutslash", + Flags::Seen | Flags::Flagged, + false, + "/home/foo/Maildir/test/cur/123456:2,FS" + }, + + TestCase{ /* fail: path must be below maildir */ + "/home/foo/Maildir/test/cur/123456:2,FR", + "/home/bar/Maildir", + "/test2", + Flags::Flagged | Flags::Replied, + false, + "/home/foo/Maildir/test2/cur/123456:2,FR" + }, + TestCase{ /* fail: New cannot be combined */ + "/home/foo/Maildir/test/new/123456", + "/home/foo/Maildir", + {}, + Flags::New | Flags::Replied, + false, + "/home/foo/Maildir/test/cur/123456:2," + }, + }; + + for (auto&& testcase: testcases) { + const auto res = maildir_determine_target( + testcase.old_path, + testcase.root_maildir, + testcase.target_maildir, + testcase.new_flags, + testcase.new_name); + g_assert_false(!!res); + } +} + + + +static void +test_maildir_get_new_path_01(void) +{ + struct { + std::string oldpath; + Flags flags; + std::string newpath; + } paths[] = {{"/home/foo/Maildir/test/cur/123456:2,FR", + Flags::Replied, + "/home/foo/Maildir/test/cur/123456:2,R"}, + {"/home/foo/Maildir/test/cur/123456:2,FR", + Flags::New, + "/home/foo/Maildir/test/new/123456"}, + {"/home/foo/Maildir/test/new/123456:2,FR", + (Flags::Seen | Flags::Replied), + "/home/foo/Maildir/test/cur/123456:2,RS"}, + {"/home/foo/Maildir/test/new/1313038887_0.697", + (Flags::Seen | Flags::Flagged | Flags::Passed), + "/home/foo/Maildir/test/cur/1313038887_0.697:2,FPS"}, + {"/home/foo/Maildir/test/new/1313038887_0.697:2,", + (Flags::Seen | Flags::Flagged | Flags::Passed), + "/home/foo/Maildir/test/cur/1313038887_0.697:2,FPS"}, + /* note the ':2,' suffix on the new message is + * removed */ + + {"/home/foo/Maildir/trash/new/1312920597.2206_16.cthulhu", + Flags::Seen, + "/home/foo/Maildir/trash/cur/1312920597.2206_16.cthulhu:2,S"}}; + + for (int i = 0; i != G_N_ELEMENTS(paths); ++i) { + const auto newpath{maildir_determine_target(paths[i].oldpath, + "/home/foo/Maildir", + {}, paths[i].flags, false)}; + assert_valid_result(newpath); + assert_equal(*newpath, paths[i].newpath); + } +} + +static void +test_maildir_get_new_path_02(void) +{ + struct { + std::string oldpath; + Flags flags; + std::string targetdir; + std::string newpath; + std::string root_maildir; + } paths[] = {{"/home/foo/Maildir/test/cur/123456:2,FR", + Flags::Replied, + "/blabla", + "/home/foo/Maildir/blabla/cur/123456:2,R", + "/home/foo/Maildir"}, + {"/home/bar/Maildir/test/cur/123456:2,FR", + Flags::New, + "/coffee", + "/home/bar/Maildir/coffee/new/123456", + "/home/bar/Maildir" + }, + {"/home/cuux/Maildir/test/new/123456", + (Flags::Seen | Flags::Replied), + "/tea", + "/home/cuux/Maildir/tea/cur/123456:2,RS", + "/home/cuux/Maildir"}, + {"/home/boy/Maildir/test/new/1313038887_0.697:2,", + (Flags::Seen | Flags::Flagged | Flags::Passed), + "/stuff", + "/home/boy/Maildir/stuff/cur/1313038887_0.697:2,FPS", + "/home/boy/Maildir"}}; + + for (int i = 0; i != G_N_ELEMENTS(paths); ++i) { + auto newpath{maildir_determine_target(paths[i].oldpath, + paths[i].root_maildir, + paths[i].targetdir, + paths[i].flags, + false)}; + assert_valid_result(newpath); + assert_equal(*newpath, paths[i].newpath); + } +} + + +static void +test_maildir_get_new_path_custom_real(bool change_name) +{ + struct { + std::string oldpath; + Flags flags; + std::string targetdir; + std::string newpath; + std::string root_maildir; + } paths[] = {{"/home/foo/Maildir/test/cur/123456:2,FR", + Flags::Replied, + "/blabla", + "/home/foo/Maildir/blabla/cur/123456:2,R", + "/home/foo/Maildir"}, + {"/home/foo/Maildir/test/cur/123456:2,hFeRllo123", + Flags::Flagged, + "/blabla", + "/home/foo/Maildir/blabla/cur/123456:2,F", + "/home/foo/Maildir"}, + {"/home/foo/Maildir/test/cur/123456:2,abc", + Flags::Passed, + "/blabla", + "/home/foo/Maildir/blabla/cur/123456:2,P", + "/home/foo/Maildir"}}; + + for (int i = 0; i != G_N_ELEMENTS(paths); ++i) { + auto newpath{maildir_determine_target(paths[i].oldpath, + paths[1].root_maildir, + paths[i].targetdir, + paths[i].flags, + change_name)}; + assert_valid_result(newpath); + if (change_name) + g_assert_true(*newpath != paths[i].newpath); // weak test + else + assert_equal(*newpath, paths[i].newpath); + } +} + + +static void +test_maildir_get_new_path_custom(void) +{ + return test_maildir_get_new_path_custom_real(false); +} + + +static void +test_maildir_get_new_path_custom_change_name(void) +{ + return test_maildir_get_new_path_custom_real(true); +} + + +static void +test_maildir_from_path(void) +{ + unsigned u; + + struct { + std::string path, exp; + } cases[] = {{"/home/foo/Maildir/test/cur/123456:2,FR", "/test"}, + {"/home/foo/Maildir/lala/new/1313038887_0.697:2,", "/lala"}}; + + for (u = 0; u != G_N_ELEMENTS(cases); ++u) { + auto mdir{maildir_from_path(cases[u].path, "/home/foo/Maildir")}; + assert_valid_result(mdir); + assert_equal(*mdir, cases[u].exp); + } +} + +static void +test_maildir_link() +{ + TempDir tmpdir; + + assert_valid_result(maildir_mkdir(tmpdir.path() + "/foo")); + assert_valid_result(maildir_mkdir(tmpdir.path() + "/bar")); + + const auto srcpath1 = tmpdir.path() + "/foo/cur/msg1"; + const auto srcpath2 = tmpdir.path() + "/foo/new/msg2"; + + { + std::ofstream stream(srcpath1); + stream.write("cur", 3); + g_assert_true(stream.good()); + stream.close(); + } + + { + std::ofstream stream(srcpath2); + stream.write("new", 3); + g_assert_true(stream.good()); + stream.close(); + } + + assert_valid_result(maildir_link(srcpath1, tmpdir.path() + "/bar", false)); + assert_valid_result(maildir_link(srcpath2, tmpdir.path() + "/bar", false)); + + const auto dstpath1 = tmpdir.path() + "/bar/cur/msg1"; + const auto dstpath2 = tmpdir.path() + "/bar/new/msg2"; + + g_assert_true(g_access(dstpath1.c_str(), F_OK) == 0); + g_assert_true(g_access(dstpath2.c_str(), F_OK) == 0); + + g_assert_false(!!maildir_clear_links("/nonexistent/bla/foo/xuux")); + + assert_valid_result(maildir_clear_links(tmpdir.path() + "/bar")); + g_assert_false(g_access(dstpath1.c_str(), F_OK) == 0); + g_assert_false(g_access(dstpath2.c_str(), F_OK) == 0); +} + + +static void +test_maildir_move(bool assume_remote) +{ + TempDir tmpdir; + + assert_valid_result(maildir_mkdir(tmpdir.path() + "/foo")); + assert_valid_result(maildir_mkdir(tmpdir.path() + "/bar")); + + const auto srcpath1{join_paths(tmpdir.path(), "/foo/cur/msg1")}; + const auto srcpath2{join_paths(tmpdir.path(), "/foo/new/msg2")}; + + { + std::ofstream stream(srcpath1); + stream.write("cur", 3); + g_assert_true(stream.good()); + stream.close(); + } + + { + std::ofstream stream(srcpath2); + stream.write("new", 3); + g_assert_true(stream.good()); + stream.close(); + } + + const auto dstpath = tmpdir.path() + "/test1"; + + assert_valid_result(maildir_move_message(srcpath1, dstpath, assume_remote)); + assert_valid_result(maildir_move_message(srcpath2, dstpath, assume_remote)); + + assert_valid_result(maildir_move_message(dstpath, dstpath)); // self-move is okay. +} + +static void +test_maildir_move_vanilla() +{ + test_maildir_move(false/*!assume_remote*/); +} + +static void +test_maildir_move_remote() +{ + test_maildir_move(true/*assume_remote*/); +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + /* mu_util_maildir_mkmdir */ + g_test_add_func("/maildir/mkdir-01", test_maildir_mkdir_01); + g_test_add_func("/maildir/mkdir-02", test_maildir_mkdir_02); + g_test_add_func("/maildir/mkdir-03", test_maildir_mkdir_03); + g_test_add_func("/maildir/mkdir-04", test_maildir_mkdir_04); + g_test_add_func("/maildir/mkdir-05", test_maildir_mkdir_05); + + g_test_add_func("/maildir/determine-target-ok", test_determine_target_ok); + g_test_add_func("/maildir/determine-target-fail", test_determine_target_fail); + + // /* get/set flags */ + g_test_add_func("/maildir/get-new-path-01", test_maildir_get_new_path_01); + g_test_add_func("/maildir/get-new-path-02", test_maildir_get_new_path_02); + g_test_add_func("/maildir/get-new-path-custom", test_maildir_get_new_path_custom); + g_test_add_func("/maildir/get-new-path-custom-change-name", + test_maildir_get_new_path_custom_change_name); + + g_test_add_func("/maildir/from-path", test_maildir_from_path); + + g_test_add_func("/maildir/link", test_maildir_link); + g_test_add_func("/maildir/move-vanilla", test_maildir_move_vanilla); + g_test_add_func("/maildir/move-remote", test_maildir_move_remote); + + return g_test_run(); +} diff --git a/lib/tests/test-mu-msg-fields.cc b/lib/tests/test-mu-msg-fields.cc new file mode 100644 index 0000000..5f5df16 --- /dev/null +++ b/lib/tests/test-mu-msg-fields.cc @@ -0,0 +1,126 @@ +/* +** Copyright (C) 2008-2020 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#if HAVE_CONFIG_H +#include "config.h" +#endif /*HAVE_CONFIG_H*/ + +#include <glib.h> +#include <stdlib.h> +#include <unistd.h> +#include <time.h> + +#include <locale.h> + +#include "utils/mu-test-utils.hh" +#include "mu-message-fields.hh" + +static void +test_mu_msg_field_body(void) +{ + Field::Id field; + + field = Field::Id::BodyText; + + g_assert_cmpstr(mu_msg_field_name(field), ==, "body"); + g_assert_cmpuint(mu_msg_field_shortcut(field), ==, 'b'); + g_assert_cmpuint(mu_msg_field_xapian_prefix(field), ==, 'B'); + + g_assert_cmpuint(mu_msg_field_is_numeric(field), ==, FALSE); +} + +static void +test_mu_msg_field_subject(void) +{ + Field::Id field; + + field = Field::Id::Subject; + + g_assert_cmpstr(mu_msg_field_name(field), ==, "subject"); + g_assert_cmpuint(mu_msg_field_shortcut(field), ==, 's'); + g_assert_cmpuint(mu_msg_field_xapian_prefix(field), ==, 'S'); + + g_assert_cmpuint(mu_msg_field_is_numeric(field), ==, FALSE); +} + +static void +test_mu_msg_field_to(void) +{ + Field::Id field; + + field = Field::Id::To; + + g_assert_cmpstr(mu_msg_field_name(field), ==, "to"); + g_assert_cmpuint(mu_msg_field_shortcut(field), ==, 't'); + g_assert_cmpuint(mu_msg_field_xapian_prefix(field), ==, 'T'); + + g_assert_cmpuint(mu_msg_field_is_numeric(field), ==, FALSE); +} + +static void +test_mu_msg_field_prio(void) +{ + Field::Id field; + + field = Field::Id::Priority; + + g_assert_cmpstr(mu_msg_field_name(field), ==, "prio"); + g_assert_cmpuint(mu_msg_field_shortcut(field), ==, 'p'); + g_assert_cmpuint(mu_msg_field_xapian_prefix(field), ==, 'P'); + + g_assert_cmpuint(mu_msg_field_is_numeric(field), ==, TRUE); +} + +static void +test_mu_msg_field_flags(void) +{ + Field::Id field; + + field = Field::Id::Flags; + + g_assert_cmpstr(mu_msg_field_name(field), ==, "flag"); + g_assert_cmpuint(mu_msg_field_shortcut(field), ==, 'g'); + g_assert_cmpuint(mu_msg_field_xapian_prefix(field), ==, 'G'); + + g_assert_cmpuint(mu_msg_field_is_numeric(field), ==, TRUE); +} + +int +main(int argc, char* argv[]) +{ + g_test_init(&argc, &argv, NULL); + + /* mu_msg_str_date */ + g_test_add_func("/mu-msg-fields/mu-msg-field-body", test_mu_msg_field_body); + g_test_add_func("/mu-msg-fields/mu-msg-field-subject", test_mu_msg_field_subject); + g_test_add_func("/mu-msg-fields/mu-msg-field-to", test_mu_msg_field_to); + g_test_add_func("/mu-msg-fields/mu-msg-field-prio", test_mu_msg_field_prio); + g_test_add_func("/mu-msg-fields/mu-msg-field-flags", test_mu_msg_field_flags); + + /* FIXME: add tests for mu_msg_str_flags; but note the + * function simply calls mu_msg_field_str */ + + g_log_set_handler( + NULL, + (GLogLevelFlags)(G_LOG_LEVEL_MASK | G_LOG_FLAG_FATAL | G_LOG_FLAG_RECURSION), + (GLogFunc)black_hole, + NULL); + + return g_test_run(); +} diff --git a/lib/tests/test-mu-msg.cc b/lib/tests/test-mu-msg.cc new file mode 100644 index 0000000..1e5d82d --- /dev/null +++ b/lib/tests/test-mu-msg.cc @@ -0,0 +1,355 @@ +/* +** Copyright (C) 2008-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include <glib.h> +#include <stdlib.h> +#include <unistd.h> +#include <time.h> +#include <array> +#include <string> + +#include <locale.h> + +#include "utils/mu-test-utils.hh" +#include "utils/mu-result.hh" +#include "utils/mu-utils.hh" + +#include <message/mu-message.hh> + +using namespace Mu; + +using ExpectedContacts = const std::vector<std::pair<std::string, std::string>>; + +static void +assert_contacts_equal(const Contacts& contacts, + const ExpectedContacts& expected) +{ + g_assert_cmpuint(contacts.size(), ==, expected.size()); + + size_t n{}; + for (auto&& contact: contacts) { + if (g_test_verbose()) + mu_message("{{ \"{}\", \"{}\"}},\n", + contact.name, contact.email); + assert_equal(contact.name, expected.at(n).first); + assert_equal(contact.email, expected.at(n).second); + ++n; + } + mu_print("\n"); +} + + +static void +test_mu_msg_01(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/1220863042.12663_1.mindcrime!2,S") + .value()}; + + assert_contacts_equal(msg.to(), {{ "Donald Duck", "gcc-help@gcc.gnu.org" }}); + assert_contacts_equal(msg.from(), {{ "Mickey Mouse", "anon@example.com" }}); + + assert_equal(msg.subject(), "gcc include search order"); + assert_equal(msg.message_id(), + "3BE9E6535E3029448670913581E7A1A20D852173@" + "emss35m06.us.lmco.com"); + assert_equal(msg.header("Mailing-List").value_or(""), + "contact gcc-help-help@gcc.gnu.org; run by ezmlm"); + g_assert_true(msg.priority() == Priority::Normal); + g_assert_cmpuint(msg.date(), ==, 1217530645); + + assert_contacts_equal(msg.all_contacts(), { + { "", "gcc-help-owner@gcc.gnu.org"}, + { "Mickey Mouse", "anon@example.com" }, + { "Donald Duck", "gcc-help@gcc.gnu.org" } + }); + +} + +static void +test_mu_msg_02(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/1220863087.12663_19.mindcrime!2,S") + .value()}; + + assert_equal(msg.to().at(0).email, "help-gnu-emacs@gnu.org"); + assert_equal(msg.subject(), "Re: Learning LISP; Scheme vs elisp."); + assert_equal(msg.from().at(0).email, "anon@example.com"); + assert_equal(msg.message_id(), "r6bpm5-6n6.ln1@news.ducksburg.com"); + assert_equal(msg.header("Errors-To").value_or(""), + "help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org"); + g_assert_true(msg.priority() /* 'low' */ + == Priority::Low); + g_assert_cmpuint(msg.date(), ==, 1218051515); + mu_println("flags: {}", Mu::to_string(msg.flags())); + g_assert_true(msg.flags() == (Flags::Seen|Flags::MailingList)); + + assert_contacts_equal(msg.all_contacts(), { + { "", "help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org"}, + { "", "anon@example.com"}, + { "", "help-gnu-emacs@gnu.org"}, + }); + +} + +static void +test_mu_msg_03(void) +{ + //const GSList* params; + + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/1283599333.1840_11.cthulhu!2,") + .value()}; + + assert_equal(msg.to().at(0).display_name(), "Bilbo Baggins <bilbo@anotherexample.com>"); + assert_equal(msg.subject(), "Greetings from Lothlórien"); + assert_equal(msg.from().at(0).display_name(), "Frodo Baggins <frodo@example.com>"); + g_assert_true(msg.priority() == Priority::Normal); + g_assert_cmpuint(msg.date(), ==, 0); + assert_equal(msg.body_text().value_or(""), + "\nLet's write some fünkÿ text\nusing umlauts.\n\nFoo.\n"); + + // params = mu_msg_get_body_text_content_type_parameters(msg, MU_MSG_OPTION_NONE); + // g_assert_cmpuint(g_slist_length((GSList*)params), ==, 2); + + // assert_equal((char*)params->data, "charset"); + // params = g_slist_next(params); + // assert_equal((char*)params->data, "UTF-8"); + g_assert_true(msg.flags() == (Flags::Unread)); +} + +static void +test_mu_msg_04(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/mail5").value()}; + + assert_equal(msg.to().at(0).display_name(), "George Custer <gac@example.com>"); + assert_equal(msg.subject(), "pics for you"); + assert_equal(msg.from().at(0).display_name(), "Sitting Bull <sb@example.com>"); + g_assert_true(msg.priority() /* 'low' */ + == Priority::Normal); + g_assert_cmpuint(msg.date(), ==, 0); + g_assert_true(msg.flags() == + (Flags::HasAttachment|Flags::Unread)); + g_assert_true(msg.flags() == + (Flags::HasAttachment|Flags::Unread)); +} + +static void +test_mu_msg_multimime(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/multimime!2,FS").value()}; + + /* ie., are text parts properly concatenated? */ + assert_equal(msg.subject(), "multimime"); + assert_equal(msg.body_text().value_or(""), "abcdef"); + g_assert_true(msg.flags() == (Flags::HasAttachment|Flags::Flagged|Flags::Seen)); +} + +static void +test_mu_msg_flags(void) +{ + std::array<std::pair<std::string, Flags>, 2> tests= {{ + {MU_TESTMAILDIR4 "/multimime!2,FS", + (Flags::Flagged | Flags::Seen | + Flags::HasAttachment)}, + {MU_TESTMAILDIR4 "/special!2,Sabc", + (Flags::Seen)} + }}; + + for (auto&& test: tests) { + auto msg = Message::make_from_path(test.first); + assert_valid_result(msg); + g_assert_true(msg->flags() == test.second); + } +} + +static void +test_mu_msg_umlaut(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/1305664394.2171_402.cthulhu!2,") + .value()}; + + assert_contacts_equal(msg.to(), { { "Helmut Kröger", "hk@testmu.xxx"}}); + assert_contacts_equal(msg.from(), { { "Mü", "testmu@testmu.xx"}}); + + assert_equal(msg.subject(), "Motörhead"); + assert_equal(msg.from().at(0).display_name(), "Mü <testmu@testmu.xx>"); + g_assert_true(msg.priority() == Priority::Normal); + g_assert_cmpuint(msg.date(), ==, 0); +} + +static void +test_mu_msg_references(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/1305664394.2171_402.cthulhu!2,") + .value()}; + + std::array<std::string, 4> expected_refs = { + "non-exist-01@msg.id", + "non-exist-02@msg.id", + "non-exist-03@msg.id", + "non-exist-04@msg.id" + }; + + assert_equal_seq_str(msg.references(), expected_refs); + assert_equal(msg.thread_id(), expected_refs[0]); +} + +static void +test_mu_msg_references_dups(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/1252168370_3.14675.cthulhu!2,S") + .value()}; + + std::array<std::string, 6> expected_refs = { + "439C1136.90504@euler.org", + "4399DD94.5070309@euler.org", + "20051209233303.GA13812@gauss.org", + "439B41ED.2080402@euler.org", + "439A1E03.3090604@euler.org", + "20051211184308.GB13513@gauss.org" + }; + + assert_equal_seq_str(msg.references(), expected_refs); + assert_equal(msg.thread_id(), expected_refs[0]); +} + +static void +test_mu_msg_references_many(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR2 "/bar/cur/181736.eml") + .value()}; + + std::array<std::string, 11> expected_refs = { + "e9065dac-13c1-4103-9e31-6974ca232a89@t15g2000prt.googlegroups.com", + "87hbblwelr.fsf@sapphire.mobileactivedefense.com", + "pql248-4va.ln1@wilbur.25thandClement.com", + "ikns6r$li3$1@Iltempo.Update.UU.SE", + "8762s0jreh.fsf@sapphire.mobileactivedefense.com", + "ikqqp1$jv0$1@Iltempo.Update.UU.SE", + "87hbbjc5jt.fsf@sapphire.mobileactivedefense.com", + "ikr0na$lru$1@Iltempo.Update.UU.SE", + "tO8cp.1228$GE6.370@news.usenetserver.com", + "ikr6ks$nlf$1@Iltempo.Update.UU.SE", + "8ioh48-8mu.ln1@leafnode-msgid.gclare.org.uk" + }; + + assert_equal_seq_str(msg.references(), expected_refs); + assert_equal(msg.thread_id(), expected_refs[0]); +} + +static void +test_mu_msg_tags(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/mail1").value()}; + + assert_contacts_equal(msg.to(), {{ "Julius Caesar", "jc@example.com" }}); + assert_contacts_equal(msg.from(), {{ "John Milton", "jm@example.com" }}); + + assert_equal(msg.subject(),"Fere libenter homines id quod volunt credunt"); + + g_assert_true(msg.priority() == Priority::High); + g_assert_cmpuint(msg.date(), ==, 1217530645); + + std::array<std::string, 4> expected_tags = { + "Paradise", + "losT", + "john", + "milton" + }; + assert_equal_seq_str(msg.tags(), expected_tags); +} + +static void +test_mu_msg_comp_unix_programmer(void) +{ + auto msg{Message::make_from_path(MU_TESTMAILDIR4 "/181736.eml").value()}; + + g_assert_true(msg.to().empty()); + assert_equal(msg.subject(), + "Re: Are writes \"atomic\" to readers of the file?"); + assert_equal(msg.from().at(0).display_name(), "Jimbo Foobarcuux <jimbo@slp53.sl.home>"); + assert_equal(msg.message_id(), "oktdp.42997$Te.22361@news.usenetserver.com"); + + auto refs = join(msg.references(), ','); + assert_equal(refs, + "e9065dac-13c1-4103-9e31-6974ca232a89@t15g2000prt" + ".googlegroups.com," + "87hbblwelr.fsf@sapphire.mobileactivedefense.com," + "pql248-4va.ln1@wilbur.25thandClement.com," + "ikns6r$li3$1@Iltempo.Update.UU.SE," + "8762s0jreh.fsf@sapphire.mobileactivedefense.com," + "ikqqp1$jv0$1@Iltempo.Update.UU.SE," + "87hbbjc5jt.fsf@sapphire.mobileactivedefense.com," + "ikr0na$lru$1@Iltempo.Update.UU.SE," + "tO8cp.1228$GE6.370@news.usenetserver.com," + "ikr6ks$nlf$1@Iltempo.Update.UU.SE," + "8ioh48-8mu.ln1@leafnode-msgid.gclare.org.uk"); + + //"jimbo@slp53.sl.home (Jimbo Foobarcuux)"; + g_assert_true(msg.priority() == Priority::Normal); + g_assert_cmpuint(msg.date(), ==, 1299603860); +} + +static void +test_mu_str_prio_01(void) +{ + g_assert_true(priority_name(Priority::Low) == "low"); + g_assert_true(priority_name(Priority::Normal) == "normal"); + g_assert_true(priority_name(Priority::High) == "high"); +} + +G_GNUC_UNUSED static gboolean +ignore_error(const char* log_domain, GLogLevelFlags log_level, const gchar* msg, gpointer user_data) +{ + return FALSE; /* don't abort */ +} + + +int +main(int argc, char* argv[]) +{ + int rv; + + g_test_init(&argc, &argv, NULL); + + /* mu_msg_str_date */ + g_test_add_func("/mu-msg/mu-msg-01", test_mu_msg_01); + g_test_add_func("/mu-msg/mu-msg-02", test_mu_msg_02); + g_test_add_func("/mu-msg/mu-msg-03", test_mu_msg_03); + g_test_add_func("/mu-msg/mu-msg-04", test_mu_msg_04); + g_test_add_func("/mu-msg/mu-msg-multimime", test_mu_msg_multimime); + + g_test_add_func("/mu-msg/mu-msg-flags", test_mu_msg_flags); + + g_test_add_func("/mu-msg/mu-msg-tags", test_mu_msg_tags); + g_test_add_func("/mu-msg/mu-msg-references", test_mu_msg_references); + g_test_add_func("/mu-msg/mu-msg-references_dups", test_mu_msg_references_dups); + g_test_add_func("/mu-msg/mu-msg-references_many", test_mu_msg_references_many); + + g_test_add_func("/mu-msg/mu-msg-umlaut", test_mu_msg_umlaut); + g_test_add_func("/mu-msg/mu-msg-comp-unix-programmer", test_mu_msg_comp_unix_programmer); + + g_test_add_func("/mu-str/mu-str-prio-01", test_mu_str_prio_01); + + rv = g_test_run(); + + return rv; +} diff --git a/lib/tests/test-mu-store-query.cc b/lib/tests/test-mu-store-query.cc new file mode 100644 index 0000000..5f28286 --- /dev/null +++ b/lib/tests/test-mu-store-query.cc @@ -0,0 +1,913 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include "config.h" + +#include "utils/mu-result.hh" +#include <array> +#include <thread> +#include <string> +#include <string_view> +#include <fstream> +#include <unordered_map> + +#include <mu-store.hh> +#include <mu-maildir.hh> +#include <utils/mu-utils.hh> +#include <utils/mu-utils-file.hh> +#include <utils/mu-test-utils.hh> +#include <message/mu-message.hh> + +#include "mu-query-parser.hh" + +using namespace Mu; + + +/// map of some (unique) path-tail to the message-text +using TestMap = std::unordered_map<std::string, std::string>; + +static Store +make_test_store(const std::string& test_path, const TestMap& test_map, + Option<const Config&> conf={}) +{ + const auto maildir{join_paths(test_path, "/Maildir/")}; + // note the trailing '/' + g_test_bug("2513"); + + /* write messages to disk */ + for (auto&& item: test_map) { + + /* create the directory for the message */ + const auto msgpath{join_paths(maildir, item.first)}; + auto dir = to_string_gchar(g_path_get_dirname(msgpath.c_str())); + if (g_test_verbose()) + mu_message("create maildir {}", dir.c_str()); + + g_assert_cmpuint(g_mkdir_with_parents(dir.c_str(), 0700), ==, 0); + + /* write the file */ + std::ofstream stream(msgpath); + stream.write(item.second.data(), item.second.size()); + g_assert_true(stream.good()); + stream.close(); + } + + auto store = Store::make_new(test_path, maildir, conf); + assert_valid_result(store); + + /* index the messages */ + g_assert_true(store->indexer().start({},true/*block*/)); + if (test_map.size() > 0) + g_assert_false(store->empty()); + + g_assert_cmpuint(store->size(),==,test_map.size()); + + /* and we have a fully-ready store */ + return std::move(store.value()); +} + +static void +test_simple() +{ + const TestMap test_msgs = {{ + +// "sqlite-msg" "Simple mailing list message. +{ +"basic/cur/sqlite-msg:2,S", +R"(Return-Path: <sqlite-dev-bounces@sqlite.org> +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 32F276963F + for <xxxx@localhost>; Mon, 4 Aug 2008 21:49:34 +0300 (EEST) +Message-Id: <83B5AF40-DBFA-4578-A043-04C80276E195@sqlabs.net> +From: "Foo Example" <foo@example.com> +To: sqlite-dev@sqlite.org +Cc: "Bank of America" <bank@example.com> +Bcc: Aku Ankka <donald.duck@duckstad.nl> +Mime-Version: 1.0 (Apple Message framework v926) +Date: Mon, 4 Aug 2008 11:40:49 +0200 +X-Mailer: Apple Mail (2.926) +Subject: [sqlite-dev] VM optimization inside sqlite3VdbeExec +Precedence: list +Reply-To: sqlite-dev@sqlite.org +List-Id: <sqlite-dev.sqlite.org> +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit +Sender: sqlite-dev-bounces@sqlite.org + +Inside sqlite3VdbeExec there is a very big switch statement. +In order to increase performance with few modifications to the +original code, why not use this technique ? +http://docs.freebsd.org/info/gcc/gcc.info.Labels_as_Values.html + +With a properly defined "instructions" array, instead of the switch +statement you can use something like: +goto * instructions[pOp->opcode]; + +I said: "Aujourd'hui!" +)"}, +}}; + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + + // matches + for (auto&& expr: { + "Inside", + "from:foo@example.com", + "from:Foo", + "from:\"Foo Example\"", + "from:/Foo.*Example/", + "recip:\"Bank Of America\"", + "cc:bank@example.com", + "cc:bank", + "cc:america", + "bcc:donald.duck@duckstad.nl", + "bcc:donald.duck", + "bcc:duckstad.nl", + "bcc:aku", + "bcc:ankka", + "bcc:\"aku ankka\"", + "date:2008-08-01..2008-09-01", + "prio:low", + "to:sqlite-dev@sqlite.org", + "list:sqlite-dev.sqlite.org", + "aujourd'hui", +#ifdef HAVE_CLD2 + "lang:en", +#endif /*HAVE_CLD2*/ + }) { + + if (g_test_verbose()) + mu_message("query: '{}'\n", expr, + make_xapian_query(store, expr)->get_description()); + + auto qr = store.run_query(expr); + assert_valid_result(qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 1); + } + + auto qr = store.run_query("statement"); + assert_valid_result(qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 1); + + assert_equal(qr->begin().subject().value_or(""), + "[sqlite-dev] VM optimization inside sqlite3VdbeExec"); + g_assert_true(qr->begin().references().empty()); + //g_assert_cmpuint(qr->begin().date().value_or(0), ==, 123454); +} + +static void +test_spam_address_components() +{ + const TestMap test_msgs = {{ + +// "sqlite-msg" "Simple mailing list message. +{ +"spam/cur/spam-msg:2,S", +R"(Message-Id: <abcde@foo.bar> +From: "Foo Example" <bar@example.com> +To: example@example.com +Subject: ***SPAM*** this is a test + +Boo! +)"}, +}}; + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + + g_test_bug("2278"); + g_test_bug("2281"); + + // matches both + for (auto&& expr: { + "SPAM", + "spam", + "/.*SPAM.*/", + "subject:SPAM", + "from:bar@example.com", + "subject:\\*\\*\\*SPAM\\*\\*\\*", + "bar", + "example.com" + }) { + + if (g_test_verbose()) + g_message("query: '%s'", expr); + auto qr = store.run_query(expr); + assert_valid_result(qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 1); + } +} + + +static void +test_dups_related() +{ + const TestMap test_msgs = {{ +/* parent */ +{ +"inbox/cur/msg1:2,S", +R"(Message-Id: <abcde@foo.bar> +From: "Foo Example" <bar@example.com> +Date: Sat, 06 Aug 2022 11:01:54 -0700 +To: example@example.com +Subject: test1 + +Parent +)"}, +/* child (dup vv) */ +{ +"boo/cur/msg2:1,S", +R"(Message-Id: <edcba@foo.bar> +In-Reply-To: <abcde@foo.bar> +From: "Foo Example" <bar@example.com> +Date: Sat, 06 Aug 2022 13:01:54 -0700 +To: example@example.com +Subject: Re: test1 + +Child +)"}, +/* child (dup ^^) */ +{ +"inbox/cur/msg2:1,S", +R"(Message-Id: <edcba@foo.bar> +In-Reply-To: <abcde@foo.bar> +From: "Foo Example" <bar@example.com> +Date: Sat, 06 Aug 2022 14:01:54 -0700 +To: example@example.com +Subject: Re: test1 + +Child +)"}, +}}; + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + { + // direct matches + auto qr = store.run_query("test1", Field::Id::Date, + QueryFlags::None); + g_assert_true(!!qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 3); + } + + { + // skip duplicate messages; which one is skipped is arbitrary. + auto qr = store.run_query("test1", Field::Id::Date, + QueryFlags::SkipDuplicates); + g_assert_true(!!qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 2); + } + + { + // no related + auto qr = store.run_query("Parent", Field::Id::Date); + g_assert_true(!!qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 1); + } + + { + // find related messages + auto qr = store.run_query("Parent", Field::Id::Date, + QueryFlags::IncludeRelated); + g_assert_true(!!qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 3); + } + + { + // find related messages, skip dups. the leader message + // should _not_ be skipped. + auto qr = store.run_query("test1 AND maildir:/inbox", + Field::Id::Date, + QueryFlags::IncludeRelated| + QueryFlags::SkipDuplicates); + g_assert_true(!!qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 2); + + // ie the /boo is to be skipped, since it's not in the leader + // set. + for (auto&& m: *qr) + assert_equal(m.message()->maildir(), "/inbox"); + } + + { + // find related messages, find parent from child. + auto qr = store.run_query("Child and maildir:/inbox", + Field::Id::Date, + QueryFlags::IncludeRelated); + g_assert_true(!!qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 3); + + } + + { + // find related messages, find parent from child. + // leader message wins + auto qr = store.run_query("Child and maildir:/inbox", + Field::Id::Date, + QueryFlags::IncludeRelated| + QueryFlags::SkipDuplicates| + QueryFlags::Descending); + g_assert_true(!!qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 2); + + // ie the /boo is to be skipped, since it's not in the leader + // set. + for (auto&& m: *qr) + assert_equal(m.message()->maildir(), "/inbox"); + } +} + + +static void +test_related_missing_root() +{ + const TestMap test_msgs = {{ +{ +"inbox/cur/msg1:2,S", +R"(Content-Type: text/plain; charset=utf-8 +References: <EZrZOnVCsYfFcX3Ls0VFoRnJdCGV4GM5YtO739l-iOB2ADNH7cIJWb0DaO5Of3BWDUEKq18Rz3a7rNoI96bNwQ==@protonmail.internalid> +To: "Joerg Roedel" <joro@8bytes.org>, "Suman Anna" <s-anna@ti.com> +Reply-To: "Dan Carpenter" <dan.carpenter@oracle.com> +From: "Dan Carpenter" <dan.carpenter@oracle.com> +Subject: [PATCH] iommu/omap: fix buffer overflow in debugfs +Date: Thu, 4 Aug 2022 17:32:39 +0300 +Message-Id: <YuvYh1JbE3v+abd5@kili> +List-Id: <kernel-janitors.vger.kernel.org> +Precedence: bulk + +There are two issues here: +)"}, +{ +"inbox/cur/msg2:2,S", +R"(Content-Transfer-Encoding: quoted-printable +Content-Type: text/plain; charset=utf-8 +References: <YuvYh1JbE3v+abd5@kili> + <9pEUi_xoxa7NskF7EK_qfrlgjXzGsyw9K7cMfYbo-KI6fnyVMKTpc8E2Fu94V8xedd7cMpn0LlBrr9klBMflpw==@protonmail.internalid> +Reply-To: "Laurent Pinchart" <laurent.pinchart@ideasonboard.com> +From: "Laurent Pinchart" <laurent.pinchart@ideasonboard.com> +Subject: Re: [PATCH] iommu/omap: fix buffer overflow in debugfs +List-Id: <kernel-janitors.vger.kernel.org> +Message-Id: <YuvzKJM66k+ZPD9c@pendragon.ideasonboard.com> +Precedence: bulk +In-Reply-To: <YuvYh1JbE3v+abd5@kili> + +Hi Dan, + +Thank you for the patch. +)"}, +{ +"inbox/cur/msg3:2,S", +R"(Content-Transfer-Encoding: quoted-printable +Content-Type: text/plain; charset=utf-8 +References: <YuvYh1JbE3v+abd5@kili> + <G6TStg8J52Q-uSMTR7wRQdPeloxpZMiEQT_F8_JIDYM25eEPeHGgrNKO0fuO78MiQgD9Mz4BDtsZlZgmPKFe4Q==@protonmail.internalid> +To: "Dan Carpenter" <dan.carpenter@oracle.com>, "Joerg Roedel" + <joro@8bytes.org>, "Suman Anna" <s-anna@ti.com> +Reply-To: "Robin Murphy" <robin.murphy@arm.com> +From: "Robin Murphy" <robin.murphy@arm.com> +Subject: Re: [PATCH] iommu/omap: fix buffer overflow in debugfs +List-Id: <kernel-janitors.vger.kernel.org> +Message-Id: <90a760c4-6e88-07b4-1f20-8b10414e49aa@arm.com> +Precedence: bulk +In-Reply-To: <YuvYh1JbE3v+abd5@kili> +Date: Thu, 4 Aug 2022 17:31:39 +0100 + +On 04/08/2022 3:32 pm, Dan Carpenter wrote: +> There are two issues here: +)"}, +{ +"inbox/new/msg4", +R"(Content-Transfer-Encoding: quoted-printable +Content-Type: text/plain; charset=utf-8 +References: <YuvYh1JbE3v+abd5@kili> + <90a760c4-6e88-07b4-1f20-8b10414e49aa@arm.com> + <T4CDWjUrgtI5n4mh1JEdW6RLYzqbPE9-yDrhEVwDM22WX-198fBwcnLd-4_xR1gvsVSHQps9fp_pZevTF0ZmaA==@protonmail.internalid> +To: "Robin Murphy" <robin.murphy@arm.com> +Reply-To: "Dan Carpenter" <dan.carpenter@oracle.com> +From: "Dan Carpenter" <dan.carpenter@oracle.com> +Subject: Re: [PATCH] iommu/omap: fix buffer overflow in debugfs +List-Id: <kernel-janitors.vger.kernel.org> +Date: Fri, 5 Aug 2022 09:37:02 +0300 +In-Reply-To: <90a760c4-6e88-07b4-1f20-8b10414e49aa@arm.com> +Precedence: bulk +Message-Id: <20220805063702.GH3438@kadam> + +On Thu, Aug 04, 2022 at 05:31:39PM +0100, Robin Murphy wrote: +> On 04/08/2022 3:32 pm, Dan Carpenter wrote: +> > There are two issues here: +)"}, +}}; + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + { + auto qr = store.run_query("fix buffer overflow in debugfs", + Field::Id::Date, QueryFlags::IncludeRelated); + g_assert_true(!!qr); + g_assert_cmpuint(qr->size(), ==, 4); + } + + { + auto qr = store.run_query("fix buffer overflow in debugfs and flag:unread", + Field::Id::Date, QueryFlags::None); + g_assert_true(!!qr); + g_assert_cmpuint(qr->size(), ==, 1); + assert_equal(qr->begin().message_id().value_or(""), "20220805063702.GH3438@kadam"); + assert_equal(qr->begin().thread_id().value_or(""), "YuvYh1JbE3v+abd5@kili"); + } + + { + /* this one failed earlier, because the 'protonmail' id is the + * first reference, which means it does _not_ have the same + * thread-id as the rest; however, we filter these + * fake-message-ids now.*/ + g_test_bug("2312"); + + auto qr = store.run_query("fix buffer overflow in debugfs and flag:unread", + Field::Id::Date, QueryFlags::IncludeRelated); + g_assert_true(!!qr); + g_assert_cmpuint(qr->size(), ==, 4); + } +} + + +static void +test_body_matricula() +{ + const TestMap test_msgs = {{ +{ +"basic/cur/matricula-msg:2,S", +R"(From: XXX <XX@XX.com> +Subject: + =?iso-8859-1?Q?EF_-_Pago_matr=EDcula_de_la_matr=EDcula_de_inscripci=F3n_a?= +Date: Thu, 4 Aug 2022 14:29:41 +0000 +Message-ID: + <VE1PR03MB5471882920DE08CFE44D97A0FE9F9@VE1PR03MB5471.eurprd03.prod.outlook.com> +Accept-Language: es-AR, es-ES, en-US +Content-Language: es-AR +X-MS-Has-Attach: yes +Content-Type: multipart/mixed; + boundary="_004_VE1PR03MB5471882920DE08CFE44D97A0FE9F9VE1PR03MB5471eurp_" +MIME-Version: 1.0 +X-OriginatorOrg: ef.com +X-MS-Exchange-CrossTenant-AuthAs: Internal +X-MS-Exchange-CrossTenant-AuthSource: VE1PR03MB5471.eurprd03.prod.outlook.com + +--_004_VE1PR03MB5471882920DE08CFE44D97A0FE9F9VE1PR03MB5471eurp_ +Content-Type: multipart/alternative; + boundary="_000_VE1PR03MB5471882920DE08CFE44D97A0FE9F9VE1PR03MB5471eurp_" + +--_000_VE1PR03MB5471882920DE08CFE44D97A0FE9F9VE1PR03MB5471eurp_ +Content-Type: text/plain; charset="iso-8859-1" +Content-Transfer-Encoding: quoted-printable + +Buenas tardes Familia, + + +Espero que est=E9n muy bien. + + + +Ya cargamos en sistema su pre inscripci=F3n para el curso + + +Quedamos atentos ante cualquier consulta que surja. + +Saludos, +)"}, +}}; + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + + /* i.e., non-utf8 text parts were not converted */ + g_test_bug("2333"); + + // matches + for (auto&& expr: { + "subject:matrícula", + "subject:matricula", + "body:atentos", + "body:inscripción" + }) { + + if (g_test_verbose()) + g_message("query: '%s'", expr); + auto qr = store.run_query(expr); + assert_valid_result(qr); + g_assert_false(qr->empty()); + g_assert_cmpuint(qr->size(), ==, 1); + } +} + + + +static void +test_duplicate_refresh_real(bool rename) +{ + g_test_bug("2327"); + + const TestMap test_msgs = {{ + "inbox/new/msg", + { R"(Message-Id: <abcde@foo.bar> +From: "Foo Example" <bar@example.com> +Date: Wed, 26 Oct 2022 11:01:54 -0700 +To: example@example.com +Subject: Rainy night in Helsinki + +Boo! +)"}, + }}; + + /* create maildir with message */ + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + g_debug("%s", store.root_maildir().c_str()); + /* ensure we have a proper maildir, with new/, cur/ */ + auto mres = maildir_mkdir(store.root_maildir() + "/inbox"); + assert_valid_result(mres); + g_assert_cmpuint(store.size(), ==, 1U); + + /* + * find the one msg with a query + */ + auto qr = store.run_query("Helsinki", Field::Id::Date, QueryFlags::None); + g_assert_true(!!qr); + g_assert_cmpuint(qr->size(), ==, 1); + const auto old_path = qr->begin().path().value(); + const auto old_docid = qr->begin().doc_id(); + assert_equal(qr->begin().message()->path(), old_path); + g_assert_true(::access(old_path.c_str(), F_OK) == 0); + + + /* + * mark as read, i.e. move to cur/; ensure it really moved. + */ + auto move_opts{rename ? Store::MoveOptions::ChangeName : Store::MoveOptions::None}; + auto moved_msgs = store.move_message(old_docid, Nothing, Flags::Seen, move_opts); + assert_valid_result(moved_msgs); + + g_assert_true(moved_msgs->size() == 1); + auto&& moved_msg_opt = store.find_message(moved_msgs->at(0).first); + g_assert_true(!!moved_msg_opt); + const auto&moved_msg = std::move(*moved_msg_opt); + const auto new_path = moved_msg.path(); + if (!rename) + assert_equal(new_path, store.root_maildir() + "/inbox/cur/msg:2,S"); + g_assert_cmpuint(store.size(), ==, 1); + g_assert_false(::access(old_path.c_str(), F_OK) == 0); + g_assert_true(::access(new_path.c_str(), F_OK) == 0); + + /* also ensure that the cached sexp for the message has been updated; + * that's what mu4e uses */ + const auto moved_sexp{moved_msg.sexp()}; + g_assert_true(moved_sexp.plistp()); + g_assert_true(!!moved_sexp.get_prop(":path")); + assert_equal(moved_sexp.get_prop(":path").value().string(), new_path); + + /* + * find new message with query, ensure it's really that new one. + */ + auto qr2 = store.run_query("Helsinki", Field::Id::Date, QueryFlags::None); + g_assert_true(!!qr2); + g_assert_cmpuint(qr2->size(), ==, 1); + assert_equal(qr2->begin().path().value(), new_path); + + /* index the messages */ + auto res = store.indexer().start({}); + g_assert_true(res); + while(store.indexer().is_running()) { + using namespace std::chrono_literals; + std::this_thread::sleep_for(100ms); + } + g_assert_cmpuint(store.size(), ==, 1); + + /* + * ensure query still has the right results + */ + auto qr3 = store.run_query("Helsinki", Field::Id::Date, QueryFlags::None); + g_assert_true(!!qr3); + g_assert_cmpuint(qr3->size(), ==, 1); + const auto path3{qr3->begin().path().value()}; + assert_equal(path3, new_path); + assert_equal(qr3->begin().message()->path(), new_path); + g_assert_true(::access(path3.c_str(), F_OK) == 0); +} + + +static void +test_duplicate_refresh() +{ + test_duplicate_refresh_real(false/*no rename*/); +} + + +static void +test_duplicate_refresh_rename() +{ + test_duplicate_refresh_real(true/*rename*/); +} + +static void +test_term_split() +{ + g_test_bug("2365"); + + // Note the fancy quote in "foo’s bar" + const TestMap test_msgs = {{ + "inbox/new/msg", + { +R"(Message-Id: <abcde@foo.bar> +From: "Foo Example" <bar@example.com> +Date: Wed, 26 Oct 2022 11:01:54 -0700 +To: example@example.com +Subject: foo’s bar + +Boo! +)"}, + }}; + + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + /* true: match; false: no match */ + const auto cases = std::array<std::pair<const char*, bool>, 8>{{ + {"subject:foo's", true}, + {"subject:foo*", true}, + {"subject:/foo/", true}, + {"subject:/foo’s/", true}, /* <-- breaks before PR #2365 */ + {"subject:/foo.*bar/", true}, /* <-- breaks before PR #2365 */ + {"subject:/foo’s bar/", false}, /* <-- no matching, needs quoting */ + {"subject:\"/foo’s bar/\"", true}, /* <-- this works, quote the regex */ + {R"(subject:"/foo’s bar/")", true}, /* <-- this works, quote the regex */ + }}; + + for (auto&& test: cases) { + mu_debug("query: '{}'", test.first); + auto qr = store.run_query(test.first); + assert_valid_result(qr); + if (test.second) + g_assert_cmpuint(qr->size(), ==, 1); + else + g_assert_true(qr->empty()); + } +} + +static void +test_subject_kata_containers() +{ + g_test_bug("2167"); + + // Note the fancy quote in "foo’s bar" + const TestMap test_msgs = {{ + "inbox/new/msg", + { +R"(Message-Id: <abcde@foo.bar> +From: "Foo Example" <bar@example.com> +Date: Wed, 26 Oct 2022 11:01:54 -0700 +To: example@example.com +Subject: kata-containers + +voodoo-containers + +Boo! +)"}, + }}; + + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + /* true: match; false: no match */ + const auto cases = std::vector<std::pair<const char*, bool>>{{ + {"subject:kata", true}, + {"subject:containers", true}, + {"subject:kata-containers", true}, + {"subject:\"kata containers\"", true}, + {"voodoo-containers", true}, + {"voodoo containers", true} + }}; + + for (auto&& test: cases) { + mu_debug("query: '{}'", test.first); + auto qr = store.run_query(test.first); + assert_valid_result(qr); + if (test.second) + g_assert_cmpuint(qr->size(), ==, 1); + else + g_assert_true(qr->empty()); + } +} + +static void +test_related_dup_threaded() +{ + // test message sent to self, and copy of received msg. + + const auto test_msg = R"(From: "Edward Mallory" <ed@leviathan.gb> +To: "Laurence Oliphant <oli@hotmail.com> +Subject: Boo +Date: Wed, 07 Dec 2022 18:38:06 +0200 +Message-ID: <875yentbhg.fsf@djcbsoftware.nl> +MIME-Version: 1.0 +Content-Type: text/plain + +Boo! +)"; + const TestMap test_msgs = { + {"sent/cur/msg1", test_msg }, + {"inbox/cur/msg1", test_msg }, + {"inbox/cur/msg2", test_msg }}; + + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + + g_assert_cmpuint(store.size(), ==, 3); + + + // normal query should give 2 + { + auto qr = store.run_query("maildir:/inbox", Field::Id::Date, + QueryFlags::None); + assert_valid_result(qr); + g_assert_cmpuint(qr->size(), ==, 2); + } + + // a related query should give 3 + { + auto qr = store.run_query("maildir:/inbox", Field::Id::Date, + QueryFlags::IncludeRelated); + assert_valid_result(qr); + g_assert_cmpuint(qr->size(), ==, 3); + } + + // a related/threading query should give 3. + { + auto qr = store.run_query("maildir:/inbox", Field::Id::Date, + QueryFlags::IncludeRelated | QueryFlags::Threading); + assert_valid_result(qr); + g_assert_cmpuint(qr->size(), ==, 3); + } +} + + +static void +test_html() +{ + // test message sent to self, and copy of received msg. + + const auto test_msg = R"(From: Test <test@example.com> +To: abc@example.com +Date: Mon, 23 May 2011 10:53:45 +0200 +Subject: vla +MIME-Version: 1.0 +Content-Type: multipart/alternative; + boundary="_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d" +Message-ID: <10374608.109906.11909.20115aabbccdd.MSGID@mailinglijst.nl> + +--_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d +Content-Type: text/plain; charset="iso-8859-15" +Content-Transfer-Encoding: quoted-printable + +text + +--_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d +Content-Type: text/html; charset="iso-8859-15" +Content-Transfer-Encoding: quoted-printable + +html + +--_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d-- +)"; + const TestMap test_msgs = {{"inbox/cur/msg1", test_msg }}; + + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, {})}; + g_assert_cmpuint(store.size(), ==, 1); + + { + auto qr = store.run_query("body:text", Field::Id::Date, + QueryFlags::None); + assert_valid_result(qr); + g_assert_cmpuint(qr->size(), ==, 1); + } + + { + auto qr = store.run_query("body:html", Field::Id::Date, + QueryFlags::None); + assert_valid_result(qr); + g_assert_cmpuint(qr->size(), ==, 1); + } +} + + +static void +test_ngrams() +{ + g_test_bug("2167"); + + // Note the fancy quote in "foo’s bar" + const TestMap test_msgs = {{ + "inbox/new/msg", + { +R"(From: "Bob" <bob@builder.com> +Subject: スポンサーシップ募集 +To: "Chase" <chase@ppatrol.org> +Message-Id: 112342343e9dfo.fsf@builder.com + + 中文 + +https://trac.xapian.org/ticket/719 + + サーバがダウンしました +)"}}}; + + MemDb mdb; + Config conf{mdb}; + conf.set<Config::Id::SupportNgrams>(true); + + TempDir tdir; + auto store{make_test_store(tdir.path(), test_msgs, conf)}; + + /* true: match; false: no match */ + const auto cases = std::vector<std::pair<std::string_view, bool>>{{ + {"body:中文", true}, + {"body:中", true}, + {"body:文", true}, + {"body:し", true}, + {"body:サー", true}, + {"body:サーバがダウンしました", true}, // fail + {"中文", true}, + {"中", true}, + {"文", true}, + {"subject:スポン", true }, + {"subject:スポンサーシップ募集", true }, + {"subject:シップ", true }, // XXX should match + {"サーバがダウンしました", true}, // okay + {"body:サーバがダウンしました", true}, // okay + {"subject:スポンサーシップ募集", true}, // okay + {"subject:シップx", true }, // XXX should match + }}; + + for (auto&& test: cases) { + auto qr = store.run_query(std::string{test.first}); + assert_valid_result(qr); + if (test.second) + g_assert_cmpuint(qr->size(), ==, 1); + else + g_assert_true(qr->empty()); + } +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/store/query/simple", + test_simple); + g_test_add_func("/store/query/spam-address-components", + test_spam_address_components); + g_test_add_func("/store/query/dups-related", + test_dups_related); + g_test_add_func("/store/query/related-missing-root", + test_related_missing_root); + g_test_add_func("/store/query/body-matricula", + test_body_matricula); + g_test_add_func("/store/query/duplicate-refresh", + test_duplicate_refresh); + g_test_add_func("/store/query/duplicate-refresh-rename", + test_duplicate_refresh_rename); + g_test_add_func("/store/query/term-split", + test_term_split); + g_test_add_func("/store/query/kata_containers", + test_subject_kata_containers); + g_test_add_func("/store/query/related-dup-threaded", + test_related_dup_threaded); + g_test_add_func("/store/query/html", + test_html); + g_test_add_func("/store/query/ngrams", + test_ngrams); + + return g_test_run(); +} diff --git a/lib/tests/test-mu-store.cc b/lib/tests/test-mu-store.cc new file mode 100644 index 0000000..da7f120 --- /dev/null +++ b/lib/tests/test-mu-store.cc @@ -0,0 +1,590 @@ +/* +** Copyright (C) 2008-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include <glib.h> +#include <stdlib.h> +#include <thread> +#include <array> +#include <unistd.h> +#include <time.h> +#include <fstream> + +#include <locale.h> + +#include "utils/mu-test-utils.hh" +#include "mu-store.hh" +#include "utils/mu-result.hh" +#include <utils/mu-utils.hh> +#include <utils/mu-utils-file.hh> +#include "mu-maildir.hh" + +using namespace Mu; + +using namespace std::chrono_literals; + +static std::string MuTestMaildir = Mu::canonicalize_filename(MU_TESTMAILDIR, "/"); +static std::string MuTestMaildir2 = Mu::canonicalize_filename(MU_TESTMAILDIR2, "/"); + +static void +test_store_ctor_dtor() +{ + TempDir tempdir; + auto store{Store::make_new(tempdir.path(), "/tmp")}; + assert_valid_result(store); + + g_assert_true(store->empty()); + g_assert_cmpuint(0, ==, store->size()); + + g_assert_cmpuint(MU_STORE_SCHEMA_VERSION, ==, + store->config().get<Config::Id::SchemaVersion>()); +} + +static void +test_store_reinit() +{ + TempDir tempdir; + { + MemDb mdb; + Config conf{mdb}; + conf.set<Config::Id::MaxMessageSize>(1234567); + conf.set<Config::Id::BatchSize>(7654321); + conf.set<Config::Id::PersonalAddresses>( + StringVec{ "foo@example.com", "bar@example.com" }); + + auto store{Store::make_new(tempdir.path(), MuTestMaildir, conf)}; + assert_valid_result(store); + + g_assert_true(store->empty()); + g_assert_cmpuint(0, ==, store->size()); + + g_assert_cmpuint(MU_STORE_SCHEMA_VERSION, ==, + store->config().get<Config::Id::SchemaVersion>()); + + const auto msgpath{MuTestMaildir + "/cur/1283599333.1840_11.cthulhu!2,"}; + const auto id = store->add_message(msgpath); + assert_valid_result(id); + g_assert_true(store->contains_message(msgpath)); + g_assert_cmpuint(store->size(), ==, 1); + } + + //now let's reinitialize it. + { + auto store{Store::make(tempdir.path(), + Store::Options::Writable|Store::Options::ReInit)}; + + assert_valid_result(store); + g_assert_true(store->empty()); + + assert_equal(store->path(), tempdir.path()); + assert_equal(store->root_maildir(), MuTestMaildir); + + g_assert_cmpuint(store->config().get<Config::Id::BatchSize>(),==,7654321); + g_assert_cmpuint(store->config().get<Config::Id::MaxMessageSize>(),==,1234567); + + const auto addrs{store->config().get<Config::Id::PersonalAddresses>()}; + g_assert_cmpuint(addrs.size(),==,2); + g_assert_true(seq_some(addrs, [](auto&& a){return a=="foo@example.com";})); + g_assert_true(seq_some(addrs, [](auto&& a){return a=="bar@example.com";})); + + const auto msgpath{MuTestMaildir + "/cur/1283599333.1840_11.cthulhu!2,"}; + const auto id = store->add_message(msgpath); + assert_valid_result(id); + g_assert_true(store->contains_message(msgpath)); + g_assert_cmpuint(store->size(), ==, 1); + } +} + + + +static void +test_store_add_count_remove() +{ + TempDir tempdir{false}; + + auto store{Store::make_new(tempdir.path() + "/xapian", MuTestMaildir)}; + assert_valid_result(store); + + assert_equal(store->path(), tempdir.path() + "/xapian"); + assert_equal(store->root_maildir(), MuTestMaildir); + + const auto msgpath{MuTestMaildir + "/cur/1283599333.1840_11.cthulhu!2,"}; + const auto id1 = store->add_message(msgpath); + assert_valid_result(id1); + + g_assert_cmpuint(store->size(), ==, 1); + g_assert_true(store->contains_message(msgpath)); + + const auto id2 = store->add_message(MuTestMaildir2 + "/bar/cur/mail3"); + g_assert_false(!!id2); // wrong maildir. + + const auto msg3path{MuTestMaildir + "/cur/1252168370_3.14675.cthulhu!2,S"}; + const auto id3 = store->add_message(msg3path); + assert_valid_result(id3); + + g_assert_cmpuint(store->size(), ==, 2); + g_assert_true(store->contains_message(msg3path)); + + store->remove_message(id1.value()); + g_assert_cmpuint(store->size(), ==, 1); + g_assert_false( + store->contains_message(MuTestMaildir + "/cur/1283599333.1840_11.cthulhu!2,")); + + store->remove_message(msg3path); + g_assert_true(store->empty()); + g_assert_false(store->contains_message(msg3path)); +} + + +static void +test_message_mailing_list() +{ + constexpr const char *test_message_1 = +R"(Return-Path: <sqlite-dev-bounces@sqlite.org> +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 32F276963F + for <xxxx@localhost>; Mon, 4 Aug 2008 21:49:34 +0300 (EEST) +Message-Id: <83B5AF40-DBFA-4578-A043-04C80276E195@sqlabs.net> +From: anon@example.com +To: sqlite-dev@sqlite.org +Mime-Version: 1.0 (Apple Message framework v926) +Date: Mon, 4 Aug 2008 11:40:49 +0200 +X-Mailer: Apple Mail (2.926) +Subject: Capybaras United +Precedence: list +Reply-To: sqlite-dev@sqlite.org +List-Id: <sqlite-dev.sqlite.org> +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit +Sender: sqlite-dev-bounces@sqlite.org +Content-Length: 639 + +Inside sqlite3VdbeExec there is a very big switch statement. +In order to increase performance with few modifications to the +original code, why not use this technique ? +http://docs.freebsd.org/info/gcc/gcc.info.Labels_as_Values.html + +With a properly defined "instructions" array, instead of the switch +statement you can use something like: +goto * instructions[pOp->opcode]; +)"; + TempDir tempdir; + auto store{Store::make_new(tempdir.path(), "/home/test/Maildir")}; + assert_valid_result(store); + + const auto msgpath{"/home/test/Maildir/inbox/cur/1649279256.107710_1.evergrey:2,S"}; + auto message{Message::make_from_text(test_message_1, msgpath)}; + assert_valid_result(message); + + const auto docid = store->add_message(*message); + assert_valid_result(docid); + g_assert_cmpuint(store->size(),==, 1); + + /* ensure 'update' dtrt, i.e., nothing. */ + const auto docid2 = store->add_message(*message); + assert_valid_result(docid2); + g_assert_cmpuint(store->size(),==, 1); + g_assert_cmpuint(*docid,==,*docid2); + + auto msg2{store->find_message(*docid)}; + g_assert_true(!!msg2); + assert_equal(message->path(), msg2->path()); + + g_assert_true(store->contains_message(message->path())); + + const auto qr = store->run_query("to:sqlite-dev@sqlite.org"); + g_assert_true(!!qr); + g_assert_cmpuint(qr->size(), ==, 1); +} + + +static void +test_message_attachments(void) +{ + constexpr const char* msg_text = +R"(Return-Path: <foo@example.com> +Received: from pop.gmail.com [256.85.129.309] + by evergrey with POP3 (fetchmail-6.4.29) + for <djcb@localhost> (single-drop); Thu, 24 Mar 2022 20:12:40 +0200 (EET) +Sender: "Foo, Example" <foo@example.com> +User-agent: mu4e 1.7.11; emacs 29.0.50 +From: "Foo Example" <foo@example.com> +To: bar@example.com +Subject: =?utf-8?B?w6R0dMOkY2htZcOxdHM=?= +Date: Thu, 24 Mar 2022 20:04:39 +0200 +Organization: ACME Inc. +Message-Id: <3144HPOJ0VC77.3H1XTAG2AMTLH@"@WILSONB.COM> +MIME-Version: 1.0 +X-label: @NextActions operation:mindcrime Queensrÿche +Content-Type: multipart/mixed; boundary="=-=-=" + +--=-=-= +Content-Type: text/plain + +Hello, +--=-=-= +Content-Type: image/jpeg +Content-Disposition: attachment; filename=file-01.bin +Content-Transfer-Encoding: base64 + +AAECAw== +--=-=-= +Content-Type: audio/ogg +Content-Disposition: inline; filename=/tmp/file-02.bin +Content-Transfer-Encoding: base64 + +BAUGBw== +--=-=-= +Content-Type: message/rfc822 +Content-Disposition: attachment; + filename="message.eml" + +From: "Fnorb" <fnorb@example.com> +To: Bob <bob@example.com> +Subject: news for you +Date: Mon, 28 Mar 2022 22:53:26 +0300 + +Attached message! + +--=-=-= +Content-Type: text/plain + +World! +--=-=-=-- +)"; + + TempDir tempdir; + auto store{Store::make_new(tempdir.path(), "/home/test/Maildir")}; + assert_valid_result(store); + + auto message{Message::make_from_text( + msg_text, + "/home/test/Maildir/inbox/cur/1649279256.abcde_1.evergrey:2,S")}; + assert_valid_result(message); + + const auto docid = store->add_message(*message); + assert_valid_result(docid); + + auto msg2{store->find_message(*docid)}; + g_assert_true(!!msg2); + assert_equal(message->path(), msg2->path()); + + g_assert_true(store->contains_message(message->path())); + + // for (auto&& term = msg2->document().xapian_document().termlist_begin(); + // term != msg2->document().xapian_document().termlist_end(); ++term) + // g_message(">>> %s", (*term).c_str()); + + const auto stats{store->statistics()}; + g_assert_cmpuint(stats.size,==,store->size()); + g_assert_cmpuint(stats.last_index,==,0); + g_assert_cmpuint(stats.last_change,>=,::time({})); +} + + +static void +test_index_move() +{ + const std::string msg_text = +R"(From: Valentine Michael Smith <mike@example.com> +To: Raul Endymion <raul@example.com> +Cc: emacs-devel@gnu.org +Subject: Re: multi-eq hash tables +Date: Tue, 03 May 2022 20:58:02 +0200 +Message-ID: <87h766tzzz.fsf@gnus.org> +MIME-Version: 1.0 +Content-Type: text/plain +Precedence: list +List-Id: "Emacs development discussions." <emacs-devel.gnu.org> +List-Post: <mailto:emacs-devel@gnu.org> + +Raul Endymion <raul@example.com> writes: + +> Maybe we should introduce something like: +> +> (define-hash-table-test shallow-equal +> (lambda (x1 x2) (while (and (consp x1) (consp x2) (eql (car x1) (car x2))) +> (setq x1 (cdr x1)) (setq x2 (cdr x2))) +> (equal x1 x2))) +> ...) + +Yes, that would be excellent. +)"; + + TempDir tempdir2; + + { // create a message file. + const auto res1 = maildir_mkdir(tempdir2.path() + "/Maildir/a"); + assert_valid_result(res1); + + std::ofstream output{tempdir2.path() + "/Maildir/a/new/msg"}; + output.write(msg_text.c_str(), msg_text.size()); + output.close(); + g_assert_true(output.good()); + } + + // Index it into a store. + TempDir tempdir; + auto store{Store::make_new(tempdir.path(), tempdir2.path() + "/Maildir")}; + assert_valid_result(store); + + store->indexer().start({}); + size_t n{}; + while (store->indexer().is_running()) { + std::this_thread::sleep_for(100ms); + g_assert_cmpuint(n++,<=,25); + } + g_assert_true(!store->indexer().is_running()); + const auto& prog{store->indexer().progress()}; + g_assert_cmpuint(prog.updated,==,1); + g_assert_cmpuint(store->size(), ==, 1); + g_assert_false(store->empty()); + + // Find the message + auto qr = store->run_query("path:" + tempdir2.path() + "/Maildir/a/new/msg"); + assert_valid_result(qr); + g_assert_cmpuint(qr->size(),==,1); + + const auto msg = qr->begin().message(); + g_assert_true(!!msg); + + // Check the message + const auto oldpath{msg->path()}; + assert_equal(msg->subject(), "Re: multi-eq hash tables"); + g_assert_true(msg->docid() != 0); + g_debug("%s", msg->sexp().to_string().c_str()); + + // Move the message from new->cur + std::this_thread::sleep_for(1s); /* ctime should change */ + const auto msgs3 = store->move_message(msg->docid(), {}, Flags::Seen); + assert_valid_result(msgs3); + g_assert_true(msgs3->size() == 1); + auto&& msg3_opt{store->find_message(msgs3->at(0).first/*id*/)}; + g_assert_true(!!msg3_opt); + auto&& msg3{std::move(*msg3_opt)}; + + assert_equal(msg3.maildir(), "/a"); + assert_equal(msg3.path(), tempdir2.path() + "/Maildir/a/cur/msg:2,S"); + g_assert_true(::access(msg3.path().c_str(), R_OK)==0); + g_assert_false(::access(oldpath.c_str(), R_OK)==0); + + g_debug("%s", msg3.sexp().to_string().c_str()); g_assert_cmpuint(store->size(), ==, 1); +} + + + +static void +test_store_move_dups() +{ + const std::string msg_text = +R"(From: Valentine Michael Smith <mike@example.com> +To: Raul Endymion <raul@example.com> +Subject: Re: multi-eq hash tables +Date: Tue, 03 May 2022 20:58:02 +0200 +Message-ID: <87h766tzzz.fsf@gnus.org> + +Yes, that would be excellent. +)"; + TempDir tempdir2; + + // create a message file + dups + const auto res1 = maildir_mkdir(tempdir2.path() + "/Maildir/a"); + assert_valid_result(res1); + const auto res2 = maildir_mkdir(tempdir2.path() + "/Maildir/b"); + assert_valid_result(res2); + + auto msg1_path = join_paths(tempdir2.path(), "Maildir/a/new/msg123"); + auto msg2_path = join_paths(tempdir2.path(), "Maildir/a/cur/msgabc:2,S"); + auto msg3_path = join_paths(tempdir2.path(),"Maildir/b/cur/msgdef:2,RS"); + + TempDir tempdir; + auto store{Store::make_new(tempdir.path(), + join_paths(tempdir2.path() , "Maildir"))}; + assert_valid_result(store); + + std::vector<Store::Id> ids; + for (auto&& p: {msg1_path, msg2_path, msg3_path}) { + std::ofstream output{p}; + output.write(msg_text.c_str(), msg_text.size()); + output.close(); + auto res = store->add_message(p); + assert_valid_result(res); + ids.emplace_back(*res); + } + g_assert_cmpuint(store->size(), ==, 3); + + // mark main message (+ dups) as seen + auto mres = store->move_message(ids.at(0), {}, + Flags::Seen | Flags::Flagged | Flags::Passed, + Store::MoveOptions::DupFlags); + assert_valid_result(mres); + mu_info("found {} matches", mres->size()); + for (auto&& m: *mres) + mu_info("id: {}: {}", m.first, m.second); + + // al three dups should have been updated + g_assert_cmpuint(mres->size(), ==, 3); + auto&& id_msgs{store->find_messages(Store::id_vec(*mres))}; + + // first should be the original + g_assert_cmpuint(id_msgs.at(0).first, ==, ids.at(0)); + { // Message 1 + const Message& msg = id_msgs.at(0).second; + assert_equal(msg.path(), tempdir2.path() + "/Maildir/a/cur/msg123:2,FPS"); + g_assert_true(msg.flags() == (Flags::Seen|Flags::Flagged|Flags::Passed)); + } + // note: Seen and Passed should be added to msg2/3, but Flagged shouldn't + // msg3 should loose its R flag. + + auto check_msg2 = [&](const Message& msg) { + assert_equal(msg.path(), join_paths(tempdir2.path(), "/Maildir/a/cur/msgabc:2,PS")); + }; + auto check_msg3 = [&](const Message& msg) { + assert_equal(msg.path(), join_paths(tempdir2.path(), "/Maildir/b/cur/msgdef:2,PS")); + }; + + if (id_msgs.at(1).first == ids.at(1)) { + check_msg2(id_msgs.at(1).second); + check_msg3(id_msgs.at(2).second); + } else { + check_msg2(id_msgs.at(2).second); + check_msg3(id_msgs.at(1).second); + } +} + +static void +test_store_circular_symlink(void) +{ + allow_warnings(); + + g_test_bug("2517"); + + auto testhome{unwrap(make_temp_dir())}; + auto dbpath{runtime_path(RuntimePath::XapianDb, testhome)}; + + /* create a writable copy */ + const auto testmdir = join_paths(testhome, "test-maildir"); + auto cres1 = run_command({CP_PROGRAM, "-r", MU_TESTMAILDIR, testmdir}); + assert_valid_command(cres1); + // create a symink + auto cres2 = run_command({LN_PROGRAM, "-s", testmdir, join_paths(testmdir, "testlink")}); + assert_valid_command(cres2); + + auto&& store = unwrap(Store::make_new(dbpath, testmdir)); + store.indexer().start({}); + size_t n{}; + while (store.indexer().is_running()) { + std::this_thread::sleep_for(100ms); + g_assert_cmpuint(n++,<=,25); + } + // there will be a lot of dups.... + g_assert_false(store.empty()); + + remove_directory(testhome); +} + +static void +test_store_maildirs() +{ + allow_warnings(); + + TempDir tdir; + auto store = Store::make_new(tdir.path(), MU_TESTMAILDIR2); + assert_valid_result(store); + g_assert_true(store->empty()); + + const auto mdirs = store->maildirs(); + + g_assert_cmpuint(mdirs.size(), ==, 3); + g_assert(seq_some(mdirs, [](auto&& m){return m == "/Foo";})); + g_assert(seq_some(mdirs, [](auto&& m){return m == "/bar";})); + g_assert(seq_some(mdirs, [](auto&& m){return m == "/wom_bat";})); +} + + +static void +test_store_parse() +{ + allow_warnings(); + + TempDir tdir; + auto store = Store::make_new(tdir.path(), MU_TESTMAILDIR2); + assert_valid_result(store); + g_assert_true(store->empty()); + + // Xapian internal format (get_description()) is _not_ guaranteed + // to be the same between versions + const auto&& pq1{store->parse_query("subject:\"hello world\"", false)}; + const auto&& pq2{store->parse_query("subject:\"hello world\"", true)}; + + assert_equal(pq1, "(or (subject \"hello world\") (subject (phrase \"hello world\")))"); + + /* LCOV_EXCL_START*/ + if (pq2 != "Query((Shello world OR (Shello PHRASE 2 Sworld)))") { + g_test_skip("incompatible xapian descriptions"); + return; + } + /* LCOV_EXCL_STOP*/ + + assert_equal(pq2, "Query((Shello world OR (Shello PHRASE 2 Sworld)))"); +} + +static void +test_store_fail() +{ + { + const auto store = Store::make("/root/non-existent-path/12345"); + g_assert_false(!!store); + } + + { + const auto store = Store::make_new("/../../root/non-existent-path/12345", + "/../../root/non-existent-path/54321"); + g_assert_false(!!store); + } +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/store/ctor-dtor", test_store_ctor_dtor); + g_test_add_func("/store/reinit", test_store_reinit); + g_test_add_func("/store/add-count-remove", test_store_add_count_remove); + g_test_add_func("/store/message/mailing-list", + test_message_mailing_list); + g_test_add_func("/store/message/attachments", + test_message_attachments); + g_test_add_func("/store/move-dups", test_store_move_dups); + + g_test_add_func("/store/maildirs", test_store_maildirs); + g_test_add_func("/store/parse", test_store_parse); + + g_test_add_func("/store/index/index-move", test_index_move); + g_test_add_func("/store/index/circular-symlink", test_store_circular_symlink); + + g_test_add_func("/store/index/fail", test_store_fail); + + return g_test_run(); +} diff --git a/lib/tests/test-query.cc b/lib/tests/test-query.cc new file mode 100644 index 0000000..fd9ff1d --- /dev/null +++ b/lib/tests/test-query.cc @@ -0,0 +1,99 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include <config.h> + +#include <vector> +#include <glib.h> + +#include <iostream> +#include <sstream> +#include <unistd.h> + +#include "mu-store.hh" +#include "mu-query.hh" +#include "utils/mu-result.hh" +#include "utils/mu-utils.hh" +#include "utils/mu-test-utils.hh" + +using namespace Mu; + +static void +test_query() +{ + allow_warnings(); + TempDir temp_dir; + + auto store = Store::make_new(temp_dir.path(), std::string{MU_TESTMAILDIR}); + assert_valid_result(store); + + auto&& idx{store->indexer()}; + g_assert_true(idx.start(Indexer::Config{})); + while (idx.is_running()) { + g_usleep(1000); + } + + auto dump_matches = [](const QueryResults& res) { + size_t n{}; + for (auto&& item : res) { + if (g_test_verbose()) { + std::cout << item.query_match() << '\n'; + mu_debug("{:02d} {} {}", + ++n, + item.path().value_or("<none>"), + item.message_id().value_or("<none>")); + } + } + }; + + g_assert_cmpuint(store->size(), ==, 19); + + { + const auto res = store->run_query("", {}, QueryFlags::None); + g_assert_true(!!res); + g_assert_cmpuint(res->size(), ==, 19); + dump_matches(*res); + + g_assert_cmpuint(store->count_query(""), ==, 19); + + } + + { + const auto res = store->run_query("", Field::Id::Path, QueryFlags::None, 11); + g_assert_true(!!res); + g_assert_cmpuint(res->size(), ==, 11); + dump_matches(*res); + } +} + +int +main(int argc, char* argv[]) try { + + mu_test_init(&argc, &argv); + + g_test_add_func("/query", test_query); + + return g_test_run(); + +} catch (const std::runtime_error& re) { + std::cerr << re.what() << "\n"; + return 1; +} catch (...) { + std::cerr << "caught exception\n"; + return 1; +} diff --git a/lib/utils/meson.build b/lib/utils/meson.build new file mode 100644 index 0000000..3263a94 --- /dev/null +++ b/lib/utils/meson.build @@ -0,0 +1,66 @@ +## Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +thirdparty=join_paths('..', '..', 'thirdparty') + +srcs = [ + 'mu-command-handler.cc', + 'mu-html-to-text.cc', + 'mu-lang-detector.cc', + 'mu-logger.cc', + 'mu-option.cc', + 'mu-readline.cc', + 'mu-sexp.cc', + 'mu-utils-file.cc', + 'mu-utils.cc', +] + +if not get_option('tests').disabled() + test_srcs = [ 'mu-test-utils.cc' ] +else + test_srcs = [] +endif + +lib_mu_utils=static_library('mu-utils', + [ srcs, test_srcs ], dependencies: [ + glib_dep, + gio_dep, + gio_unix_dep, + config_h_dep, + readline_dep, + cld2_dep +], include_directories: + include_directories(['.', '..', thirdparty]), +install: false) + +lib_mu_utils_dep = declare_dependency( + link_with: lib_mu_utils, + compile_args: '-DFMT_HEADER_ONLY', + include_directories: + include_directories(['.', '..', thirdparty])) + +# +# tools +# +html2text = executable('mu-html2text', + 'mu-html-to-text.cc', + dependencies: [ lib_mu_utils_dep, glib_dep ], + cpp_args: ['-DBUILD_HTML_TO_TEXT'], + install: false) + +if not get_option('tests').disabled() + subdir('tests') +endif diff --git a/lib/utils/mu-async-queue.hh b/lib/utils/mu-async-queue.hh new file mode 100644 index 0000000..afabef5 --- /dev/null +++ b/lib/utils/mu-async-queue.hh @@ -0,0 +1,184 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef __MU_ASYNC_QUEUE_HH__ +#define __MU_ASYNC_QUEUE_HH__ + +#include <deque> +#include <mutex> +#include <chrono> +#include <condition_variable> + +namespace Mu { + +constexpr std::size_t UnlimitedAsyncQueueSize{0}; + +template <typename ItemType, /**< the type of Item to queue */ + std::size_t MaxSize = UnlimitedAsyncQueueSize, /**< maximum size for the queue */ + typename Allocator = std::allocator<ItemType>> /**< allocator for the items */ + +class AsyncQueue { + public: + using value_type = ItemType; + using allocator_type = Allocator; + using size_type = std::size_t; + using reference = value_type&; + using const_reference = const value_type&; + using pointer = typename std::allocator_traits<allocator_type>::pointer; + using const_pointer = typename std::allocator_traits<allocator_type>::const_pointer; + + using Timeout = std::chrono::steady_clock::duration; + + /** + * Push an item to the end of the queue by moving it + * + * @param item the item to move to the end of the queue + * @param timeout and optional timeout + * + * @return true if the item was pushed; false otherwise. + */ + bool push(const value_type& item, Timeout timeout = {}) { + return push(std::move(value_type(item)), timeout); + } + + /** + * Push an item to the end of the queue by moving it + * + * @param item the item to move to the end of the queue + * @param timeout and optional timeout + * + * @return true if the item was pushed; false otherwise. + */ + bool push(value_type&& item, Timeout timeout = {}) { + std::unique_lock lock{m_}; + + if (!unlimited()) { + const auto rv = cv_full_.wait_for(lock, timeout, [&]() { + return !full_unlocked(); + }) && !full_unlocked(); + if (!rv) + return false; + } + + q_.emplace_back(std::move(item)); + cv_empty_.notify_one(); + + return true; + } + + /** + * Pop an item from the queue + * + * @param receives the value if the function returns true + * @param timeout optional time to wait for an item to become available + * + * @return true if an item was popped (into val), false otherwise. + */ + bool pop(value_type& val, Timeout timeout = {}) { + std::unique_lock lock{m_}; + + if (timeout != Timeout{}) { + const auto rv = cv_empty_.wait_for(lock, timeout, [&]() { + return !q_.empty(); + }) && !q_.empty(); + if (!rv) + return false; + + } else if (q_.empty()) + return false; + + val = std::move(q_.front()); + q_.pop_front(); + cv_full_.notify_one(); + + return true; + } + + /** + * Clear the queue + * + */ + void clear() { + std::unique_lock lock{m_}; + q_.clear(); + cv_full_.notify_one(); + } + + /** + * Size of the queue + * + * + * @return the size + */ + size_type size() const { + std::unique_lock lock{m_}; + return q_.size(); + } + + /** + * Maximum size of the queue if specified through the template + * parameter; otherwise the (theoretical) max_size of the inner + * container. + * + * @return the maximum size + */ + size_type max_size() const { return unlimited() ? q_.max_size() : MaxSize; } + + /** + * Is the queue empty? + * + * @return true or false + */ + bool empty() const { + std::unique_lock lock{m_}; + return q_.empty(); + } + + /** + * Is the queue full? Returns false unless a maximum size was specified + * (as a template argument) + * + * @return true or false. + */ + bool full() const { + if (unlimited()) + return false; + + std::unique_lock lock{m_}; + return full_unlocked(); + } + + /** + * Is this queue (theoretically) unlimited in size? + * + * @return true or false + */ + constexpr static bool unlimited() { return MaxSize == UnlimitedAsyncQueueSize; } + +private: + bool full_unlocked() const { return q_.size() >= max_size(); } + + std::deque<ItemType, Allocator> q_; + mutable std::mutex m_; + std::condition_variable cv_full_, cv_empty_; +}; + +} // namespace Mu + +#endif /* __MU_ASYNC_QUEUE_HH__ */ diff --git a/lib/utils/mu-command-handler.cc b/lib/utils/mu-command-handler.cc new file mode 100644 index 0000000..927df0b --- /dev/null +++ b/lib/utils/mu-command-handler.cc @@ -0,0 +1,273 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-command-handler.hh" +#include "mu-error.hh" +#include "mu-utils.hh" + +#include <iostream> +#include <algorithm> + +using namespace Mu; + +Option<std::vector<std::string>> +Command::string_vec_arg(const std::string& name) const +{ + auto&& val{arg_val(name, Sexp::Type::List)}; + if (!val) + return Nothing; + + std::vector<std::string> vec; + for (const auto& item : val->list()) { + if (!item.stringp()) { + // mu_warning("command: non-string in string-list for {}: {}", + // name, to_string()); + return Nothing; + } else + vec.emplace_back(item.string()); + } + + return vec; +} + +static Result<void> +validate(const CommandHandler::CommandInfoMap& cmap, + const CommandHandler::CommandInfo& cmd_info, + const Command& cmd) +{ + // all required parameters must be present + for (auto&& arg : cmd_info.args) { + + const auto& argname{arg.first}; + const auto& arginfo{arg.second}; + + // calls use keyword-parameters, e.g. + // + // (my-function :bar 1 :cuux "fnorb") + // + // so, we're looking for the odd-numbered parameters. + const auto param_it = cmd.find_arg(argname); + const auto&& param_val = std::next(param_it); + // it's an error when a required parameter is missing. + if (param_it == cmd.cend()) { + if (arginfo.required) + return Err(Error::Code::Command, + "missing required parameter {} in command '{}'", + argname, cmd.to_string()); + continue; // not required + } + + // the types must match, but the 'nil' symbol is acceptable as "no value" + if (param_val->type() != arginfo.type && !(param_val->nilp())) + return Err(Error::Code::Command, + "parameter {} expects type {}, but got {} in command '{}'", + argname, to_string(arginfo.type), + to_string(param_val->type()), cmd.to_string()); + } + + // all parameters must be known + for (auto it = cmd.cbegin() + 1; it != cmd.cend() && it + 1 != cmd.cend(); it += 2) { + const auto& cmdargname{it->symbol()}; + if (std::none_of(cmd_info.args.cbegin(), cmd_info.args.cend(), + [&](auto&& arg) { return cmdargname == arg.first; })) + return Err(Error::Code::Command, + "unknown parameter '{} 'in command '{}'", + cmdargname.name.c_str(), cmd.to_string().c_str()); + } + + return Ok(); + +} + +Result<void> +CommandHandler::invoke(const Command& cmd, bool do_validate) const +{ + const auto cmit{cmap_.find(cmd.name())}; + if (cmit == cmap_.cend()) + return Err(Error::Code::Command, + "unknown command '{}'", cmd.to_string().c_str()); + + const auto& cmd_info{cmit->second}; + if (do_validate) { + if (auto&& res = validate(cmap_, cmd_info, cmd); !res) + return Err(res.error()); + } + + if (cmd_info.handler) + cmd_info.handler(cmd); + + return Ok(); +} + + +// LCOV_EXCL_START +#ifdef BUILD_TESTS + +#include "mu-test-utils.hh" + + +static void +test_args() +{ + const auto cmd = Command::make_parse(R"((foo :bar 123 :cuux "456" :boo nil :bah true))"); + assert_valid_result(cmd); + + assert_equal(cmd->name(), "foo"); + g_assert_true(cmd->find_arg(":bar") != cmd->cend()); + g_assert_true(cmd->find_arg(":bxr") == cmd->cend()); + + g_assert_cmpint(cmd->number_arg(":bar").value_or(-1), ==, 123); + g_assert_cmpint(cmd->number_arg(":bor").value_or(-1), ==, -1); + + assert_equal(cmd->string_arg(":cuux").value_or(""), "456"); + assert_equal(cmd->string_arg(":caax").value_or(""), ""); // not present + assert_equal(cmd->string_arg(":bar").value_or("abc"), "abc"); // wrong type + + g_assert_false(cmd->boolean_arg(":boo")); + g_assert_true(cmd->boolean_arg(":bah")); +} + +using CommandInfoMap = CommandHandler::CommandInfoMap; +using ArgMap = CommandHandler::ArgMap; +using ArgInfo = CommandHandler::ArgInfo; +using CommandInfo = CommandHandler::CommandInfo; + +static Result<void> +call(const CommandInfoMap& cmap, const std::string& str) try { + + if (const auto cmd{Command::make_parse(str)}; !cmd) + return Err(Error::Code::Internal, "invalid s-expression '{}'", str); + else + return CommandHandler(cmap).invoke(*cmd); + +} catch (const Error& err) { + return Err(Error{err}); +} + +static void +test_command() +{ + allow_warnings(); + + CommandInfoMap ci_map; + ci_map.emplace( + "my-command", + CommandInfo{ArgMap{{":param1", ArgInfo{Sexp::Type::String, true, "some string"}}, + {":param2", ArgInfo{Sexp::Type::Number, false, "some integer"}}}, + "My command,", + {}}); + ci_map.emplace( + "another-command", + CommandInfo{ + ArgMap{ + {":queries", ArgInfo{Sexp::Type::List, false, + "queries for which to get read/unread numbers"}}, + {":symbol", ArgInfo{Sexp::Type::Symbol, true, + "some boring symbol"}}, + {":bool", ArgInfo{Sexp::Type::Symbol, true, + "some even more boring boolean symbol"}}, + {":symbol2", ArgInfo{Sexp::Type::Symbol, false, + "some even more boring symbol"}}, + {":bool2", ArgInfo{Sexp::Type::Symbol, false, + "some boring boolean symbol"}}, + }, + "get unread/totals information for a list of queries", + [&](const auto& params) { + const auto queries{params.string_vec_arg(":queries") + .value_or(std::vector<std::string>{})}; + g_assert_cmpuint(queries.size(),==,3); + g_assert_true(params.bool_arg(":bool").value_or(false) == true); + assert_equal(params.symbol_arg(":symbol").value_or("boo"), "sym"); + + g_assert_false(!!params.bool_arg(":bool2")); + g_assert_false(!!params.bool_arg(":symbol2")); + + }}); + + CommandHandler handler(std::move(ci_map)); + const auto cmap{handler.info_map()}; + + assert_valid_result(call(cmap, "(my-command :param1 \"hello\")")); + assert_valid_result(call(cmap, "(my-command :param1 \"hello\" :param2 123)")); + g_assert_false(!!call(cmap, "(my-command :param1 \"hello\" :param2 123 :param3 xxx)")); + assert_valid_result(call(cmap, "(another-command :queries (\"foo\" \"bar\" \"cuux\") " + ":symbol sym :bool true)")); +} + +static void +test_command2() +{ + allow_warnings(); + + CommandInfoMap cmap; + cmap.emplace("bla", + CommandInfo{ArgMap{ + {":foo", ArgInfo{Sexp::Type::Number, false, "foo"}}, + {":bar", ArgInfo{Sexp::Type::String, false, "bar"}}, + }, "yeah", + [&](const auto& params) {}}); + + g_assert_true(call(cmap, "(bla :foo nil)")); + g_assert_false(call(cmap, "(bla :foo nil :bla nil)")); +} + +static void +test_command_fail() +{ + allow_warnings(); + + CommandInfoMap cmap; + + cmap.emplace( + "my-command", + CommandInfo{ArgMap{{":param1", ArgInfo{Sexp::Type::String, true, "some string"}}, + {":param2", ArgInfo{Sexp::Type::Number, false, "some integer"}}}, + "My command,", + {}}); + + g_assert_false(call(cmap, "(my-command)")); + g_assert_false(call(cmap, "(my-command2)")); + g_assert_false(call(cmap, "(my-command :param1 123 :param2 123)")); + g_assert_false(call(cmap, "(my-command :param1 \"hello\" :param2 \"123\")")); + + g_assert_false(call(cmap, "(my-command")); + + g_assert_false(!!Command::make_parse(R"((foo :bar 123 :cuux "456" :boo nil :bah))")); +} + + +int +main(int argc, char* argv[]) try { + + mu_test_init(&argc, &argv); + + g_test_add_func("/utils/command-parser/args", test_args); + g_test_add_func("/utils/command-parser/command", test_command); + g_test_add_func("/utils/command-parser/command2", test_command2); + g_test_add_func("/utils/command-parser/command-fail", test_command_fail); + + return g_test_run(); + +} catch (const std::runtime_error& re) { + std::cerr << re.what() << "\n"; + return 1; +} + +#endif /*BUILD_TESTS*/ +// LCOV_EXCL_STOP diff --git a/lib/utils/mu-command-handler.hh b/lib/utils/mu-command-handler.hh new file mode 100644 index 0000000..755af53 --- /dev/null +++ b/lib/utils/mu-command-handler.hh @@ -0,0 +1,298 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#ifndef MU_COMMAND_HANDLER_HH__ +#define MU_COMMAND_HANDLER_HH__ + +#include <vector> +#include <string> +#include <ostream> +#include <stdexcept> +#include <unordered_map> +#include <functional> +#include <algorithm> + +#include "utils/mu-error.hh" +#include "utils/mu-sexp.hh" +#include "utils/mu-option.hh" + +namespace Mu { + +/// +/// Commands are s-expressions with the follow properties: + +/// 1) a command is a list with a command-name as its first argument +/// 2) the rest of the parameters are pairs of colon-prefixed symbol and a value of some +/// type (ie. 'keyword arguments') +/// 3) each command is described by its CommandInfo structure, which defines the type +/// 4) calls to the command must include all required parameters +/// 5) all parameters must be of the specified type; however the symbol 'nil' is allowed +/// for specify a non-required parameter to be absent; this is for convenience on the +/// call side. + +struct Command: public Sexp { + + static Result<Command> make(Sexp&& sexp) try { + return Ok(Command{std::move(sexp)}); + } catch (const Error& e) { + return Err(e); + } + + static Result<Command> make_parse(const std::string& cmdstr) try { + if (auto&& sexp{Sexp::parse(cmdstr)}; !sexp) + return Err(sexp.error()); + else + return Ok(Command(std::move(*sexp))); + } catch (const Error& e) { + return Err(e); + } + + /** + * Get name of the command (first element) in a command exp + * + * @return name + */ + const std::string& name() const { + return cbegin()->symbol().name; + } + + /** + * Find the argument with the given name. + * + * @param arg name + * + * @return iterator point at the argument, or cend + */ + const_iterator find_arg(const std::string& arg) const { + return find_prop(arg, cbegin() + 1, cend()); + } + + /** + * Get a string argument + * + * @param name of the argument + * + * @return ref to string, or Nothing if not found + */ + Option<const std::string&> string_arg(const std::string& name) const { + if (auto&& val{arg_val(name, Sexp::Type::String)}; !val) + return Nothing; + else + return val->string(); + } + + /** + * Get a string-vec argument + * + * @param name of the argument + * + * @return ref to string-vec, or Nothing if not found or some error. + */ + Option<std::vector<std::string>> string_vec_arg(const std::string& name) const; + + /** + * Get a symbol argument + * + * @param name of the argument + * + * @return ref to symbol name, or Nothing if not found + */ + Option<const std::string&> symbol_arg(const std::string& name) const { + if (auto&& val{arg_val(name, Sexp::Type::Symbol)}; !val) + return Nothing; + else + return val->symbol().name; + } + + /** + * Get a number argument + * + * @param name of the argument + * + * @return number or Nothing if not found + */ + Option<int> number_arg(const std::string& name) const { + if (auto&& val{arg_val(name, Sexp::Type::Number)}; !val) + return Nothing; + else + return static_cast<int>(val->number()); + } + + /* + * helpers + */ + + /** + * Get a boolean argument + * + * @param name of the argument + * + * @return true if there's a non-nil symbol value for the given + * name; false otherwise. + */ + Option<bool> bool_arg(const std::string& name) const { + if (auto&& symb{symbol_arg(name)}; !symb) + return Nothing; + else + return symb.value() == "nil" ? false : true; + } + + /** + * Treat any argument as a boolean + * + * @param name name of the argument + * + * @return false if the the argument is absent or the symbol false; + * otherwise true. + */ + bool boolean_arg(const std::string& name) const { + auto&& it{find_arg(name)}; + return (it == cend() || std::next(it)->nilp()) ? false : true; + } + +private: + explicit Command(Sexp&& s){ + *this = std::move(static_cast<Command&&>(s)); + if (!listp() || empty() || !cbegin()->symbolp() || + !plistp(cbegin() + 1, cend())) + throw Error(Error::Code::Command, + "expected command, got '{}'", to_string()); + } + + + Option<const Sexp&> arg_val(const std::string& name, Sexp::Type type) const { + if (auto&& it{find_arg(name)}; it == cend()) { + //std::cerr << "--> %s name found " << name << '\n'; + return Nothing; + } else if (auto&& val{it + 1}; val->type() != type) { + //std::cerr << "--> type " << Sexp::type_name(it->type()) << '\n'; + return Nothing; + } else + return *val; + } +}; + +struct CommandHandler { + + /// Information about a function argument + struct ArgInfo { + ArgInfo(Sexp::Type typearg, bool requiredarg, std::string&& docarg) + : type{typearg}, required{requiredarg}, docstring{std::move(docarg)} {} + const Sexp::Type type; /**< Sexp::Type of the argument */ + const bool required; /**< Is this argument required? */ + const std::string docstring; /**< Documentation */ + }; + + /// The arguments for a function, which maps their names to the information. + using ArgMap = std::unordered_map<std::string, ArgInfo>; + + // A handler function + using Handler = std::function<void(const Command&)>; + + /// Information about some command + struct CommandInfo { + CommandInfo(ArgMap&& argmaparg, std::string&& docarg, Handler&& handlerarg) + : args{std::move(argmaparg)}, docstring{std::move(docarg)}, + handler{std::move(handlerarg)} {} + const ArgMap args; + const std::string docstring; + const Handler handler; + + /** + * Get a sorted list of argument names, for display. Required args come + * first, then alphabetical. + * + * @return vec with the sorted names. + */ /* LCOV_EXCL_START */ + std::vector<std::string> sorted_argnames() const { + // sort args -- by required, then alphabetical. + std::vector<std::string> names; + for (auto&& arg : args) + names.emplace_back(arg.first); + std::sort(names.begin(), names.end(), [&](const auto& name1, const auto& name2) { + const auto& arg1{args.find(name1)->second}; + const auto& arg2{args.find(name2)->second}; + if (arg1.required != arg2.required) + return arg1.required; + else + return name1 < name2; + }); + return names; + } + /* LCOV_EXCL_STOP */ + + }; + + /// All commands, mapping their name to information about them. + using CommandInfoMap = std::unordered_map<std::string, CommandInfo>; + + CommandHandler(const CommandInfoMap& cmap): cmap_{cmap} {} + CommandHandler(CommandInfoMap&& cmap): cmap_{std::move(cmap)} {} + + const CommandInfoMap& info_map() const { return cmap_; } + + /** + * Invoke some command + * + * A command uses keyword arguments, e.g. something like: (foo :bar 1 + * :cuux "fnorb") + * + * @param cmd a Sexp describing a command call + * @param validate whether to validate before invoking. Useful during + * development. + * + * Return Ok() or some Error + */ + Result<void> invoke(const Command& cmd, bool validate=true) const; + +private: + const CommandInfoMap cmap_; +}; + +/* LCOV_EXCL_START */ +static inline std::ostream& +operator<<(std::ostream& os, const CommandHandler::ArgInfo& info) +{ + os << info.type << " (" << (info.required ? "required" : "optional") << ")"; + + return os; +} +/* LCOV_EXCL_STOP */ + +static inline std::ostream& +operator<<(std::ostream& os, const CommandHandler::CommandInfo& info) +{ + for (auto&& arg : info.args) + os << " " << arg.first << " " << arg.second << '\n' + << " " << arg.second.docstring << "\n"; + + return os; +} + +static inline std::ostream& +operator<<(std::ostream& os, const CommandHandler::CommandInfoMap& map) +{ + for (auto&& c : map) + os << c.first << '\n' << c.second; + + return os; +} + +} // namespace Mu + +#endif /* MU_COMMAND_HANDLER_HH__ */ diff --git a/lib/utils/mu-error.cc b/lib/utils/mu-error.cc new file mode 100644 index 0000000..1d098fc --- /dev/null +++ b/lib/utils/mu-error.cc @@ -0,0 +1,64 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + + +#if BUILD_TESTS + +#include "mu-error.hh" +#include "mu-test-utils.hh" + +using namespace Mu; + +static void +test_fill_error() +{ + const Error err{Error::Code::Internal, "boo!"}; + GError *gerr{}; + + err.fill_g_error(&gerr); + + assert_equal(gerr->message, "boo!"); + g_assert_cmpint(gerr->code, ==, static_cast<int>(err.code())); + + g_clear_error(&gerr); +} + +static void +test_add_hint() +{ + Error err(Error::Code::Internal, "baa!"); + err.add_hint("hello"); + + assert_equal(err.hint(), "hello"); +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/error/fill-error", test_fill_error); + g_test_add_func("/error/add-hint", test_add_hint); + + return g_test_run(); + +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/utils/mu-error.hh b/lib/utils/mu-error.hh new file mode 100644 index 0000000..36b4178 --- /dev/null +++ b/lib/utils/mu-error.hh @@ -0,0 +1,200 @@ +/* +** Copyright (C) 2019-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_ERROR_HH__ +#define MU_ERROR_HH__ + +#include <stdexcept> +#include <string> +#include <errno.h> +#include <cstdint> + +#include "mu-utils.hh" +#include <glib.h> + +#ifndef FMT_HEADER_ONLY +#define FMT_HEADER_ONLY +#endif + +#include <fmt/format.h> +#include <fmt/core.h> + +namespace Mu { + +// calculate an error enum value. +constexpr uint32_t err_enum(uint8_t code, uint8_t rv, uint8_t cat) { + return static_cast<uint32_t>(code|(rv << 16)|cat<<24); +} + +struct Error final : public std::exception { + + // 16 lower bits are for the error code;the next 8 bits are for the return code; the upper + // byte is for flags + static constexpr uint8_t SoftError = 1; + + enum struct Code: uint32_t { + Ok = err_enum(0,0,0), + + // used by mu4e. + NoMatches = err_enum(4,2,SoftError), + SchemaMismatch = err_enum(110,11,0), + + // other + AccessDenied = err_enum(100,1,0), + AssertionFailure = err_enum(101,1,0), + Command = err_enum(102,1,0), + Crypto = err_enum(103,1,0), + File = err_enum(104,1,0), + Index = err_enum(105,1,0), + Internal = err_enum(106,1,0), + InvalidArgument = err_enum(107,1,0), + Message = err_enum(108,1,0), + NotFound = err_enum(109,1,0), + Parsing = err_enum(111,1,0), + Play = err_enum(112,1,0), + Query = err_enum(113,1,0), + Script = err_enum(115,1,0), + ScriptNotFound = err_enum(116,1,0), + Store = err_enum(117,1,0), + StoreLock = err_enum(118,19,0), + UnverifiedSignature = err_enum(119,1,0), + User = err_enum(120,1,0), + Xapian = err_enum(121,1,0), + + CannotReinit = err_enum(122,1,0), + }; + + /** + * Construct an error + * + * @param code the error-code + * @param args... libfmt-style format string and parameters + */ + template<typename...T> + Error(Code code, fmt::format_string<T...> frm, T&&... args): + code_{code}, + what_{fmt::format(frm, std::forward<T>(args)...)} {} + + /** + * Construct an error + * + * @param code the error-code + * @param gerr a GError (or {}); the error is _consumed_ by this function + * @param args... libfmt-style format string and parameters + */ + template<typename...T> + Error(Code code, GError **gerr, fmt::format_string<T...> frm, T&&... args): + code_{code}, + what_{fmt::format(frm, std::forward<T>(args)...) + + fmt::format(": {}", (gerr && *gerr) ? (*gerr)->message : + "something went wrong")} + { g_clear_error(gerr); } + + /** + * Get the descriptive message for this error. + * + * @return + */ + virtual const char* what() const noexcept override { return what_.c_str(); } + + /** + * Get the error-code for this error + * + * @return the error-code + */ + Code code() const noexcept { return code_; } + + /** + * Get the error number (e.g. for reporting to mu4e) for some error. + * + * @param c error code + * + * @return the error number + */ + static constexpr uint32_t error_number(Code c) noexcept { + return static_cast<uint32_t>(c) & 0xffff; + } + + /** + * Is this is a 'soft error'? + * + * @return true or false + */ + constexpr bool is_soft_error() const { + return !!((static_cast<uint32_t>(code_)>>24) & SoftError); + } + + constexpr uint8_t exit_code() const { + return ((static_cast<uint32_t>(code_) >> 16) & 0xff); + } + + /** + * Fill a GError with the error information + * + * @param err GError** (or NULL) + */ + void fill_g_error(GError **err) const noexcept{ + g_set_error(err, error_quark(), static_cast<int>(code_), + "%s", what_.c_str()); + } + + /** + * Add an end-user hint + * + * @param args... libfmt-style format string and parameters + * + * @return the error + */ + template<typename...T> + Error& add_hint(fmt::format_string<T...> frm, T&&... args) { + hint_ = fmt::format(frm, std::forward<T>(args)...); + return *this; + } + + /** + * Get the hint + * + * @return the hint, empty for no hint. + */ + const std::string& hint() const { return hint_; } + +private: + static inline GQuark error_quark (void) { + static GQuark error_domain = 0; + if (G_UNLIKELY(error_domain == 0)) + error_domain = g_quark_from_static_string("mu-error-quark"); + return error_domain; + } + + const Code code_; + const std::string what_; + std::string hint_; +}; + +static inline auto +format_as(const Error& err) { + return mu_format("<{} ({}:{})>", + err.what(), + Error::error_number(err.code()), + err.exit_code()); +} + +} // namespace Mu + +#endif /* MU_ERROR_HH__ */ diff --git a/lib/utils/mu-html-to-text.cc b/lib/utils/mu-html-to-text.cc new file mode 100644 index 0000000..08f1f4d --- /dev/null +++ b/lib/utils/mu-html-to-text.cc @@ -0,0 +1,598 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-utils.hh" +#include "mu-option.hh" +#include "mu-regex.hh" + +#include <string> +#include <array> +#include <string_view> +#include <algorithm> + +using namespace Mu; + + +static bool +starts_with(std::string_view haystack, std::string_view needle) +{ + if (needle.size() > haystack.size()) + return false; + + for (auto&& c = 0U; c != needle.size(); ++c) + if (::tolower(haystack[c]) != ::tolower(needle[c])) + return false; + + return true; +} + +static bool +matches(std::string_view haystack, std::string_view needle) +{ + if (needle.size() != haystack.size()) + return false; + else + return starts_with(haystack, needle); +} + + + +/** + * HTML parsing context + * + */ +class Context { +public: + /** + * Construct a parsing context + * + * @param html some html to parse + */ + Context(const std::string& html): html_{html}, pos_{} {} + + /** + * Are we done with the html blob, i.e, has it been fully scraped? + * + * @return true or false + */ + bool done() const { + return pos_ >= html_.size(); + } + + /** + * Get the current position + * + * @return position + */ + size_t position() const { + return pos_; + } + + /** + * Get the size of the HTML + * + * @return size + */ + size_t size() const { + return html_.size(); + } + + /** + * Advance the position by _n_ characters. + * + * @param n number by which to advance. + */ + void advance(size_t n=1) { + if (pos_ + n > html_.size()) + throw std::range_error("out of range"); + pos_ += n; + } + + /** + * Are we looking at the given string? + * + * @param str string to match (case-insensitive) + * + * @return true or false + */ + bool looking_at(std::string_view str) const { + if (pos_ >= html_.size() || pos_ + str.size() >= html_.size()) + return false; + else + return matches({html_.data()+pos_, str.size()}, str); + } + + /** + * Grab a substring-view from the html + * + * @param fpos starting position + * @param len length + * + * @return string view + */ + std::string_view substr(size_t fpos, size_t len) const { + if (fpos + len > html_.size()) + throw std::range_error(mu_format("{} + {} > {}", + fpos, len, html_.size())); + else + return { html_.data() + fpos, len }; + } + + /** + * Grab the string of alphabetic characters at the + * head (pos) of the context, and advance over it. + * + * @return the head-word or empty + */ + std::string_view eat_head_word() { + size_t start_pos{pos_}; + while (!done()) { + if (!::isalpha(html_.at(pos_))) + break; + ++pos_; + } + return {html_.data() + start_pos, pos_ - start_pos}; + } + + + /** + * Get the scraped data; only available when done() + + * @return scraped data + */ + std::string scraped() { + return cleanup(raw_scraped_); + } + + /** + * Get the raw scrape buffer, where we can append + * scraped data. + * + * @return the buffer + */ + std::string& raw_scraped() { + return raw_scraped_; + } + + + /** + * Get a reference to the HTML + * + * @return html + */ + const std::string& html() const { return html_; } + +private: + + /** + * Cleanup some raw scraped html: remove superfluous + * whitespace, avoid too long lines. + * + * @param unclean + * + * @return cleaned up string. + */ + std::string cleanup(const std::string unclean) const { + // reduce whitespace and avoid too long lines; + // makes it easier to debug. + bool was_wspace{}; + size_t col{}; + std::string clean; + clean.reserve(unclean.size()/2); + for(auto&& c: unclean) { + auto wspace = c == ' ' || c == '\t' || c == '\n'; + if (wspace) { + was_wspace = true; + continue; + } + ++col; + if (was_wspace) { + if (col > 80) { + clean += '\n'; + col = 0; + } else if (!clean.empty()) + clean += ' '; + was_wspace = false; + } + clean += c; + } + return clean; + } + + + const std::string& html_; // no copy! + size_t pos_{}; + std::string raw_scraped_; +}; + + +G_GNUC_UNUSED static auto +format_as(const Context& ctx) +{ + return mu_format("<{}:{}: '{}'>", + ctx.position(), ctx.size(), + ctx.substr(ctx.position(), + std::min(static_cast<size_t>(8), + ctx.size() - ctx.position()))); +} + + +static void +skip_quoted(Context& ctx, std::string_view quote) +{ + while(!ctx.done()) { + if (ctx.looking_at(quote)) // closing quote + return; + ctx.advance(); + } +} + + +// attempt to skip over <script> / <style> blocks +static void +skip_script_style(Context& ctx, std::string_view tag) +{ + // <script> or <style> must be ignored + + bool escaped{}; + bool quoted{}, squoted{}; + bool inl_comment{}; + bool endl_comment{}; + + auto end_tag_str = mu_format("</{}>", tag); + auto end_tag = std::string_view(end_tag_str.data()); + + while (!ctx.done()) { + + if (inl_comment) { + if (ctx.looking_at("*/")) { + inl_comment = false; + ctx.advance(2); + } else + ctx.advance(); + continue; + } + + if (endl_comment) { + endl_comment = ctx.looking_at("\n"); + ctx.advance(); + continue; + } + + if (ctx.looking_at("\\")) { + escaped = !escaped; + ctx.advance(); + continue; + } + + if (ctx.looking_at("\"") && !escaped && squoted) { + quoted = !quoted; + ctx.advance(); + continue; + } + + if (ctx.looking_at("'") && !escaped && !quoted) { + squoted = !squoted; + ctx.advance(); + continue; + } + + + if (ctx.looking_at("/*")) { + inl_comment = true; + ctx.advance(2); + continue; + } + + if (ctx.looking_at("//")) { + endl_comment = true; + ctx.advance(2); + continue; + } + + if (!quoted && !squoted && ctx.looking_at(end_tag)) { + ctx.advance(end_tag.size()); + break; /* we're done, finally! */ + } + + ctx.advance(); + } +} + +// comment block; ignore completely +// pos will be immediately after the '<!-- +static void +comment(Context& ctx) +{ + constexpr std::string_view comment_endtag{"-->"}; + while (!ctx.done()) { + + if (ctx.looking_at(comment_endtag)) { + ctx.advance(comment_endtag.size()); + ctx.raw_scraped() += ' '; + return; + } + ctx.advance(); + } +} + +static bool // do we need a SPC separator for this tag? +needs_separator(std::string_view tagname) +{ + constexpr std::array<const char*, 7> nosep_tags = { + "b", "em", "i", "s", "strike", "tt", "u" + }; + return !seq_some(nosep_tags, [&](auto&& t){return matches(tagname, t);}); +} + +static bool // do we need to skip the element completely? +is_skip_element(std::string_view tagname) +{ + constexpr std::array<const char*, 4> skip_tags = { + "script", "style", "head", "meta" + }; + return seq_some(skip_tags, [&](auto&& t){return matches(tagname, t);}); +} + +// skip the end-tag +static void +end_tag(Context& ctx) +{ + while (!ctx.done()) { + if (ctx.looking_at(">")) { + ctx.advance(); + return; + } + ctx.advance(); + } +} + +// skip the whole element +static void +skip_element(Context& ctx, std::string_view tagname) +{ + // do something special? +} + + +// the start of a tag, i.e., pos will be just after the '<' +static void +tag(Context& ctx) +{ + // some elements we want to skip completely, + // for others just the tags. + constexpr std::string_view comment_start {"!--"}; + if (ctx.looking_at(comment_start)) { + ctx.advance(comment_start.size()); + comment(ctx); + return; + } + + if (ctx.looking_at("/")) { + ctx.advance(); + end_tag(ctx); + return; + } + + auto tagname = ctx.eat_head_word(); + if (tagname == "script" ||tagname == "style") { + skip_script_style(ctx, tagname); + return; + } + else if (is_skip_element(tagname)) + skip_element(ctx, tagname); + + const auto needs_sepa = needs_separator(tagname); + while (!ctx.done()) { + + if (ctx.looking_at("\"")) + skip_quoted(ctx, "\""); + + if (ctx.looking_at("'")) + skip_quoted(ctx, "'"); + + if (ctx.looking_at(">")) { + ctx.advance(); + if (needs_sepa) + ctx.raw_scraped() += ' '; + return; + } + ctx.advance(); + } +} + + +static void +html_escape_char(Context& ctx) +{ + // we only care about a few accented chars, and add them unaccented, lowercase, since that's + // we do for indexing anyway. + constexpr std::array<const char*, 11> escs = { + "breve", + "caron", + "circ", + "cute", + "grave", + "horn"/*thorn*/, + "macr", + "slash", + "strok", + "tilde", + "uml", + }; + + auto unescape=[escs](std::string_view esc)->char { + if (esc.empty()) + return ' '; + auto first{static_cast<char>(::tolower(esc.at(0)))}; + auto rest=esc.substr(1); + if (seq_some(escs, [&](auto&& e){return starts_with(rest, e);})) + return first; + else + return ' '; + }; + + size_t start_pos{ctx.position()}; + while (!ctx.done()) { + if (ctx.looking_at(";")) { + auto esc = ctx.substr(start_pos, ctx.position() - start_pos); + ctx.raw_scraped() += unescape(esc); + ctx.advance(); + return; + } + ctx.advance(); + } +} + + +// a block of text to be scraped +static void +text(Context& ctx) +{ + size_t start_pos{ctx.position()}; + while (!ctx.done()) { + + if (ctx.looking_at("&")) { + + ctx.raw_scraped() += ctx.substr(start_pos, + ctx.position() - start_pos); + ctx.advance(); + html_escape_char(ctx); + start_pos = ctx.position(); + + } else if (ctx.looking_at("<")) { + ctx.raw_scraped() += ctx.substr(start_pos, + ctx.position() - start_pos); + ctx.advance(); + tag(ctx); + start_pos = ctx.position(); + + } else + ctx.advance(); + } + + ctx.raw_scraped() += ctx.substr(start_pos, ctx.size() - start_pos); +} + +static Context *CTX{}; + +std::string +Mu::html_to_text(const std::string& html) +{ + Context ctx{html}; + CTX = &ctx; + + text(ctx); + + CTX = {}; + return ctx.scraped(); +} + +#ifdef BUILD_TESTS +#include "mu-test-utils.hh" + +static void +test_1() +{ + static std::vector<std::pair<std::string, std::string>> + tests = { + { "<!-- Hello -->A", "A" }, + { "A<!-- Test -->B", "A B" }, + { "A<i>a</i><b>p</b>", "Aap"}, + { "N&ocute;Ôt", "Noot"}, + { + "foo<!-- bar --><i>c</i>uu<bla>x</bla>" + "<!--hello -->world<!--", + "foo cuu x world" + } + }; + + for (auto&& test: tests) + assert_equal(html_to_text(test.first), test.second); +} + +static void +test_2() +{ + static std::vector<std::pair<std::string, std::string>> + tests = { + { R"(<i>hello, <b bar="/b">world!</b>)", + "hello, world!"}, + }; + + for (auto&& test: tests) + assert_equal(html_to_text(test.first), test.second); +} + + +static void +test_3() +{ + static std::vector<std::pair<std::string, std::string>> + tests = { + {R"(<i>hello, </i><script language="javascript"> + function foo() { + alert("Stroopwafel!"); // test + } + </script>world!)", + "hello, world!"}, + }; + + for (auto&& test: tests) + assert_equal(html_to_text(test.first), test.second); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/html-to-text/test-1", test_1); + g_test_add_func("/html-to-text/test-2", test_2); + g_test_add_func("/html-to-text/test-3", test_3); + + return g_test_run(); +} + + +#endif /*BUILD_TESTS*/ + + +#ifdef BUILD_HTML_TO_TEXT + +#include "mu-utils-file.hh" + +// simple tool that reads html on stdin and outputs text on stdout +// e.g. curl --silent https://www.example.com | build/lib/utils/mu-html2text + +int +main (int argc, char *argv[]) +{ + auto res = read_from_stdin(); + if (!res) { + mu_printerrln("error reading from stdin: {}", res.error().what()); + return 1; + } + + mu_println("{}", html_to_text(*res)); + + return 0; +} + +#endif /*BUILD_HTML_TO_TEXT*/ diff --git a/lib/utils/mu-lang-detector.cc b/lib/utils/mu-lang-detector.cc new file mode 100644 index 0000000..75af37e --- /dev/null +++ b/lib/utils/mu-lang-detector.cc @@ -0,0 +1,100 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include "config.h" +#include "mu-lang-detector.hh" + +using namespace Mu; + +#ifndef HAVE_CLD2 +// Dummy implementation +Option<Language> Mu::detect_language(const std::string& txt) { return Nothing; } +#else +#include <cld2/public/compact_lang_det.h> +#include <cld2/public/encodings.h> + +Option<Language> +Mu::detect_language(const std::string& txt) +{ + bool is_reliable; + const auto lang = CLD2::DetectLanguage( + txt.c_str(), txt.length(), + true/*plain-text*/, + &is_reliable); + + if (lang == CLD2::UNKNOWN_LANGUAGE || !is_reliable) + return {}; + + Mu::Language res = { + CLD2::LanguageName(lang), + CLD2::LanguageCode(lang) + }; + if (!res.name || !res.code) + return {}; + else + return Some(std::move(res)); +} +#endif /*HAVE_CLD2*/ + +#ifdef BUILD_TESTS +#include <vector> +#include "mu-test-utils.hh" + +static void +test_lang_detector() +{ + using Case = std::tuple<std::string,std::string, std::string>; + using Cases = std::vector<Case>; + + const Cases tests = {{ + { "hello world, this is a bit of English", + "ENGLISH", "en" }, + { "En nu een paar Nederlandse woorden", + "DUTCH", "nl" }, + { "Hyvää huomenta! Puhun vähän suomea", + "FINNISH", "fi" }, + { "So eine Arbeit wird eigentlich nie fertig, man muß sie für " + "fertig erklären, wenn man nach Zeit und Umständen das " + "möglichste getan hat.", + "GERMAN", "de"} + }}; + + for (auto&& test: tests) { + const auto res = detect_language(std::get<0>(test)); +#ifndef HAVE_CLD2 + g_assert_false(!!res); +#else + g_assert_true(!!res); + assert_equal(std::get<1>(test), res->name); + assert_equal(std::get<2>(test), res->code); +#endif + + } +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/utils/lang-detector", test_lang_detector); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/utils/mu-lang-detector.hh b/lib/utils/mu-lang-detector.hh new file mode 100644 index 0000000..0b692bc --- /dev/null +++ b/lib/utils/mu-lang-detector.hh @@ -0,0 +1,46 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_LANG_DETECTOR_HH__ +#define MU_LANG_DETECTOR_HH__ + +#include <string> +#include "mu-option.hh" + +namespace Mu { + +struct Language { + const char *name; /**< Language name, e.g. "Dutch" */ + const char *code; /**< Language code, e.g. "nl" */ +}; + +/** + * Detect the language of text + * + * @param txt some text (UTF-8) + * + * @return either a Language or nothing; the latter + * also if we cannot not reliably determine a single language + */ +Option<Language> detect_language(const std::string& txt); + +} // namespace Mu + + +#endif /* MU_LANG_DETECTOR_HH__ */ diff --git a/lib/utils/mu-logger.cc b/lib/utils/mu-logger.cc new file mode 100644 index 0000000..c9f516d --- /dev/null +++ b/lib/utils/mu-logger.cc @@ -0,0 +1,239 @@ +/* +** Copyright (C) 2020-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#define G_LOG_USE_STRUCTURED +#include <glib.h> +#include <glib/gstdio.h> +#include <fcntl.h> +#include <sys/types.h> +#include <sys/stat.h> + +#include <iostream> +#include <fstream> +#include <cstring> + +#include <thread> +#include <mutex> + +#include "mu-logger.hh" + +using namespace Mu; + +static bool MuLogInitialized = false; +static Mu::Logger::Options MuLogOptions; +static std::ofstream MuStream; +static auto MaxLogFileSize = 1000 * 1024; +static std::mutex logger_mtx; + +static std::string MuLogPath; + +static bool +maybe_open_logfile() +{ + if (MuStream.is_open()) + return true; + + const auto logdir{to_string_gchar(g_path_get_dirname(MuLogPath.c_str()))}; + if (g_mkdir_with_parents(logdir.c_str(), 0700) != 0) { + mu_printerrln("creating {} failed: {}", logdir, g_strerror(errno)); + return false; + } + + MuStream.open(MuLogPath, std::ios::out | std::ios::app); + if (!MuStream.is_open()) { + mu_printerrln("opening {} failed: {}", MuLogPath, g_strerror(errno)); + return false; + } + + MuStream.sync_with_stdio(false); + return true; +} + +static bool +maybe_rotate_logfile() +{ + static unsigned n = 0; + + if (n++ % 1000 != 0) + return true; + + GStatBuf statbuf; + if (g_stat(MuLogPath.c_str(), &statbuf) == -1 || statbuf.st_size <= MaxLogFileSize) + return true; + + const auto old = MuLogPath + ".old"; + g_unlink(old.c_str()); // opportunistic + + if (MuStream.is_open()) + MuStream.close(); + + if (g_rename(MuLogPath.c_str(), old.c_str()) != 0) + mu_printerrln("failed to rename {} -> {}: {}", MuLogPath, old, g_strerror(errno)); + + return maybe_open_logfile(); +} + +static GLogWriterOutput +log_file(GLogLevelFlags level, const GLogField* fields, gsize n_fields, gpointer user_data) +{ + std::lock_guard lock{logger_mtx}; + + if (!maybe_open_logfile()) + return G_LOG_WRITER_UNHANDLED; + + char timebuf[22]; + time_t now{::time(NULL)}; + ::strftime(timebuf, sizeof(timebuf), "%F %T", ::localtime(&now)); + + char* msg = g_log_writer_format_fields(level, fields, n_fields, FALSE); + if (msg && msg[0] == '\n') // hmm... seems lines start with '\n'r + msg[0] = ' '; + + MuStream << timebuf << ' ' << msg << std::endl; + + g_free(msg); + + return maybe_rotate_logfile() ? G_LOG_WRITER_HANDLED : G_LOG_WRITER_UNHANDLED; +} + +static GLogWriterOutput +log_stdouterr(GLogLevelFlags level, const GLogField* fields, gsize n_fields, gpointer user_data) +{ + return g_log_writer_standard_streams(level, fields, n_fields, user_data); +} + +static GLogWriterOutput +log_journal(GLogLevelFlags level, const GLogField* fields, gsize n_fields, gpointer user_data) +{ + return g_log_writer_journald(level, fields, n_fields, user_data); +} + + +Result<Logger> +Mu::Logger::make(const std::string& path, Mu::Logger::Options opts) +{ + if (MuLogInitialized) + return Err(Error::Code::Internal, "logging already initialized"); + + return Ok(Logger(path, opts)); +} + +Mu::Logger::Logger(const std::string& path, Mu::Logger::Options opts) +{ + if (g_getenv("MU_LOG_STDOUTERR")) + opts |= Logger::Options::StdOutErr; + + MuLogOptions = opts; + MuLogPath = path; + + g_log_set_writer_func( + [](GLogLevelFlags level, const GLogField* fields, gsize n_fields, gpointer user_data) { + // filter out debug-level messages? + if (level == G_LOG_LEVEL_DEBUG && + (none_of(MuLogOptions & Options::Debug))) + return G_LOG_WRITER_HANDLED; + + // log criticals to stdout / err or if asked + if (level == G_LOG_LEVEL_CRITICAL || + any_of(MuLogOptions & Options::StdOutErr)) { + log_stdouterr(level, fields, n_fields, user_data); + } + + // log to the journal, or, if not available to a file. + if (any_of(MuLogOptions & Options::File) || + log_journal(level, fields, n_fields, user_data) != G_LOG_WRITER_HANDLED) + return log_file(level, fields, n_fields, user_data); + else + return G_LOG_WRITER_HANDLED; + }, + NULL, + NULL); + + g_message("logging initialized; debug: %s, stdout/stderr: %s", + any_of(opts & Options::Debug) ? "yes" : "no", + any_of(opts & Options::StdOutErr) ? "yes" : "no"); + + MuLogInitialized = true; +} + +Logger::~Logger() +{ + if (!MuLogInitialized) + return; + + if (MuStream.is_open()) + MuStream.close(); + + MuLogInitialized = false; +} + + +#ifdef BUILD_TESTS +#include <vector> +#include <atomic> + +#include "mu-test-utils.hh" +#include "mu-utils-file.hh" + +static void +test_logger_threads(void) +{ + TempDir temp_dir; + const auto testpath{join_paths(temp_dir.path(), "test.log")}; + mu_message("log-file: {}", testpath); + + auto logger = Logger::make(testpath, Logger::Options::File | Logger::Options::Debug); + assert_valid_result(logger); + + const auto thread_num = 16; + std::atomic<bool> running = true; + + std::vector<std::thread> threads; + + /* log to the logger file from many threass */ + for (auto n = 0; n != thread_num; ++n) + threads.emplace_back( + std::thread([&running]{ + while (running) { + //mu_debug("log message from thread <{}>", n); + std::this_thread::yield(); + } + })); + + using namespace std::chrono_literals; + std::this_thread::sleep_for(1s); + running = false; + + for (auto n = 0; n != 16; ++n) + if (threads[n].joinable()) + threads[n].join(); +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/utils/logger", test_logger_threads); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/utils/mu-logger.hh b/lib/utils/mu-logger.hh new file mode 100644 index 0000000..6024e28 --- /dev/null +++ b/lib/utils/mu-logger.hh @@ -0,0 +1,74 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_LOGGER_HH__ +#define MU_LOGGER_HH__ + +#include <string> +#include <utils/mu-utils.hh> +#include <utils/mu-result.hh> + +namespace Mu { + +/** + * RAII object for handling logging (through g_(debug|warning|...)) + * + */ +struct Logger { + + /** + * Logging options + * + */ + enum struct Options { + None = 0, /**< Nothing specific */ + StdOutErr = 1 << 1, /**< Log to stdout/stderr */ + File = 1 << 2, /**< Force logging to file, even if journal available */ + Debug = 1 << 3, /**< Include debug-level logs */ + }; + + /** + * Initialize the logging sub-system. + * + * Note that the path is only used if structured logging fails -- + * practically, it goes to the file if there's no systemd/journald. + * + * if the environment variable MU_LOG_STDOUTERR is set, + * LogOptions::StdoutErr is implied. + * + * @param path path to the log file + * @param opts logging options + */ + static Result<Logger> make(const std::string& path, Options opts=Options::None); + + /** + * DTOR + * + */ + ~Logger(); + +private: + Logger(const std::string& path, Options opts); +}; + +MU_ENABLE_BITOPS(Logger::Options); + +} // namespace Mu + +#endif /* MU_LOGGER_HH__ */ diff --git a/lib/utils/mu-option.cc b/lib/utils/mu-option.cc new file mode 100644 index 0000000..e096117 --- /dev/null +++ b/lib/utils/mu-option.cc @@ -0,0 +1,106 @@ +/* +** Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-option.hh" +#include <glib.h> + +using namespace Mu; + +Mu::Option<std::string> +Mu::to_string_opt_gchar(gchar*&& str) +{ + auto res = to_string_opt(str); + g_free(str); + + return res; +} + +#if BUILD_TESTS +#include "mu-test-utils.hh" + +static Option<int> +get_opt_int(bool b) +{ + if (b) + return Some(123); + else + return Nothing; +} + +static void +test_option() +{ + { + const auto oi{get_opt_int(true)}; + g_assert_true(!!oi); + g_assert_cmpint(oi.value(), ==, 123); + } + + { + const auto oi{get_opt_int(false)}; + g_assert_false(!!oi); + g_assert_false(oi.has_value()); + g_assert_cmpint(oi.value_or(456), ==, 456); + } +} + +static void +test_unwrap() +{ + { + auto&& oi{get_opt_int(true)}; + g_assert_cmpint(unwrap(std::move(oi)), ==, 123); + } + + auto ex{0}; + try { + auto&& oi{get_opt_int(false)}; + unwrap(std::move(oi)); + } catch(...) { + ex = 1; + } + + g_assert_cmpuint(ex, ==, 1); +} + +static void +test_opt_gchar() +{ + auto o1{to_string_opt_gchar(g_strdup("boo!"))}; + auto o2{to_string_opt_gchar(nullptr)}; + + g_assert_false(!!o2); + g_assert_true(o1.value() == "boo!"); +} + + + +int +main(int argc, char* argv[]) +{ + g_test_init(&argc, &argv, NULL); + + g_test_add_func("/option/option", test_option); + g_test_add_func("/option/unwrap", test_unwrap); + g_test_add_func("/option/opt-gchar", test_opt_gchar); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/utils/mu-option.hh b/lib/utils/mu-option.hh new file mode 100644 index 0000000..32b1bee --- /dev/null +++ b/lib/utils/mu-option.hh @@ -0,0 +1,77 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_OPTION__ +#define MU_OPTION__ + +#include <tl/optional.hpp> +#include <stdexcept> +#include <string> + +namespace Mu { + +/// Either a value of type T, or None +template <typename T> using Option = tl::optional<T>; + +template <typename T> +Option<T> +Some(T&& t) +{ + return std::move(t); +} +constexpr auto Nothing = tl::nullopt; // 'None' is already taken. + +template<typename T> T +unwrap(Option<T>&& res) +{ + if (!!res) + return std::move(res.value()); + else + throw std::runtime_error("failure is not an option"); +} + + +/** + * Maybe create a string from a const char pointer. + * + * @param str a char pointer or NULL + * + * @return option with either the string or nothing if str was NULL. + */ +Option<std::string> +static inline to_string_opt(const char* str) { + if (str) + return std::string{str}; + else + return Nothing; +} + +/** + * Like maybe_string that takes a const char*, but additionally, + * g_free() the string. + * + * @param str char pointer or NULL (consumed) + * + * @return option with either the string or nothing if str was NULL. + */ +Option<std::string> to_string_opt_gchar(char*&& str); + + +} // namespace Mu +#endif /*MU_OPTION__*/ diff --git a/lib/utils/mu-readline.cc b/lib/utils/mu-readline.cc new file mode 100644 index 0000000..edf6a52 --- /dev/null +++ b/lib/utils/mu-readline.cc @@ -0,0 +1,136 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include "config.h" + +#include "mu-utils.hh" +#include "mu-readline.hh" + +#include <string> +#include <unistd.h> + +#ifdef HAVE_LIBREADLINE +#if defined(HAVE_READLINE_READLINE_H) +#include <readline/readline.h> +#elif defined(HAVE_READLINE_H) +#include <readline.h> +#else /* !defined(HAVE_READLINE_H) */ +extern char* readline(); +#endif /* !defined(HAVE_READLINE_H) */ +char* cmdline = NULL; +#else /* !defined(HAVE_READLINE_READLINE_H) */ +/* no readline */ +#endif /* HAVE_LIBREADLINE */ + +#ifdef HAVE_READLINE_HISTORY +#if defined(HAVE_READLINE_HISTORY_H) +#include <readline/history.h> +#elif defined(HAVE_HISTORY_H) +#include <history.h> +#else /* !defined(HAVE_HISTORY_H) */ +extern void add_history(); +extern int write_history(); +extern int read_history(); +#endif /* defined(HAVE_READLINE_HISTORY_H) */ +/* no history */ +#endif /* HAVE_READLINE_HISTORY */ + +#if defined(HAVE_LIBREADLINE) && defined(HAVE_READLINE_HISTORY) +#define HAVE_READLINE (1) +#else +#define HAVE_READLINE (0) +#endif + +using namespace Mu; + +static bool is_a_tty{}; +static std::string hist_path; +static size_t max_lines{}; + +// LCOV_EXCL_START + +bool +Mu::have_readline() +{ + return HAVE_READLINE != 0; +} + +void +Mu::setup_readline(const std::string& histpath, size_t maxlines) +{ + is_a_tty = !!::isatty(::fileno(stdout)); + hist_path = histpath; + max_lines = maxlines; + +#if HAVE_READLINE + rl_bind_key('\t', rl_insert); // default (filenames) is not useful + using_history(); + read_history(hist_path.c_str()); + + if (max_lines > 0) + stifle_history(max_lines); +#endif /*HAVE_READLINE*/ +} + +void +Mu::shutdown_readline() +{ +#if HAVE_READLINE + if (!is_a_tty) + return; + + write_history(hist_path.c_str()); + if (max_lines > 0) + history_truncate_file(hist_path.c_str(), max_lines); +#endif /*HAVE_READLINE*/ +} + +std::string +Mu::read_line(bool& do_quit) +{ +#if HAVE_READLINE + if (is_a_tty) { + auto buf = readline(";; mu% "); + if (!buf) { + do_quit = true; + return {}; + } + std::string line{buf}; + ::free(buf); + return line; + } +#endif /*HAVE_READLINE*/ + + std::string line; + mu_print(";; mu> "); + if (!std::getline(std::cin, line)) + do_quit = true; + + return line; +} + +void +Mu::save_line(const std::string& line) +{ +#if HAVE_READLINE + if (is_a_tty) + add_history(line.c_str()); +#endif /*HAVE_READLINE*/ +} + +// LCOV_EXCL_STOP diff --git a/lib/utils/mu-readline.hh b/lib/utils/mu-readline.hh new file mode 100644 index 0000000..ca0455f --- /dev/null +++ b/lib/utils/mu-readline.hh @@ -0,0 +1,61 @@ +/* +** Copyright (C) 2020 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include <string> + +namespace Mu { + +/** + * Setup readline when available and on tty. + * + * @param histpath path to the history file + * @param max_lines maximum number of history to save + */ +void setup_readline(const std::string& histpath, size_t max_lines); + +/** + * Shutdown readline + * + */ +void shutdown_readline(); + +/** + * Read a command line + * + * @param do_quit recceives whether we should quit. + * + * @return the string read or empty + */ +std::string read_line(bool& do_quit); + +/** + * Save a line to history (or do nothing when readline is not active) + * + * @param line a line. + */ +void save_line(const std::string& line); + + +/** + * Do we have the non-shim readline? + * + * @return true or failse + */ +bool have_readline(); + +} // namespace Mu diff --git a/lib/utils/mu-regex.cc b/lib/utils/mu-regex.cc new file mode 100644 index 0000000..8127695 --- /dev/null +++ b/lib/utils/mu-regex.cc @@ -0,0 +1,114 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "mu-regex.hh" +#include <iostream> + +using namespace Mu; + +#if BUILD_TESTS +#include "mu-test-utils.hh" + +// No need for extensive regex test, we just rely on GRegex. + +static void +test_regex_match() +{ + auto rx = Regex::make("a.*b.c"); + assert_valid_result(rx); + + assert_equal(mu_format("{}", *rx), "/a.*b.c/"); + + g_assert_true(rx->matches("axxxxxbqc")); + g_assert_false(rx->matches("axxxxxbqqc")); + + { // unset matches nothing. + Regex rx2; + g_assert_false(rx2.matches("")); + } +} + + +static void +test_regex_match2() +{ + Regex rx; + { + std::string foo = "h.llo"; + rx = unwrap(Regex::make(foo.c_str())); + } + + std::string hei = "hei"; + + g_assert_true(rx.matches("hallo")); + g_assert_false(rx.matches(hei)); +} + + +static void +test_regex_replace() +{ + { + auto rx = Regex::make("f.o"); + assert_valid_result(rx); + assert_equal(rx->replace("foobar", "cuux").value_or("error"), "cuuxbar"); + } + + { + auto rx = Regex::make("f.o", G_REGEX_MULTILINE); + assert_valid_result(rx); + assert_equal(rx->replace("foobar\nfoobar", "cuux").value_or("error"), + "cuuxbar\ncuuxbar"); + } +} + + +static void +test_regex_fail() +{ + allow_warnings(); + + { // unset rx can't replace / error. + Regex rx; + assert_equal(mu_format("{}", rx), "//"); + g_assert_false(!!rx.replace("foo", "bar")); + } + + { + auto rx = Regex::make("("); + g_assert_false(!!rx); + + } + +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/regex/match", test_regex_match); + g_test_add_func("/regex/match2", test_regex_match2); + g_test_add_func("/regex/replace", test_regex_replace); + g_test_add_func("/regex/fail", test_regex_fail); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/utils/mu-regex.hh b/lib/utils/mu-regex.hh new file mode 100644 index 0000000..c5fd4a0 --- /dev/null +++ b/lib/utils/mu-regex.hh @@ -0,0 +1,193 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#ifndef MU_REGEX_HH__ +#define MU_REGEX_HH__ + +#include <glib.h> +# +#include <utils/mu-result.hh> +#include <utils/mu-utils.hh> + +namespace Mu { +/** + * RAII wrapper around a GRegex which in itself is a wrapper around PCRE. We use + * PCRE rather than std::regex because it is much faster. + */ +struct Regex { +#if !GLIB_CHECK_VERSION(2,74,0) /* backward compat */ +#define G_REGEX_DEFAULT (static_cast<GRegexCompileFlags>(0)) +#define G_REGEX_MATCH_DEFAULT (static_cast<GRegexMatchFlags>(0)) +#endif + /** + * Trivial constructor + * + * @return + */ + Regex() noexcept: rx_{} {} + + /** + * Construct a new Regex object + * + * @param ptrn PRRE regular expression pattern + * @param cflags compile flags + * @param mflags match flags + * + * @return a Regex object or an error. + */ + static Result<Regex> make(const std::string& ptrn, + GRegexCompileFlags cflags = G_REGEX_DEFAULT, + GRegexMatchFlags mflags = G_REGEX_MATCH_DEFAULT) noexcept try { + return Regex(ptrn.c_str(), cflags, mflags); + } catch (const Error& err) { + return Err(err); + } + + + /** + * Copy CTOR + * + * @param other some other Regex + */ + Regex(const Regex& other) noexcept: rx_{} { *this = other; } + + /** + * Move CTOR + * + * @param other some other Regex + */ + Regex(Regex&& other) noexcept: rx_{} { *this = std::move(other); } + + + /** + * DTOR + */ + ~Regex() noexcept { g_clear_pointer(&rx_, g_regex_unref); } + + /** + * Cast to the the underlying GRegex* + * + * @return a GRegex* + */ + operator const GRegex*() const noexcept { return rx_; } + + /** + * Doe this object contain a valid GRegex*? + * + * @return true or false + */ + operator bool() const noexcept { return !!rx_; } + + /** + * operator= + * + * @param other copy some other object to this one + * + * @return *this + */ + Regex& operator=(const Regex& other) noexcept { + if (this != &other) { + g_clear_pointer(&rx_, g_regex_unref); + if (other.rx_) + rx_ = g_regex_ref(other.rx_); + } + return *this; + } + + /** + * operator= + * + * @param other move some other object to this one + * + * @return *this + */ + Regex& operator=(Regex&& other) noexcept { + if (this != &other) { + g_clear_pointer(&rx_, g_regex_unref); + rx_ = other.rx_; + other.rx_ = nullptr; + } + return *this; + } + + /** + * Does this regexp match the given string? An unset Regex matches + * nothing. + * + * @param str string to test + * @param mflags match flags + * + * @return true or false + */ + bool matches(const std::string& str, + GRegexMatchFlags mflags=G_REGEX_MATCH_DEFAULT) const noexcept { + if (!rx_) + return false; + else + return g_regex_match(rx_, str.c_str(), mflags, nullptr); + // strangely, valgrind reports some memory error related to + // the str.c_str(). It *seems* like a false alarm. + } + + /** + * Replace all occurrences of @this regexp in some string with a + * replacement string + * + * @param str some string + * @param repl replacement string + * + * @return string or error + */ + Result<std::string> replace(const std::string& str, const std::string& repl) const { + GError *gerr{}; + + if (!rx_) + return Err(Error::Code::InvalidArgument, "missing regexp"); + else if (auto&& s{g_regex_replace(rx_, str.c_str(), str.length(), 0, + repl.c_str(), G_REGEX_MATCH_DEFAULT, &gerr)}; !s) + return Err(Error::Code::InvalidArgument, &gerr, "error in Regex::replace"); + else + return Ok(to_string_gchar(std::move(s))); + } + + const GRegex* g_regex() const { return rx_; } + +private: + Regex(const char *ptrn, GRegexCompileFlags cflags, GRegexMatchFlags mflags) { + GError *err{}; + if (rx_ = g_regex_new(ptrn, cflags, mflags, &err); !rx_) + throw Error{Error::Code::InvalidArgument, &err, + "invalid regexp: '{}'", ptrn}; + } + + GRegex *rx_{}; +}; + +static inline std::string format_as(const Regex& rx) { + if (auto&& grx{rx.g_regex()}; !grx) + return "//"; + else + return mu_format("/{}/", g_regex_get_pattern(grx)); + +} + + +} // namespace Mu + + +#endif /* MU_REGEX_HH__ */ diff --git a/lib/utils/mu-result.hh b/lib/utils/mu-result.hh new file mode 100644 index 0000000..887f8ad --- /dev/null +++ b/lib/utils/mu-result.hh @@ -0,0 +1,145 @@ +/* +** Copyright (C) 2019-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_RESULT_HH__ +#define MU_RESULT_HH__ + +#include <tl/expected.hpp> +#include "utils/mu-error.hh" + +namespace Mu { +/** + * A little Rust-envy...a Result is _either_ some value of type T, _or_ a Mu::Error + */ +template <typename T> using Result = tl::expected<T, Error>; + +/** + * Ok() is not typically strictly needed (unlike Err), but imitates Rust's Ok + * and it helps the reader. + * + * @param t the value to return + * + * @return a success Result<T> + */ +template <typename T> Result<T> +Ok(T&& t) +{ + return std::move(t); +} + +/** + * Implementation of Ok() for void results. + * + * @return a success Result<void> + */ +static inline Result<void> +Ok() +{ + return {}; +} + +/** + * Return an error + * + * @param err the error + * + * @return error + */ +template<typename T> Result<T> +Err(Error&& err) +{ + return tl::unexpected(std::move(err)); +} +template<typename T> Result<T> +Err(const Error& err) +{ + return tl::unexpected(err); +} + +static inline tl::unexpected<Error> +Err(Error&& err) +{ + return tl::unexpected(std::move(err)); +} + +static inline tl::unexpected<Error> +Err(const Error& err) +{ + return tl::unexpected(err); +} + +template<typename T> +static inline tl::unexpected<Error> +Err(const Result<T>& res) +{ + return res.error(); +} + +template<typename T> +static inline tl::unexpected<Error> +Err(Result<T>&& res) +{ + return std::move(res.error()); +} + +/* + * convenience + */ +template <typename ...T> +tl::unexpected<Error> +Err(Error::Code code, fmt::format_string<T...> frm, T&&... args) +{ + return Err(Error{code, frm, std::forward<T>(args)...}); +} + +template <typename ...T> +tl::unexpected<Error> +Err(Error::Code code, GError **err, fmt::format_string<T...> frm, T&&... args) +{ + return Err(Error{code, err, frm, std::forward<T>(args)...}); +} + + +template<typename T> T +unwrap(Result<T>&& res) +{ + if (!!res) + return std::move(res.value()); + else + throw res.error(); +} + +/** + * Assert that some result has a value (for unit tests) + * + * @param R some result + */ +#define assert_valid_result(R) do { \ + auto&& res__ = R; \ + if(!res__) { \ + mu_printerrln("{}:{}: error-result: {}", \ + __FILE__, __LINE__, \ + (res__).error().what()); \ + g_assert_true(!!res__); \ + } \ +} while(0) + +}// namespace Mu + +#endif /* MU_RESULT_HH__ */ diff --git a/lib/utils/mu-sexp.cc b/lib/utils/mu-sexp.cc new file mode 100644 index 0000000..47510d1 --- /dev/null +++ b/lib/utils/mu-sexp.cc @@ -0,0 +1,522 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + + +#include "mu-sexp.hh" +#include "mu-utils.hh" + +#include <atomic> +#include <sstream> +#include <array> + +using namespace Mu; + +template<typename...T> static Mu::Error +parsing_error(size_t pos, fmt::format_string<T...> frm, T&&... args) +{ + const auto&& msg{fmt::format(frm, std::forward<T>(args)...)}; + if (pos == 0) + return Mu::Error(Error::Code::Parsing, "{}", msg); + else + return Mu::Error(Error::Code::Parsing, "{}: {}", pos, msg); +} + +static size_t +skip_whitespace(const std::string& s, size_t pos) +{ + while (pos != s.size()) { + if (s[pos] == ' ' || s[pos] == '\t' || s[pos] == '\n') + ++pos; + else + break; + } + return pos; +} + +static Result<Sexp> parse(const std::string& expr, size_t& pos); + +static Result<Sexp> +parse_list(const std::string& expr, size_t& pos) +{ + if (expr[pos] != '(') // sanity check. + return Err(parsing_error(pos, "expected: '(' but got '{}", expr[pos])); + + Sexp lst{}; + + ++pos; + while (expr[pos] != ')' && pos != expr.size()) { + if (auto&& item = parse(expr, pos); item) + lst.add(std::move(*item)); + else + return Err(item.error()); + } + + if (expr[pos] != ')') + return Err(parsing_error(pos, "expected: ')' but got '{}'", expr[pos])); + ++pos; + return Ok(std::move(lst)); +} + +static Result<Sexp> +parse_string(const std::string& expr, size_t& pos) +{ + if (expr[pos] != '"') // sanity check. + return Err(parsing_error(pos, "expected: '\"'' but got '{}", expr[pos])); + + bool escape{}; + std::string str; + for (++pos; pos != expr.size(); ++pos) { + auto kar = expr[pos]; + if (escape && (kar == '"' || kar == '\\')) { + str += kar; + escape = false; + continue; + } + + if (kar == '"') + break; + else if (kar == '\\') + escape = true; + else + str += kar; + } + + if (escape || expr[pos] != '"') + return Err(parsing_error(pos, "unterminated string '{}'", str)); + + ++pos; + return Ok(Sexp{std::move(str)}); +} + + +static Result<Sexp> +parse_integer(const std::string& expr, size_t& pos) +{ + if (!isdigit(expr[pos]) && expr[pos] != '-') // sanity check. + return Err(parsing_error(pos, "expected: <digit> but got '{}", expr[pos])); + + std::string num; // negative number? + if (expr[pos] == '-') { + num = "-"; + ++pos; + } + + for (; isdigit(expr[pos]); ++pos) + num += expr[pos]; + + return Ok(Sexp{::atoi(num.c_str())}); +} + +static Result<Sexp> +parse_symbol(const std::string& expr, size_t& pos) +{ + if (!isalpha(expr[pos]) && expr[pos] != ':') // sanity check. + return Err(parsing_error(pos, "expected: <alpha>|: but got '{}", expr[pos])); + + std::string symb(1, expr[pos]); + for (++pos; isalnum(expr[pos]) || expr[pos] == '-'; ++pos) + symb += expr[pos]; + + return Ok(Sexp{Sexp::Symbol{symb}}); +} + +static Result<Sexp> +parse(const std::string& expr, size_t& pos) +{ + pos = skip_whitespace(expr, pos); + + if (pos == expr.size()) + return Err(parsing_error(pos, "expected: character '{}", expr[pos])); + + const auto kar = expr[pos]; + const auto sexp = std::invoke([&]() -> Result<Sexp> { + if (kar == '(') + return parse_list(expr, pos); + else if (kar == '"') + return parse_string(expr, pos); + else if (isdigit(kar) || kar == '-') + return parse_integer(expr, pos); + else if (isalpha(kar) || kar == ':') + return parse_symbol(expr, pos); + else + return Err(parsing_error(pos, "unexpected character '{}", kar)); + }); + + if (sexp) + pos = skip_whitespace(expr, pos); + + return sexp; +} + +Result<Sexp> +Sexp::parse(const std::string& expr) +{ + size_t pos{}; + auto res = ::parse(expr, pos); + if (!res) + return res; + else if (pos != expr.size()) + return Err(parsing_error(pos, "trailing data starting with '{}'", expr[pos])); + else + return res; +} + +std::string +Sexp::to_string(Format fopts) const +{ + std::stringstream sstrm; + const auto splitp{any_of(fopts & Format::SplitList)}; + const auto typeinfop{any_of(fopts & Format::TypeInfo)}; + + if (listp()) { + sstrm << '('; + bool first{true}; + for(auto&& elm: list()) { + sstrm << (first ? "" : " ") << elm.to_string(fopts); + first = false; + } + sstrm << ')'; + if (splitp) + sstrm << '\n'; + } else if (stringp()) + sstrm << quote(string()); + else if (numberp()) + sstrm << number(); + else if (symbolp()) + sstrm << symbol().name; + + if (typeinfop) + sstrm << '<' << Sexp::type_name(type()) << '>'; + + return sstrm.str(); +} + +// LCOV_EXCL_START + +std::string +Sexp::to_json_string(Format fopts) const +{ + std::stringstream sstrm; + + switch (type()) { + case Type::List: { + // property-lists become JSON objects + if (plistp()) { + sstrm << "{"; + auto it{list().begin()}; + bool first{true}; + while (it != list().end()) { + sstrm << (first ? "" : ",") << quote(it->symbol().name) << ":"; + ++it; + sstrm << it->to_json_string(); + ++it; + first = false; + } + sstrm << "}"; + if (any_of(fopts & Format::SplitList)) + sstrm << '\n'; + } else { // other lists become arrays. + sstrm << '['; + bool first{true}; + for (auto&& child : list()) { + sstrm << (first ? "" : ", ") << child.to_json_string(); + first = false; + } + sstrm << ']'; + if (any_of(fopts & Format::SplitList)) + sstrm << '\n'; + } + break; + } + case Type::String: + sstrm << quote(string()); + break; + case Type::Symbol: + if (nilp()) + sstrm << "false"; + else if (symbol() == "t") + sstrm << "true"; + else + sstrm << quote(symbol().name); + break; + case Type::Number: + sstrm << number(); + break; + default: + break; + } + + return sstrm.str(); +} + + + +Sexp& +Sexp::del_prop(const std::string& pname) +{ + if (auto kill_it = find_prop(pname, begin(), end()); kill_it != cend()) + list().erase(kill_it, kill_it + 2); + return *this; +} + + +Sexp::const_iterator +Sexp::find_prop(const std::string& s, + Sexp::const_iterator b, Sexp::const_iterator e) const +{ + for (auto&& it = b; it != e && it+1 != e; it += 2) + if (it->symbolp() && it->symbol() == s) + return it; + return e; +} + +Sexp::iterator +Sexp::find_prop(const std::string& s, + Sexp::iterator b, Sexp::iterator e) +{ + for (auto&& it = b; it != e && it+1 != e; it += 2) + if (it->symbolp() && it->symbol() == s) + return it; + return e; +} + + +bool +Sexp::plistp(Sexp::const_iterator b, Sexp::const_iterator e) const +{ + if (b == e) + return true; + else if (b + 1 == e) + return false; + else + return b->symbolp() && plistp(b + 2, e); +} + + +// LCOV_EXCL_STOP + +#if BUILD_TESTS + +#include "mu-test-utils.hh" + +static void +test_list() +{ + { + Sexp s; + g_assert_true(s.listp()); + g_assert_true(s.to_string() == "()"); + g_assert_true(Sexp::type_name(s.type()) == "list"); + g_assert_true(s.empty()); + } + + { + Sexp::List items = { + Sexp("hello"), + Sexp(123), + Sexp::Symbol("world") + }; + const Sexp s{std::move(items)}; + g_assert_false(s.empty()); + g_assert_cmpuint(s.size(),==,3); + g_assert_true(s.to_string() == "(\"hello\" 123 world)"); + + + /* copy */ + Sexp s2 = s; + g_assert_true(s2.to_string() == "(\"hello\" 123 world)"); + + /* move */ + Sexp s3 = std::move(s2); + g_assert_true(s3.to_string() == "(\"hello\" 123 world)"); + + s3.clear(); + g_assert_true(s3.empty()); + } + +} + +static void +test_string() +{ + { + Sexp s("hello"); + g_assert_true(s.stringp()); + g_assert_true(s.string()=="hello"); + g_assert_true(s.to_string()=="\"hello\""); + g_assert_true(Sexp::type_name(s.type()) == "string"); + } + + { + // Sexp s(std::string_view("hel\"lo")); + // g_assert_true(s.is_string()); + // g_assert_cmpstr(s.string().c_str(),==,"hel\"lo"); + // g_assert_cmpstr(s.to_string().c_str(),==,"\"hel\\\"lo\""); + } +} + +static void +test_number() +{ + { + Sexp s(123); + g_assert_true(s.numberp()); + g_assert_cmpint(s.number(),==,123); + g_assert_true(s.to_string() == "123"); + g_assert_true(Sexp::type_name(s.type()) == "number"); + } + + { + Sexp s(true); + g_assert_true(s.numberp()); + g_assert_cmpint(s.number(),==,1); + g_assert_true(s.to_string()=="1"); + } +} + +static void +test_symbol() +{ + { + Sexp s{Sexp::Symbol("hello")}; + g_assert_true(s.symbolp()); + g_assert_true(s.symbol()=="hello"); + g_assert_true (s.to_string()=="hello"); + g_assert_true(Sexp::type_name(s.type()) == "symbol"); + } + + { + Sexp s{"hello"_sym}; + g_assert_true(s.symbolp()); + g_assert_true(s.symbol()=="hello"); + g_assert_true (s.to_string()=="hello"); + } + +} + +static void +test_multi() +{ + Sexp s{"abc", 123, Sexp::Symbol{"def"}}; + g_assert_true(s.to_string() == "(\"abc\" 123 def)"); +} + + +static void +test_add() +{ + { + Sexp s{"abc", 123}; + s.add("def"_sym); + g_assert_true(s.to_string() == "(\"abc\" 123 def)"); + } +} + +static void +test_add_multi() +{ + { + Sexp s{"abc", 123}; + s.add("def"_sym, 456, Sexp{"boo", 2}); + g_assert_true(s.to_string() == "(\"abc\" 123 def 456 (\"boo\" 2))"); + } + + { + Sexp s{"abc", 123}; + Sexp t{"boo", 2}; + s.add("def"_sym, 456, t); + g_assert_true(s.to_string() == "(\"abc\" 123 def 456 (\"boo\" 2))"); + } + +} + +static void +test_plist() +{ + Sexp s; + s.put_props("hello", "world"_sym, "foo", 123, "bar"_sym, "cuux"); + g_assert_true(s.to_string() == R"((hello world foo 123 bar "cuux"))"); + + s.put_props("hello", 12345); + g_assert_true(s.to_string() == R"((foo 123 bar "cuux" hello 12345))"); +} + + +static void +check_parse(const std::string& expr, const std::string& expected) +{ + auto sexp = Sexp::parse(expr); + assert_valid_result(sexp); + assert_equal(to_string(*sexp), expected); +} + +static void +test_parser() +{ + check_parse(":foo-123", ":foo-123"); + check_parse("foo", "foo"); + check_parse(R"(12345)", "12345"); + check_parse(R"(-12345)", "-12345"); + check_parse(R"((123 bar "cuux"))", "(123 bar \"cuux\")"); + + check_parse(R"("foo\"bar\"cuux")", "\"foo\\\"bar\\\"cuux\""); + + check_parse(R"("foo +bar")", + "\"foo\nbar\""); +} + +static void +test_parser_fail() +{ + g_assert_false(!!Sexp::parse("\"")); + g_assert_false(!!Sexp::parse("123abc")); + g_assert_false(!!Sexp::parse("(")); + g_assert_false(!!Sexp::parse(")")); + g_assert_false(!!Sexp::parse("(hello (boo))))")); + + g_assert_true(Sexp::type_name(static_cast<Sexp::Type>(-1)) == "<error>"); +} + + +int +main(int argc, char* argv[]) +try { + mu_test_init(&argc, &argv); + + g_test_add_func("/sexp/list", test_list); + g_test_add_func("/sexp/string", test_string); + g_test_add_func("/sexp/number", test_number); + g_test_add_func("/sexp/symbol", test_symbol); + g_test_add_func("/sexp/multi", test_multi); + g_test_add_func("/sexp/add", test_add); + g_test_add_func("/sexp/add-multi", test_add_multi); + g_test_add_func("/sexp/plist", test_plist); + g_test_add_func("/sexp/parser", test_parser); + g_test_add_func("/sexp/parser-fail", test_parser_fail); + + return g_test_run(); + +} catch (const std::runtime_error& re) { + mu_printerrln("{}", re.what()); + return 1; +} + + +#endif /*BUILD_TESTS*/ diff --git a/lib/utils/mu-sexp.hh b/lib/utils/mu-sexp.hh new file mode 100644 index 0000000..8127cbf --- /dev/null +++ b/lib/utils/mu-sexp.hh @@ -0,0 +1,326 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_SEXP_HH__ +#define MU_SEXP_HH__ + +#include "mu-utils.hh" + +#include <stdexcept> +#include <vector> +#include <string> +#include <string_view> +#include <iostream> +#include <variant> +#include <cinttypes> +#include <ostream> +#include <cassert> + +#include <utils/mu-result.hh> +#include <utils/mu-option.hh> + +namespace Mu { + +/** + * A structure somewhat similar to a Lisp s-expression and which can be + * constructed from/to an s-expressing string representation. + * + * A sexp is either an atom (String, Number, Symbol) or a List. + */ +struct Sexp { + /** + * Types + * + */ + using List = std::vector<Sexp>; + using String = std::string; + using Number = int64_t; + struct Symbol { // distinguish from String. + Symbol(const std::string& s): name{s} {} + Symbol(std::string&& s): name(std::move(s)) {} + Symbol(const char* str): Symbol(std::string{str}) {} + Symbol(std::string_view sv): Symbol(std::string{sv}) {} + operator const std::string&() const {return name; } + std::string name; + + bool operator==(const Symbol& rhs) const { + return this == &rhs ? true : rhs.name == name; + } + bool operator!=(const Symbol& rhs) const { return *this == rhs ? false : true; } + }; + enum struct Type { List, String, Number, Symbol }; + using ValueType = std::variant<List, String, Number, Symbol>; + + /** + * Is some Sexp of the given type? + * + * @return true or false + */ + constexpr bool stringp() const { return std::holds_alternative<String>(value); } + constexpr bool numberp() const { return std::holds_alternative<Number>(value); } + constexpr bool listp() const { return std::holds_alternative<List>(value); } + constexpr bool symbolp() const { return std::holds_alternative<Symbol>(value); } + constexpr bool symbolp(const Sexp::Symbol& sym) const {return symbolp() && symbol() == sym; } + constexpr bool nilp() const { return symbolp(nil_sym); } + + // Get the specific variant type. + const List& list() const { return std::get<List>(value); } + List& list() { return std::get<List>(value); } + const String& string() const { return std::get<String>(value); } + String& string() { return std::get<String>(value); } + const Number& number() const { return std::get<Number>(value); } + Number& number() { return std::get<Number>(value); } + const Symbol& symbol() const { return std::get<Symbol>(value); } + Symbol& symbol() { return std::get<Symbol>(value); } + + /** + * Constructors + */ + Sexp():value{List{}} {} // default: an empty list. + // Copy & move ctors + Sexp(const Sexp& other):value{other.value}{} + Sexp(Sexp&& other):value{std::move(other.value)}{} + // From various types + Sexp(const List& lst): value{lst} {} + Sexp(List&& lst): value{std::move(lst)} {} + Sexp(const String& str): value{str} {} + Sexp(String&& str): value{std::move(str)} {} + Sexp(const char *str): Sexp{std::string{str}} {} + Sexp(std::string_view sv): Sexp{std::string{sv}} {} + + template<typename N, typename = std::enable_if_t<std::is_integral_v<N>> > + Sexp(N n):value{static_cast<Number>(n)} {} + + Sexp(const Symbol& sym): value{sym} {} + Sexp(Symbol&& sym): value{std::move(sym)} {} + + template<typename S, typename T, typename... Args> + Sexp(S&& s, T&& t, Args&&... args): value{List()} { + auto& l{std::get<List>(value)}; + l.emplace_back(Sexp(std::forward<S>(s))); + l.emplace_back(Sexp(std::forward<T>(t))); + (l.emplace_back(Sexp(std::forward<Args>(args))), ...); + } + + /** + * Copy-assignment + * + * @param rhs another sexp + * + * @return the sexp + */ + Sexp& operator=(const Sexp& rhs) { + if (this != &rhs) + value = rhs.value; + return *this; + } + + /** + * Move-assignment + * + * @param rhs another sexp + * + * @return the sexp + */ + Sexp& operator=(Sexp&& rhs) { + if (this != &rhs) + value = std::move(rhs.value); + return *this; + } + + /** + * Get the type of value + * + * @return type + */ + constexpr Type type() const { return static_cast<Type>(value.index()); } + /** + * Get the name for some type + * + * @param t type + * + * @return name + */ + static constexpr std::string_view type_name(Type t) { + switch(t) { + case Type::String: + return "string"; + case Type::Number: + return "number"; + case Type::Symbol: + return "symbol"; + case Type::List: + return "list"; + default: + return "<error>"; + } + } + + /** + * Parse sexp from string + * + * @param str a string + * + * @return either an Sexp or an error + */ + static Result<Sexp> parse(const std::string& str); + + + /** + * List specific functionality + * + */ + using iterator = List::iterator; + using const_iterator = List::const_iterator; + + iterator begin() { return list().begin(); } + const_iterator begin() const { return list().begin(); } + const_iterator cbegin() const { return list().cbegin(); } + + iterator end() { return list().end(); } + const_iterator end() const { return list().end(); } + const_iterator cend() const { return list().cend(); } + + bool empty() const { return list().empty(); } + size_t size() const { return list().size(); } + void clear() { list().clear(); } + + /// Adding to lists + Sexp& add(const Sexp& s) { list().emplace_back(s); return *this; } + Sexp& add(Sexp&& s) { list().emplace_back(std::move(s)); return *this; } + Sexp& add() { return *this; } + + template <typename V1, typename V2, typename... Args> + Sexp& add(V1&& v1, V2&& v2, Args... args) { + return add(std::forward<V1>(v1)) + .add(std::forward<V2>(v2)) + .add(std::forward<Args>(args)...); + } + + /// Adding list elements + Sexp& add_list(Sexp&& l) { for (auto&& e: l) add(std::move(e)); return *this;}; + + /// Some convenience for the query parser + Sexp& front() { return list().front(); } + const Sexp& front() const { return list().front(); } + void pop_front() { list().erase(list().begin()); } + + Option<Sexp&> head() { if (listp()&&!empty()) return front(); else return Nothing; } + Option<const Sexp&> head() const { if (listp()&&!empty()) return front(); else return Nothing; } + + bool head_symbolp() const { + if (auto&& h{head()}; h) return h->symbolp(); else return false; + } + bool head_symbolp(const Symbol& sym) const { + if (head_symbolp()) return head()->symbolp(sym); else return false; + } + + /** + * Property lists (aka plists) + */ + + bool plistp() const { return listp() && plistp(cbegin(), cend()); } + Sexp& put_props() { return *this; } // Final case for template pack. + template <class PropType, class SexpType, typename... Args> + Sexp& put_props(PropType&& prop, SexpType&& sexp, Args... args) { + auto&& propname{std::string(prop)}; + return del_prop(propname) + .add(Symbol(std::move(propname)), + std::forward<SexpType>(sexp)) + .put_props(std::forward<Args>(args)...); + } + + /** + * Find the property value for some property by name + * + * @param p property name + * + * @return the property if found, or nothing + */ + const Option<const Sexp&> get_prop(const std::string& p) const { + if (auto&& it = find_prop(p, cbegin(), cend()); it != cend()) + return *(std::next(it)); + else + return Nothing; + } + /// Output to string + enum struct Format { + Default = 0, /**< Nothing in particular */ + SplitList = 1 << 0, /**< Insert newline after list item */ + TypeInfo = 1 << 1, /**< Show type-info */ + }; + + /** + * Get a string representation of the sexp + * + * @return str + */ + std::string to_string(Format fopts=Format::Default) const; + std::string to_json_string(Format fopts=Format::Default) const; + + Sexp& del_prop(const std::string& pname); + + /** + * Some useful constants + * + */ + static inline const auto nil_sym = Sexp::Symbol{"nil"}; + static inline const auto t_sym = Sexp::Symbol{"t"}; + +protected: + const_iterator find_prop(const std::string& s, const_iterator b, + const_iterator e) const; + bool plistp(const_iterator b, const_iterator e) const; +private: + iterator find_prop(const std::string& s,iterator b, + iterator e); + ValueType value; + + +}; + +MU_ENABLE_BITOPS(Sexp::Format); + +/** + * String-literal; allow for ":foo"_sym to be a symbol + */ +static inline Sexp::Symbol +operator"" _sym(const char* str, std::size_t n) +{ + return Sexp::Symbol{str}; +} + +static inline std::ostream& +operator<<(std::ostream& os, const Sexp::Type& stype) +{ + os << Sexp::type_name(stype); + return os; +} + + +static inline std::ostream& +operator<<(std::ostream& os, const Sexp& sexp) +{ + os << sexp.to_string(); + return os; +} + +} // namespace Mu + +#endif /* MU_SEXP_HH__ */ diff --git a/lib/utils/mu-test-utils.cc b/lib/utils/mu-test-utils.cc new file mode 100644 index 0000000..0d4a149 --- /dev/null +++ b/lib/utils/mu-test-utils.cc @@ -0,0 +1,140 @@ +/* +** Copyright (C) 2008-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include <glib.h> +#include <glib/gstdio.h> + +#include <stdlib.h> +#include <unistd.h> +#include <string.h> + +#include <langinfo.h> +#include <locale.h> + +#include "utils/mu-utils.hh" +#include "utils/mu-test-utils.hh" +#include "utils/mu-utils-file.hh" +#include "utils/mu-error.hh" + +using namespace Mu; + +/* LCOV_EXCL_START*/ +bool +Mu::mu_test_mu_hacker() +{ + return !!g_getenv("MU_HACKER"); +} +/* LCOV_EXCL_STOP*/ + + +const char* +Mu::set_tz(const char* tz) +{ + static const char* oldtz; + + oldtz = getenv("TZ"); + if (tz) + setenv("TZ", tz, 1); + else + unsetenv("TZ"); + + tzset(); + return oldtz; +} + +bool +Mu::set_en_us_utf8_locale() +{ + setenv("LC_ALL", "en_US.UTF-8", 1); + + if (auto str = setlocale(LC_ALL, "en_US.UTF-8"); !str) + return false; + + if (strcmp(nl_langinfo(CODESET), "UTF-8") != 0) + return false; + + return true; +} + +static void +black_hole(void) +{ + return; /* do nothing */ +} + +void +Mu::mu_test_init(int *argc, char ***argv) +{ + TempDir temp_dir; + + g_unsetenv("XAPIAN_CJK_NGRAM"); + g_setenv("MU_TEST", "yes", TRUE); + g_setenv("XDG_CACHE_HOME", temp_dir.path().c_str(), TRUE); + + setlocale(LC_ALL, ""); + + g_test_init(argc, argv, NULL); + + g_test_bug_base("https://github.com/djcb/mu/issues/"); + + if (!g_test_verbose()) + g_log_set_handler( + NULL, + (GLogLevelFlags)(G_LOG_LEVEL_MASK | + G_LOG_FLAG_FATAL | G_LOG_FLAG_RECURSION), + (GLogFunc)black_hole, NULL); +} + +void +Mu::allow_warnings() +{ + g_test_log_set_fatal_handler( + [](const char*, GLogLevelFlags, const char*, gpointer) { return FALSE; }, + {}); +} + +Mu::TempDir::TempDir(bool autodelete): autodelete_{autodelete} { + + if (auto res{make_temp_dir()}; !res) + throw res.error(); + else + path_ = std::move(*res); + + mu_debug("created '{}'", path_); +} + +Mu::TempDir::~TempDir() +{ + if (::access(path_.c_str(), F_OK) != 0) + return; /* nothing to do */ + + if (!autodelete_) { + mu_debug("_not_ deleting {}", path_); + return; + } + + if (auto&& res{run_command0({RM_PROGRAM, "-fr", path_})}; !res) { + /* LCOV_EXCL_START*/ + mu_warning("error removing {}: {}", path_, format_as(res.error())); + /* LCOV_EXCL_STOP*/ + } else + mu_debug("removed '{}'", path_); +} diff --git a/lib/utils/mu-test-utils.hh b/lib/utils/mu-test-utils.hh new file mode 100644 index 0000000..051230a --- /dev/null +++ b/lib/utils/mu-test-utils.hh @@ -0,0 +1,167 @@ +/* +** Copyright (C) 2008-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_TEST_UTILS_HH__ +#define MU_TEST_UTILS_HH__ + +#include <initializer_list> +#include <string> +#include <utils/mu-utils.hh> +#include <utils/mu-result.hh> + +namespace Mu { + +/** + * mu wrapper for g_test_init. Sets environment variable MU_TEST to 1. + * + * @param argc + * @param argv + */ +void mu_test_init(int *argc, char ***argv); + + +/** + * Are we running in a MU_HACKER environment? + * + * @return true or false + */ +bool mu_test_mu_hacker(); + +/** + * set the timezone + * + * @param tz timezone + * + * @return the old timezone + */ +const char* set_tz(const char* tz); + +/** + * switch the locale to en_US.utf8, return TRUE if it succeeds + * + * @return true if the switch succeeds, false otherwise + */ +bool set_en_us_utf8_locale(); + + +/** + * For unit tests, assert two std::string's are equal. + * + * @param s1 string1 + * @param s2 string2 + */ +#define assert_equal(s1__,s2__) do { \ + std::string s1s__(s1__), s2s__(s2__); \ + g_assert_cmpstr(s1s__.c_str(), ==, s2s__.c_str()); \ + } while(0) + + +#define assert_equal_seq(seq1__, seq2__) do { \ + g_assert_cmpuint(seq1__.size(), ==, seq2__.size()); \ + size_t n__{}; \ + for (auto&& item__: seq1__) { \ + g_assert_true(item__ == seq2__.at(n__)); \ + ++n__; \ + } \ + } while(0) + +#define assert_equal_seq_str(seq1__, seq2__) do { \ + g_assert_cmpuint(seq1__.size(), ==, seq2__.size()); \ + size_t n__{}; \ + for (auto&& item__: seq1__) { \ + assert_equal(item__, seq2__.at(n__)); \ + ++n__; \ + } \ + } while(0) + +#define assert_valid_command(RCO) do { \ + assert_valid_result(RCO); \ + if ((RCO)->exit_code != 0 && !(RCO)->standard_err.empty()) \ + mu_printerrln("{}:{}: {}", \ + __FILE__, __LINE__, (RCO)->standard_err); \ + g_assert_cmpuint((RCO)->exit_code, ==, 0); \ +} while (0) + +/** + * For unit-tests, allow warnings in the current function. + * + */ +void allow_warnings(); + + +/** + * For unit-tests, a RAII tempdir. + * + */ +struct TempDir { + /** + * Construct a temporary directory + */ + TempDir(bool autodelete=true); + + /** + * DTOR; removes the temporary directory + * + * + * @return + */ + ~TempDir(); + + /** + * Path to the temporary directory + * + * @return the path. + */ + const std::string& path() const { return path_; } +private: + std::string path_; + const bool autodelete_; +}; + +static inline auto format_as(const TempDir& td) { + return td.path(); +} + + +/** + * Temporary (RAII) timezone + */ +struct TempTz { + TempTz(const char* tz) { + if (timezone_available(tz)) + old_tz_ = set_tz(tz); + else + old_tz_ = {}; + mu_debug("timezone '{}' {}available", tz, old_tz_ ? "": "not "); + } + ~TempTz() { + if (old_tz_) { + mu_debug("reset timezone to '{}'", old_tz_); + set_tz(old_tz_); + } + } + bool available() const { return !!old_tz_; } +private: + const char *old_tz_{}; +}; + +} // namepace Mu + + +#endif /* MU_TEST_UTILS_HH__ */ diff --git a/lib/utils/mu-unbroken.hh b/lib/utils/mu-unbroken.hh new file mode 100644 index 0000000..7c431d4 --- /dev/null +++ b/lib/utils/mu-unbroken.hh @@ -0,0 +1,127 @@ +// borrowed from Xapian; slightly adapted + +/* Copyright (c) 2007, 2008 Yung-chung Lin (henearkrxern@gmail.com) + * Copyright (c) 2011 Richard Boulton (richard@tartarus.org) + * Copyright (c) 2011 Brandon Schaefer (brandontschaefer@gmail.com) + * Copyright (c) 2011,2018,2019,2023 Olly Betts + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * deal in the Software without restriction, including without limitation the + * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or + * sell copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#ifndef MU_UNBROKEN_HH__ +#define MU_UNBROKEN_HH__ + +#include <algorithm> +#include <iterator> + +/** + * Does unichar p belong to a script without explicit word separators? + * + * @param p + * + * @return true or false + */ +static inline bool +is_unbroken_script(unsigned p) +{ + // Array containing the last value in each range of codepoints which + // are either all in scripts which are written without explicit word + // breaks, or all not in such scripts. + // + // We only include scripts here which ICU has dictionaries for. The + // same list is currently also used to decide which languages to do + // ngrams for, though perhaps that should use a separate list. + constexpr unsigned splits[] = { + // 0E00..0E7F; Thai, Lanna Tai, Pali + // 0E80..0EFF; Lao + 0x0E00 - 1, 0x0EFF, + // 1000..109F; Myanmar (Burmese) + 0x1000 - 1, 0x109F, + // 1100..11FF; Hangul Jamo + 0x1100 - 1, 0x11FF, + // 1780..17FF; Khmer + 0x1780 - 1, 0x17FF, + // 19E0..19FF; Khmer Symbols + 0x19E0 - 1, 0x19FF, + // 2E80..2EFF; CJK Radicals Supplement + // 2F00..2FDF; Kangxi Radicals + // 2FE0..2FFF; Ideographic Description Characters + // 3000..303F; CJK Symbols and Punctuation + // 3040..309F; Hiragana + // 30A0..30FF; Katakana + // 3100..312F; Bopomofo + // 3130..318F; Hangul Compatibility Jamo + // 3190..319F; Kanbun + // 31A0..31BF; Bopomofo Extended + // 31C0..31EF; CJK Strokes + // 31F0..31FF; Katakana Phonetic Extensions + // 3200..32FF; Enclosed CJK Letters and Months + // 3300..33FF; CJK Compatibility + // 3400..4DBF; CJK Unified Ideographs Extension A + // 4DC0..4DFF; Yijing Hexagram Symbols + // 4E00..9FFF; CJK Unified Ideographs + 0x2E80 - 1, 0x9FFF, + // A700..A71F; Modifier Tone Letters + 0xA700 - 1, 0xA71F, + // A960..A97F; Hangul Jamo Extended-A + 0xA960 - 1, 0xA97F, + // A9E0..A9FF; Myanmar Extended-B (Burmese) + 0xA9E0 - 1, 0xA9FF, + // AA60..AA7F; Myanmar Extended-A (Burmese) + 0xAA60 - 1, 0xAA7F, + // AC00..D7AF; Hangul Syllables + // D7B0..D7FF; Hangul Jamo Extended-B + 0xAC00 - 1, 0xD7FF, + // F900..FAFF; CJK Compatibility Ideographs + 0xF900 - 1, 0xFAFF, + // FE30..FE4F; CJK Compatibility Forms + 0xFE30 - 1, 0xFE4F, + // FF00..FFEF; Halfwidth and Fullwidth Forms + 0xFF00 - 1, 0xFFEF, + // 1AFF0..1AFFF; Kana Extended-B + // 1B000..1B0FF; Kana Supplement + // 1B100..1B12F; Kana Extended-A + // 1B130..1B16F; Small Kana Extension + 0x1AFF0 - 1, 0x1B16F, + // 1F200..1F2FF; Enclosed Ideographic Supplement + 0x1F200 - 1, 0x1F2FF, + // 20000..2A6DF; CJK Unified Ideographs Extension B + 0x20000 - 1, 0x2A6DF, + // 2A700..2B73F; CJK Unified Ideographs Extension C + // 2B740..2B81F; CJK Unified Ideographs Extension D + // 2B820..2CEAF; CJK Unified Ideographs Extension E + // 2CEB0..2EBEF; CJK Unified Ideographs Extension F + 0x2A700 - 1, 0x2EBEF, + // 2F800..2FA1F; CJK Compatibility Ideographs Supplement + 0x2F800 - 1, 0x2FA1F, + // 30000..3134F; CJK Unified Ideographs Extension G + // 31350..323AF; CJK Unified Ideographs Extension H + 0x30000 - 1, 0x323AF + }; + // Binary chop to find the first entry which is >= p. If it's an odd + // offset then the codepoint is in a script which needs splitting; if it's + // an even offset then it's not. + auto it = std::lower_bound(std::begin(splits), + std::end(splits), p); + + return ((it - splits) & 1); +} + + +#endif /* MU_UNBROKEN_HH__ */ diff --git a/lib/utils/mu-utils-file.cc b/lib/utils/mu-utils-file.cc new file mode 100644 index 0000000..3daea34 --- /dev/null +++ b/lib/utils/mu-utils-file.cc @@ -0,0 +1,521 @@ +/* +** Copyright (C) 2023-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include "mu-utils.hh" +#include "mu-utils-file.hh" + +#include <sys/stat.h> +#include <sys/wait.h> + +#include <glib.h> +#include <gio/gio.h> +#include <gio/gunixinputstream.h> + +#ifdef HAVE_WORDEXP_H +#include <wordexp.h> +#endif /*HAVE_WORDEXP_H*/ + +using namespace Mu; + + +bool +Mu::check_dir (const std::string& path, bool readable, bool writeable) +{ + const auto mode = F_OK | (readable ? R_OK : 0) | (writeable ? W_OK : 0); + + if (::access (path.c_str(), mode) != 0) + return false; + + struct stat statbuf{}; + if (::stat (path.c_str(), &statbuf) != 0) + return false; + + return S_ISDIR(statbuf.st_mode) ? true : false; +} + +uint8_t +Mu::determine_dtype (const std::string& path, bool use_lstat) +{ + int res; + struct stat statbuf{}; + + if (use_lstat) + res = ::lstat(path.c_str(), &statbuf); + else + res = ::stat(path.c_str(), &statbuf); + + if (res != 0) { + mu_warning ("{}stat failed on {}: {}", + use_lstat ? "l" : "", path, g_strerror(errno)); + return DT_UNKNOWN; + } + + /* we only care about dirs, regular files and links */ + if (S_ISREG (statbuf.st_mode)) + return DT_REG; + else if (S_ISDIR (statbuf.st_mode)) + return DT_DIR; + else if (S_ISLNK (statbuf.st_mode)) + return DT_LNK; + + return DT_UNKNOWN; +} + +std::string +Mu::canonicalize_filename(const std::string& path, const std::string& relative_to) +{ + auto str{to_string_opt_gchar( + g_canonicalize_filename( + path.c_str(), + relative_to.empty() ? nullptr : relative_to.c_str())).value()}; + + // remove trailing '/'... is this needed? + if (str[str.length()-1] == G_DIR_SEPARATOR) + str.erase(str.length() - 1); + + return str; +} + +std::string +Mu::basename(const std::string& path) +{ + return to_string_gchar(g_path_get_basename(path.c_str())); +} + +std::string +Mu::dirname(const std::string& path) +{ + return to_string_gchar(g_path_get_dirname(path.c_str())); +} + +Result<std::string> +Mu::make_temp_dir() +{ + GError *err{}; + if (auto tmpdir{g_dir_make_tmp("mu-tmp-XXXXXX", &err)}; !tmpdir) + return Err(Error::Code::File, &err, + "failed to create temporary directory"); + else + return Ok(to_string_gchar(std::move(tmpdir))); +} + + +Result<void> +Mu::remove_directory(const std::string& path) +{ + /* ugly */ + GError *err{}; + const auto cmd{mu_format("/bin/rm -rf '{}'", path)}; + if (!g_spawn_command_line_sync(cmd.c_str(), NULL, + NULL, NULL, &err)) + return Err(Error::Code::File, &err, "failed to remove {}", path); + else + return Ok(); +} + + + + +std::string +Mu::runtime_path(Mu::RuntimePath path, const std::string& muhome) +{ + auto [mu_cache, mu_config] = + std::invoke([&]()->std::pair<std::string, std::string> { + if (muhome.empty()) + return { join_paths(g_get_user_cache_dir(), "mu"), + join_paths(g_get_user_config_dir(), "mu")}; + else + return { muhome, muhome }; + }); + + switch (path) { + case Mu::RuntimePath::Cache: + return mu_cache; + case Mu::RuntimePath::XapianDb: + return join_paths(mu_cache, "xapian"); + case Mu::RuntimePath::LogFile: + return join_paths(mu_cache, "mu.log"); + case Mu::RuntimePath::Bookmarks: + return join_paths(mu_config, "bookmarks"); + case Mu::RuntimePath::Config: + return mu_config; + case Mu::RuntimePath::Scripts: + return join_paths(mu_config, "scripts"); + /*LCOV_EXCL_START*/ + default: + throw std::logic_error("unknown path"); + /*LCOV_EXCL_STOP*/ + } +} + +/* LCOV_EXCL_START*/ +static gpointer +cancel_wait(gpointer data) +{ + guint timeout, deadline; + GCancellable *cancel; + + cancel = (GCancellable*)data; + timeout = GPOINTER_TO_UINT(g_object_get_data(G_OBJECT(cancel), "timeout")); + deadline = g_get_monotonic_time() + 1000 * timeout; + + while (g_get_monotonic_time() < deadline && !g_cancellable_is_cancelled(cancel)) { + g_usleep(50 * 1000); /* 50 ms */ + g_thread_yield(); + } + + g_cancellable_cancel(cancel); + + return NULL; +} + +static void +cancel_wait_free(gpointer data) +{ + GThread *thread; + GCancellable *cancel; + + cancel = (GCancellable*)data; + thread = (GThread*)g_object_get_data(G_OBJECT(cancel), "thread"); + + g_cancellable_cancel(cancel); + g_thread_join(thread); +} + +GCancellable* +Mu::g_cancellable_new_with_timeout(guint timeout) +{ + GCancellable *cancel; + + cancel = g_cancellable_new(); + + g_object_set_data(G_OBJECT(cancel), "timeout", GUINT_TO_POINTER(timeout)); + g_object_set_data(G_OBJECT(cancel), "thread", + g_thread_new("cancel-wait", cancel_wait, cancel)); + g_object_set_data_full(G_OBJECT(cancel), "cancel", cancel, cancel_wait_free); + + return cancel; +} +/* LCOV_EXCL_STOP*/ + +/* LCOV_EXCL_START*/ +Result<std::string> +Mu::read_from_stdin() +{ + g_autoptr(GOutputStream) outmem = g_memory_output_stream_new_resizable(); + g_autoptr(GInputStream) input = g_unix_input_stream_new(STDIN_FILENO, TRUE); + //g_autoptr(GCancellable) cancel{maybe_cancellable_timeout(timeout)}; + + GError *err{}; + auto bytes = g_output_stream_splice(outmem, input, + static_cast<GOutputStreamSpliceFlags> + (G_OUTPUT_STREAM_SPLICE_CLOSE_SOURCE | + G_OUTPUT_STREAM_SPLICE_CLOSE_TARGET), + {}, &err); + + if (bytes < 0) + return Err(Error::Code::File, &err, "error reading from pipe"); + + return Ok(std::string{ + static_cast<const char*>(g_memory_output_stream_get_data( + G_MEMORY_OUTPUT_STREAM(outmem))), + g_memory_output_stream_get_size(G_MEMORY_OUTPUT_STREAM(outmem))}); +} +/* LCOV_EXCL_STOP*/ + + +/* + * Set the child to a group leader to avoid being killed when the + * parent group is killed. + */ +/*LCOV_EXCL_START*/ +static void +maybe_setsid (G_GNUC_UNUSED gpointer user_data) +{ +#if HAVE_SETSID + setsid(); +#endif /*HAVE_SETSID*/ +} +/*LCOV_EXCL_STOP*/ + +Result<Mu::CommandOutput> +Mu::run_command(std::initializer_list<std::string> args, bool try_setsid) +{ + std::vector<char*> argvec{}; + for (auto&& arg: args) + argvec.push_back(g_strdup(arg.c_str())); + argvec.push_back({}); + + { + std::vector<std::string> qargs{}; + for(auto&& arg: args) + qargs.emplace_back("'" + arg + "'"); + mu_debug("run-command: {}", fmt::join(qargs, " ")); + } + + GError *err{}; + int wait_status{}; + gchar *std_out{}, *std_err{}; + auto res = g_spawn_sync({}, + static_cast<char**>(argvec.data()), + {}, + (GSpawnFlags)(G_SPAWN_SEARCH_PATH), + try_setsid ? maybe_setsid : nullptr, {}, + &std_out, &std_err, &wait_status, &err); + + for (auto& a: argvec) + g_free(a); + + if (!res) + return Err(Error::Code::File, &err, "failed to execute command"); + else + return Ok(Mu::CommandOutput{ + WEXITSTATUS(wait_status), + to_string_gchar(std::move(std_out/*consumed*/)), + to_string_gchar(std::move(std_err/*consumed*/))}); +} + +Result<Mu::CommandOutput> +Mu::run_command0(std::initializer_list<std::string> args, bool try_setsid) +{ + if (auto&& res{run_command(args, try_setsid)}; !res) + return res; + else if (res->exit_code != 0) + return Err(Error::Code::File, "command returned {}: {}", + res->exit_code, + res->standard_err.empty() ? + std::string{"something went wrong"}: + res->standard_err); + else + return Ok(std::move(*res)); +} + + +Mu::Option<std::string> +Mu::program_in_path(const std::string& name) +{ + if (char *path = g_find_program_in_path(name.c_str()); path) + return to_string_gchar(std::move(path)/*consumes*/); + else + return Nothing; +} + + +/* LCOV_EXCL_START*/ +constexpr auto default_open_program = +#ifdef __APPLE__ + "open" +#else + "xdg-open" +#endif /*!__APPLE__*/ + ; + +Mu::Result<void> +Mu::play (const std::string& path) +{ + /* check nativity */ + GFile *gf = g_file_new_for_path(path.c_str()); + auto is_native = g_file_is_native(gf); + g_object_unref(gf); + if (!is_native) + return Err(Error::Code::File, "'{}' is not a native file", path); + + auto mpp{g_getenv ("MU_PLAY_PROGRAM")}; + const std::string prog{mpp ? mpp : default_open_program}; + + const auto program_path{program_in_path(prog)}; + if (!program_path) + return Err(Error::Code::File, "cannot find '{}' in path", prog); + else if (auto&& res{run_command({*program_path, path}, true/*try-setsid*/)}; !res) + return Err(std::move(res.error())); + else + return Ok(); +} +/* LCOV_EXCL_STOP*/ + + +Result<std::string> +expand_path_real(const std::string& str) +{ +#ifndef HAVE_WORDEXP_H + return Ok(std::string{str}); +#else + int res; + wordexp_t result{}; + + res = wordexp(str.c_str(), &result, 0); + if (res != 0) + return Err(Error::Code::File, "cannot expand {}; err={}", str, res); + else if (auto&n = result.we_wordc; n != 1) { + wordfree(&result); + return Err(Error::Code::File, "expected 1 expansions, but got {} for {}", n, str); + } + + std::string expanded{result.we_wordv[0]}; + wordfree(&result); + + return Ok(std::move(expanded)); + +#endif /*HAVE_WORDEXP_H*/ +} + + +Result<std::string> +Mu::expand_path(const std::string& str) +{ + if (auto&& res{expand_path_real(str)}; res) + return res; + + // failed... try quoting. + auto qstr{to_string_gchar(g_shell_quote(str.c_str()))}; + return expand_path_real(qstr); +} + + + +#ifdef BUILD_TESTS + +/* + * Tests. + * + */ + +#include <glib/gstdio.h> +#include <fcntl.h> +#include <sys/types.h> +#include <sys/stat.h> + +#include "utils/mu-test-utils.hh" + +static void +test_check_dir_01(void) +{ + if (g_access("/usr/bin", F_OK) == 0) { + g_assert_cmpuint( + check_dir("/usr/bin", true, false) == true, + ==, + g_access("/usr/bin", R_OK) == 0); + } +} + +static void +test_check_dir_02(void) +{ + if (g_access("/tmp", F_OK) == 0) { + g_assert_cmpuint( + check_dir("/tmp", false, true) == true, + ==, + g_access("/tmp", W_OK) == 0); + } +} + +static void +test_check_dir_03(void) +{ + if (g_access(".", F_OK) == 0) { + g_assert_cmpuint( + check_dir(".", true, true) == true, + ==, + g_access(".", W_OK | R_OK) == 0); + } +} + +static void +test_check_dir_04(void) +{ + /* not a dir, so it must be false */ + g_assert_cmpuint( + check_dir("test-util.c", true, true), + ==, + false); +} + +static void +test_determine_dtype_with_lstat(void) +{ + g_assert_cmpuint( + determine_dtype(MU_TESTMAILDIR, true), ==, DT_DIR); + g_assert_cmpuint( + determine_dtype(MU_TESTMAILDIR2, true), ==, DT_DIR); + g_assert_cmpuint( + determine_dtype(MU_TESTMAILDIR2 "/Foo/cur/mail5", true), + ==, DT_REG); +} + + +static void +test_program_in_path(void) +{ + g_assert_true(!!program_in_path("ls")); +} + +static void +test_join_paths() +{ + + assert_equal(join_paths(), ""); + assert_equal(join_paths("a"), "a"); + assert_equal(join_paths("a", "b"), "a/b"); + assert_equal(join_paths("/a/b///c/d//", "e"), "/a/b/c/d/e"); +} + +static void +test_runtime_paths() +{ + TempDir tdir; + + assert_equal(runtime_path(RuntimePath::Cache, tdir.path()), tdir.path()); + assert_equal(runtime_path(RuntimePath::XapianDb, tdir.path()), + join_paths(tdir.path(), "xapian")); + assert_equal(runtime_path(RuntimePath::Bookmarks, tdir.path()), + join_paths(tdir.path(), "bookmarks")); + assert_equal(runtime_path(RuntimePath::Config, tdir.path()), tdir.path()); + assert_equal(runtime_path(RuntimePath::Scripts, tdir.path()), + join_paths(tdir.path(), "scripts")); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + /* check_dir */ + g_test_add_func("/utils/check-dir-01", + test_check_dir_01); + g_test_add_func("/utils/check-dir-02", + test_check_dir_02); + g_test_add_func("/utils/check-dir-03", + test_check_dir_03); + g_test_add_func("/utils/check-dir-04", + test_check_dir_04); + g_test_add_func("/utils/determine-dtype-with-lstat", + test_determine_dtype_with_lstat); + g_test_add_func("/utils/program-in-path", + test_program_in_path); + g_test_add_func("/utils/join-paths", + test_join_paths); + g_test_add_func("/utils/runtime-paths", + test_runtime_paths); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/lib/utils/mu-utils-file.hh b/lib/utils/mu-utils-file.hh new file mode 100644 index 0000000..7eb3ba5 --- /dev/null +++ b/lib/utils/mu-utils-file.hh @@ -0,0 +1,275 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_UTILS_FILE_HH__ +#define MU_UTILS_FILE_HH__ + +#include <string> +#include <cinttypes> +#include <sys/stat.h> + +#include <gio/gio.h> +#include <utils/mu-option.hh> +#include <utils/mu-result.hh> +#include <utils/mu-regex.hh> + +namespace Mu { + +/** + * Check if the directory has the given attributes + * + * @param path path to dir + * @param readable is it readable? false means "don't care" + * @param writeable is it writable? false means "don't care" + * + * @return true if is is a directory with given attributes; false otherwise. + */ +bool check_dir(const std::string& path, bool readable=false, bool writeable=false); + +/** + * See g_canonicalize_filename + * + * @param filename + * @param relative_to + * + * @return + */ +std::string canonicalize_filename(const std::string& path, const std::string& relative_to=""); + +/** + * Expand the filesystem path (as per wordexp(3)) + * + * @param str a filesystem path string + * + * @return the expanded string or some error + */ +Result<std::string> expand_path(const std::string& str); + + +/** + * Get the basename for path, i.e. without leading directory component, + * @see g_path_get_basename + * + * @param path + * + * @return the basename + */ +std::string basename(const std::string& path); + + +/** + * Get the dirname for path, i.e. without leading directory component, + * @see g_path_get_dirname + * + * @param path + * + * @return the dirname + */ +std::string dirname(const std::string& path); + + +/* + * for OSs without support for direntry->d_type, like Solaris + */ +#ifndef DT_UNKNOWN +enum { + DT_UNKNOWN = 0, +#define DT_UNKNOWN DT_UNKNOWN + DT_FIFO = 1, +#define DT_FIFO DT_FIFO + DT_CHR = 2, +#define DT_CHR DT_CHR + DT_DIR = 4, +#define DT_DIR DT_DIR + DT_BLK = 6, +#define DT_BLK DT_BLK + DT_REG = 8, +#define DT_REG DT_REG + DT_LNK = 10, +#define DT_LNK DT_LNK + DT_SOCK = 12, +#define DT_SOCK DT_SOCK + DT_WHT = 14 +#define DT_WHT DT_WHT +}; +#endif /*DT_UNKNOWN*/ + + /** + * get the d_type (as in direntry->d_type) for the file at path, using either + * stat(3) or lstat(3) + * + * @param path full path + * @param use_lstat whether to use lstat (otherwise use stat) + * + * @return DT_REG, DT_DIR, DT_LNK, or DT_UNKNOWN (other values are not supported + * currently) + */ +uint8_t determine_dtype(const std::string& path, bool use_lstat=false); + + +/** + * Well-known runtime paths + * + */ +enum struct RuntimePath { + XapianDb, + Cache, + LogFile, + Config, + Scripts, + Bookmarks +}; + +/** + * Get some well-known Path for internal use when don't have + * access to the command-line + * + * @param path the RuntimePath to find + * @param muhome path to muhome directory, or empty for the default. + * + * @return the path name + */ +std::string runtime_path(RuntimePath path, const std::string& muhome=""); + +/** + * Join path components into a path (with '/') + * + * @param s a string-convertible value + * @param args 0 or more string-convertible values + * + * @return the path + */ +static inline std::string join_paths() { return {}; } +template<typename S> std::string join_paths_(S&& s) { return std::string{s}; } +template<typename S, typename...Args> +std::string join_paths_(S&& s, Args...args) { + + static std::string sepa{"/"}; + auto&& str{std::string{std::forward<S>(s)}}; + if (auto&& rest{join_paths_(std::forward<Args>(args)...)}; !rest.empty()) + str += (sepa + rest); + return str; +} + +template<typename S, typename...Args> +std::string join_paths(S&& s, Args...args) { + + constexpr auto sepa = '/'; + auto path = join_paths_(std::forward<S>(s), std::forward<Args>(args)...); + + auto c{0U}; + while (c < path.size()) { + + if (path[c] != sepa) { + ++c; + continue; + } + + while (path[++c] == '/') { + path.erase(c, 1); + --c; + } + } + + return path; +} + + +/** + * Like g_cancellable_new(), but automatically cancels itself + * after timeout + * + * @param timeout timeout in millisecs + * + * @return A GCancellable* instances; free with g_object_unref() when + * no longer needed. + */ +GCancellable* g_cancellable_new_with_timeout(guint timeout); + +/** + * Read for standard input + * + * @return data from standard input or an error. + */ +Result<std::string> read_from_stdin(); + +/** + * Create a randomly-named temporary directory + * + * @return name of the temporary directory or an error. + */ +Result<std::string> make_temp_dir(); + + +/** + * Remove a directory, recursively. Does not have to be empty. + * + * @param path path to directory + * + * @return Ok() or an error. + */ +Result<void> remove_directory(const std::string& path); + +/** + * Run some system command. + * + * @param args a list of commmand line arguments (like argv) + * @param try_setsid whether to try setsid(2) (see its manpage for details) if this + * system supports it. + * + * @return Ok(exit code) or an error. Note that exit-code != 0 is _not_ + * considered an error from the perspective of run_command, but is for + * run_command0 + */ +struct CommandOutput { + int exit_code; + std::string standard_out; + std::string standard_err; +}; +Result<CommandOutput> run_command(std::initializer_list<std::string> args, + bool try_setsid=false); +Result<CommandOutput> run_command0(std::initializer_list<std::string> args, + bool try_setsid=false); + +/** + * Try to 'play' (ie., open with it's associated program) a file. On MacOS, the + * the program 'open' is used for this; on other platforms 'xdg-open' to do the + * actual opening. In addition you can set it to another program by setting thep + * MU_PLAY_PROGRAM environment variable + * + * This requires a 'native' file, see g_file_is_native() + * + * @param path full path of the file to open + * + * @return Ok() if succeeded, some error otherwise. + */ +Result<void> play(const std::string& path); + +/** + * Find program in PATH + * + * @param name the name of the program + * + * @return either the full path to program, or Nothing if not found. + */ +Option<std::string> program_in_path(const std::string& name); + +} // namespace Mu + +#endif /* MU_UTILS_FILE_HH__ */ diff --git a/lib/utils/mu-utils.cc b/lib/utils/mu-utils.cc new file mode 100644 index 0000000..6d36dc1 --- /dev/null +++ b/lib/utils/mu-utils.cc @@ -0,0 +1,713 @@ +/* +** Copyright (C) 2017-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This library is free software; you can redistribute it and/or +** modify it under the terms of the GNU Lesser General Public License +** as published by the Free Software Foundation; either version 2.1 +** of the License, or (at your option) any later version. +** +** This library is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +** Lesser General Public License for more details. +** +** You should have received a copy of the GNU Lesser General Public +** License along with this library; if not, write to the Free +** Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA +** 02110-1301, USA. +*/ + +#ifndef _XOPEN_SOURCE +#define _XOPEN_SOURCE +#include <stdexcept> +#endif /*_XOPEN_SOURCE*/ + +#include <array> + +#include <time.h> + +#define GNU_SOURCE +#include <stdio.h> +#include <stdint.h> +#include <unistd.h> + +#include <string.h> +#include <iostream> +#include <algorithm> +#include <numeric> +#include <functional> +#include <cinttypes> +#include <charconv> +#include <limits> + +#include <glib.h> +#include <glib/gprintf.h> + +#include "mu-utils.hh" +#include "mu-unbroken.hh" + +#include "mu-error.hh" +#include "mu-option.hh" + +using namespace Mu; + +namespace { + +static gunichar +unichar_tolower(gunichar uc) +{ + if (!g_unichar_isalpha(uc)) + return uc; + + if (g_unichar_get_script(uc) != G_UNICODE_SCRIPT_LATIN) + return g_unichar_tolower(uc); + + switch (uc) { + case 0x00e6: + case 0x00c6: return 'e'; /* æ */ + case 0x00f8: return 'o'; /* ø */ + case 0x0110: + case 0x0111: + return 'd'; /* đ */ + /* todo: many more */ + default: return g_unichar_tolower(uc); + } +} + +/** + * gx_utf8_flatten: + * @str: a UTF-8 string + * @len: the length of @str, or -1 if it is %NULL-terminated + * + * Flatten some UTF-8 string; that is, downcase it and remove any diacritics. + * + * Returns: (transfer full): a flattened string, free with g_free(). + */ +static char* +gx_utf8_flatten(const gchar* str, gssize len) +{ + GString* gstr; + char * norm, *cur; + + g_return_val_if_fail(str, NULL); + + norm = g_utf8_normalize(str, len, G_NORMALIZE_ALL); + if (!norm) + return NULL; + + gstr = g_string_sized_new(strlen(norm)); + + for (cur = norm; cur && *cur; cur = g_utf8_next_char(cur)) { + gunichar uc; + + uc = g_utf8_get_char(cur); + if (g_unichar_combining_class(uc) != 0) + continue; + + g_string_append_unichar(gstr, unichar_tolower(uc)); + } + + g_free(norm); + + return g_string_free(gstr, FALSE); +} + +} // namespace + +bool +Mu::contains_unbroken_script(const char *str) +{ + while (str && *str) { + auto uc = g_utf8_get_char(str); + if (is_unbroken_script(uc)) + return true; + str = g_utf8_next_char(str); + } + + return false; +} + +std::string // gx_utf8_flatten +Mu::utf8_flatten(const char* str) +{ + if (!str) + return {}; + + if (contains_unbroken_script(str)) + return std::string{str}; + + // the pure-ascii case + if (g_str_is_ascii(str)) { + auto l = g_ascii_strdown(str, -1); + std::string s{l}; + g_free(l); + return s; + } + + // seems we need the big guns + char* flat = gx_utf8_flatten(str, -1); + if (!flat) + return {}; + + std::string s{flat}; + g_free(flat); + + return s; +} + + +/* turn \0-terminated buf into ascii (which is a utf8 subset); convert + * any non-ascii into '.' + */ +static char* +asciify_in_place (char *buf) +{ + char *c; + + g_return_val_if_fail (buf, NULL); + + for (c = buf; c && *c; ++c) { + if ((!isprint(*c) && !isspace (*c)) || !isascii(*c)) + *c = '.'; + } + + return buf; +} + +static char* +utf8ify (const char *buf) +{ + char *utf8; + + g_return_val_if_fail (buf, NULL); + + utf8 = g_strdup (buf); + + if (!g_utf8_validate (buf, -1, NULL)) + asciify_in_place (utf8); + + return utf8; +} + + +std::string +Mu::utf8_clean(const std::string& dirty) +{ + g_autoptr(GString) gstr = g_string_sized_new(dirty.length()); + g_autofree char *cstr = utf8ify(dirty.c_str()); + + for (auto cur = cstr; cur && *cur; cur = g_utf8_next_char(cur)) { + const gunichar uc = g_utf8_get_char(cur); + if (g_unichar_iscntrl(uc)) + g_string_append_c(gstr, ' '); + else + g_string_append_unichar(gstr, uc); + } + + return std::string{g_strstrip(gstr->str)}; +} + + +std::string +Mu::utf8_wordbreak(const std::string& txt) +{ + g_autoptr(GString) gstr = g_string_sized_new(txt.length()); + + bool spc{}; + for (auto cur = txt.c_str(); cur && *cur; cur = g_utf8_next_char(cur)) { + const gunichar uc = g_utf8_get_char(cur); + + if (g_unichar_iscntrl(uc)) { + g_string_append_c(gstr, ' '); + continue; + } + // inspired by Xapian's termgenerator. + + switch(uc) { + case '\'': + case '&': + case 0xb7: + case 0x5f4: + case 0x2019: + case 0x201b: + case 0x2027: + case ',': + case '.': + case ';': + case '+': + case '#': + case '-': + case 0x037e: // GREEK QUESTION MARK + case 0x0589: // ARMENIAN FULL STOP + case 0x060D: // ARABIC DATE SEPARATOR + case 0x07F8: // NKO COMMA + case 0x2044: // FRACTION SLASH + case 0xFE10: // PRESENTATION FORM FOR VERTICAL COMMA + case 0xFE13: // PRESENTATION FORM FOR VERTICAL COLON + case 0xFE14: // PRESENTATION FORM FOR VERTICAL SEMICOLON + if (spc) + break; + spc = true; + g_string_append_c(gstr, ' '); + break; + default: + spc = false; + g_string_append_unichar(gstr, uc); + break; + } + } + + return std::string{g_strstrip(gstr->str)}; +} + + +std::string +Mu::remove_ctrl(const std::string& str) +{ + char prev{'\0'}; + std::string result; + result.reserve(str.length()); + + for (auto&& c : str) { + if (::iscntrl(c) || c == ' ') { + if (prev != ' ') + result += prev = ' '; + } else + result += prev = c; + } + + return result; +} + +std::vector<std::string> +Mu::split(const std::string& str, const std::string& sepa) +{ + std::vector<std::string> vec; + size_t b = 0, e = 0; + + /* special cases */ + if (str.empty()) + return vec; + else if (sepa.empty()) { + for (auto&& c: str) + vec.emplace_back(1, c); + return vec; + } + + while (true) { + if (e = str.find(sepa, b); e != std::string::npos) { + vec.emplace_back(str.substr(b, e - b)); + b = e + sepa.length(); + } else { + vec.emplace_back(str.substr(b)); + break; + } + } + + return vec; +} + +std::vector<std::string> +Mu::split(const std::string& str, char sepa) +{ + std::vector<std::string> vec; + size_t b = 0, e = 0; + + /* special case */ + if (str.empty()) + return vec; + + while (true) { + if (e = str.find(sepa, b); e != std::string::npos) { + vec.emplace_back(str.substr(b, e - b)); + b = e + sizeof(sepa); + } else { + vec.emplace_back(str.substr(b)); + break; + } + } + + return vec; +} + +std::string +Mu::join(const std::vector<std::string>& svec, const std::string& sepa) +{ + if (svec.empty()) + return {}; + + + /* calculate the overall size beforehand, to avoid re-allocations. */ + size_t value_len = + std::accumulate(svec.cbegin(), svec.cend(), 0, + [](size_t size, const std::string& s) { + return size + s.size(); + }) + (svec.size() - 1) * sepa.length(); + + std::string value; + value.reserve(value_len); + + std::accumulate(svec.cbegin(), svec.cend(), std::ref(value), + [&](std::string& s1, const std::string& s2)->std::string& { + if (s1.empty()) + s1 = s2; + else { + s1.append(sepa); + s1.append(s2); + } + return s1; + }); + + return value; +} + +std::string +Mu::quote(const std::string& str) +{ + std::string res{"\""}; + + for (auto&& k : str) { + switch (k) { + case '"': res += "\\\""; break; + case '\\': res += "\\\\"; break; + default: res += k; + } + } + + return res + "\""; +} + +static Option<::time_t> +delta_ymwdhMs(const std::string& expr) +{ + char* endptr; + auto num = strtol(expr.c_str(), &endptr, 10); + if (num <= 0 || num > 9999 || !endptr || !*endptr) + return Nothing; + + int years, months, weeks, days, hours, minutes, seconds; + years = months = weeks = days = hours = minutes = seconds = 0; + + switch (endptr[0]) { + case 's': seconds = num; break; + case 'M': minutes = num; break; + case 'h': hours = num; break; + case 'd': days = num; break; + case 'w': weeks = num; break; + case 'm': months = num; break; + case 'y': years = num; break; + default: + return Nothing; + } + + GDateTime *then, *now = g_date_time_new_now_local(); + if (weeks != 0) + then = g_date_time_add_weeks(now, -weeks); + else + then = + g_date_time_add_full(now, -years, -months, -days, -hours, -minutes, -seconds); + + auto t = std::max<::time_t>(0, g_date_time_to_unix(then)); + + g_date_time_unref(then); + g_date_time_unref(now); + + return t; +} + +static Option<::time_t> +special_date_time(const std::string& d, bool is_first) +{ + if (d == "now") + return ::time({}); + + if (d == "today") { + GDateTime *dt, *midnight; + dt = g_date_time_new_now_local(); + + if (!is_first) { + GDateTime* tmp = dt; + dt = g_date_time_add_days(dt, 1); + g_date_time_unref(tmp); + } + + midnight = g_date_time_add_full(dt, + 0, + 0, + 0, + -g_date_time_get_hour(dt), + -g_date_time_get_minute(dt), + -g_date_time_get_second(dt)); + time_t t = MAX(0, (gint64)g_date_time_to_unix(midnight)); + g_date_time_unref(dt); + g_date_time_unref(midnight); + + return t; + } + + return Nothing; +} + +// if a date has a month day greater than the number of days in that month, +// change it to a valid date point to the last second in that month +static void +fixup_month(struct tm* tbuf) +{ + decltype(tbuf->tm_mday) max_days; + const auto month = tbuf->tm_mon + 1; + const auto year = tbuf->tm_year + 1900; + + switch (month) { + case 2: + if (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)) + max_days = 29; + else + max_days = 28; + break; + case 4: + case 6: + case 9: + case 11: + max_days = 30; + break; + default: + max_days = 31; + break; + } + + if (tbuf->tm_mday > max_days) { + tbuf->tm_mday = max_days; + tbuf->tm_hour = 23; + tbuf->tm_min = 59; + tbuf->tm_sec = 59; + } + } + + +Option<::time_t> +Mu::parse_date_time(const std::string& dstr, bool is_first, bool utc) +{ + struct tm tbuf{}; + GDateTime *dtime{}; + gint64 t; + + /* one-sided dates */ + if (dstr.empty()) + return is_first ? 0 : G_MAXINT64; + else if (dstr == "today" || dstr == "now") + return special_date_time(dstr, is_first); + else if (dstr.find_first_of("ymdwhMs") != std::string::npos) + return delta_ymwdhMs(dstr); + + constexpr char UserDateMin[] = "19700101000000"; + constexpr char UserDateMax[] = "29991231235959"; + + std::string date(is_first ? UserDateMin : UserDateMax); + std::copy_if(dstr.begin(), dstr.end(), date.begin(), [](auto c) { return isdigit(c); }); + + if (!::strptime(date.c_str(), "%Y%m%d%H%M%S", &tbuf) && + !::strptime(date.c_str(), "%Y%m%d%H%M", &tbuf) && + !::strptime(date.c_str(), "%Y%m%d%H", &tbuf) && + !::strptime(date.c_str(), "%Y%m%d", &tbuf) && + !::strptime(date.c_str(), "%Y%m", &tbuf) && + !::strptime(date.c_str(), "%Y", &tbuf)) + return Nothing; + + fixup_month(&tbuf); + dtime = utc ? + g_date_time_new_utc(tbuf.tm_year + 1900, + tbuf.tm_mon + 1, + tbuf.tm_mday, + tbuf.tm_hour, + tbuf.tm_min, + tbuf.tm_sec) : + g_date_time_new_local(tbuf.tm_year + 1900, + tbuf.tm_mon + 1, + tbuf.tm_mday, + tbuf.tm_hour, + tbuf.tm_min, + tbuf.tm_sec); + + t = g_date_time_to_unix(dtime); + g_date_time_unref(dtime); + + return to_time_t(t); +} + + +Option<int64_t> +Mu::parse_size(const std::string& val, bool is_first) +{ + int64_t size{-1}; + std::string str; + GRegex* rx; + GMatchInfo* minfo; + + /* one-sided ranges */ + if (val.empty()) + return is_first ? 0 : std::numeric_limits<int64_t>::max(); + + rx = g_regex_new("^(\\d+)(b|k|kb|m|mb|g|gb)?$", + G_REGEX_CASELESS, (GRegexMatchFlags)0, NULL); + minfo = NULL; + if (g_regex_match(rx, val.c_str(), (GRegexMatchFlags)0, &minfo)) { + + char* s; + s = g_match_info_fetch(minfo, 1); + size = atoll(s); + g_free(s); + + s = g_match_info_fetch(minfo, 2); + switch (s ? g_ascii_tolower(s[0]) : 0) { + case 'k': size *= 1024; break; + case 'm': size *= (1024 * 1024); break; + case 'g': size *= (1024 * 1024 * 1024); break; + default: break; + } + + g_free(s); + } + + g_regex_unref(rx); + g_match_info_unref(minfo); + + if (size < 0) + return Nothing; + else + return size; + +} + +std::string +Mu::to_lexnum(int64_t val) +{ + char buf[18]; /* 1 byte prefix + hex + \0 */ + buf[0] = 'f' + ::snprintf(buf + 1, sizeof(buf) - 1, "%" PRIx64, val); + return buf; +} + +int64_t +Mu::from_lexnum(const std::string& str) +{ + int64_t val{}; + std::from_chars(str.c_str() + 1, str.c_str() + str.size(), val, 16); + + return val; +} + +bool +Mu::locale_workaround() try +{ + // quite horrible... but some systems break otherwise with + // https://github.com/djcb/mu/issues/2252 + + try { + std::locale::global(std::locale("")); + } catch (const std::runtime_error& re) { + g_setenv("LC_ALL", "C", 1); + std::locale::global(std::locale("")); + } + + return true; + +} catch (...) { + return false; +} + +bool +Mu::timezone_available(const std::string& tz) +{ + const auto old_tz = g_getenv("TZ"); + + g_setenv("TZ", tz.c_str(), TRUE); + + auto tzone = g_time_zone_new_local (); + bool have_tz = g_strcmp0(g_time_zone_get_identifier(tzone), tz.c_str()) == 0; + g_time_zone_unref (tzone); + + if (old_tz) + g_setenv("TZ", old_tz, TRUE); + else + g_unsetenv("TZ"); + + return have_tz; +} + +std::string +Mu::summarize(const std::string& str, size_t max_lines) +{ + size_t nl_seen; + unsigned i,j; + gboolean last_was_blank; + + if (str.empty()) + return {}; + + /* len for summary <= original len */ + char *summary = g_new (gchar, str.length() + 1); + + /* copy the string up to max_lines lines, replace CR/LF/tab with + * single space */ + for (i = j = 0, nl_seen = 0, last_was_blank = TRUE; + nl_seen < max_lines && i < str.length(); ++i) { + + if (str[i] == '\n' || str[i] == '\r' || + str[i] == '\t' || str[i] == ' ' ) { + + if (str[i] == '\n') + ++nl_seen; + + /* no double-blanks or blank at end of str */ + if (!last_was_blank && str[i+1] != '\0') + summary[j++] = ' '; + + last_was_blank = TRUE; + } else { + + summary[j++] = str[i]; + last_was_blank = FALSE; + } + } + + summary[j] = '\0'; + + return to_string_gchar(std::move(summary)/*consumes*/); +} + + + + +static bool +locale_is_utf8 (void) +{ + const gchar *dummy; + static int is_utf8 = -1; + if (G_UNLIKELY(is_utf8 == -1)) + is_utf8 = g_get_charset(&dummy) ? 1 : 0; + + return !!is_utf8; +} + +bool +Mu::fputs_encoded (const std::string& str, FILE *stream) +{ + g_return_val_if_fail (stream, false); + + /* g_get_charset return TRUE when the locale is UTF8 */ + if (locale_is_utf8()) + return ::fputs (str.c_str(), stream) == EOF ? false: true; + + /* charset is _not_ utf8, so we need to convert it */ + char *conv{}; + if (g_utf8_validate (str.c_str(), -1, NULL)) + conv = g_locale_from_utf8 (str.c_str(), -1, {}, {}, {}); + + /* conversion failed; this happens because is some cases GMime may gives + * us non-UTF-8 strings from e.g. wrongly encoded message-subjects; if + * so, we escape the string */ + conv = conv ? conv : g_strescape (str.c_str(), "\n\t"); + int rv = conv ? ::fputs (conv, stream) : EOF; + g_free (conv); + + return (rv == EOF) ? false: true; +} diff --git a/lib/utils/mu-utils.hh b/lib/utils/mu-utils.hh new file mode 100644 index 0000000..783351f --- /dev/null +++ b/lib/utils/mu-utils.hh @@ -0,0 +1,640 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This library is free software; you can redistribute it and/or +** modify it under the terms of the GNU Lesser General Public License +** as published by the Free Software Foundation; either version 2.1 +** of the License, or (at your option) any later version. +** +** This library is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +** Lesser General Public License for more details. +** +** You should have received a copy of the GNU Lesser General Public +** License along with this library; if not, write to the Free +** Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA +** 02110-1301, USA. +*/ + +#ifndef MU_UTILS_HH__ +#define MU_UTILS_HH__ + +#include <string> +#include <string_view> +#include <sstream> +#include <vector> +#include <chrono> +#include <memory> +#include <cstdarg> +#include <glib.h> +#include <ostream> +#include <iostream> +#include <type_traits> +#include <algorithm> +#include <numeric> + +#include "mu-option.hh" + +#ifndef FMT_HEADER_ONLY +#define FMT_HEADER_ONLY +#endif /*FMT_HEADER_ONLY*/ +#include <fmt/format.h> +#include <fmt/core.h> +#include <fmt/chrono.h> +#include <fmt/ostream.h> + +namespace Mu { + +/* + * Separator characters used in various places; importantly, + * they are not used in UTF-8 + */ +constexpr const auto SepaChar1 = '\xfe'; +constexpr const auto SepaChar2 = '\xff'; + +/* + * Logging/printing/formatting functions connect libfmt with the Glib logging + * system. We wrap so perhaps at some point (C++23?) we can use std:: instead. + */ + +/* + * Debug/error/warning logging + * + * The 'noexcept' means that they _wilL_ terminate the program + * when the formatting fails (ie. a bug) + */ + +template<typename...T> +void mu_debug(fmt::format_string<T...> frm, T&&... args) noexcept { + g_log("mu", G_LOG_LEVEL_DEBUG, "%s", + fmt::format(frm, std::forward<T>(args)...).c_str()); +} +template<typename...T> +void mu_info(fmt::format_string<T...> frm, T&&... args) noexcept { + g_log("mu", G_LOG_LEVEL_INFO, "%s", + fmt::format(frm, std::forward<T>(args)...).c_str()); +} +template<typename...T> +void mu_message(fmt::format_string<T...> frm, T&&... args) noexcept { + g_log("mu", G_LOG_LEVEL_MESSAGE, "%s", + fmt::format(frm, std::forward<T>(args)...).c_str()); +} +template<typename...T> +void mu_warning(fmt::format_string<T...> frm, T&&... args) noexcept { + g_log("mu", G_LOG_LEVEL_WARNING, "%s", + fmt::format(frm, std::forward<T>(args)...).c_str()); +} +/* LCOV_EXCL_START*/ +template<typename...T> +void mu_critical(fmt::format_string<T...> frm, T&&... args) noexcept { + g_log("mu", G_LOG_LEVEL_CRITICAL, "%s", + fmt::format(frm, std::forward<T>(args)...).c_str()); +} +template<typename...T> +void mu_error(fmt::format_string<T...> frm, T&&... args) noexcept { + g_log("mu", G_LOG_LEVEL_ERROR, "%s", + fmt::format(frm, std::forward<T>(args)...).c_str()); +} +/* LCOV_EXCL_STOP*/ + +/* + * Printing; add our wrapper functions, one day we might be able to use std:: + */ + +template<typename...T> +void mu_print(fmt::format_string<T...> frm, T&&... args) noexcept { + fmt::print(frm, std::forward<T>(args)...); +} +template<typename...T> +void mu_println(fmt::format_string<T...> frm, T&&... args) noexcept { + fmt::println(frm, std::forward<T>(args)...); +} + +template<typename...T> +void mu_printerr(fmt::format_string<T...> frm, T&&... args) noexcept { + fmt::print(stderr, frm, std::forward<T>(args)...); +} +template<typename...T> +void mu_printerrln(fmt::format_string<T...> frm, T&&... args) noexcept { + fmt::println(stderr, frm, std::forward<T>(args)...); +} + + +/* stream */ +template<typename...T> +void mu_print(std::ostream& os, fmt::format_string<T...> frm, T&&... args) noexcept { + fmt::print(os, frm, std::forward<T>(args)...); +} +template<typename...T> +void mu_println(std::ostream& os, fmt::format_string<T...> frm, T&&... args) noexcept { + fmt::println(os, frm, std::forward<T>(args)...); +} + +/* + * Fprmatting + */ +template<typename...T> +std::string mu_format(fmt::format_string<T...> frm, T&&... args) noexcept { + return fmt::format(frm, std::forward<T>(args)...); +} + +template<typename Range> +auto mu_join(Range&& range, std::string_view sepa) { + return fmt::join(std::forward<Range>(range), sepa); +} + +template <typename T=::time_t> +std::tm mu_time(T t={}, bool use_utc=false) { + ::time_t tt{static_cast<::time_t>(t)}; + return use_utc ? fmt::gmtime(tt) : fmt::localtime(tt); +} + +using StringVec = std::vector<std::string>; + +/** + * Does the string contain script without explicit word separators? + * + * @param str a string + * + * @return true or false + */ +bool contains_unbroken_script(const char* str); +static inline bool contains_unbroken_script(const std::string& str) { + return contains_unbroken_script(str.c_str()); +} + +/** + * Flatten a string -- down-case and fold diacritics. + * + * @param str a string + * + * @return a flattened string + */ +std::string utf8_flatten(const char* str); +inline std::string +utf8_flatten(const std::string& s) { + return utf8_flatten(s.c_str()); +} + +/** + * Replace all control characters with spaces, and remove leading and trailing space. + * + * @param dirty an unclean string + * + * @return a cleaned-up string. + */ +std::string utf8_clean(const std::string& dirty); + + +/** + * Replace all wordbreak chars (as recognized by Xapian by single SPC) + * + * @param txt text + * + * @return string + */ +std::string utf8_wordbreak(const std::string& txt); + + +/** + * Remove ctrl characters, replacing them with ' '; subsequent + * ctrl characters are replaced by a single ' ' + * + * @param str a string + * + * @return the string without control characters + */ +std::string remove_ctrl(const std::string& str); + +/** + * Split a string in parts. As a special case, splitting an empty string + * yields an empty vector (not a vector with a single empty element) + * + * @param str a string + * @param sepa the separator + * + * @return the parts. + */ +std::vector<std::string> split(const std::string& str, const std::string& sepa); + +/** + * Split a string in parts. As a special case, splitting an empty string + * yields an empty vector (not a vector with a single empty element) + * + * @param str a string + * @param sepa the separator + * + * @return the parts. + */ +std::vector<std::string> split(const std::string& str, char sepa); + +/** + * Join the strings in svec into a string, separated by sepa + * + * @param svec a string vector + * @param sepa separator + * + * @return string + */ +std::string join(const std::vector<std::string>& svec, const std::string& sepa); +static inline std::string join(const std::vector<std::string>& svec, char sepa) { + return join(svec, std::string(1, sepa)); +} + +/** + * write a string (assumed to be in utf8-format) to a stream, + * converted to the current locale + * + * @param str a string + * @param stream a stream + * + * @return true if printing worked, false otherwise + */ +bool fputs_encoded (const std::string& str, FILE *stream); + +/** + * print a fmt-style formatted string (assumed to be in utf8-format) to stdout, + * converted to the current locale + * + * @param a standard fmt-style format string, followed by a parameter list + * + * @return true if printing worked, false otherwise + */ +template<typename...T> +static inline bool mu_print_encoded(fmt::format_string<T...> frm, T&&... args) noexcept { + return fputs_encoded(fmt::format(frm, std::forward<T>(args)...), + stdout); +} + +/** + * Convert an int64_t to a time_t, clamping it within the range. + * + * This is only doing anything when using a 32-bit time_t value. This doesn't + * solve the 3038 problem, but at least allows for clearly marking where we + * convert + * + * @param t some 64-bit value that encodes a Unix time. + * + * @return a time_t value + */ +constexpr ::time_t time_t_min = 0; +constexpr ::time_t time_t_max = std::numeric_limits<::time_t>::max(); +constexpr ::time_t to_time_t(int64_t t) { + return std::clamp(t, + static_cast<int64_t>(time_t_min), + static_cast<int64_t>(time_t_max)); +} + + +/** + * Parse a date string to the corresponding time_t + * * + * @param date the date expressed a YYYYMMDDHHMMSS or any n... of the first + * characters, using the local timezone. Non-digits are ignored, + * so 2018-05-05 is equivalent to 20180505. + * @param first whether to fill out incomplete dates to the start (@true) or the + * end (@false); ie. either 1972 -> 197201010000 or 1972 -> 197212312359 + * @param use_utc interpret @param date as UTC + * + * @return the corresponding time_t or Nothing if parsing failed. + */ +Option<::time_t> parse_date_time(const std::string& date, bool first, bool use_utc=false); + +/** + * Crudely convert HTML to plain text. This attempts to scrape the + * human-readable text from html-email so we can use it for indexing. + * + * @param html html + * + * @return plain text + */ +std::string html_to_text(const std::string& html); + +/** + * Hack to avoid locale crashes + * + * @return true if setting locale worked; false otherwise + */ +bool locale_workaround(); + + +/** + * Is the given timezone available? For tests + * + * @param tz a timezone, such as Europe/Helsinki + * + * @return true or false + */ +bool timezone_available(const std::string& tz); + + +// https://stackoverflow.com/questions/19053351/how-do-i-use-a-custom-deleter-with-a-stdunique-ptr-member +template <auto fn> +struct deleter_from_fn { + template <typename T> + constexpr void operator()(T* arg) const { + fn(arg); + } +}; +template <typename T, auto fn> +using deletable_unique_ptr = std::unique_ptr<T, deleter_from_fn<fn>>; + + + +using Clock = std::chrono::steady_clock; +using Duration = Clock::duration; + +template <typename Unit> +constexpr int64_t +to_unit(Duration d) +{ + using namespace std::chrono; + return duration_cast<Unit>(d).count(); +} + +constexpr int64_t +to_s(Duration d) +{ + return to_unit<std::chrono::seconds>(d); +} +constexpr int64_t +to_ms(Duration d) +{ + return to_unit<std::chrono::milliseconds>(d); +} +constexpr int64_t +to_us(Duration d) +{ + return to_unit<std::chrono::microseconds>(d); +} + +struct StopWatch { + using Clock = std::chrono::steady_clock; + StopWatch(const std::string name) : start_{Clock::now()}, name_{name} {} + ~StopWatch() { + const auto us{static_cast<double>(to_us(Clock::now() - start_))}; + /* LCOV_EXCL_START*/ + if (us > 2000000) + mu_debug("sw: {}: finished after {:.1f} s", name_, us / 1000000); + /* LCOV_EXCL_STOP*/ + else if (us > 2000) + mu_debug("sw: {}: finished after {:.1f} ms", name_, us / 1000); + else + mu_debug("sw: {}: finished after {} us", name_, us); + } +private: + Clock::time_point start_; + std::string name_; +}; + +/** + * Convert a size string to a size in bytes + * + * @param sizestr the size string + * @param first + * + * @return the size or Nothing if parsing failed + */ +Option<int64_t> parse_size(const std::string& sizestr, bool first); + +/** + * Convert a size into a size in bytes string + * + * @param size the size + * @param first + * + * @return the size expressed as a string with the decimal number of bytes + */ +std::string size_to_string(int64_t size); + +/** + * get a crude 'summary' of the string, ie. the first /n/ lines of the strings, + * with all newlines removed, replaced by single spaces + * + * @param str the source string + * @param max_lines the maximum number of lines to include in the summary + * + * @return a newly allocated string with the summary. use g_free to free it. + */ +std::string summarize(const std::string& str, size_t max_lines); + + +/** + * Quote & escape a string for " and \ + * + * @param str a string + * + * @return quoted string + */ +std::string quote(const std::string& str); + + +/** + * Convert any ostreamable<< value to a string + * + * @param t the value + * + * @return a std::string + */ +template <typename T> +static inline std::string +to_string(const T& val) +{ + std::stringstream sstr; + sstr << val; + + return sstr.str(); +} +/** + * Convert to std::string to a std::string_view + * Careful with the lifetimes! + * + * @param s a string + * + * @return a string_view + */ +static inline std::string_view +to_string_view(const std::string& s) +{ + return std::string_view{s.data(), s.size()}; +} + +/** + * Consume a gchar and return a std::string + * + * @param str a gchar* (consumed/freed) + * + * @return a std::string, empty if gchar was {} + */ +static inline std::string +to_string_gchar(gchar*&& str) +{ + std::string s(str?str:""); + g_free(str); + return s; +} + + +/* + * Lexnums are lexicographically sortable string representations of non-negative + * integers. Start with 'f' + length of hex-representation number, followed by + * the hex representation itself. So, + * + * 0 -> 'g0' + * 1 -> 'g1' + * 10 -> 'ga' + * 16 -> 'h10' + * + * etc. + */ +std::string to_lexnum(int64_t val); +int64_t from_lexnum(const std::string& str); + +/** + * Like std::find_if, but using sequence instead of a range. + * + * @param seq some std::find_if compatible sequence + * @param pred a predicate + * + * @return an iterator + */ +template<typename Sequence, typename UnaryPredicate> +typename Sequence::const_iterator seq_find_if(const Sequence& seq, UnaryPredicate pred) { + return std::find_if(seq.cbegin(), seq.cend(), pred); +} + +/** + * Is pred(element) true for at least one element of sequence? + * + * @param seq sequence + * @param pred a predicate + * + * @return true or false + */ +template<typename Sequence, typename UnaryPredicate> +bool seq_some(const Sequence& seq, UnaryPredicate pred) { + return seq_find_if(seq, pred) != seq.cend(); +} + +/** + * Create a sequence that has all element of seq for which pred is true + * + * @param seq sequence + * @param pred false + * + * @return sequence + */ +template<typename Sequence, typename UnaryPredicate> +Sequence seq_filter(const Sequence& seq, UnaryPredicate pred) { + Sequence res; + std::copy_if(seq.begin(), seq.end(), std::back_inserter(res), pred); + return res; +} + +/** + * Create a sequence that has all element of seq for which pred is false + * + * @param seq sequence + * @param pred false + * + * @return sequence + */ +template<typename Sequence, typename UnaryPredicate> +Sequence seq_remove(const Sequence& seq, UnaryPredicate pred) { + Sequence res; + std::remove_copy_if(seq.begin(), seq.end(), std::back_inserter(res), pred); + return res; +} + +template<typename Sequence, typename Compare> +void seq_sort(Sequence& seq, Compare cmp) { std::sort(seq.begin(), seq.end(), cmp); } + + +/** + * Like std::accumulate, but using a sequence instead of a range. + * + * @param seq some std::accumulate compatible sequence + * @param init the initial value + * @param op binary operation to calculate the next element + * + * @return the result value. + */ +template<typename Sequence, typename ResultType, typename BinaryOp> +ResultType seq_fold(const Sequence& seq, ResultType init, BinaryOp op) { + return std::accumulate(seq.cbegin(), seq.cend(), init, op); +} + +template<typename Sequence, typename UnaryOp> +void seq_for_each(const Sequence& seq, UnaryOp op) { + std::for_each(seq.cbegin(), seq.cend(), op); +} + +struct MaybeAnsi { + explicit MaybeAnsi(bool use_color) : color_{use_color} {} + + enum struct Color { + Black = 30, + Red = 31, + Green = 32, + Yellow = 33, + Blue = 34, + Magenta = 35, + Cyan = 36, + White = 37, + + BrightBlack = 90, + BrightRed = 91, + BrightGreen = 92, + BrightYellow = 93, + BrightBlue = 94, + BrightMagenta = 95, + BrightCyan = 96, + BrightWhite = 97, + }; + + std::string fg(Color c) const { return ansi(c, true); } + std::string bg(Color c) const { return ansi(c, false); } + + std::string reset() const { return color_ ? "\x1b[0m" : ""; } + +private: + std::string ansi(Color c, bool fg = true) const + { + return color_ ? mu_format("\x1b[{}m", + static_cast<int>(c) + (fg ? 0 : 10)) : ""; + } + + const bool color_; +}; + +#define MU_COLOR_RED "\x1b[31m" +#define MU_COLOR_GREEN "\x1b[32m" +#define MU_COLOR_YELLOW "\x1b[33m" +#define MU_COLOR_BLUE "\x1b[34m" +#define MU_COLOR_MAGENTA "\x1b[35m" +#define MU_COLOR_CYAN "\x1b[36m" +#define MU_COLOR_DEFAULT "\x1b[0m" + + +/// Allow using enum structs as bitflags +#define MU_TO_NUM(ET, ELM) std::underlying_type_t<ET>(ELM) +#define MU_TO_ENUM(ET, NUM) static_cast<ET>(NUM) +#define MU_ENABLE_BITOPS(ET) \ + constexpr ET operator&(ET e1, ET e2) { \ + return MU_TO_ENUM(ET, MU_TO_NUM(ET, e1) & MU_TO_NUM(ET, e2)); \ + } \ + constexpr ET operator|(ET e1, ET e2) { \ + return MU_TO_ENUM(ET, MU_TO_NUM(ET, e1) | MU_TO_NUM(ET, e2)); \ + } \ + constexpr ET operator~(ET e) { return MU_TO_ENUM(ET, ~(MU_TO_NUM(ET, e))); } \ + constexpr bool any_of(ET e) { return MU_TO_NUM(ET, e) != 0; } \ + constexpr bool none_of(ET e) { return MU_TO_NUM(ET, e) == 0; } \ + constexpr bool one_of(ET e1, ET e2) { return (e1 & e2) == e2; } \ + constexpr ET& operator&=(ET& e1, ET e2) { return e1 = e1 & e2; } \ + constexpr ET& operator|=(ET& e1, ET e2) { return e1 = e1 | e2; } \ + static_assert(1==1) // require a semicolon + +} // namespace Mu + +#endif /* MU_UTILS_HH__ */ diff --git a/lib/utils/tests/meson.build b/lib/utils/tests/meson.build new file mode 100644 index 0000000..9c2883b --- /dev/null +++ b/lib/utils/tests/meson.build @@ -0,0 +1,83 @@ +## Copyright (C) 2021-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + +################################################################################ +# tests + + +# +# tests +# +test('test-sexp', + executable('test-sexp', '../mu-sexp.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_utils_dep])) + +test('test-regex', + executable('test-regex', '../mu-regex.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_utils_dep])) + +test('test-command-handler', + executable('test-command-handler', '../mu-command-handler.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_utils_dep])) + +test('test-utils-file', + executable('test-utils-file', '../mu-utils-file.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, gio_unix_dep,config_h_dep, lib_mu_utils_dep])) + +test('test-logger', + executable('test-logger', '../mu-logger.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_utils_dep, thread_dep ])) + +test('test-option', + executable('test-option', '../mu-option.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_utils_dep ])) + +test('test-lang-detector', + executable('test-lang-detector', '../mu-lang-detector.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [ config_h_dep, glib_dep, lib_mu_utils_dep ])) + +test('test-html-to-text', + executable('test-html-to-text', '../mu-html-to-text.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_utils_dep])) + +test('test-error', + executable('test-error', '../mu-error.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_utils_dep])) + +test('test-mu-utils', + executable('test-mu-utils', + 'test-utils.cc', + install: false, + dependencies: [glib_dep, lib_mu_utils_dep])) diff --git a/lib/utils/tests/test-utils.cc b/lib/utils/tests/test-utils.cc new file mode 100644 index 0000000..fe0d075 --- /dev/null +++ b/lib/utils/tests/test-utils.cc @@ -0,0 +1,343 @@ +/* +** Copyright (C) 2017-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This library is free software; you can redistribute it and/or +** modify it under the terms of the GNU Lesser General Public License +** as published by the Free Software Foundation; either version 2.1 +** of the License, or (at your option) any later version. +** +** This library is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +** Lesser General Public License for more details. +** +** You should have received a copy of the GNU Lesser General Public +** License along with this library; if not, write to the Free +** Software Foundation, 51 Franklin Street, Fifth Floor, Boston, MA +** 02110-1301, USA. +*/ + +#include <vector> +#include <glib.h> + +#include <iostream> +#include <sstream> +#include <functional> +#include <array> + +#include "mu-utils.hh" +#include "mu-test-utils.hh" +#include "mu-error.hh" + +using namespace Mu; + + +struct Case { + const std::string expr; + bool is_first{}; + const std::string expected; +}; +using CaseVec = std::vector<Case>; +using ProcFunc = std::function<std::string(std::string, bool)>; + +static void +test_cases(const CaseVec& cases, ProcFunc proc) +{ + for (const auto& casus : cases) { + const auto res = proc(casus.expr, casus.is_first); + //mu_println("'{}'\n'{}'", casus.expected, res); + assert_equal(casus.expected, res); + } +} + +static void +test_date_basic() +{ + const auto hki = "Europe/Helsinki"; + + // ensure we have the needed TZ or skip the test. + if (!timezone_available(hki)) { + g_test_skip("timezone Europe/Helsinki not available"); + return; + } + + g_setenv("TZ", hki, TRUE); + std::vector<std::tuple<const char*, bool/*is_first*/, ::time_t>> cases = {{ + {"2015-09-18T09:10:23", true, 1442556623}, + {"1972-12-14T09:10:23", true, 93165023}, + {"1972-12-14T09:10", true, 93165000}, + {"1854-11-18T17:10:23", true, 0}, + + {"2000-02-31T09:10:23", true, 951861599}, + {"2000-02-29T23:59:59", true, 951861599}, + + {"20220602", true, 1654117200}, + {"20220605", false, 1654462799}, + + {"202206", true, 1654030800}, + {"202206", false, 1656622799}, + + {"2016", true, 1451599200}, + {"2016", false, 1483221599}, + + // {"fnorb", true, -1}, + // {"fnorb", false, -1}, + {"", false, time_t_max}, + {"", true, time_t_min} + }}; + + for (auto& test: cases) { + if (g_test_verbose()) + g_debug("checking %s", std::get<0>(test)); + g_assert_cmpuint(parse_date_time(std::get<0>(test), + std::get<1>(test)).value_or(-1),==, + std::get<2>(test)); + } +} + +static void +test_date_ymwdhMs(void) +{ + struct testcase { + std::string expr; + int64_t diff; + int tolerance; + }; + + std::array<testcase, 7> cases = {{ + {"7s", 7, 1}, + {"3M", 3 * 60, 1}, + {"3h", 3 * 60 * 60, 1}, + {"21d", 21 * 24 * 60 * 60, 3600 + 1}, + {"2w", 2 * 7 * 24 * 60 * 60, 3600 + 1}, + {"2y", 2 * 365 * 24 * 60 * 60, 24 * 3600 + 1}, + {"3m", 3 * 30 * 24 * 60 * 60, 3 * 24 * 3600 + 1} + }}; + + for (auto&& tcase: cases) { + const auto date = parse_date_time(tcase.expr, true); + g_assert_true(date); + const auto diff = ::time({}) - *date; + if (g_test_verbose()) + std::cerr << tcase.expr << ' ' << diff << ' ' << tcase.diff << '\n'; + + g_assert_true(tcase.diff - diff <= tcase.tolerance); + } + + // note: perhaps it'd be nice if we'd detect this error; + // currently we're being rather tolerant + // g_assert_false(!!parse_date_time("25q", false)); +} + +static void +test_parse_size() +{ + constexpr std::array<std::tuple<const char*, bool, int64_t>, 6> cases = {{ + { "456", false, 456 }, + { "", false, G_MAXINT64 }, + { "", true, 0 }, + { "2K", false, 2048 }, + { "2M", true, 2097152 }, + { "5G", true, 5368709120 } + }}; + for(auto&& test: cases) { + g_assert_cmpint(parse_size(std::get<0>(test), std::get<1>(test)) + .value_or(-1), ==, std::get<2>(test)); + } + + g_assert_false(!!parse_size("-1", true)); + g_assert_false(!!parse_size("scoobydoobydoo", false)); +} + +static void +test_flatten() +{ + CaseVec cases = { + {"Менделе́ев", true, "менделеев"}, + {"", true, ""}, + {"Ångström", true, "angstrom"}, + {"đodø", true, "dodo"}, + + // don't touch combining characters in CJK etc. + {"スポンサーシップ募集",true, "スポンサーシップ募集"} + }; + + test_cases(cases, [](auto s, auto f) { return utf8_flatten(s); }); +} + +static void +test_remove_ctrl() +{ + CaseVec cases = { + {"Foo\n\nbar", true, "Foo bar"}, + {"", false, ""}, + {" ", false, " "}, + {"Hello World ", false, "Hello World "}, + {"Ångström", false, "Ångström"}, + }; + + test_cases(cases, [](auto s, auto f) { return remove_ctrl(s); }); +} + +static void +test_clean() +{ + CaseVec cases = { + {"\t a\t\nb ", true, "a b"}, + {"", true, ""}, + {"Ångström", true, "Ångström"}, + {"\345\245", true, ".."}, + }; + + test_cases(cases, [](auto s, auto f) { return utf8_clean(s); }); +} + + +static void +test_word_break() +{ + CaseVec cases = { + {"aap+noot&mies", true, "aap noot mies"}, + {"hallo", true, "hallo"}, + {" foo-bar###cuux,fnorb ", true, "foo bar cuux fnorb"}, + {"eyes\nof\tMedusa", true, "eyes of Medusa"}, + }; + + test_cases(cases, [](auto s, auto f) { return utf8_wordbreak(s); }); +} + + +static void +test_format() +{ + g_assert_true(mu_format("hello {}", "world") == "hello world"); + g_assert_true(mu_format("hello {}, {}", "world", 123) == "hello world, 123"); +} + +static void +test_split() +{ + using svec = std::vector<std::string>; + auto assert_equal_svec=[](const svec& sv1, const svec& sv2) { + g_assert_cmpuint(sv1.size(),==,sv2.size()); + for (auto i = 0U; i != sv1.size(); ++i) + g_assert_cmpstr(sv1[i].c_str(),==,sv2[i].c_str()); + }; + + // string sepa + assert_equal_svec(split("axbxc", "x"), {"a", "b", "c"}); + assert_equal_svec(split("axbxcx", "x"), {"a", "b", "c", ""}); + assert_equal_svec(split("", "boo"), {}); + assert_equal_svec(split("ayybyyc", "yy"), {"a", "b", "c"}); + assert_equal_svec(split("abc", ""), {"a", "b", "c"}); + assert_equal_svec(split("", "boo"), {}); + + // char sepa + assert_equal_svec(split("axbxc", 'x'), {"a", "b", "c"}); + assert_equal_svec(split("axbxcx", 'x'), {"a", "b", "c", ""}); +} + +static void +test_join() +{ + assert_equal(join({"a", "b", "c"}, "x"), "axbxc"); + assert_equal(join({"a", "b", "c"}, ""), "abc"); + assert_equal(join({},"foo"), ""); + assert_equal(join({"d", "e", "f"}, "foo"), "dfooefoof"); +} + + +enum struct Bits { None = 0, Bit1 = 1 << 0, Bit2 = 1 << 1 }; +MU_ENABLE_BITOPS(Bits); + +static void +test_define_bitmap() +{ + g_assert_cmpuint((guint)Bits::None, ==, (guint)0); + g_assert_cmpuint((guint)Bits::Bit1, ==, (guint)1); + g_assert_cmpuint((guint)Bits::Bit2, ==, (guint)2); + + g_assert_cmpuint((guint)(Bits::Bit1 | Bits::Bit2), ==, (guint)3); + g_assert_cmpuint((guint)(Bits::Bit1 & Bits::Bit2), ==, (guint)0); + + g_assert_cmpuint((guint)(Bits::Bit1 & (~Bits::Bit2)), ==, (guint)1); + + { + Bits b{Bits::Bit1}; + b |= Bits::Bit2; + g_assert_cmpuint((guint)b, ==, (guint)3); + } + + { + Bits b{Bits::Bit1}; + b &= Bits::Bit1; + g_assert_cmpuint((guint)b, ==, (guint)1); + } +} + +static void +test_to_from_lexnum() +{ + assert_equal(to_lexnum(0), "g0"); + assert_equal(to_lexnum(100), "h64"); + assert_equal(to_lexnum(12345), "j3039"); + + g_assert_cmpuint(from_lexnum(to_lexnum(0)), ==, 0); + g_assert_cmpuint(from_lexnum(to_lexnum(7777)), ==, 7777); + g_assert_cmpuint(from_lexnum(to_lexnum(9876543)), ==, 9876543); +} + +static void +test_locale_workaround() +{ + g_assert_true(locale_workaround()); + + g_setenv("LC_ALL", "BOO", 1); + + g_assert_true(locale_workaround()); +} + + +static void +test_summarize(void) +{ + const char *txt = + "Khiron was fortified and made the seat of a pargana during " + "the reign of Asaf-ud-Daula.\n\the headquarters had previously " + "been at Satanpur since its foundation and fortification by " + "the Bais raja Sathna.\n\nKhiron was also historically the seat " + "of a taluqdari estate belonging to a Janwar dynasty.\n" + "There were also several Kayasth qanungo families, " + "including many descended from Rai Sahib Rai, who had been " + "a chakladar under the Nawabs of Awadh."; + + const auto summ = summarize(txt, 3); + g_assert_cmpstr(summ.c_str(), ==, + "Khiron was fortified and made the seat of a pargana " + "during the reign of Asaf-ud-Daula. he headquarters had " + "previously been at Satanpur since its foundation and " + "fortification by the Bais raja Sathna. "); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/utils/date-basic", test_date_basic); + g_test_add_func("/utils/date-ymwdhMs", test_date_ymwdhMs); + g_test_add_func("/utils/parse-size", test_parse_size); + g_test_add_func("/utils/flatten", test_flatten); + g_test_add_func("/utils/remove-ctrl", test_remove_ctrl); + g_test_add_func("/utils/clean", test_clean); + g_test_add_func("/utils/word-break", test_word_break); + g_test_add_func("/utils/format", test_format); + g_test_add_func("/utils/summarize", test_summarize); + g_test_add_func("/utils/split", test_split); + g_test_add_func("/utils/join", test_join); + g_test_add_func("/utils/define-bitmap", test_define_bitmap); + g_test_add_func("/utils/to-from-lexnum", test_to_from_lexnum); + g_test_add_func("/utils/locale-workaround", test_locale_workaround); + + return g_test_run(); +} diff --git a/man/author.inc b/man/author.inc new file mode 100644 index 0000000..db14d93 --- /dev/null +++ b/man/author.inc @@ -0,0 +1,7 @@ +* AUTHOR + +Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +# Local Variables: +# mode: org +# End: diff --git a/man/bugs.inc b/man/bugs.inc new file mode 100644 index 0000000..882e6a5 --- /dev/null +++ b/man/bugs.inc @@ -0,0 +1,7 @@ +* REPORTING BUGS + +Please report bugs at <https://github.com/djcb/mu/issues>. + +# Local Variables: +# mode: org +# End: diff --git a/man/common-options.inc b/man/common-options.inc new file mode 100644 index 0000000..ec83e3f --- /dev/null +++ b/man/common-options.inc @@ -0,0 +1,31 @@ +* COMMON OPTIONS + +** -d, --debug +makes mu generate extra debug information, useful for debugging the program +itself. By default, debug information goes to the log file, ~/.cache/mu/mu.log. +It can safely be deleted when mu is not running. When running with --debug +option, the log file can grow rather quickly. See the note on logging below. + +** -q, --quiet +causes mu not to output informational messages and progress information to +standard output, but only to the log file. Error messages will still be sent to +standard error. Note that mu index is much faster with --quiet, so it is +recommended you use this option when using mu from scripts etc. + +** --log-stderr +causes mu to not output log messages to standard error, in addition to sending +them to the log file. + +** --nocolor +do not use ANSI colors. The environment variable ~NO_COLOR~ can be used as an +alternative to ~--nocolor~. + +** -V, --version +prints mu version and copyright information. + +** -h, --help +lists the various command line options. + +# Local Variables: +# mode: org +# End: diff --git a/man/copyright.inc.in b/man/copyright.inc.in new file mode 100644 index 0000000..2e02670 --- /dev/null +++ b/man/copyright.inc.in @@ -0,0 +1,12 @@ +* COPYRIGHT + +This manpage is part of ~mu~ @VERSION@. + +Copyright © 2008-@YEAR@ Dirk-Jan C. Binnema. License GPLv3+: GNU GPL version 3 +or later <https://gnu.org/licenses/gpl.html>. This is free software: you are +free to change and redistribute it. There is NO WARRANTY, to the extent +permitted by law. + +# Local Variables: +# mode: org +# End: diff --git a/man/exit-code.inc b/man/exit-code.inc new file mode 100644 index 0000000..07c8138 --- /dev/null +++ b/man/exit-code.inc @@ -0,0 +1,14 @@ +* EXIT CODE + +This command returns 0 upon successful completion, or a non-zero exit code +otherwise. + + 0. success + 2. no matches found. Try a different query + 11. database schema mismatch. You need to re-initialize ~mu~, see *mu-init(1)* + 19. failed to acquire lock. Some other program has exclusive access to the mu database + 99. caught an exception + +# Local Variables: +# mode: org +# End: diff --git a/man/meson.build b/man/meson.build new file mode 100644 index 0000000..de4ba29 --- /dev/null +++ b/man/meson.build @@ -0,0 +1,102 @@ +## Copyright (C) 2021-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# +# generate org include files +# +man_data=configuration_data() +man_data.set('VERSION', meson.project_version()) +man_data.set('YEAR', mu_year) +incs=[ + 'author.inc', + 'bugs.inc', + 'common-options.inc', + 'copyright.inc.in', + 'exit-code.inc', + 'muhome.inc', + 'prefooter.inc', +] +foreach inc: incs + # configure the .in ones + if inc.substring(-3) == '.in' + configure_file(input: inc, + output: '@BASENAME@', + configuration: man_data) + else # and copy the rest + configure_file(input: inc, output:'@BASENAME@.inc', + copy:true) + endif +endforeach + +# man-pages is org-format. +man_orgs=[ + 'mu.1.org', + 'mu-add.1.org', + 'mu-bookmarks.5.org', + 'mu-cfind.1.org', + 'mu-easy.7.org', + 'mu-extract.1.org', + 'mu-find.1.org', + 'mu-help.1.org', + 'mu-index.1.org', + 'mu-info.1.org', + 'mu-init.1.org', + 'mu-mkdir.1.org', + 'mu-move.1.org', + 'mu-query.7.org', + 'mu-remove.1.org', + 'mu-server.1.org', + 'mu-verify.1.org', + 'mu-view.1.org' +] + +foreach src : man_orgs + # meson makes in tricky to use the results of e.g. configure_file + # in custom_commands..., so this is admittedly a little hacky. + org = join_paths(meson.current_build_dir(), src) + man = '@BASENAME@' + section = src.substring(-5, -4) + + # we fill in some man-page details: + # @SECTION_ID@: the man-page section + # @MAN_DATE@: date of the generation (not yet supported by ox-man) + conf_data = configuration_data() + conf_data.set('SECTION_ID', section) + conf_data.set('MAN_DATE', mu_month_year) + configure_file(input: src, output:'@BASENAME@.org', + configuration: conf_data) + + expr_tmpl = ''.join([ + '(progn', + ' (require \'ox-man)', + ' (setq org-export-with-sub-superscripts \'{})', + ' (org-export-to-file \'man "@0@"))']) + expr = expr_tmpl.format(org.substring(0,-4)) + sectiondir = join_paths(mandir, 'man' + section) + + custom_target(src + '-to-man', + build_by_default: true, + input: src, + output: '@BASENAME@', + install: true, + install_dir: sectiondir, + depend_files: incs, + command: [emacs, + '--no-init-file', + '--batch', + org, + '--eval', expr]) +endforeach diff --git a/man/mu-add.1.org b/man/mu-add.1.org new file mode 100644 index 0000000..f2b644c --- /dev/null +++ b/man/mu-add.1.org @@ -0,0 +1,29 @@ +#+TITLE: MU ADD +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-add - add one or more messages to the database + +* SYNOPSIS + +*mu [common-options] add [options] <file> [<files>]* + +* DESCRIPTION + +~mu add~ is the command to add specific message files to the database. Each file +must be specified with an absolute path. + +* ADD OPTIONS + +#+include: "muhome.inc" :minlevel 2 + +#+include: "common-options.inc" :minlevel 1 + +#+include: "exit-code.inc" :minlevel 1 + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO + +*mu(1)*, *mu-index(1)*, *mu-remove(1)* diff --git a/man/mu-bookmarks.5.org b/man/mu-bookmarks.5.org new file mode 100644 index 0000000..b7d275e --- /dev/null +++ b/man/mu-bookmarks.5.org @@ -0,0 +1,36 @@ +#+TITLE: MU BOOKMARKS +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-bookmarks - file with bookmarks (shortcuts) for mu search expressions + +* DESCRIPTION + +Bookmarks are named shortcuts for search queries. They allow using a convenient +name for often-used queries. The bookmarks are also visible as shortcuts in the +mu experimental user interfaces, =mug= and =mug2=. + +The bookmarks file is read from =<muhome>/bookmarks=. On Unix this would typically +be w be =~/.config/mu/bookmarks=, but this can be influenced using the ~--muhome~ +parameter for *mu-find(1)*. + +The bookmarks file is a typical key=value *.ini*-file, which is best shown by +means of an example: + +#+begin_example +[mu] +inbox=maildir:/inbox # inbox +oldhat=maildir:/archive subject:hat # archived with subject containing 'hat' +#+end_example + +The *[mu]* group header is required. For practical uses of bookmarks, see +*mu-find(1)*. + +#+include: "author.inc" :minlevel 1 + +#+include: "copyright.inc" :minlevel 1 + +* SEE ALSO + +*mu(1)*, *mu-find(1)* diff --git a/man/mu-cfind.1.org b/man/mu-cfind.1.org new file mode 100644 index 0000000..0c14dc6 --- /dev/null +++ b/man/mu-cfind.1.org @@ -0,0 +1,161 @@ +#+TITLE: MU CFIND +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-cfind - find contacts in the *mu* database and export them +for use in other programs. + +* SYNOPSIS + +*mu [common-options] cfind [options] [<pattern>]* + +* DESCRIPTION + +*mu cfind* is the *mu* command for finding =contacts= (name and e-mail address of +people who were either an e-mail's sender or receiver). There are different +output formats available, for importing the contacts into other programs. + +* SEARCHING CONTACTS + +When you index your messages (see *mu index*), *mu* creates a list of unique e-mail +addresses found and the accompanying name, and caches this list. In case the +same e-mail address is used with different names, the most recent non-empty name +is used. + +*mu cfind* starts a search for contacts that match a =regular expression=. For +example: + +#+begin_example +$ mu cfind '@gmail\.com' +#+end_example + +would find all contacts with a gmail-address, while + +#+begin_example +$ mu cfind Mary +#+end_example + +lists all contacts with Mary in either name or e-mail address. + +If you do not specify a search expression, *mu cfind* returns the full list of +contacts. Note, *mu cfind* uses a cache with the e-mail information, which is +populated during the indexing process. + +The regular expressions are basic case-insensitive PCRE, see *pcre(3)*. + +* CFIND OPTIONS + +** --format=plain|mutt-alias|mutt-ab|wl|org-contact|bbdb|csv +sets the output format to the given value. The following are available: + +#+ATTR_MAN: :disable-caption t +| --format= | description | +|-------------+-----------------------------------| +| plain | default, simple list | +| mutt-alias | mutt alias-format | +| mutt-ab | mutt external address book format | +| wl | wanderlust addressbook format | +| org-contact | org-mode org-contact format | +| bbdb | BBDB format | +| csv | comma-separated values [1] | +| json | JSON format | + + +[1] *CSV* is not fully standardized, but *mu cfind* follows some common practices: +any double-quote is replaced by a double-double quote (thus, "hello" become +""hello"", and fields with commas are put in double-quotes. Normally, this +should only apply to name fields. + +** --personal,-p only show addresses seen in messages where one of `my' e-mail +addresses was seen in one of the address fields; this is to exclude addresses +only seen in mailing-list messages. See the ~--my-address~ parameter to *mu init*. + +** --after=<timestamp> only show addresses last seen after +=<timestamp>=. =<timestamp>= is a UNIX *time_t* value, the number of +seconds since 1970-01-01 (in UTC). + +From the command line, you can use the *date* command to get this value. For +example, only consider addresses last seen after 2020-06-01, you could specify +#+begin_example + --after=`date +%s --date='2020-06-01'` +#+end_example + +#+include: "muhome.inc" :minlevel 2 + +#+include: "common-options.inc" :minlevel 1 + +* JSON FORMAT + +With ~--format=json~, the matching contacts come out as a JSON array, e.g., +#+begin_example +[ + { + "email" : "syb@example.com", + "name" : "Sybil Gerard", + "display" : "Sybil Gerard <syb@example.com>", + "last-seen" : 1075982687, + "last-seen-iso" : "2004-02-05T14:04:47Z", + "personal" : false, + "frequency" : 14 + }, + { + "email" : "ed@example.com", + "name" : "Mallory, Edward", + "display" : "\"Mallory, Edward\" <ed@example.com>", + "last-seen" : 1425991805, + "last-seen-iso" : "2015-03-10T14:50:05Z", + "personal" : true, + "frequency" : 2 + } +] +#+end_example + +Each contact has the following fields: + +#+ATTR_MAN: :disable-caption t +| property | description | +|---------------+--------------------------------------------------------------------------| +| ~email~ | the email-address | +| ~name~ | the name (or ~none~) | +| ~display~ | the combination name and e-mail address for display purposes | +| ~last-seen~ | date of most recent message with this contact (Unix time) | +| ~last-seen-iso~ | ~last-seen~ represented as an ISO-8601 timestamp | +| ~personal~ | whether the email was seen in a message together with a personal address | +| ~frequency~ | approximation of the number of times this contact was seen in messages | + +The JSON format is useful for further processing, e.g. using the *jq(1)* tool: + +List display names, sorted by their last-seen date: +#+begin_example +$ mu cfind --format=json --personal | jq -r '.[] | ."last-seen-iso" + " " + .display' | sort +#+end_example + +* INTEGRATION WITH MUTT + +You can use *mu cfind* as an external address book server for *mutt*. +For this to work, add the following to your =muttrc=: + +#+begin_example +set query_command = "mu cfind --format=mutt-ab '%s'" +#+end_example + +Now, in mutt, you can search for e-mail addresses using the *query*-command, +which is (by default) accessible by pressing *Q*. + +* ENCODING + +*mu cfind* output is encoded according to the current locale except for +=--format=bbdb=. This is hard-coded to UTF-8, and as such specified in the +output-file, so emacs/bbdb can handle things correctly, without guessing. + +#+include: "exit-code.inc" :minlevel 1 + +#+include: "bugs.inc" :minlevel 1 + +#+include: "author.inc" :minlevel 1 + +#+include: "copyright.inc" :minlevel 1 + +* SEE ALSO +*mu(1)*, *mu-index(1)*, *mu-find(1)*, *pcre(3)*, *jq(1)* diff --git a/man/mu-easy.7.org b/man/mu-easy.7.org new file mode 100644 index 0000000..662a314 --- /dev/null +++ b/man/mu-easy.7.org @@ -0,0 +1,303 @@ +#+TITLE: MU EASY +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-easy - a quick introduction to mu + +* DESCRIPTION + +*mu* is a set of tools for dealing with e-mail messages in Maildirs. There are +many options, which are all described in the man pages for the various +sub-commands. This man pages jumps over all of the details and gives examples of +some common use cases. If the use cases described here do not precisely do what +you want, please check the more extensive information in the man page about the +sub-command you are using -- for example, the *mu-index(1)* or *mu-find(1)* man +pages. + +*NOTE*: the *index* command (and therefore, the ones that depend on that, such as +*find*), require that you store your mail in the Maildir-format. If you don't do +so, you can still use the other commands, but you won't be able to index/search +your mail. + +By default, *mu* uses colorized output when it thinks your terminal is capable of +doing so. If you don't like color, you can use the *--nocolor* command-line +option, or set either the *MU_NOCOLOR* or the *NO_COLOR* environment variable to +non-empty. + +* SETTING THINGS UP + +The first time you run the mu commands, you need to initialize it. This is done +with the *init* command. + +#+begin_example +$ mu init +#+end_example + +This uses the defaults (see *mu-init(1)* for details on how to change that). + + +* INDEXING YOUR E-MAIL + +Before you can search e-mails, you'll first need to index them: + +#+begin_example +$ mu index +#+end_example + +The process can take a few minutes, depending on the amount of mail you have, +the speed of your computer, hard drive etc. Usually, indexing should be able to +reach a speed of a few hundred messages per second. + +*mu index* guesses the top-level Maildir to do its job; if it guesses wrong, you +can use the =--maildir= option to specify the top-level directory that should be +processed. See the *mu-index(1)* man page for more details. + +Normally, *mu index* visits all the directories under the top-level Maildir; +however, you can exclude certain directories (say, the `trash' or `spam' +folders) by creating a file called =.noindex= in the directory. When *mu* sees such +a file, it will exclude this directory and its sub-directories from indexing. +Also see *.noupdate* in the *mu-index(1)* manpage. + +* SEARCHING YOUR E-MAIL + +After you have indexed your mail, you can start searching it. By default, the +search results are printed on standard output. Alternatively, the output can +take the form of Maildir with symbolic links to the found messages. This enables +integration with e-mail clients; see the *mu-find(1)* man page for details, the +syntax of the search parameters and so on. Here, we just give some examples for +common cases. + +You can use the *mu fields* command to get information about all possible fields +and flags. + +First, let's search for all messages sent to Julius (Caesar) regarding fruit: + +#+begin_example +$ mu find t:julius fruit +#+end_example + +This should return something like: + +#+begin_example +2008-07-31T21:57:25 EEST John Milton <jm@example.com> Fere libenter homines id quod volunt credunt +#+end_example + +This means there is a message to `julius' with `fruit' somewhere in the message. +In this case, it's a message from John Milton. Note that the date format depends +on your the language/locale you are using. + +How do we know that the message was sent to Julius Caesar? Well, it's not +visible from the results above, because the default fields that are shown are +date/sender/subject. However, we can change this using the =--fields= parameter +(try *mu fields* to see all the details): + +#+begin_example +$ mu find --fields="t s" t:julius fruit +#+end_example + +In other words, display the `To:'-field (t) and the subject (s). This should +return something like: +#+begin_example +Julius Caesar <jc@example.com> Fere libenter homines id quod volunt credunt +#+end_example + +This is the same message found before, only with some different fields +displayed. + +By default, *mu* uses the logical ~AND~ for the search parameters -- that is, it +displays messages that match all the parameters. However, we can use logical ~OR~ +as well: + +#+begin_example +$ mu find t:julius OR f:socrates +#+end_example + +In other words, display messages that are either sent to Julius Caesar *or* are +from Socrates. This could return something like: + +#+begin_example +2008-07-31T21:57:25 EEST Socrates <soc@example.com> cool stuff +2008-07-31T21:57:25 EEST John Milton <jm@example.com> Fere libenter homines id quod volunt credunt +#+end_example + +What if we want to see some of the body of the message? You can get a `summary' +of the first lines of the message using the =--summary-len= option, which will +`summarize' the first =n= lines of the message: + +#+begin_example +$ mu find --summary-len=3 napoleon m:/archive +#+end_example + +#+begin_example +1970-01-01T02:00:00 EET Napoleon Bonaparte <nb@example.com> rock on dude +Summary: Le 24 février 1815, la vigie de Notre-Dame de la Garde signala le +trois-mâts le Pharaon, venant de Smyrne, Trieste et Naples. Comme +d'habitude, un pilote côtier partit aussitôt du port, rasa le château +#+end_example + +The summary consists of the first /n/ lines of the message with all superfluous +whitespace removed. + +Also note the *m:/archive* parameter in the query. This means that we only match +messages in a maildir called ~'/archive'~. + +* MORE QUERIES + +Let's list a few more queries that may be interesting; please note that +searches for message flags, priority and date ranges are only available in mu +version 0.9 or later. + +Get all important messages which are signed: +#+begin_example + *$ mu find flag:signed prio:high * +#+end_example + +Get all messages from Jim without an attachment: +#+begin_example + *$ mu find from:jim AND NOT flag:attach* +#+end_example + +Get all messages where Jack is in one of the contact fields: +#+begin_example + *$ mu find contact:jack* +#+end_example +This uses the special contact: pseudo-field which matches (*from*, +*to*, *cc* and *bcc*). + +Get all messages in the Sent Items folder about yoghurt: +#+begin_example + *$mu find maildir:'/Sent Items' yoghurt* +#+end_example +Note how we need to quote search terms that include spaces. + + +Get all unread messages where the subject mentions Ångström: +#+begin_example + *$ mu find subject:Ångström flag:unread* +#+end_example +which is equivalent to: +#+begin_example + *$ mu find subject:angstrom flag:unread* +#+end_example +because does mu is case-insensitive and accent-insensitive. + +Get all unread messages between March 2002 and August 2003 about some bird (or +a Swedish rock band): +#+begin_example + *$ mu find date:20020301..20030831 nightingale flag:unread* +#+end_example + +Get all messages received today: +#+begin_example + *$ mu find date:today..now* +#+end_example + +Get all messages we got in the last two weeks about emacs: +#+begin_example + *$ mu find date:2w..now emacs* +#+end_example + +Another powerful feature (since 0.9.6) are wildcard searches, where you can +search for the last =n= characters in a word. For example, you can search +for: +#+begin_example + *$ mu find 'subject:soc*'* +#+end_example +and get mails about soccer, Socrates, society, and so on. Note, it's important +to quote the search query, otherwise the shell will interpret +the `*'. + +You can also search for messages with a certain attachment using their +filename, for example: + +#+begin_example + *$ mu find 'file:pic*'* +#+end_example +will get you all messages with an attachment starting with `pic'. + +If you want to find attachments with a certain MIME-type, you can use the +following: + +Get all messages with PDF attachments: +#+begin_example + *$ mu find mime:application/pdf* +#+end_example + +or even: + +Get all messages with image attachments: +#+begin_example + *$ mu find 'mime:image/*'* +#+end_example + + +Note that (1) the `*' wildcard can only be used as the rightmost thing in a +search query, and (2) that you need to quote the search term, because +otherwise your shell will interpret the `*' (expanding it to all files in the +current directory -- probably not what you want). + +* DISPLAYING MESSAGES + +We might also want to display the complete messages instead of the header +information. This can be done using *mu view* command. Note that this +command does not use the database; you simply provide it the path to a +message. + +Therefore, if you want to display some message from a search query, you'll +need its path. To get the path (think *l*ocation) for our first example we +can use: + +#+begin_example +$ mu find --fields="l" t:julius fruit +#+end_example + +And we'll get something like: +#+begin_example +/home/someuser/Maildir/archive/cur/1266188485_0.6850.cthulhu:2, +#+end_example + +We can now display this message: + +#+begin_example +$ mu view /home/someuser/Maildir/archive/cur/1266188485_0.6850.cthulhu:2, +From: John Milton <jm@example.com> +To: Julius Caesar <jc@example.com> +Subject: Fere libenter homines id quod volunt credunt +Date: 2008-07-31T21:57:25 EEST + +OF Mans First Disobedience, and the Fruit +Of that Forbidden Tree, whose mortal taste +Brought Death into the World, and all our woe, +[...] +#+end_example + +* FINDING CONTACTS + +While *mu find* searches for messages, there is also *mu cfind* to find =contacts=, +that is, names + addresses. Without any search expression, *mu cfind* lists all of +your contacts. + +#+begin_example +$ mu cfind julius +#+end_example + +will find all contacts with `julius' in either name or e-mail address. Note that +*mu cfind* accepts a =regular expression= (as per *pcre(3)*) + +*mu cfind* also supports a =--format==-parameter, which sets the output to some +specific format, so the results can be imported into another program. For +example, to export your contact information to a *mutt* address book file, you can +use something like: + +#+begin_example +$ mu cfind --format=mutt-alias > ~/mutt-aliases +#+end_example + +Then, you can use them in *mutt* if you add something like *source ~/mutt-aliases* +to your =muttrc=. + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO +*mu(1)*, *mu-init(1)*, *mu-index(1)*, *mu-find(1)*, *mu-mfind(1)*, *mu-mkdir(1)*, *mu-view(1)*, *mu-extract(1)* diff --git a/man/mu-extract.1.org b/man/mu-extract.1.org new file mode 100644 index 0000000..596a663 --- /dev/null +++ b/man/mu-extract.1.org @@ -0,0 +1,108 @@ +#+TITLE: MU EXTRACT +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-extract - display and save message parts +(attachments), and open them with other tools. + +* SYNOPSIS + +*mu [common-options] extract [options] [<file>]* + +*mu [common-options] extract [options] <file> <pattern>* + +* DESCRIPTION + +*mu extract* is the *mu* sub-command for extracting MIME-parts (e.g., attachments) +from mail messages. The sub-command works on message files, and does not require +the message to be indexed in the database. + +For attachments, the file name used when saving it is the name of the attachment +in the message. If there is no such name, or when saving non-attachment +MIME-parts, a name is derived from the message-id of the message. + +If you specify a regular express pattern as the second argument, all attachments +with filenames matching that pattern will be extracted. The regular expressions +are basic PCRE, and are case-sensitive by default; see *pcre(3)* for more details. + +Without any options, *mu extract* simply outputs the list of leaf MIME-parts in +the message. Only `leaf' MIME-parts (including RFC822 attachments) are +considered, *multipart/** etc. are ignored. + +Without a filename parameter, ~mu extract~ reads a message from standard-input. In +that case, you cannot use the second, ~<pattern>~ parameter as this would be +ambiguous; instead, use the ~--matches~ option. + +* EXTRACT OPTIONS + +** -a, --save-attachments +save all MIME-parts that look like attachments. + +** --save-all +save all non-multipart MIME-parts. + +** --parts=<parts> +only consider the following numbered parts (comma-separated list). The numbers +for the parts can be seen from running *mu extract* without any options but only +the message file. + +** --target-dir=<dir> +save the parts in the target directory rather than the current working +directory. + +** --overwrite +overwrite existing files with the same name; by default overwriting is not +allowed. + +** -u,--uncooked +by default, ~mu~ transforms the attachment filenames a bit (such as by replacing +spaces by dashes); with this option, leave that to the minimum for creating +a legal filename in the target directory. + +** --matches=<pattern> +Attachments with filenames matching the pattern will be extracted. The regular +expressions are basic PCRE, and are case-sensitive by default; see *pcre(3)* for +more details. + +** --play +Try to `play' (open) the attachment with the default application for the +particular file type. On MacOS, this uses the *open* program, on other platforms +it uses *xdg-open*. You can choose a different program by setting the +*MU_PLAY_PROGRAM* environment variable. + +#+include: "common-options.inc" :minlevel 1 + +* EXAMPLES + +To display information about all the MIME-parts in a message file: +#+begin_example +$ mu extract msgfile +#+end_example + +To extract MIME-part 3 and 4 from this message, overwriting existing files with +the same name: +#+begin_example +$ mu extract --parts=3,4 --overwrite msgfile +#+end_example + +To extract all files ending in `.jpg' (case-insensitive): +#+begin_example +$ mu extract msgfile '.*\.jpg' +#+end_example + +To extract an mp3-file, and play it in the default mp3-playing application: +#+begin_example +$ mu extract --play msgfile 'whoopsididitagain.mp3' +#+end_example + +when reading from standard-input, you need ~--matches~, so: +#+begin_example +$ cat msgfile | mu extract --play --matches 'whoopsididitagain.mp3' +#+end_example + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO + +*mu(1)* diff --git a/man/mu-find.1.org b/man/mu-find.1.org new file mode 100644 index 0000000..a8fc9fe --- /dev/null +++ b/man/mu-find.1.org @@ -0,0 +1,314 @@ +#+TITLE: MU FIND +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-find - find e-mail messages in the *mu* database. + +* SYNOPSIS + +*mu [common-options] find [options] <search expression>* + +* DESCRIPTION + +*mu find* is the *mu* command for searching e-mail message that were stored earlier +using *mu index(1)*. + +* SEARCHING MAIL + +*mu find* starts a search for messages in the database that match some search +pattern. The search patterns are described in detail in *mu-query(7)*. + +For example: + +#+begin_example +$ mu find subject:snow and date:2009.. +#+end_example + +would find all messages in 2009 with `snow' in the subject field, e.g: + +#+begin_example +2009-03-05 17:57:33 EET Lucia <lucia@example.com> running in the snow +2009-03-05 18:38:24 EET Marius <marius@foobar.com> Re: running in the snow +#+end_example + +Note, this the default, plain-text output, which is the default, so you don't +have to use *--format=plain*. For other types of output (such as symlinks, XML or +s-expressions), see the discussion in the *OPTIONS*-section below about *--format*. + +The search pattern is taken as a command-line parameter. If the search +parameter consists of multiple parts (as in the example) they are +treated as if there were a logical *and* between them. + +For details on the possible queries, see *mu-query(7)*. + +* FIND OPTIONS + +Note, some of the important options are described in the *mu*(1) man-page +and not here, as they apply to multiple mu-commands. + +The *find*-command has various options that influence the way *mu* displays the +results. If you don't specify anything, the defaults are ~fields="d f s"~, +~--sortfield=date~ and ~--reverse~. + +** -f, --fields=<fields> +specifies a string that determines which fields are shown in the output. This +string consists of a number of characters (such as 's' for subject or 'f' for +from), which will replace with the actual field in the output. Fields that are +not known will be output as-is, allowing for some simple formatting. + +For example: + +#+begin_example +$ mu find subject:snow --fields "d f s" +#+end_example + +lists the date, subject and sender of all messages with `snow' in the their +subject. + +The table of replacement characters is superset of the list mentions for search +parameters, such as: +#+begin_example + t *t*o: recipient + d Sent *d*ate of the message + f Message sender (*f*rom:) + g Message flags (fla*g*s) + l Full path to the message (*l*ocation) + s Message *s*ubject + i Message-*i*d + m *m*aildir +#+end_example + +For the complete list, try the command: ~mu info fields~. + +The message flags are described in *mu-query(7)*. As an example, a message which +is `seen', has an attachment and is signed would have `asz' as its corresponding +output string, while an encrypted new message would have `nx'. + +** -s, --sortfield=<field> and -z,--reverse +specify the field to sort the search results by and the direction (i.e., +`reverse' means that the sort should be reverted - Z-A). Examples include: + +#+begin_example + cc,c Cc (carbon-copy) recipient(s) + date,d Message sent date + from,f Message sender + maildir,m Maildir + msgid,i Message id + prio,p Nessage priority + subject,s Message subject + to,t To:-recipient(s) +#+end_example + +For the complete list, try the command: ~mu info fields~. + +Thus, for example, to sort messages by date, you could specify: + +#+begin_example +$ mu find fahrrad --fields "d f s" --sortfield=date --reverse +#+end_example + +Note, if you specify a sortfield, by default, messages are sorted in reverse +(descending) order (e.g., from lowest to highest). This is usually a good +choice, but for dates it may be more useful to sort in the opposite direction. + +** -n, --maxnum=<number> +If > 0, display maximally that number of entries. If not specified, all matching +entries are displayed. + +** --summary-len=<number> +If > 0, use that number of lines of the message to provide a summary. + +** --format=<plain|links|xml|sexp> + +output results in the specified format: + +- The default is *plain*, i.e normal output with one line per message. +- *links* outputs the results as a maildir with symbolic links to the found + messages. This enables easy integration with mail-clients (see below for more + information). +- *xml* formats the search results as XML. +- *sexp* formats the search results as an s-expression as used in Lisp programming + environments + +** --linksdir=<dir> and -c, --clearlinks +when using ~-format=links~, output the results as a maildir with symbolic links to +the found messages. This enables easy integration with mail-clients (see below +for more information). *mu* will create the maildir if it does not exist yet. + +If you specify ~--clearlinks~, existing symlinks will be cleared from the target +directories; this allows for re-use of the same maildir. However, this option +will delete any symlink it finds, so be careful. + +#+begin_example +$ mu find grolsch --format=links --linksdir=~/Maildir/search --clearlinks +#+end_example + +stores links to found messages in =~/Maildir/search=. If the directory does not +exist yet, it will be created. Note: when *mu* creates a Maildir for these links, +it automatically inserts a =.noindex= file, to exclude the directory from *mu +index*. + +** --after=<timestamp> +only show messages whose message files were last modified (*mtime*) after +=<timestamp>=. =<timestamp>= is a UNIX *time_t* value, the number of seconds since +1970-01-01 (in UTC). + +From the command line, you can use the *date* command to get this value. For +example, only consider messages modified (or created) in the last 5 minutes, you +could specify +#+begin_example + --after=`date +%s --date='5 min ago'` +#+end_example +This is assuming the GNU *date* command. + +** --exec=<command> +the ~--exec~ coption causes the =command= to be executed on each matched message; +for example, to see the raw text of all messages matching `milkshake', you could +use: +#+begin_example +$ mu find milkshake --exec='less' +#+end_example +which is roughly equivalent to: +#+begin_example +$ mu find milkshake --fields="l" | xargs less +#+end_example + +** -b, --bookmark=<bookmark> +use a bookmarked search query. Using this option, a query from your bookmark +file will be prepended to other search queries. See *mu-bookmarks(5)* for the +details of the bookmarks file. + + +** -u, --skip-dups +whenever there are multiple messages with the same message-id field, only show +the first one. This is useful if you have copies of the same message, which is a +common occurrence when using e.g. Gmail together with *offlineimap*. + +** -r, --include-related +include messages being referred to by the matched messages -- i.e.. include +messages that are part of the same message thread as some matched messages. This +is useful if you want Gmail-style `conversations'. + +** -t, --threads +show messages in a `threaded' format -- that is, with indentation and arrows +showing the conversation threads in the list of matching messages. When using +this, sorting is chronological (by date), based on the newest message in a +thread. + +Messages in the threaded list are indented based on the depth in the discussion, +and are prefix with a kind of arrow with thread-related information about the +message, as in the following table: +#+begin_example +| | normal | orphan | duplicate | +|-------------+--------+--------+-----------| +| first child | `-> | `*> | `=> | +| other | |-> | |*> | |=> | +#+end_example + +Here, an `orphan' is a message without a parent message (in the list of +matches), and a duplicate is a message whose message-id was already seen before; +not this may not really be the same message, if the message-id was copied. + +The algorithm used for determining the threads is based on Jamie Zawinksi's +description: http://www.jwz.org/doc/threading.html + +** -a,--analyze +instead of executing the query, analyze it by show the parse-tree s-expression +and a stringified version of the Xapian query. This can help users to determine +how ~mu~ interprets some query. + +The output of this command are differ between versions, but should be helpful +nevertheless. + +#+include: "muhome.inc" :minlevel 2 + +#+include: "common-options.inc" :minlevel 1 + +* INTEGRATION + +It is possible to integrate *mu find* with some mail clients + +** *mutt* + +For *mutt* you can use the following in your =muttrc=; pressing the F8 key will +start a search, and F9 will take you to the results. + +#+begin_example +# mutt macros for mu +macro index <F8> "<shell-escape>mu find --clearlinks --format=links --linksdir=~/Maildir/search " \\ + "mu find" +macro index <F9> "<change-folder-readonly>~/Maildir/search" \\ + "mu find results" +#+end_example + + +** *Wanderlust* + +*Sam B* suggested the following on the *mu*-mailing list. First add the following to +your Wanderlust configuration file: + +#+begin_example +(require 'elmo-search) +(elmo-search-register-engine + 'mu 'local-file + :prog "/usr/local/bin/mu" ;; or wherever you've installed it + :args '("find" pattern "--fields" "l") :charset 'utf-8) + +(setq elmo-search-default-engine 'mu) +;; for when you type "g" in folder or summary. +(setq wl-default-spec "[") +#+end_example + +Now, you can search using the *g* key binding; you can also create permanent +virtual folders when the messages matching some expression by adding something +like the following to your =folders= file. + +#+begin_example +VFolders { + [date:today..now]!mu "Today" + [size:1m..100m]!mu "Big" + [flag:unread]!mu "Unread" +} +#+end_example + +After restarting Wanderlust, the virtual folders should appear. + +* ENCODING + +*mu find* output is encoded according to the locale for =--format=plain= (the +default format), and UTF-8 for all other formats (=sexp=, =xml=). + + +* PERFORMANCE + +Some notes on performance, comparing the timings between some recent releases; +taking the total number for 10 test runs. + +1. time (repeat 10 mu find "" -n 50000 > /dev/null) +2. time (repeat 10 mu find "" -n 50000 --include-related --threads > /dev/null) + + +#+ATTR_MAN: :disable-caption t +| release | time 1 (sec) | time 2 (sec) | +|---------------+--------------+--------------| +| 1.4 | 8.9s | 59.3s | +| 1.6 | 8.3s | 27.5s | +| 1.8 | 8.7s | 29.3s | +| 1.10 | 9.8s | 30.6s | +| 1.11 (master) | 10.1s | 29.5s | + + + + +#+include: "exit-code.inc" :minlevel 1 + +#+include: "bugs.inc" :minlevel 1 + +#+include: "author.inc" :minlevel 1 + +#+include: "copyright.inc" :minlevel 1 + +* SEE ALSO + +*mu(1)*, *mu-index(1)*, *mu-query(7)*, *mu-info(1)* diff --git a/man/mu-help.1.org b/man/mu-help.1.org new file mode 100644 index 0000000..2a67dc2 --- /dev/null +++ b/man/mu-help.1.org @@ -0,0 +1,20 @@ +#+TITLE: MU HELP +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-help - show help information about mu commands. + +* SYNOPSIS + +*mu [common-options] help [<command>]* + +* DESCRIPTION + +*mu help* provides help information about mu commands. + +#+include: "common-options.inc" :minlevel 1 + +#+include: "exit-code.inc" :minlevel 1 + +#+include: "prefooter.inc" :minlevel 1 diff --git a/man/mu-index.1.org b/man/mu-index.1.org new file mode 100644 index 0000000..80452e0 --- /dev/null +++ b/man/mu-index.1.org @@ -0,0 +1,218 @@ +#+TITLE: MU INDEX +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-index - index e-mail messages stored in Maildirs + +* SYNOPSIS + +*mu [common-options] index* + +* DESCRIPTION + +*mu index* is the *mu* command for scanning the contents of Maildir directories and +storing the results in a Xapian database. The data can then be queried using +*mu-find(1)*. + +Before the first time you run *mu index*, you must run *mu init* to initialize the +database. + +*index* understands Maildirs as defined by Daniel Bernstein for *qmail(7)*. In +addition, it understands recursive Maildirs (Maildirs within Maildirs), +Maildir++. It also supports VFAT-based Maildirs which use =!= or =;= as the +separators instead of =:=. + +E-mail messages which are not stored in something resembling a maildir +leaf-directory (=cur= and =new=) are ignored, as are the cache directories for +=notmuch= and =gnus=, and any dot-directory. + +Symlinks are followed, and the directories can be spread over multiple +filesystems; however note that moving files around is much faster when multiple +filesystems are not involved. Be careful to avoid self-referential symlinks! + +If there is a file called =.noindex= in a directory, the contents of that +directory and all of its subdirectories will be ignored. This can be useful to +exclude certain directories from the indexing process, for example directories +with spam-messages. + +If there is a file called =.noupdate= in a directory, the contents of that +directory and all of its subdirectories will be ignored. This can be useful to +speed up things you have some maildirs that never change. + +=.noupdate= does not affect already-indexed message: you can still search for +them. =.noupdate= is ignored when you start indexing with an empty database (such +as directly after =mu init=). + +There also the option *--lazy-check* which can greatly speed up indexing; see +below for details. + +The first run of *mu index* may take a few minutes if you have a lot of mail (tens +of thousands of messages). Fortunately, such a full scan needs to be done only +once; after that it suffices to index the changes, which goes much faster. See +the `PERFORMANCE (i,ii,iii)' below for more information. + +The optional `phase two' of the indexing-process is the removal of messages from +the database for which there is no longer a corresponding file in the Maildir. +If you do not want this, you can use ~-n~, ~--nocleanup~. + +When *mu index* catches one of the signals *SIGINT*, *SIGHUP* or *SIGTERM* (e.g., when +you press Ctrl-C during the indexing process), it attempts to shutdown +gracefully; it tries to save and commit data, and close the database etc. If it +receives another signal (e.g., when pressing Ctrl-C once more), *mu index* will +terminate immediately. + +* INDEX OPTIONS + +** --lazy-check + +in lazy-check mode, *mu* does not consider messages for which the time-stamp +(ctime) of the directory they reside in has not changed since the previous +indexing run. This is much faster than the non-lazy check, but won't update +messages that have change (rather than having been added or removed), since +merely editing a message does not update the directory time-stamp. Of course, +you can run *mu-index* occasionally without ~--lazy-check~, to pick up such +messages. + +** --nocleanup + +disable the database cleanup that *mu* does by default after indexing. + +** --reindex + +perform a complete reindexing of all the messages in the maildir. + +#+include: "muhome.inc" :minlevel 2 + +#+include: "common-options.inc" :minlevel 1 + +* ENCRYPTION + +*mu index* does _not_ decrypt messages, and only the metadata (such as headers) of +encrypted messages makes it to the database. *mu view* and *mu4e* can decrypt +messages, but those work with the message directly and the information is not +added to the database. + +* PERFORMANCE + +** indexing in ancient times (2009?) + +As a non-scientific benchmark, a simple test on the author's machine (a Thinkpad +X61s laptop using Linux 2.6.35 and an ext3 file system) with no existing +database, and a maildir with 27273 messages: + +#+begin_example +$ sudo sh -c 'sync && echo 3 > /proc/sys/vm/drop_caches' +$ time mu index --quiet +66,65s user 6,05s system 27% cpu 4:24,20 total +#+end_example +(about 103 messages per second) + +A second run, which is the more typical use case when there is a database +already, goes much faster: + +#+begin_example +$ sudo sh -c 'sync && echo 3 > /proc/sys/vm/drop_caches' +$ time mu index --quiet +0,48s user 0,76s system 10% cpu 11,796 total +#+end_example +(more than 56818 messages per second) + +Note that each test flushes the caches first; a more common use case might be to +run *mu index* when new mail has arrived; the cache may stay quite `warm' in that +case: + +#+begin_example + $ time mu index --quiet + 0,33s user 0,40s system 80% cpu 0,905 total +#+end_example +which is more than 30000 messages per second. + +** indexing in 2012 + +As per June 2012, we did the same non-scientific benchmark, this time with an +Intel i5-2500 CPU @ 3.30GHz, an ext4 file system and a maildir with 22589 +messages. We start without an existing database. + +#+begin_example + $ sudo sh -c 'sync && echo 3 > /proc/sys/vm/drop_caches' + $ time mu index --quiet + 27,79s user 2,17s system 48% cpu 1:01,47 total +#+end_example +(about 813 messages per second) + +A second run, which is the more typical use case when there is a database +already, goes much faster: + +#+begin_example +$ sudo sh -c 'sync && echo 3 > /proc/sys/vm/drop_caches' +$ time mu index --quiet +0,13s user 0,30s system 19% cpu 2,162 total +#+end_example +(more than 173000 messages per second) + +** indexing in 2016 + +As per July 2016, we did the same non-scientific benchmark, again with the Intel +i5-2500 CPU @ 3.30GHz, an ext4 file system. This time, the maildir contains +72525 messages. + +#+begin_example +$ sudo sh -c 'sync && echo 3 > /proc/sys/vm/drop_caches' +$ time mu index --quiet +40,34s user 2,56s system 64% cpu 1:06,17 total +#+end_example +(about 1099 messages per second). + +** indexing in 2022 + +A few years later and it is June 2022. There's a lot more happening during +indexing, but indexing became multi-threaded and machines are faster; e.g. this +is with an AMD Ryzen Threadripper 1950X (16 cores) @ 3.399GHz. + +The instructions are a little different since we have a proper repeatable +benchmark now. After building, + +#+begin_example + $ sudo sh -c 'sync && echo 3 > /proc/sys/vm/drop_caches' +% THREAD_NUM=4 build/lib/tests/bench-indexer -m perf +# random seed: R02Sf5c50e4851ec51adaf301e0e054bd52b +1..1 +# Start of bench tests +# Start of indexer tests +indexed 5000 messages in 20 maildirs in 3763ms; 752 μs/message; 1328 messages/s (4 thread(s)) +ok 1 /bench/indexer/4-cores +# End of indexer tests +# End of bench tests +#+end_example + +Things are again a little faster, even though the index does a lot more now +(text-normalizatian, and pre-generating message-sexps). A faster machine helps, +too! + +** recent releases + +Indexing the the same 93000-message mail corpus with the last few releases: + +#+ATTR_MAN: :disable-caption t +| release | time (sec) | notes | +|---------------+------------+------------------------------------------| +| 1.4 | 160s | | +| 1.6 | 178s | | +| 1.8 | 97s | | +| 1.10 | 120s | adds html indexing, sexp-caching | +| 1.11 (master) | 96s | adds language-guessing, batch-size=50000 | +| | | | + +Quite some variation! + +Over time new features / refactoring can change the timings quite a bit. At +least for now, the latest code is both the fastest and the most featureful! + +#+include: "exit-code.inc" :minlevel 1 + +#+include: "prefooter.inc" + +* SEE ALSO + +*maildir(5)*, *mu(1)*, *mu-init(1)*, *mu-find(1)*, *mu-cfind(1)* diff --git a/man/mu-info.1.org b/man/mu-info.1.org new file mode 100644 index 0000000..0d182d7 --- /dev/null +++ b/man/mu-info.1.org @@ -0,0 +1,32 @@ +#+TITLE: MU INFO +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-info - show information + +* SYNOPSIS + +*mu [common options] info [<topic>]* + +* DESCRIPTION + +~mu info~ is the ~mu~ command for getting information about various topics: + +- *mu*: general mu build information (default) +- *store*: information about the message store +- *fields*: table with all the query fields and flags +- *maildirs*: list all maildirs under the store's root-maildir + +Note that while running (e.g. ~mu4e~), some of the ~store~ information can be +delayed due to database caching. + +#+include: "common-options.inc" :minlevel 1 + +#+include: "exit-code.inc" :minlevel 1 + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO + +*mu(1)* diff --git a/man/mu-init.1.org b/man/mu-init.1.org new file mode 100644 index 0000000..6c3d4c9 --- /dev/null +++ b/man/mu-init.1.org @@ -0,0 +1,100 @@ +#+TITLE: MU INIT +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-init - initialize the mu message database + +* SYNOPSIS + +*mu [common-options] init [options]* + +* DESCRIPTION + +*mu init* is the subcommand for setting up the mu message database. After *mu init* +has completed, you can run *mu index* + +* INIT OPTIONS + +** -m, --maildir=<maildir> + +use =<maildir>= as the root-maildir. + +By default, *mu* uses the *MAILDIR* environment; if it is not set, it uses =~/Maildir= +if it is an existing directory. If neither of those can be used, the ~--maildir~ +option is required; it must be an absolute path (but ~~/~ expansion is +performed). + +** --my-address=<email-address-or-regex> + +specifies that some e-mail address is `my-address' (the option can be used +multiple times). Any message in which at least one of the contact fields +contains such an address is considered a `personal' messages; this can then be +used for filtering in *mu-find(1)*, *mu-cfind(1)* and *mu4e*, e.g. to filter-out +mailing list messages. + +=<email-address-or-regex>= can be either a plain e-mail address (such as +*foo@example.com*), or a basic PCRE regular-expression (see *pcre(3)* for details), +wrapped in */* (such as =/foo-.*@example\\.com/=). Depending on your shell, the +argument may need to be quoted. + +** --ignored-address=<email-address-or-regex> + +specifies that some e-mail address is to be ignored from the contacts-cache (the +option can be used multiple times). Such addresses then cannot be found with +*mu-cfind(1)* or in the Mu4e contacts cache. + +=<my-email-address>= can be either a plain e-mail address or a regexp, just like +for the =--my-address= option. + +** --max-message-size=<size> + +specifies the maximum size for an e-mail message. Usually, the default of +100000000 bytes should be fine. + +** --batch-size=<size> + +the number of changes after which they are committed to the database; decreasing +the value reduces the memory requirements, at the cost of make indexing +substantially slower. Usually, the default of 250000 should be fine. + +Batch-size 0 is interpreted as `use the default'. + +** --support-ngrams + +whether to enable support for using ngrams in indexing and query parsing; this +can be useful for languages without explicit word breaks, such as +Chinese/Japanese/Korean. See *NGRAM SUPPORT* below for details. + +** --reinit + +reinitialize the database from an earlier version; that is, create a new empty +database with the existing settings. This cannot be combined with the other ~init~ +options. + +#+include: "muhome.inc" :minlevel 2 + +* NGRAM SUPPORT + +*mu*'s underlying Xapian database supports `ngrams', which improve searching for +languages/scripts that do not have explicit word breaks, such as Chinese, +Japanese and Korean. It is fairly intrusive, and influences both indexing and +query-parsing; it is not enabled by default, and is recommended only if you need +to search for messages written in such languages. + +When enabled, *mu* automatically uses ngrams automatically. Xapian environment +variables such as ~XAPIAN_CJK_NGRAM~ are ignored. + +#+include: "exit-code.inc" :minlevel 1 + + +* EXAMPLE +#+begin_example +$ mu init --maildir=~/Maildir --my-address=alice@example.com --my-address=bob@example.com --ignored-address='/.*reply.*/' +#+end_example + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO + +*mu-index(1)*, *mu-find(1)*, *mu-cfind(1)*, *pcre(3)* diff --git a/man/mu-mkdir.1.org b/man/mu-mkdir.1.org new file mode 100644 index 0000000..f10a714 --- /dev/null +++ b/man/mu-mkdir.1.org @@ -0,0 +1,42 @@ +#+TITLE: MU MKDIR +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-mkdir - create a new Maildir + +* SYNOPSIS + +*mu [common-options] mkdir [options] <dir> [<dirs>]* + +* DESCRIPTION + +*mu mkdir* is the command for creating Maildirs as per *maildir(5)*. A maildir is a +a directory with subdirectories ~new~, ~cur~ and ~tmp~. + +The command does not use the mu database. + +If creation fails for any reason, *no* attempt is made to remove any parts that +were created. This is for safety reasons. + +* MKDIR OPTIONS + +** --mode=<mode> +set the file access mode for the new maildir(s) as in *chmod(1)*. The default +is 0755. + +#+include: "common-options.inc" :minlevel 1 + +* EXAMPLE + +#+begin_example +$ mu mkdir tom dick harry +#+end_example + +creates three maildirs, =tom=, =dick= and =harry=. + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO + +*maildir(5)*, *chmod(1)* diff --git a/man/mu-move.1.org b/man/mu-move.1.org new file mode 100644 index 0000000..d43a3fa --- /dev/null +++ b/man/mu-move.1.org @@ -0,0 +1,117 @@ +#+TITLE: MU MOVE +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-move - move a message file or change its flags + +* SYNOPSIS + +*mu [common-options] move [options] <src> [--flags=<flags>] [<target>]* + +* DESCRIPTION + +*mu move* is the command for moving messages in a Maildir or changing their flags. + +For any change, both the message file in the file system as well as its +representation in the database are updated accordingly. + +The source message file and target-maildir must reside under the root-maildir +for mu's database (see *mu info store*). + +* MOVE OPTIONS + +** --flags=<flags> + +specify the new message flags. See *FLAGS* for details. + +** --change-name + +change the basename of the message file when moving; this can be useful when +using some external tools such as *mbsync(1)* which otherwise get confused + +** --update-dups + +update the flags of duplicate messages too, where "duplicate messages" are +defined as all message that share the same message-id. Note that the +Draft/Flagged/Trashed flags are deliberately _not_ changed if you change those on +the source message. + +** --dry-run,-n + +print the target filename(s), but don't change anything. + +Note that with the ~--change-name~, the target name is not constant, so you cannot +use a dry-run to predict the exact name when doing a `real' run. + +#+include: "common-options.inc" :minlevel 1 + +* FLAGS + +(Note: if you are not familiar with Maildirs, please refer to the *maildir(5)* +man-page, or see http://cr.yp.to/proto/maildir.html) + +The message flags specify the Maildir-metadata for a message and are represented +by uppercase letters at the end of the message file name for all `non-new' +messages, i.e. messages that live in the ~cur/~ sub-directory of a Maildir. + +#+ATTR_MAN: :disable-caption t +| Flag | Meaning | +|------+------------------------------------| +| D | Draft message | +| F | Flagged message | +| P | Passed message (i.e., `forwarded') | +| R | Replied message | +| S | Seen message | +| T | Trashed; to be deleted later | + +New messages (in the ~new/~ sub-directory) do not have flags encoded in their +file-name; but we *mu* uses `N' in the ~--flags~ to represent that: + +#+ATTR_MAN: :disable-caption t +| Flag | Meaning | +|------+---------| +| N | New | + +Thus, changing flags means changing the letters at the end of the message +file-name, except when setting or removing the `N' (new) flag. Setting or +un-setting the New flag causes the message is to be moved from ~cur/~ to ~new/~ or +vice-versa, respectively. When marking a message as New, it looses the other +flags. + +* ABSOLUTE AND RELATIVE FLAGS + +You can specify the flags with the ~--flags~ parameter, and do either with either +*absolute* or *relative* flags. + +Absolute flags just specify the new flags by their letters; e.g. to specify a +/Trashed/, /Seen/, /Replied/ message, you'd use ~--flags STR~. +#+end_example + +Relative flags are relative to the current flags for some message, and each of +the flags is prefixed with either ~+~ ("add this flag") or ~-~ ("remove this flag"). + +So to add the /Seen/ flag and remove the /Draft/ flag from whatever the message +already has, ~--flags +S-D~. + +You cannot combine relative and relative flags. + +* EXAMPLES + +** change some flags +#+begin_example +$ mu move /home/user/Maildir/inbox/cur/1695559560.a73985881f4611ac2.hostname!2,S --flags +F +/home/user/Maildir/inbox/cur/1695559560.a73985881f4611ac2.hostname!2,FS +#+end_example + +** move to a different maildir +#+begin_example +$ mu move /home/user/Maildir/project1/cur/1695559560.a73985881f4611ac2.hostname!2,S /project2 +/home/user/Maildir/project2/cur/1695559560.a73985881f4611ac2.hostname!2,S +#+end_example + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO + +*maildir(5)* diff --git a/man/mu-query.7.org b/man/mu-query.7.org new file mode 100644 index 0000000..49b8c46 --- /dev/null +++ b/man/mu-query.7.org @@ -0,0 +1,384 @@ +#+TITLE: MU QUERY +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-query - a language for finding messages in *mu* databases. + +* DESCRIPTION + +The mu query language is the language used by *mu find* and *mu4e* to find messages +in *mu*'s Xapian database. The language is quite similar to Xapian's default +query-parser, but is an independent implementation that is customized for the +mu/mu4e use-case. + +Here, we give a structured but informal overview of the query language and +provide examples. As a companion to this, we recommend the *mu fields* and *mu +flags* commands to get an up-to-date list of the available fields and flags. + +Furthermore, *mu find* provides the ~--analyze~ option, which shows how *mu* +interprets your query; see the *ANALYZING QUERIES* section below. + +*NOTE:* if you use queries on the command-line (say, for *mu find*), you need to +quote any characters that would otherwise be interpreted by the shell, such as +*""*, *(* and *)* and whitespace. + +* TERMS + +The basic building blocks of a query are *terms*; these are just normal words like +`banana' or `hello', or words prefixed with a field-name which makes them apply +to just that field. See *mu info fields* for all the available fields. + +Some example queries: +#+begin_example +vacation +subject:capybara +maildir:/inbox +#+end_example + +Terms without an explicit field-prefix, (like `vacation' above) are interpreted +like: +#+begin_example +to:vacation or subject:vacation or body:vacation or ... +#+end_example + +The language is case-insensitive for terms and attempts to `flatten' diacritics, +so =angtrom= matches =Ångström=. + +If terms contain whitespace, they need to be quoted: +#+begin_example +subject:"hi there" +#+end_example +This is a so-called =phrase query=, which means that we match against subjects +that contain the literal phrase "hi there". Phrase queries only work for fields +that are /indexed/, i.e., fields with *index* in the *mu info fields* search column. + +Remember that you need to escape those quotes when using this from the +command-line: +#+begin_example +mu find subject:\\"hi there\\" +#+end_example + +* LOGICAL OPERATORS + +We can combine terms with logical operators -- binary ones: *and*, *or*, *xor* and the +unary *not*, with the conventional rules for precedence and association. The +operators are case-insensitive. + +You can also group things with *(* and *)*, so you can write: +#+begin_example +(subject:beethoven or subject:bach) and not body:elvis +#+end_example + +If you do not explicitly specify an operator between terms, *and* is implied, so +the queries +#+begin_example +subject:chip subject:dale +#+end_example +#+begin_example +subject:chip AND subject:dale +#+end_example +are equivalent. For readability, we recommend the second version. + +Note that a =pure not= - e.g. searching for *not apples* is quite a `heavy' query. + +* REGULAR EXPRESSIONS AND WILDCARDS + +The language supports matching basic PCRE regular expressions, see *pcre(3)*. + +Regular expressions are enclosed in *//*. Some examples: + +#+begin_example +subject:/h.llo/ # match hallo, hello, ... +subject:/ +#+end_example + +Note the difference between `maildir:/foo' and `maildir:/foo/'; the former +matches messages in the `/foo' maildir, while the latter matches all messages in +all maildirs that match `foo', such as `/foo', `/bar/cuux/foo', `/fooishbar' +etc. + +Wildcards are another mechanism for matching where a term with a rightmost *** +(and =only= in that position) matches any term that starts with the part before +the ***; they are therefore less powerful than regular expressions, but also much +faster: +#+begin_example +foo* +#+end_example +is equivalent to +#+begin_example +/foo.*/ +#+end_example + +Regular expressions can be useful, but are relatively slow. + +* FIELDS + +We already saw a number of search fields, such as *subject:* and *body:*. For the +full table with all details, including single-char shortcuts, try the command: +~mu info fields~. + +#+ATTR_MAN: :disable-caption t +#+begin_example ++-----------+----------+----------+-----------------------------+ +| flag | shortcut | category | description | ++-----------+----------+----------+-----------------------------+ +| draft | D | file | Draft (in progress) | ++-----------+----------+----------+-----------------------------+ +| flagged | F | file | User-flagged | ++-----------+----------+----------+-----------------------------+ +| passed | P | file | Forwarded message | ++-----------+----------+----------+-----------------------------+ +| replied | R | file | Replied-to | ++-----------+----------+----------+-----------------------------+ +| seen | S | file | Viewed at least once | ++-----------+----------+----------+-----------------------------+ +| trashed | T | file | Marked for deletion | ++-----------+----------+----------+-----------------------------+ +| new | N | maildir | New message | ++-----------+----------+----------+-----------------------------+ +| signed | z | content | Cryptographically signed | ++-----------+----------+----------+-----------------------------+ +| encrypted | x | content | Encrypted | ++-----------+----------+----------+-----------------------------+ +| attach | a | content | Has at least one attachment | ++-----------+----------+----------+-----------------------------+ +| unread | u | pseudo | New or not seen message | ++-----------+----------+----------+-----------------------------+ +| list | l | content | Mailing list message | ++-----------+----------+----------+-----------------------------+ +| personal | q | content | Personal message | ++-----------+----------+----------+-----------------------------+ +| calendar | c | content | Calendar invitation | ++-----------+----------+----------+-----------------------------+ +#+end_example + +(*) The language code for the text-body if found. This works only if ~mu~ was +built with CLD2 support. + +There are also the special fields *contact:*, which matches all contact-fields +(=from=, =to=, =cc= and =bcc=), and *recip*, which matches all recipient-fields (=to=, =cc= +and =bcc=). + +Hence, for instance, +#+begin_example +contact:fnorb@example.com +#+end_example +is equivalent to +#+begin_example +(from:fnorb@example.com or to:fnorb@example.com or + cc:from:fnorb@example.com or bcc:fnorb@example.com) +#+end_example + +* DATE RANGES + +The *date:* field takes a date-range, expressed as the lower and upper bound, +separated by *..*. Either lower or upper (but not both) can be omitted to create +an open range. + +Dates are expressed in local time and using ISO-8601 format (YYYY-MM-DD +HH:MM:SS); you can leave out the right part and *mu* adds the rest, depending on +whether this is the beginning or end of the range (e.g., as a lower bound, +`2015' would be interpreted as the start of that year; as an upper bound as the +end of the year). + +You can use `/' , `.', `-', `:' and `T' to make dates more human-readable. + +Some examples: +#+begin_example +date:20170505..20170602 +date:2017-05-05..2017-06-02 +date:..2017-10-01T12:00 +date:2015-06-01.. +date:2016..2016 +#+end_example + +You can also use the special `dates' *now* and *today*: +#+begin_example +date:20170505..now +date:today.. +#+end_example + +Finally, you can use relative `ago' times which express some time before now and +consist of a number followed by a unit, with units *s* for seconds, *M* for minutes, +*h* for hours, *d* for days, *w* for week, *m* for months and *y* for years. Some +examples: + +#+begin_example +date:3m.. +date:2017.01.01..5w +#+end_example + +* SIZE RANGES + +The *size* or *z* field allows you to match =size ranges= -- that is, match messages +that have a byte-size within a certain range. Units (b (for bytes), K (for 1000 +bytes) and M (for 1000 * 1000 bytes) are supported). Some examples: + +#+begin_example +size:10k..2m +size:10m.. +#+end_example + +* FLAG FIELD + +The *flag/g* field allows you to match message flags. The following fields are +available: +#+begin_example ++-----------+----------+----------+-----------------------------+ +| flag | shortcut | category | description | ++-----------+----------+----------+-----------------------------+ +| draft | D | file | Draft (in progress) | ++-----------+----------+----------+-----------------------------+ +| flagged | F | file | User-flagged | ++-----------+----------+----------+-----------------------------+ +| passed | P | file | Forwarded message | ++-----------+----------+----------+-----------------------------+ +| replied | R | file | Replied-to | ++-----------+----------+----------+-----------------------------+ +| seen | S | file | Viewed at least once | ++-----------+----------+----------+-----------------------------+ +| trashed | T | file | Marked for deletion | ++-----------+----------+----------+-----------------------------+ +| new | N | maildir | New message | ++-----------+----------+----------+-----------------------------+ +| signed | z | content | Cryptographically signed | ++-----------+----------+----------+-----------------------------+ +| encrypted | x | content | Encrypted | ++-----------+----------+----------+-----------------------------+ +| attach | a | content | Has at least one attachment | ++-----------+----------+----------+-----------------------------+ +| unread | u | pseudo | New or not seen message | ++-----------+----------+----------+-----------------------------+ +| list | l | content | Mailing list message | ++-----------+----------+----------+-----------------------------+ +| personal | q | content | Personal message | ++-----------+----------+----------+-----------------------------+ +| calendar | c | content | Calendar invitation | ++-----------+----------+----------+-----------------------------+ +#+end_example + +Some examples: +#+begin_example +flag:attach +flag:replied +g:x +#+end_example + +Encrypted messages may be signed as well, but this is only visible after +decrypting and thus invisible to *mu*. + +* PRIORITY FIELD + +The message priority field (*prio:*) has three possible values: *low*, *normal* or +*high*. For instance, to match high-priority messages: +#+begin_example +prio:high +#+end_example + +* MAILDIR + +The Maildir field describes the directory path starting *after* the Maildir root +directory, and before the =/cur/= or =/new/= part. So, for example, if there's a +message with the file name =~/Maildir/lists/running/cur/1234.213:2,=, you could +find it (and all the other messages in that same maildir) with: +#+begin_example +maildir:/lists/running +#+end_example + +Note the starting `/'. If you want to match mails in the `root' maildir, you can +do with a single `/': +#+begin_example +maildir:/ +#+end_example + +If you have maildirs (or any fields) that include spaces, you need to quote +them, ie. +#+begin_example +maildir:"/Sent Items" +#+end_example + +And once again, note that when using the command-line, such queries must be +quoted: +#+begin_example +mu find 'maildir:"/Sent Items"' +#+end_example + +Also note that you should *not* end the maildir with a ~/~, or it can be +misinterpreted as a regular expression term; see aforementioned. + +* MORE EXAMPLES + +Here are some simple examples of *mu* queries; you can make many more complicated +queries using various logical operators, parentheses and so on, but in the +author's experience, it's usually faster to find a message with a simple query +just searching for some words. + +Find all messages with both `bee' and `bird' (in any field) +#+begin_example +bee AND bird +#+end_example + +Find all messages with either Frodo or Sam: +#+begin_example +Frodo OR Sam +#+end_example + +Find all messages with the `wombat' as subject, and `capybara' anywhere: +#+begin_example +subject:wombat and capybara +#+end_example + +Find all messages in the `Archive' folder from Fred: +#+begin_example +from:fred and maildir:/Archive +#+end_example + +Find all unread messages with attachments: +#+begin_example +flag:attach and flag:unread +#+end_example + +Find all messages with PDF-attachments: +#+begin_example +mime:application/pdf +#+end_example + +Find all messages with attached images: +#+begin_example +mime:image/* +#+end_example + +Find all messages written in Dutch or German with the word `hallo': +#+begin_example +hallo and (lang:nl or lang:de) +#+end_example + +This is only available if your *mu* has support for this; see *mu info* and check +for "cld2-support*. + +* ANALZYING QUERIES + +Despite all the excellent documentation, in some cases it can be non-obvious how +~mu~ interprets your query. For that, you can ask ~mu~ to analyze the query -- that +is, show how ~mu~ interprets the query. + +This uses the the ~--analyze~ option to *mu find*. +#+begin_example +$ mu find subject:wombat AND date:3m.. size:..2000 --analyze +,* query: + subject:wombat AND date:3m.. size:..2000 +,* parsed query: + (and (subject "wombat") (date (range "2023-05-30T06:10:09Z" "")) (size (range "" "2000"))) +,* Xapian query: + Query((Swombat AND VALUE_GE 4 n64759341 AND VALUE_LE 17 i7d0)) +#+end_example + +The ~parsed query~ is usually the most useful one for understanding how *mu* +interprets your query. + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO + +*mu-find(1)*, *mu-info(1), *pcre(3)* diff --git a/man/mu-remove.1.org b/man/mu-remove.1.org new file mode 100644 index 0000000..ff2c24c --- /dev/null +++ b/man/mu-remove.1.org @@ -0,0 +1,27 @@ +#+TITLE: MU REMOVE +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-remove - remove messages from the database. + +* SYNOPSIS + +*mu [common-options] remove [options] <file> [<files>]* + +* DESCRIPTION + +*mu remove* removes specific messages from the database, each of them specified by +their filename. The files do not have to exist in the file system. + +* REMOVE OPTIONS + +#+include: "muhome.inc" :minlevel 2 + +#+include: "common-options.inc" :minlevel 1 + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO + +*mu(1)*, *mu-index(1)*, *mu-add(1)* diff --git a/man/mu-server.1.org b/man/mu-server.1.org new file mode 100644 index 0000000..1814860 --- /dev/null +++ b/man/mu-server.1.org @@ -0,0 +1,91 @@ +#+TITLE: MU-SERVER +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-server - the mu backend for the mu4e e-mail client + +* SYNOPSIS + +mu [common-options] server + +* DESCRIPTION + +*mu server* starts a simple shell in which one can query and manipulate the mu +database. The output uses s-expressions. *mu server* is not meant for use by +humans, except for debugging purposes. Instead, it is designed specifically for +the *mu4e* e-mail client. + +#+begin_example + (<command-name> :param1 value1 :param2 value2) +#+end_example + +For example, to view a certain message, the command would be: + +#+begin_example + (view :docid 12345) +#+end_example + +Parameters can be sent in any order; they must be of the correct type though. +See *lib/utils/mu-sexp-parser.hh* and *lib/utils/mu-sexp-parser.cc* in source-tree +for the details. + +* OUTPUT FORMAT + +*mu server* accepts a number of commands, and delivers its results in the form: + +#+begin_example + \\376<length>\\377<s-expr> +#+end_example + +\\376 (one byte 0xfe), followed by the length of the s-expression expressed as +an hexadecimal number, followed by another \\377 (one byte 0xff), followed by +the actual s-expression. + +By prefixing the expression with its length, it can be processed more +efficiently. The \\376 and \\377 were chosen since they never occur in valid +UTF-8 (in which the s-expressions are encoded). + +* SERVER OPTIONS + +** --commands + +List available commands (and try with ~--verbose~) + +** --eval <expression> + +Evaluate a mu4e server s-expression + +** --allow-temp-file + +If set, allow for the output of some commands to use temp-files rather than +directly through the emacs process input/output. This is noticeably faster for +commands with a lot of output, esp. when the the temp-file uses a in-memory +file-system. + +* PERFORMANCE + +As an indication for the relative performance, we can simulate something ~mu4e~ +does; we take overall time of 50 such requests: + +#+begin_src sh +time build/mu/mu server --allow-temp-file --eval '(find :query "\"\"" :include-related t :threads t :maxnum 50000)' >/dev/null +#+end_src +(and ~--allow-temp-file~ for 1.11) + +#+ATTR_MAN: :disable-caption t +| release | time (sec) | +|---------------+------------| +| 1.8 | 8.6s | +| 1.10 | 5.7s | +| 1.11 (master) | 2.8s | + + +#+include: "muhome.inc" :minlevel 2 + +#+include: "common-options.inc" :minlevel 1 + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO +*mu(1)* diff --git a/man/mu-verify.1.org b/man/mu-verify.1.org new file mode 100644 index 0000000..9cc0933 --- /dev/null +++ b/man/mu-verify.1.org @@ -0,0 +1,55 @@ +#+TITLE: MU VERIFY +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-verify - verify message signatures and display information about them + +* SYNOPSIS + +*mu [common-options] verify [options] [<file> ... ]* + +* DESCRIPTION + +*mu verify* is the *mu* command for verifying message signatures (such as PGP/GPG +signatures) and displaying information about them. The sub-command works on +message files, and does not require the message to be indexed in the database. + +If no message file is provided, the command expects the message on +standard-input. + +* VERIFY OPTIONS + +** -r, --auto-retrieve +attempt to find keys online (see the *auto-key-retrieve* option in the *gnupg(1)* +documentation). + +** decrypt +attempt to decrypt the message + +#+include: "common-options.inc" :minlevel 1 + +* EXAMPLES + +To display aggregated (one-line) information about the verification status in a +message: +#+begin_example +$ mu verify msgfile +#+end_example + +To display information about all the signatures: +#+begin_example +$ mu verify --verbose msgfile +#+end_example + +If you only want to use the exit code, you can use: +#+begin_example +$ mu verify --quiet msgfile +#+end_example +which does not give any output unless there is an error. + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO + +*mu(1)* diff --git a/man/mu-view.1.org b/man/mu-view.1.org new file mode 100644 index 0000000..17351f7 --- /dev/null +++ b/man/mu-view.1.org @@ -0,0 +1,54 @@ +#+TITLE: MU VIEW +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu-view - display an e-mail message file + +* SYNOPSIS + +mu [common options] view [options] [<file> ...] + +* DESCRIPTION + +*mu view* is the *mu* command for displaying e-mail message files. It works on +message files and does =not= require the message to be indexed in the database. + +The command shows some common headers (From:, To:, Cc:, Bcc:, Subject: and +Date:), the list of attachments and either the plain-text or html body of the +message (if any), or its s-expression representation. + +If no message file is provided, the command reads the message from +standard-input. + +* VIEW OPTIONS + +** --format,-o = <format> +use the given output format, one of: + +- ~plain~ - use the plain-text body; this is the default +- ~html~ - use the HTML body +- ~sexp~ - show the S-expression representation of the message + +** --summary-len=<number> +instead of displaying the full message, output a summary based upon the first +=<number>= lines of the message. + +** --terminate +terminate messages with \\​f (=form-feed=) characters when displaying them. This is +useful when you want to further process them. + +** --decrypt +attempt to decrypt encrypted message bodies. This is only possible if *mu* +was built with crypto-support. + +** --auto-retrieve +attempt to retrieve crypto-keys automatically from the network, when needed. + +#+include: "common-options.inc" :minlevel 1 + +* BUGS + +* SEE ALSO + +*mu(1)* diff --git a/man/mu.1.org b/man/mu.1.org new file mode 100644 index 0000000..026fd32 --- /dev/null +++ b/man/mu.1.org @@ -0,0 +1,90 @@ +#+TITLE: MU +#+MAN_CLASS_OPTIONS: :section-id "@SECTION_ID@" :date "@MAN_DATE@" + +* NAME + +mu - a set of tools to deal with Maildirs and message files, in particular to +index and search e-mail messages. + +* SYNOPSIS + +~mu~ [COMMON-OPTIONS] [[COMMAND] [COMMAND-OPTIONS]] + +For information about the common options, see *COMMON OPTIONS*. + +* DESCRIPTION + +~mu~ is the general command shows help about the specific commands: + +- ~add~: add specific messages to the database. +- ~cfind~: find contacts +- ~extract~: extract attachments and other MIME-parts +- ~find~: find messages in the database +- ~help~: get help for some command +- ~index~: (re)index the messages in a Maildir +- ~info~: show information about the mu database +- ~init~: initialize the mu database +- ~mkdir~: create a new Maildir +- ~remove~: remove specific messages from the database +- ~server~: start a server process (for ~mu4e~-internal use) +- ~view~: view a specific message + +Each of the commands have their own manpage ~mu-<command~>~. + +~mu~ is a set of tools for dealing with Maildirs and the e-mail messages +in them. + +~mu~'s main purpose is to enable searching of e-mail messages. It +does so by periodically scanning a Maildir directory tree and +analyzing the e-mail messages found (this is called `indexing'). The +results of this analysis are stored in a database, which can then be +queried. + +In addition to indexing and searching, ~mu~ also offers +functionality for viewing messages, extracting attachments and +creating maildirs, and searching and exporting contact information. + +~mu~ can be used from the command line or can be integrated with various +e-mail clients. + +This manpage gives a general overview of the available commands +(~index~, ~find~, etc.); each ~mu~ command has its own +man-page as well. + +* COLORS + +Some ~mu~ commands support colorized output, and do so by default. If you don't +want colors, you can use ~--nocolor~. + +* ENCODING + +~mu~'s output is in the current locale, with the exceptions of the output +specifically meant for output to UTF8-encoded files. In practice, this means +that the output of commands ~index~, ~view~, ~extract~ is always encoded according to +the current locale. + +The same is true for ~find~ and ~cfind~, with some exceptions, where +the output is always UTF-8, regardless of the locale: + +- For ~cfind~ the exception is ~--format=bbdb~. This is hard-coded to UTF-8, and as + such specified in the output-file, so emacs/bbdb can handle it correctly + without guessing. +- For ~find~ the output is encoded according the locale for ~--format=plain~ (the + default), and UTF-8 for all other formats. + +* DATABASE AND FILE + +Commands ~mu index~ and ~find~ and ~cfind~ work with the database, while the other +ones work on individual mail files. Hence, running ~view~, ~mkdir~ and ~extract~ does +not require the mu database. + +#+include: "common-options.inc" :minlevel 1 + +#+include: "exit-code.inc" :minlevel 1 + +#+include: "prefooter.inc" :minlevel 1 + +* SEE ALSO +~mu-add(1)~, ~mu-cfind(1)~, ~mu-extract(1)~, ~mu-find(1)~, ~mu-help(1)~, ~mu-index(1)~, +~mu-info(1)~, ~mu-init(1)~, ~mu-mkdir(1)~, ~mu-remove(1)~, ~mu-server(1)~, ~mu-view(1)~, +~mu-query(7)~, ~mu-easy(1)~ diff --git a/man/muhome.inc b/man/muhome.inc new file mode 100644 index 0000000..8b312a2 --- /dev/null +++ b/man/muhome.inc @@ -0,0 +1,12 @@ +** --muhome +use a non-default directory to store and read the database, write the logs, etc. +By default, ~mu~ uses the XDG Base Directory Specification (e.g. on GNU/Linux this +defaults to =~/.cache/mu= and =~/.config/mu=). Earlier versions of ~mu~ defaulted to +=~/.mu=, which now requires =--muhome=~/.mu=. + +The environment variable ~MUHOME~ can be used as an alternative to ~--muhome~. The +latter has precedence. + +# Local Variables: +# mode: org +# End: diff --git a/man/prefooter.inc b/man/prefooter.inc new file mode 100644 index 0000000..79c6e40 --- /dev/null +++ b/man/prefooter.inc @@ -0,0 +1,9 @@ +#+include: "bugs.inc" :minlevel 1 + +#+include: "author.inc" :minlevel 1 + +#+include: "copyright.inc" :minlevel 1 + +# Local Variables: +# mode: org +# End: diff --git a/meson.build b/meson.build new file mode 100644 index 0000000..718d68f --- /dev/null +++ b/meson.build @@ -0,0 +1,306 @@ +## Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +################################################################################ +# project setup +project('mu', ['c', 'cpp'], + version: '1.12.5', + meson_version: '>= 0.56.0', + license: 'GPL-3.0-or-later', + default_options : [ + 'buildtype=debugoptimized', + 'warning_level=3', + 'c_std=c11', + 'cpp_std=c++17']) + +# hard-code the date here (for reproduciblity); we derive the dates used in e.g. +# documentation from this. +mu_date='2024-04-15' + +# installation paths +prefixdir = get_option('prefix') +bindir = prefixdir / get_option('bindir') +datadir = prefixdir / get_option('datadir') +mandir = prefixdir / get_option('mandir') +infodir = prefixdir / get_option('infodir') + +# allow for configuring lispdir, as with autotools. +if get_option('lispdir') == '' + mu4e_lispdir= datadir / join_paths('emacs', 'site-lisp', 'mu4e') +else + mu4e_lispdir= get_option('lispdir') / 'mu4e' +endif + +################################################################################ +# compilers / flags +# + +# compilers +cc = meson.get_compiler('c') +cxx= meson.get_compiler('cpp') + +extra_flags = [ + '-Wno-unused-parameter', + '-Wno-cast-function-type', + '-Wformat-security', + '-Wformat=2', + '-Wstack-protector', + '-fstack-protector-strong', + '-Wno-switch-enum', + # assuming these are false alarm... (in fmt, with gcc13): + '-Wno-array-bounds', + '-Wno-stringop-overflow',] + +if (cxx.get_id() == 'clang') + extra_flags += [ + '-Wc11-extensions', + '-Wno-keyword-macro', + '-Wno-deprecated-volatile', + '-Wno-#warnings'] +endif + +extra_cpp_flags= [ + '-Wno-volatile' +] + +if get_option('buildtype') == 'debug' + extra_flags += [ + '-D_GLIBCXX_ASSERTIONS', + '-ggdb', + '-g3'] +endif + +# extra arguments, if available +foreach extra_arg : extra_flags + if cc.has_argument (extra_arg) + add_project_arguments([extra_arg], language: 'c') + endif +endforeach + +foreach extra_arg : extra_flags + extra_cpp_flags + if cxx.has_argument (extra_arg) + add_project_arguments([extra_arg], language: 'cpp') + endif +endforeach + +# some clang don't have charconv, but we need it. +# https://github.com/djcb/mu/issues/2347 +cxx.check_header('charconv', required:true) + + +build_aux = join_paths(meson.current_source_dir(), 'build-aux') +################################################################################ +# derived date values (based on 'mu-date'); used in docs +# we can't use the 'date' because MacOS 'date' is incompatible with GNU's. +pdate=find_program(join_paths(build_aux, 'date.py')) +env = environment() +env.set('LANG', 'C') +mu_day_month_year = run_command(pdate, mu_date, '%d %B %Y', + check:true, capture:true, + env: env).stdout().strip() +mu_month_year = run_command(pdate, mu_date, '%B %Y', + check:true, capture:true, + env: env).stdout().strip() +mu_year = run_command(pdate, mu_date, '%Y', + check:true, capture:true, env: env).stdout().strip() + +################################################################################ +# config.h setup +# +config_h_data=configuration_data() +config_h_data.set('MU_STORE_SCHEMA_VERSION', 500) +config_h_data.set_quoted('PACKAGE_VERSION', meson.project_version()) +config_h_data.set_quoted('PACKAGE_STRING', meson.project_name() + ' ' + + meson.project_version()) +config_h_data.set_quoted('VERSION', meson.project_version()) +config_h_data.set_quoted('PACKAGE_NAME', meson.project_name()) + +add_project_arguments(['-DHAVE_CONFIG_H'], language: 'c') +add_project_arguments(['-DHAVE_CONFIG_H'], language: 'cpp') +config_h_dep=declare_dependency( + include_directories: include_directories(['.'])) + + +# +# d_type, d_ino are not available universally, so let's check +# (we use them for optimizations in mu-scanner +# +if cxx.has_member('struct dirent', 'd_ino', prefix : '#include<dirent.h>') + config_h_data.set('HAVE_DIRENT_D_INO', 1) +endif + +if cxx.has_member('struct dirent', 'd_type', prefix : '#include<dirent.h>') + config_h_data.set('HAVE_DIRENT_D_TYPE', 1) +endif + + +functions=[ + 'setsid' +] +foreach f : functions + if cc.has_function(f) + define = 'HAVE_' + f.underscorify().to_upper() + config_h_data.set(define, 1) + endif +endforeach + +if cc.has_function('wordexp') + config_h_data.set('HAVE_WORDEXP_H',1) +else + message('no wordexp, no command-line option expansion') +endif + +if not get_option('tests').disabled() + # only needed for tests + cp=find_program('cp') + ln=find_program('ln') + rm=find_program('rm') + + config_h_data.set_quoted('CP_PROGRAM', cp.full_path()) + config_h_data.set_quoted('RM_PROGRAM', rm.full_path()) + config_h_data.set_quoted('LN_PROGRAM', ln.full_path()) + + testmaildir=join_paths(meson.current_source_dir(), 'testdata') + config_h_data.set_quoted('MU_TESTMAILDIR', join_paths(testmaildir, 'testdir')) + config_h_data.set_quoted('MU_TESTMAILDIR2', join_paths(testmaildir, 'testdir2')) + config_h_data.set_quoted('MU_TESTMAILDIR4', join_paths(testmaildir, 'testdir4')) + config_h_data.set_quoted('MU_TESTMAILDIR_CJK', join_paths(testmaildir, 'cjk')) +endif + + +################################################################################ +# hard dependencies +# +glib_dep = dependency('glib-2.0', version: '>= 2.60') +gobject_dep = dependency('gobject-2.0', version: '>= 2.60') +gio_dep = dependency('gio-2.0', version: '>= 2.60') +gio_unix_dep = dependency('gio-unix-2.0', version: '>= 2.60') +gmime_dep = dependency('gmime-3.0', version: '>= 3.2') +thread_dep = dependency('threads') + +# we need Xapian 1.4 +xapian_dep = dependency('xapian-core', version:'>= 1.4', required:true) +xapver = xapian_dep.version() +if xapver.version_compare('>= 1.4.6') + message('xapian ' + xapver + ' supports c++ move-semantics') + config_h_data.set('HAVE_XAPIAN_MOVE_SEMANTICS', 1) +endif +if xapver.version_compare('>= 1.4.23') + message('xapian ' + xapver + ' supports ngrams') + config_h_data.set('HAVE_XAPIAN_FLAG_NGRAMS', 1) +endif + +# optionally, use Compact Language Detector2 if we can find it. +cld2_dep = meson.get_compiler('cpp').find_library('cld2', required: get_option('cld2')) +if not get_option('cld2').disabled() and cld2_dep.found() + config_h_data.set('HAVE_CLD2', 1) +else + message('CLD2 not found or disabled; no support for language detection') +endif + +# soft dependencies +guile_dep = dependency('guile-3.0', required: get_option('guile')) +# allow for a custom guile-extension-dir +if guile_dep.found() + custom_guile_xd=get_option('guile-extension-dir') + if custom_guile_xd == '' + guile_extension_dir = guile_dep.get_variable(pkgconfig: 'extensiondir') + else + guile_extension_dir = custom_guile_xd + endif + config_h_data.set_quoted('MU_GUILE_EXTENSION_DIR', guile_extension_dir) + message('Using guile-extension-dir: ' + guile_extension_dir) +endif + +makeinfo=find_program(['makeinfo'], required:false) +if not makeinfo.found() + message('makeinfo (texinfo) not found; not building info documentation') +else + install_info=find_program(['install-info'], required:false) + if not install_info.found() + message('install-info not found') + else + install_info_script=join_paths(build_aux, 'meson-install-info.sh') + endif +endif + +# readline. annoyingly, macos has an incompatible libedit claiming to be +# readline. this is only a dev/debug convenience for the mu4e repl. +readline_dep=[] +if get_option('readline').enabled() + readline_dep = dependency('readline', version:'>= 8.0') + config_h_data.set('HAVE_LIBREADLINE', 1) + config_h_data.set('HAVE_READLINE_READLINE_H', 1) + config_h_data.set('HAVE_READLINE_HISTORY', 1) + config_h_data.set('HAVE_READLINE_HISTORY_H', 1) +endif + + +################################################################################ +# write out version.texi (for texinfo builds in mu4e, guile) +version_texi_data=configuration_data() +version_texi_data.set('VERSION', meson.project_version()) +version_texi_data.set('EDITION', meson.project_version()) + +# derived date values +version_texi_data.set('UPDATED', mu_day_month_year) +version_texi_data.set('UPDATEDMONTH', mu_month_year) +version_texi_data.set('UPDATEDYEAR', mu_year) + +configure_file(input: join_paths(build_aux, 'version.texi.in'), + output: 'version.texi', + configuration: version_texi_data) + +################################################################################ +# install some data files +install_data('NEWS.org', install_dir : join_paths(datadir,'doc', 'mu')) + +################################################################################ +# subdirs +subdir('lib') +subdir('mu') + + +# emacs -- needed for mu4e compilation +emacs_name=get_option('emacs') +emacs_min_version='26.3' +emacs=find_program([emacs_name], version: '>='+emacs_min_version, required:false) +if emacs.found() + subdir('man') + subdir('mu4e') +else + message('emacs not found; not pre-compiling mu4e / generating manpages') +endif + +if not get_option('guile').disabled() and guile_dep.found() + config_h_data.set('BUILD_GUILE', 1) + config_h_data.set_quoted('GUILE_BINARY', + guile_dep.get_variable(pkgconfig: 'guile')) + #message('guile is disabled for now') + subdir('guile') +endif + +config_h_data.set_quoted('MU_PROGRAM', mu.full_path()) +################################################################################ + +################################################################################ +# write-out config.h +configure_file(output : 'config.h', configuration : config_h_data) + +if gmime_dep.version() == '3.2.13' + warning('gmime version 3.2.13 detected, which as a decoding bug') + warning('See: https://github.com/jstedfast/gmime/issues/133') +endif diff --git a/meson_options.txt b/meson_options.txt new file mode 100644 index 0000000..93bc7db --- /dev/null +++ b/meson_options.txt @@ -0,0 +1,49 @@ +## Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +option('tests', + type : 'feature', + value: 'auto', + description: 'build unit tests') + +option('guile', + type : 'feature', + value: 'auto', + description: 'build the guile scripting support (requires guile-3.x)') + +option('cld2', + type : 'feature', + value: 'auto', + description: 'Compact Language Detector2') + +# by default, this uses guile_dep.get_variable(pkgconfig: 'extensiondir') +option('guile-extension-dir', + type: 'string', + description: 'custom install path for the guile extension module') + +option('readline', + type: 'feature', + value: 'auto', + description: 'enable readline support for the mu4e repl') + +option('emacs', + type: 'string', + value: 'emacs', + description: 'name/path of the emacs executable') + +option('lispdir', + type: 'string', + description: 'path under which to install emacs-lisp files') diff --git a/mu/meson.build b/mu/meson.build new file mode 100644 index 0000000..0ada1e5 --- /dev/null +++ b/mu/meson.build @@ -0,0 +1,43 @@ +## Copyright (C) 2021-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +mu = executable( + 'mu', [ + 'mu.cc', + 'mu-options.cc', + 'mu-cmd-add.cc', + 'mu-cmd-cfind.cc', + 'mu-cmd-extract.cc', + 'mu-cmd-find.cc', + 'mu-cmd-info.cc', + 'mu-cmd-init.cc', + 'mu-cmd-index.cc', + 'mu-cmd-mkdir.cc', + 'mu-cmd-move.cc', + 'mu-cmd-remove.cc', + 'mu-cmd-script.cc', + 'mu-cmd-server.cc', + 'mu-cmd-verify.cc', + 'mu-cmd-view.cc', + 'mu-cmd.cc' +], + dependencies: [ glib_dep, gmime_dep, lib_mu_dep, thread_dep, config_h_dep ], + cpp_args: ['-DMU_SCRIPTS_DIR="'+ join_paths(datadir, 'mu', 'scripts') + '"'], + install: true) +# +if not get_option('tests').disabled() + subdir('tests') +endif diff --git a/mu/mu-cmd-add.cc b/mu/mu-cmd-add.cc new file mode 100644 index 0000000..46dcf9a --- /dev/null +++ b/mu/mu-cmd-add.cc @@ -0,0 +1,125 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-cmd.hh" + +using namespace Mu; + +Result<void> +Mu::mu_cmd_add(Mu::Store& store, const Options& opts) +{ + for (auto&& file: opts.add.files) { + const auto docid{store.add_message(file)}; + if (!docid) + return Err(docid.error()); + else + mu_debug("added message @ {}, docid={}", file, *docid); + } + + return Ok(); +} + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +static void +test_add_ok() +{ + auto testhome{unwrap(make_temp_dir())}; + auto dbpath{runtime_path(RuntimePath::XapianDb, testhome)}; + + { + unwrap(Store::make_new(dbpath, MU_TESTMAILDIR)); + } + + { + auto res = run_command({MU_PROGRAM, "add", mu_format("--muhome={}", testhome), + MU_TESTMAILDIR "/cur/1220863042.12663_1.mindcrime!2,S"}); + assert_valid_command(res); + } + + { + auto&& store = Store::make(dbpath); + assert_valid_result(store); + g_assert_cmpuint(store->size(),==,1); + } + + { // re-add the same + auto res = run_command({MU_PROGRAM, "add", mu_format("--muhome={}",testhome), + MU_TESTMAILDIR "/cur/1220863042.12663_1.mindcrime!2,S"}); + assert_valid_command(res); + } + + { + auto&& store = Store::make(dbpath); + assert_valid_result(store); + g_assert_cmpuint(store->size(),==,1); + } + + + remove_directory(testhome); +} + +static void +test_add_fail() +{ + auto testhome{unwrap(make_temp_dir())}; + auto dbpath{runtime_path(RuntimePath::XapianDb, testhome)}; + + { + unwrap(Store::make_new(dbpath, MU_TESTMAILDIR2)); + } + + { // wrong maildir + auto res = run_command({MU_PROGRAM, "add", mu_format("--muhome={}", testhome), + MU_TESTMAILDIR "/cur/1220863042.12663_1.mindcrime!2,S"}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code,!=,0); + } + + + { // non-existent + auto res = run_command({MU_PROGRAM, "add", mu_format("--muhome={}", testhome), + "/foo/bar/non-existent"}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code,!=,0); + } + + remove_directory(testhome); +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/cmd/add/ok", test_add_ok); + g_test_add_func("/cmd/add/fail", test_add_fail); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd-cfind.cc b/mu/mu-cmd-cfind.cc new file mode 100644 index 0000000..9c61595 --- /dev/null +++ b/mu/mu-cmd-cfind.cc @@ -0,0 +1,535 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-cmd.hh" + +#include <cstdint> +#include <string> +#include <functional> +#include <unordered_map> + +#include <utils/mu-utils.hh> +#include <utils/mu-regex.hh> +#include <utils/mu-option.hh> + +using namespace Mu; + +enum struct ItemType { Header, Footer, Normal }; +using OutputFunc = std::function<void(ItemType itype, Option<const Contact&>, const Options&)>; +using OptContact = Option<const Contact&>; +using Format = Options::Cfind::Format; + +// simplistic guess of first & last names, for setting +// some initial value. +static std::pair<std::string, std::string> +guess_first_last_name(const std::string& name) +{ + if (name.empty()) + return {}; + + const auto lastspc = name.find_last_of(' '); + if (lastspc == name.npos) + return { name, "" }; // no last name + else + return { name.substr(0, lastspc), name.substr(lastspc + 1)}; +} + + +// candidate nick and a _count_ for that given nick, to uniquify them. +static std::unordered_map<std::string, size_t> nicks; +static std::string +guess_nick(const Contact& contact) +{ + auto cleanup = [](const std::string& str) { + std::string clean; + for (auto& c: str) // XXX: support non-ascii + if (!::ispunct(c) && !::isspace(c)) + clean += c; + return clean; + }; + + auto nick = cleanup(std::invoke([&]()->std::string { + + // no name? use the user part from the addr + if (contact.name.empty()) { + const auto pos{contact.email.find('@')}; + if (pos == std::string::npos) + return contact.email; // no '@' + else + return contact.email.substr(0, pos); + } + + const auto names{guess_first_last_name(contact.name)}; + /* if there's no last name, use first name as the nick */ + if (names.second.empty()) + return names.first; + + char initial[7] = {}; + if (g_unichar_to_utf8(g_utf8_get_char(names.second.c_str()), initial) == 0) { + /* couldn't we get an initial for the last name? + * just use the first name*/ + return names.first; + } else // prepend the initial + return names.first + initial; + })); + + // uniquify. + if (auto it = nicks.find(nick); it == nicks.cend()) + nicks.emplace(nick, 0); + else { + ++it->second; + nick = mu_format("{}{}", nick, ++it->second); + } + + return nick; +} + + +static void +output_plain(ItemType itype, OptContact contact, const Options& opts) +{ + if (!contact) + return; + + const auto col1{opts.nocolor ? "" : MU_COLOR_MAGENTA}; + const auto col2{opts.nocolor ? "" : MU_COLOR_GREEN}; + const auto coldef{opts.nocolor ? "" : MU_COLOR_DEFAULT}; + + mu_print_encoded("{}{}{}{}{}{}{}\n", + col1, contact->name, coldef, + contact->name.empty() ? "" : " ", + col2, contact->email, coldef); +} + +static void +output_mutt_alias(ItemType itype, OptContact contact, const Options& opts) +{ + if (!contact) + return; + + const auto nick{guess_nick(*contact)}; + mu_print_encoded("alias {} {} <{}>\n", nick, contact->name, contact->email); + +} + +static void +output_mutt_address_book(ItemType itype, OptContact contact, const Options& opts) +{ + if (itype == ItemType::Header) + mu_print ("Matching addresses in the mu database:\n"); + + if (contact) + mu_print_encoded("{}\t{}\t\n", contact->email, contact->name); +} + +static void +output_wanderlust(ItemType itype, OptContact contact, const Options& opts) +{ + if (!contact || contact->name.empty()) + return; + + auto nick=guess_nick(*contact); + + mu_print_encoded("{} \"{}\" \"{}\"\n", contact->email, nick, contact->name); + +} + +static void +output_org_contact(ItemType itype, OptContact contact, const Options& opts) +{ + if (!contact || contact->name.empty()) + return; + + mu_print_encoded("* {}\n:PROPERTIES:\n:EMAIL: {}\n:END:\n\n", + contact->name, contact->email); +} + +static void +output_bbdb(ItemType itype, OptContact contact, const Options& opts) +{ + if (itype == ItemType::Header) + mu_println (";; -*-coding: utf-8-emacs;-*-\n" + ";;; file-version: 6"); + if (!contact) + return; + + const auto names{guess_first_last_name(contact->name)}; + const auto now{mu_format("{:%Y-%m-%d}", mu_time(::time({})))}; + const auto timestamp{mu_format("{:%Y-%m-%d}", mu_time(contact->message_date))}; + + mu_println("[\"{}\" \"{}\" nil nil nil nil (\"{}\") " + "((creation-date . \"{}\") (time-stamp . \"{}\")) nil]", + names.first, names.second, contact->email, now, timestamp); +} + +static void +output_csv(ItemType itype, OptContact contact, const Options& opts) +{ + if (!contact) + return; + + mu_print_encoded("{},{}\n", + contact->name.empty() ? "" : Mu::quote(contact->name), + Mu::quote(contact->email)); +} + +static void +output_json(ItemType itype, OptContact contact, const Options& opts) +{ + if (itype == ItemType::Header) + mu_println("["); + if (contact) { + mu_print("{}", itype == ItemType::Header ? "" : ",\n"); + mu_println (" {{"); + + const std::string name = contact->name.empty() ? "null" : Mu::quote(contact->name); + mu_print_encoded( + " \"email\" : \"{}\",\n" + " \"name\" : {},\n" + " \"display\" : {},\n" + " \"last-seen\" : {},\n" + " \"last-seen-iso\" : \"{}\",\n" + " \"personal\" : {},\n" + " \"frequency\" : {}\n", + contact->email, + name, + Mu::quote(contact->display_name()), + contact->message_date, + mu_format("{:%FT%TZ}", mu_time(contact->message_date, true/*utc*/)), + contact->personal ? "true" : "false", + contact->frequency); + mu_print(" }}"); + } + + if (itype == ItemType::Footer) + mu_println("\n]"); +} + +static OutputFunc +find_output_func(Format format) +{ +#pragma GCC diagnostic push +#pragma GCC diagnostic error "-Wswitch" + switch(format) { + case Format::Plain: + return output_plain; + case Format::MuttAlias: + return output_mutt_alias; + case Format::MuttAddressBook: + return output_mutt_address_book; + case Format::Wanderlust: + return output_wanderlust; + case Format::OrgContact: + return output_org_contact; + case Format::Bbdb: + return output_bbdb; + case Format::Csv: + return output_csv; + case Format::Json: + return output_json; + default: + mu_warning("unsupported format"); + return {}; + } +#pragma GCC diagnostic pop +} + + +Result<void> +Mu::mu_cmd_cfind(const Mu::Store& store, const Mu::Options& opts) +{ + size_t num{}; + OutputFunc output = find_output_func(opts.cfind.format); + if (!output) + return Err(Error::Code::Internal, + "missing output function"); + + // get the pattern regex, if any. + Regex rx{}; + if (!opts.cfind.rx_pattern.empty()) { + if (auto&& res = Regex::make(opts.cfind.rx_pattern, + static_cast<GRegexCompileFlags> + (G_REGEX_OPTIMIZE|G_REGEX_CASELESS)); !res) + return Err(std::move(res.error())); + else + rx = res.value(); + } + + nicks.clear(); + store.contacts_cache().for_each([&](const Contact& contact)->bool { + + if (opts.cfind.maxnum && num > *opts.cfind.maxnum) + return false; /* stop the loop */ + + if (!store.contacts_cache().is_valid(contact.email)) + return true; /* next */ + + // filter for maxnum, personal & "after" + if ((opts.cfind.personal && !contact.personal) || + (opts.cfind.after.value_or(0) > contact.message_date)) + return true; /* next */ + + // filter for regex, if any. + if (rx) { + if (!rx.matches(contact.name) && !rx.matches(contact.email)) + return true; /* next */ + } + + /* seems we have a match! display it. */ + const auto itype{num == 0 ? ItemType::Header : ItemType::Normal}; + output(itype, contact, opts); + ++num; + return true; + }); + + if (num == 0) + return Err(Error::Code::NoMatches, "no matching contacts found"); + + output(ItemType::Footer, Nothing, opts); + return Ok(); +} + + + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + + +static std::string test_mu_home; + +static void +test_mu_cfind_plain(void) +{ + auto res{run_command({MU_PROGRAM, "--nocolor", "cfind", "--muhome", test_mu_home, + "--format", "plain", "testmu\\.xxx?"})}; + assert_valid_result(res); + + /* note, output order is unspecified */ + if (res->standard_out[0] == 'H') + assert_equal(res->standard_out, + "Helmut Kröger hk@testmu.xxx\n" + "Mü testmu@testmu.xx\n"); + else + assert_equal(res->standard_out, + "Mü testmu@testmu.xx\n" + "Helmut Kröger hk@testmu.xxx\n"); +} + +static void +test_mu_cfind_bbdb(void) +{ + const auto old_tz{set_tz("Europe/Helsinki")}; + auto res{run_command({MU_PROGRAM, "--nocolor", "cfind", "--muhome", test_mu_home, + "--format", "bbdb", "testmu\\.xxx?"})}; + assert_valid_result(res); + g_assert_cmpuint(res->standard_out.size(), >, 52); + +#define frm1 \ + ";; -*-coding: utf-8-emacs;-*-\n" \ + ";;; file-version: 6\n" \ + "[\"Helmut\" \"Kröger\" nil nil nil nil (\"hk@testmu.xxx\") " \ + "((creation-date . \"{}\") " \ + "(time-stamp . \"1970-01-01\")) nil]\n" \ + "[\"Mü\" \"\" nil nil nil nil (\"testmu@testmu.xx\") " \ + "((creation-date . \"{}\") " \ + "(time-stamp . \"1970-01-01\")) nil]\n" + +#define frm2 \ + ";; -*-coding: utf-8-emacs;-*-\n" \ + ";;; file-version: 6\n" \ + "[\"Mü\" \"\" nil nil nil nil (\"testmu@testmu.xx\") " \ + "((creation-date . \"{}\") " \ + "(time-stamp . \"1970-01-01\")) nil]\n" \ + "[\"Helmut\" \"Kröger\" nil nil nil nil (\"hk@testmu.xxx\") " \ + "((creation-date . \"{}\") " \ + "(time-stamp . \"1970-01-01\")) nil]\n" + + auto&& today{mu_format("{:%F}", mu_time(::time({})))}; + std::string expected; + if (res->standard_out.at(52) == 'H') + expected = mu_format(frm1, today, today); + else + expected = mu_format(frm2, today, today); + + assert_equal(res->standard_out, expected); + set_tz(old_tz); +} + +static void +test_mu_cfind_wl(void) +{ + auto res{run_command({MU_PROGRAM, "--nocolor", "cfind", "--muhome", test_mu_home, + "--format", "wl", "testmu\\.xxx?"})}; + assert_valid_result(res); + + if (res->standard_out.at(0) == 'h') + assert_equal(res->standard_out, + "hk@testmu.xxx \"HelmutK\" \"Helmut Kröger\"\n" + "testmu@testmu.xx \"Mü\" \"Mü\"\n"); + else + assert_equal(res->standard_out, + "testmu@testmu.xx \"Mü\" \"Mü\"\n" + "hk@testmu.xxx \"HelmutK\" \"Helmut Kröger\"\n"); +} + +static void +test_mu_cfind_mutt_alias(void) +{ + auto res{run_command({MU_PROGRAM, "--nocolor", "cfind", "--muhome", test_mu_home, + "--format", "mutt-alias", "testmu\\.xxx?"})}; + assert_valid_result(res); + + if (res->standard_out.at(6) == 'H') + assert_equal(res->standard_out, + "alias HelmutK Helmut Kröger <hk@testmu.xxx>\n" + "alias Mü Mü <testmu@testmu.xx>\n"); + else + assert_equal(res->standard_out, + "alias Mü Mü <testmu@testmu.xx>\n" + "alias HelmutK Helmut Kröger <hk@testmu.xxx>\n"); +} + +static void +test_mu_cfind_mutt_ab(void) +{ + auto res{run_command({MU_PROGRAM, "--nocolor", "cfind", "--muhome", test_mu_home, + "--format", "mutt-ab", "testmu\\.xxx?"})}; + assert_valid_result(res); + + if (res->standard_out.at(39) == 'h') + assert_equal(res->standard_out, + "Matching addresses in the mu database:\n" + "hk@testmu.xxx\tHelmut Kröger\t\n" + "testmu@testmu.xx\tMü\t\n"); + else + assert_equal(res->standard_out, + "Matching addresses in the mu database:\n" + "testmu@testmu.xx\tMü\t\n" + "hk@testmu.xxx\tHelmut Kröger\t\n"); +} + +static void +test_mu_cfind_org_contact(void) +{ + auto res{run_command({MU_PROGRAM, "--nocolor", "cfind", "--muhome", test_mu_home, + "--format", "org-contact", "testmu\\.xxx?"})}; + assert_valid_result(res); + + if (res->standard_out.at(2) == 'H') + assert_equal(res->standard_out, + "* Helmut Kröger\n" + ":PROPERTIES:\n" + ":EMAIL: hk@testmu.xxx\n" + ":END:\n\n" + "* Mü\n" + ":PROPERTIES:\n" + ":EMAIL: testmu@testmu.xx\n" + ":END:\n\n"); + else + assert_equal(res->standard_out, + "* Mü\n" + ":PROPERTIES:\n" + ":EMAIL: testmu@testmu.xx\n" + ":END:\n\n" + "* Helmut Kröger\n" + ":PROPERTIES:\n" + ":EMAIL: hk@testmu.xxx\n" + ":END:\n\n"); +} + +static void +test_mu_cfind_csv(void) +{ + auto res{run_command({MU_PROGRAM, "--nocolor", "cfind", "--muhome", test_mu_home, + "--format", "csv", "testmu\\.xxx?"})}; + assert_valid_result(res); + + if (res->standard_out.at(1) == 'H') + assert_equal(res->standard_out, + "\"Helmut Kröger\",\"hk@testmu.xxx\"\n" + "\"Mü\",\"testmu@testmu.xx\"\n"); + else + assert_equal(res->standard_out, + "\"Mü\",\"testmu@testmu.xx\"\n" + "\"Helmut Kröger\",\"hk@testmu.xxx\"\n"); +} + + +static void +test_mu_cfind_json() +{ + auto res{run_command({MU_PROGRAM, "--nocolor", "cfind", "--muhome", test_mu_home, + "--format", "json", "^a@example\\.com"})}; + assert_valid_result(res); + + const auto expected = R"([ + { + "email" : "a@example.com", + "name" : null, + "display" : "a@example.com", + "last-seen" : 1463331445, + "last-seen-iso" : "2016-05-15T16:57:25Z", + "personal" : false, + "frequency" : 1 + } +] +)"; + assert_equal(res->standard_out, expected); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + if (!set_en_us_utf8_locale()) + return 0; /* don't error out... */ + + TempDir temp_dir{}; + { + test_mu_home = temp_dir.path(); + + auto res1 = run_command({MU_PROGRAM, "--quiet", "init", + "--muhome", test_mu_home, "--maildir" , MU_TESTMAILDIR}); + assert_valid_result(res1); + + auto res2 = run_command({MU_PROGRAM, "--quiet", "index", + "--muhome", test_mu_home}); + assert_valid_result(res2); + } + + g_test_add_func("/cmd/find/plain", test_mu_cfind_plain); + g_test_add_func("/cmd/find/bbdb", test_mu_cfind_bbdb); + g_test_add_func("/cmd/find/wl", test_mu_cfind_wl); + g_test_add_func("/cmd/find/mutt-alias", test_mu_cfind_mutt_alias); + g_test_add_func("/cmd/find/mutt-ab", test_mu_cfind_mutt_ab); + g_test_add_func("/cmd/find/org-contact", test_mu_cfind_org_contact); + g_test_add_func("/cmd/find/csv", test_mu_cfind_csv); + g_test_add_func("/cmd/find/json", test_mu_cfind_json); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd-extract.cc b/mu/mu-cmd-extract.cc new file mode 100644 index 0000000..28793b3 --- /dev/null +++ b/mu/mu-cmd-extract.cc @@ -0,0 +1,306 @@ +/* +** Copyright (C) 2010-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-cmd.hh" +#include "utils/mu-utils.hh" +#include "utils/mu-utils-file.hh" +#include "utils/mu-regex.hh" +#include <message/mu-message.hh> + +using namespace Mu; + +static Result<void> +save_part(const Message::Part& part, size_t idx, const Options& opts) +{ + const auto targetdir = std::invoke([&]{ + const auto tdir{opts.extract.targetdir}; + return tdir.empty() ? tdir : tdir + G_DIR_SEPARATOR_S; + }); + + /* 'uncooked' isn't really _raw_; it means only doing some _minimal_ + * cooking */ + const auto path{targetdir + + part.cooked_filename(opts.extract.uncooked) + .value_or(mu_format("part-{}", idx))}; + + if (auto&& res{part.to_file(path, opts.extract.overwrite)}; !res) + return Err(res.error()); + else if (opts.extract.play) + return play(path); + else + return Ok(); +} + +static Result<void> +save_parts(const Message& message, const std::string& filename_rx, + const Options& opts) +{ + size_t partnum{}, saved_num{}; + for (auto&& part: message.parts()) { + ++partnum; + // should we extract this part? + const auto do_extract = std::invoke([&]() { + + if (opts.extract.save_all) + return true; + else if (opts.extract.save_attachments && + part.looks_like_attachment()) + return true; + else if (seq_some(opts.extract.parts, + [&](auto&& num){return num==partnum;})) + return true; + else if (!filename_rx.empty() && part.raw_filename()) { + if (auto rx = Regex::make(filename_rx); !rx) + throw rx.error(); + else if (rx->matches(*part.raw_filename())) + return true; + } + return false; + }); + + if (!do_extract) + continue; + + if (auto res = save_part(part, partnum, opts); !res) + return res; + + ++saved_num; + } + + if (saved_num == 0) + return Err(Error::Code::File, + "no {} extracted from this message", + opts.extract.save_attachments ? "attachments" : "parts"); + else + return Ok(); +} + +#define color_maybe(C) \ + do { \ + if (color) \ + fputs((C), stdout); \ + } while (0) + +static void +show_part(const MessagePart& part, size_t index, bool color) +{ + /* index */ + mu_print(" {} ", index); + + /* filename */ + color_maybe(MU_COLOR_GREEN); + const auto fname{part.raw_filename()}; + fputs_encoded(fname.value_or("<none>"), stdout); + fputs_encoded(" ", stdout); + + /* content-type */ + color_maybe(MU_COLOR_BLUE); + const auto ctype{part.mime_type()}; + fputs_encoded(ctype.value_or("<none>"), stdout); + + /* /\* disposition *\/ */ + color_maybe(MU_COLOR_MAGENTA); + mu_print_encoded(" [{}]", part.is_attachment() ? "attachment" : "inline"); + /* size */ + if (part.size() > 0) { + color_maybe(MU_COLOR_CYAN); + mu_print(" ({} bytes)", part.size()); + } + + color_maybe(MU_COLOR_DEFAULT); + fputs("\n", stdout); +} + +static Mu::Result<void> +show_parts(const Message& message, const Options& opts) +{ + size_t index{}; + mu_println("MIME-parts in this message:"); + for (auto&& part: message.parts()) + show_part(part, ++index, !opts.nocolor); + + return Ok(); +} + +Mu::Result<void> +Mu::mu_cmd_extract(const Options& opts) +{ + auto message = std::invoke([&]()->Result<Message>{ + const auto mopts{message_options(opts.extract)}; + if (!opts.extract.message.empty()) + return Message::make_from_path(opts.extract.message, mopts); + + const auto msgtxt = read_from_stdin(); + if (!msgtxt) + return Err(msgtxt.error()); + else + return Message::make_from_text(*msgtxt, {}, mopts); + }); + + if (!message) + return Err(message.error()); + else if (opts.extract.parts.empty() && + !opts.extract.save_attachments && !opts.extract.save_all && + opts.extract.filename_rx.empty()) + return show_parts(*message, opts); /* show, don't save */ + + if (!check_dir(opts.extract.targetdir, false/*!readable*/, true/*writeable*/)) + return Err(Error::Code::File, + "target '{}' is not a writable directory", + opts.extract.targetdir); + + return save_parts(*message, opts.extract.filename_rx, opts); +} + + + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include <glib.h> +#include <glib/gstdio.h> +#include <fcntl.h> +#include <sys/types.h> +#include <sys/stat.h> + +#include <utils/mu-regex.hh> +#include "utils/mu-test-utils.hh" + + +static gint64 +get_file_size(const std::string& path) +{ + int rv; + struct stat statbuf; + + mu_info("ppatj {}", path); + + rv = stat(path.c_str(), &statbuf); + if (rv != 0) { + mu_debug ("error: {}", g_strerror (errno)); + return -1; + } + + mu_debug("{} -> {} bytes", path, statbuf.st_size); + + return statbuf.st_size; +} + +static void +test_mu_extract_02(void) +{ + TempDir temp_dir{}; + auto res= run_command({ + MU_PROGRAM, "extract", "--save-attachments", + mu_format("--target-dir='{}'", temp_dir.path()), + join_paths(MU_TESTMAILDIR2, "Foo", "cur", "mail5")}); + assert_valid_result(res); + g_assert_true(res->standard_err.empty()); + + g_assert_cmpuint(get_file_size(join_paths(temp_dir.path(), "custer.jpg")), >=, 15955); + g_assert_cmpuint(get_file_size(join_paths(temp_dir.path(), "custer.jpg")), <=, 15960); + g_assert_cmpuint(get_file_size(join_paths(temp_dir.path(), "sittingbull.jpg")), ==, 17674); +} + +static void +test_mu_extract_03(void) +{ + TempDir temp_dir{}; + auto res= run_command({ + MU_PROGRAM, "extract", "--parts=3", + mu_format("--target-dir='{}'", temp_dir.path()), + join_paths(MU_TESTMAILDIR2, "Foo", "cur", "mail5")}); + assert_valid_result(res); + g_assert_true(res->standard_err.empty()); + + g_assert_true(g_access(join_paths(temp_dir.path(), "custer.jpg").c_str(), F_OK) == 0); + g_assert_false(g_access(join_paths(temp_dir.path(), "sittingbull.jpg").c_str(), F_OK) == 0); +} + +static void +test_mu_extract_overwrite(void) +{ + TempDir temp_dir{}; + auto res= run_command({ + MU_PROGRAM, "extract", "-a", + mu_format("--target-dir='{}'", temp_dir.path()), + join_paths(MU_TESTMAILDIR2, "Foo", "cur", "mail5")}); + assert_valid_result(res); + g_assert_true(res->standard_err.empty()); + + g_assert_true(g_access(join_paths(temp_dir.path(), "custer.jpg").c_str(), F_OK) == 0); + g_assert_true(g_access(join_paths(temp_dir.path(), "sittingbull.jpg").c_str(), F_OK) == 0); + + + /* now, it should fail, because we don't allow overwrites + * without --overwrite */ + auto res2 = run_command({ + MU_PROGRAM, "extract", "-a", + mu_format("--target-dir='{}'", temp_dir.path()), + join_paths(MU_TESTMAILDIR2, "Foo", "cur", "mail5")}); + + assert_valid_result(res2); + g_assert_false(res2->standard_err.empty()); + + + auto res3 = run_command({ + MU_PROGRAM, "extract", "-a", "--overwrite", + mu_format("--target-dir='{}'", temp_dir.path()), + join_paths(MU_TESTMAILDIR2, "Foo", "cur", "mail5")}); + + assert_valid_result(res3); + g_assert_true(res3->standard_err.empty()); +} + +static void +test_mu_extract_by_name(void) +{ + TempDir temp_dir{}; + auto res= run_command({ + MU_PROGRAM, "extract", + mu_format("--target-dir='{}'", temp_dir.path()), + join_paths(MU_TESTMAILDIR2, "Foo", "cur", "mail5"), + "sittingbull.jpg"}); + assert_valid_result(res); + g_assert_true(res->standard_err.empty()); + + g_assert_true(g_access(join_paths(temp_dir.path(), "sittingbull.jpg").c_str(), F_OK) == 0); + g_assert_false(g_access(join_paths(temp_dir.path(), "custer.jpg").c_str(), F_OK) == 0); +} + + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/cmd/extract/02", test_mu_extract_02); + g_test_add_func("/cmd/extract/03", test_mu_extract_03); + g_test_add_func("/cmd/extract/overwrite", test_mu_extract_overwrite); + g_test_add_func("/cmd/extract/by-name", test_mu_extract_by_name); + + return g_test_run(); +} + +#endif diff --git a/mu/mu-cmd-find.cc b/mu/mu-cmd-find.cc new file mode 100644 index 0000000..3853856 --- /dev/null +++ b/mu/mu-cmd-find.cc @@ -0,0 +1,733 @@ + /* +** Copyright (C) 2008-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include <array> + +#include <unistd.h> +#include <stdio.h> +#include <string.h> +#include <errno.h> +#include <stdlib.h> +#include <signal.h> +#include <sys/wait.h> + +#include "message/mu-message.hh" +#include "mu-maildir.hh" +#include "mu-query-match-deciders.hh" +#include "mu-query.hh" +#include "mu-query-macros.hh" +#include "mu-query-parser.hh" +#include "message/mu-message.hh" + +#include "utils/mu-option.hh" + +#include "mu-cmd.hh" +#include "utils/mu-utils.hh" + +using namespace Mu; + +using Format = Options::Find::Format; + +struct OutputInfo { + Xapian::docid docid{}; + bool header{}; + bool footer{}; + bool last{}; + Option<QueryMatch&> match_info; +}; + +constexpr auto FirstOutput{OutputInfo{0, true, false, {}, {}}}; +constexpr auto LastOutput{OutputInfo{0, false, true, {}, {}}}; + +using OutputFunc = std::function<Result<void>(const Option<Message>& msg, const OutputInfo&, + const Options&)>; + +using Format = Options::Find::Format; + +static Result<void> +analyze_query_expr(const Store& store, const std::string& expr, const Options& opts) +{ + auto print_item=[&](auto&&title, auto&&val) { + const auto blue{opts.nocolor ? "" : MU_COLOR_BLUE}; + const auto green{opts.nocolor ? "" : MU_COLOR_GREEN}; + const auto reset{opts.nocolor ? "" : MU_COLOR_DEFAULT}; + mu_println("* {}{}{}:\n {}{}{}", blue, title, reset, green, val, reset); + }; + + print_item("query", expr); + + const auto pq{parse_query(expr, false/*don't expand*/).to_string()}; + const auto pqx{parse_query(expr, true/*do expand*/).to_string()}; + + print_item("parsed query", pq); + if (pq != pqx) + print_item("parsed query (expanded)", pqx); + + auto xq{make_xapian_query(store, expr)}; + if (!xq) + return Err(std::move(xq.error())); + + print_item("Xapian query", xq->get_description()); + + return Ok(); +} + +static Result<QueryResults> +run_query(const Store& store, const std::string& expr, const Options& opts) +{ + Mu::QueryFlags qflags{QueryFlags::SkipUnreadable}; + if (opts.find.reverse) + qflags |= QueryFlags::Descending; + if (opts.find.skip_dups) + qflags |= QueryFlags::SkipDuplicates; + if (opts.find.include_related) + qflags |= QueryFlags::IncludeRelated; + if (opts.find.threads) + qflags |= QueryFlags::Threading; + + return store.run_query(expr, + opts.find.sortfield, + qflags, opts.find.maxnum.value_or(0)); +} + +static Result<void> +exec_cmd(const Option<Message>& msg, const OutputInfo& info, const Options& opts) +{ + if (!msg) + return Ok(); + + int wait_status{}; + GError *err{}; + auto cmdline{mu_format("{} {}", opts.find.exec, + to_string_gchar(g_shell_quote(msg->path().c_str())))}; + + if (!g_spawn_command_line_sync(cmdline.c_str(), {}, {}, &wait_status, &err)) + return Err(Error::Code::File, &err/*consumed*/, + "failed to execute shell command"); + else if (WEXITSTATUS(wait_status) != 0) + return Err(Error::Code::File, + "shell command exited with exit-code {}", + WEXITSTATUS(wait_status)); + return Ok(); +} + +static Result<std::string> +resolve_bookmark(const Store& store, const Options& opts) +{ + QueryMacros macros{store.config()}; + if (auto&& res{macros.load_bookmarks(opts.runtime_path(RuntimePath::Bookmarks))}; !res) + return Err(res.error()); + else if (auto&& bm{macros.find_macro(opts.find.bookmark)}; !bm) + return Err(Error::Code::InvalidArgument, "bookmark '{}' not found", + opts.find.bookmark); + else + return Ok(std::move(*bm)); +} + +static Result<std::string> +get_query(const Store& store, const Options& opts) +{ + if (opts.find.bookmark.empty() && opts.find.query.empty()) + return Err(Error::Code::InvalidArgument, + "neither bookmark nor query"); + + std::string bookmark; + if (!opts.find.bookmark.empty()) { + const auto res = resolve_bookmark(store, opts); + if (!res) + return Err(std::move(res.error())); + bookmark = res.value() + " "; + } + + auto&& query{join(opts.find.query, " ")}; + return Ok(bookmark + query); +} + +static Result<void> +prepare_links(const Options& opts) +{ + /* note, mu_maildir_mkdir simply ignores whatever part of the + * mail dir already exists */ + if (auto&& res = maildir_mkdir(opts.find.linksdir, 0700, true); !res) + return Err(std::move(res.error())); + + if (!opts.find.clearlinks) + return Ok(); + + if (auto&& res = maildir_clear_links(opts.find.linksdir); !res) + return Err(std::move(res.error())); + + return Ok(); +} + +static Result<void> +output_link(const Option<Message>& msg, const OutputInfo& info, const Options& opts) +{ + if (info.header) + return prepare_links(opts); + else if (info.footer) + return Ok(); + + /* during test, do not create "unique names" (i.e., names with path + * hashes), so we get a predictable result */ + const auto unique_names{!g_getenv("MU_TEST")&&!g_test_initialized()}; + + if (auto&& res = maildir_link(msg->path(), opts.find.linksdir, unique_names); !res) + return Err(std::move(res.error())); + + return Ok(); +} + +static void +ansi_color_maybe(Field::Id field_id, bool color) +{ + const char* ansi; + + if (!color) + return; /* nothing to do */ + + switch (field_id) { + case Field::Id::From: ansi = MU_COLOR_CYAN; break; + + case Field::Id::To: + case Field::Id::Cc: + case Field::Id::Bcc: ansi = MU_COLOR_BLUE; break; + case Field::Id::Subject: ansi = MU_COLOR_GREEN; break; + case Field::Id::Date: ansi = MU_COLOR_MAGENTA; break; + + default: + if (field_from_id(field_id).type != Field::Type::String) + ansi = MU_COLOR_YELLOW; + else + ansi = MU_COLOR_RED; + } + + fputs(ansi, stdout); +} + +static void +ansi_reset_maybe(Field::Id field_id, bool color) +{ + if (!color) + return; /* nothing to do */ + + fputs(MU_COLOR_DEFAULT, stdout); +} + +static std::string +display_field(const Message& msg, Field::Id field_id) +{ + switch (field_from_id(field_id).type) { + case Field::Type::String: + return msg.document().string_value(field_id); + case Field::Type::Integer: + if (field_id == Field::Id::Priority) { + return to_string(msg.priority()); + } else if (field_id == Field::Id::Flags) { + return to_string(msg.flags()); + } else /* as string */ + return msg.document().string_value(field_id); + case Field::Type::TimeT: + return mu_format("{:%c}", + mu_time(msg.document().integer_value(field_id))); + case Field::Type::ByteSize: + return to_string(msg.document().integer_value(field_id)); + case Field::Type::StringList: + return join(msg.document().string_vec_value(field_id), ','); + case Field::Type::ContactList: + return to_string(msg.document().contacts_value(field_id)); + default: + g_return_val_if_reached(""); + return ""; + } +} + +static void +print_summary(const Message& msg, const Options& opts) +{ + const auto body{msg.body_text()}; + if (!body) + return; + + const auto summ{summarize(body->c_str(), opts.find.summary_len.value_or(0))}; + + mu_print("Summary: "); + fputs_encoded(summ, stdout); + mu_println(""); +} + +static void +thread_indent(const QueryMatch& info, const Options& opts) +{ + const auto is_root{any_of(info.flags & QueryMatch::Flags::Root)}; + const auto first_child{any_of(info.flags & QueryMatch::Flags::First)}; + const auto last_child{any_of(info.flags & QueryMatch::Flags::Last)}; + const auto empty_parent{any_of(info.flags & QueryMatch::Flags::Orphan)}; + const auto is_dup{any_of(info.flags & QueryMatch::Flags::Duplicate)}; + // const auto is_related{any_of(info.flags & QueryMatch::Flags::Related)}; + + /* indent */ + if (opts.debug) { + ::fputs(info.thread_path.c_str(), stdout); + ::fputs(" ", stdout); + } else + for (auto i = info.thread_level; i > 1; --i) + ::fputs(" ", stdout); + + if (!is_root) { + if (first_child) + ::fputs("\\", stdout); + else if (last_child) + ::fputs("/", stdout); + else + ::fputs(" ", stdout); + ::fputs(empty_parent ? "*> " : is_dup ? "=> " + : "-> ", + stdout); + } +} + +static void +output_plain_fields(const Message& msg, const std::string& fields, + bool color, bool threads) +{ + size_t nonempty{}; + + for (auto&& k: fields) { + const auto field_opt{field_from_shortcut(k)}; + if (!field_opt || (!field_opt->is_value() && !field_opt->is_contact())) + nonempty += printf("%c", k); + + else { + ansi_color_maybe(field_opt->id, color); + nonempty += fputs_encoded( + display_field(msg, field_opt->id), stdout); + ansi_reset_maybe(field_opt->id, color); + } + } + + if (nonempty) + fputs("\n", stdout); +} + +static Result<void> +output_plain(const Option<Message>& msg, const OutputInfo& info, + const Options& opts) +{ + if (!msg) + return Ok(); + + /* we reuse the color (whatever that may be) + * for message-priority for threads, too */ + ansi_color_maybe(Field::Id::Priority, !opts.nocolor); + if (opts.find.threads && info.match_info) + thread_indent(*info.match_info, opts); + + output_plain_fields(*msg, opts.find.fields, !opts.nocolor, opts.find.threads); + + if (opts.view.summary_len) + print_summary(*msg, opts); + + return Ok(); +} + +static Result<void> +output_sexp(const Option<Message>& msg, const OutputInfo& info, const Options& opts) +{ + if (msg) { + if (const auto sexp{msg->sexp()}; !sexp.empty()) + fputs(sexp.to_string().c_str(), stdout); + else + fputs(msg->sexp().to_string().c_str(), stdout); + fputs("\n", stdout); + } + + return Ok(); +} + +static Result<void> +output_json(const Option<Message>& msg, const OutputInfo& info, const Options& opts) +{ + if (info.header) { + mu_println("["); + return Ok(); + } + + if (info.footer) { + mu_println("]"); + return Ok(); + } + + if (!msg) + return Ok(); + + mu_println("{}{}", msg->sexp().to_json_string(), info.last ? "" : ","); + + return Ok(); +} + +static void +print_attr_xml(const std::string& elm, const std::string& str) +{ + if (str.empty()) + return; /* empty: don't include */ + + auto&& esc{to_string_opt_gchar(g_markup_escape_text(str.c_str(), -1))}; + mu_println("\t\t<{}>{}</{}>", elm, esc.value_or(""), elm); +} + +static Result<void> +output_xml(const Option<Message>& msg, const OutputInfo& info, const Options& opts) +{ + if (info.header) { + mu_println("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>"); + mu_println("<messages>"); + return Ok(); + } + + if (info.footer) { + mu_println("</messages>"); + return Ok(); + } + + mu_println("\t<message>"); + print_attr_xml("from", to_string(msg->from())); + print_attr_xml("to", to_string(msg->to())); + print_attr_xml("cc", to_string(msg->cc())); + print_attr_xml("subject", msg->subject()); + mu_println("\t\t<date>{}</date>", (unsigned)msg->date()); + mu_println("\t\t<size>{}</size>", (unsigned)msg->size()); + print_attr_xml("msgid", msg->message_id()); + print_attr_xml("path", msg->path()); + print_attr_xml("maildir", msg->maildir()); + mu_println("\t</message>"); + + return Ok(); +} + +static OutputFunc +get_output_func(const Options& opts) +{ + if (!opts.find.exec.empty()) + return exec_cmd; + + switch (opts.find.format) { + case Format::Links: + return output_link; + case Format::Plain: + return output_plain; + case Format::Xml: + return output_xml; + case Format::Sexp: + return output_sexp; + case Format::Json: + return output_json; + default: + throw Error(Error::Code::Internal, + "invalid format {}", + static_cast<size_t>(opts.find.format)); + } +} + +static Result<void> +output_query_results(const QueryResults& qres, const Options& opts) +{ + GError* err{}; + const auto output_func{get_output_func(opts)}; + if (!output_func) + return Err(Error::Code::Query, &err, "failed to find output function"); + + if (auto&& res = output_func(Nothing, FirstOutput, opts); !res) + return Err(std::move(res.error())); + + size_t n{0}; + for (auto&& item : qres) { + n++; + auto msg{item.message()}; + if (!msg) + continue; + + if (msg->changed() < opts.find.after.value_or(0)) + continue; + + if (auto&& res = output_func(msg, + {item.doc_id(), + false, + false, + n == qres.size(), /* last? */ + item.query_match()}, + opts); !res) + return Err(std::move(res.error())); + } + + if (auto&& res{output_func(Nothing, LastOutput, opts)}; !res) + return Err(std::move(res.error())); + else + return Ok(); +} + +static Result<void> +process_store_query(const Store& store, const std::string& expr, const Options& opts) +{ + auto qres{run_query(store, expr, opts)}; + if (!qres) + return Err(qres.error()); + + if (qres->empty()) + return Err(Error::Code::NoMatches, "no matches for search expression"); + + return output_query_results(*qres, opts); +} + +Result<void> +Mu::mu_cmd_find(const Store& store, const Options& opts) +{ + auto expr{get_query(store, opts)}; + if (!expr) + return Err(expr.error()); + + if (opts.find.analyze) + return analyze_query_expr(store, *expr, opts); + else + return process_store_query(store, *expr, opts); +} + + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + + +/* tests for the command line interface, uses testdir2 */ + +static std::string test_mu_home; + +auto count_nl(const std::string& s)->size_t { + size_t n{}; + for (auto&& c: s) + if (c == '\n') + ++n; + return n; +} + +static size_t +search_func(const std::string& expr, size_t expected) +{ + auto res = run_command({MU_PROGRAM, "find", "--muhome", test_mu_home, expr}); + assert_valid_result(res); + + /* we expect zero lines of error output if there is a match; otherwise + * there should be one line 'No matches found' */ + if (res->exit_code != 0) { + g_assert_cmpuint(res->exit_code, ==, 2); // no match + g_assert_true(res->standard_out.empty()); + g_assert_cmpuint(count_nl(res->standard_err), ==, 1); + return 0; + } + + return count_nl(res->standard_out); +} + +#define search(Q,EXP) do { \ + g_assert_cmpuint(search_func(Q, EXP), ==, EXP); \ +} while(0) + + +static void +test_mu_find_empty_query(void) +{ + search("\"\"", 14); +} + +static void +test_mu_find_01(void) +{ + search("f:john fruit", 1); + search("f:soc@example.com", 1); + search("t:alki@example.com", 1); + search("t:alcibiades", 1); + search("http emacs", 1); + search("f:soc@example.com OR f:john", 2); + search("f:soc@example.com OR f:john OR t:edmond", 3); + search("t:julius", 1); + search("s:dude", 1); + search("t:dantès", 1); +} + +/* index testdir2, and make sure it adds two documents */ +static void +test_mu_find_02(void) +{ + search("bull", 1); + search("g:x", 0); + search("flag:encrypted", 0); + search("flag:attach", 1); + + search("i:3BE9E6535E0D852173@emss35m06.us.lmco.com", 1); +} + +static void +test_mu_find_file(void) +{ + search("file:sittingbull.jpg", 1); + search("file:custer.jpg", 1); + search("file:custer.*", 1); + search("j:sit*", 1); +} + +static void +test_mu_find_mime(void) +{ + search("mime:image/jpeg", 1); + search("mime:text/plain", 14); + search("y:text*", 14); + search("y:image*", 1); + search("mime:message/rfc822", 2); +} + +static void +test_mu_find_text_in_rfc822(void) +{ + search("embed:dancing", 1); + search("e:curious", 1); + search("embed:with", 2); + search("e:karjala", 0); + search("embed:navigation", 1); +} + +static void +test_mu_find_maildir_special(void) +{ + search("\"maildir:/wOm_bàT\"", 3); + search("\"maildir:/wOm*\"", 3); + search("\"maildir:/wOm_*\"", 3); + search("\"maildir:wom_bat\"", 0); + search("\"maildir:/wombat\"", 0); + search("subject:atoms", 1); + search("\"maildir:/wom_bat\" subject:atoms", 1); +} + + +/* some more tests */ + +static void +test_mu_find_wrong_muhome() +{ + auto res = run_command({MU_PROGRAM, "find", "--muhome", + join_paths("/foo", "bar", "nonexistent"), "f:socrates"}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code,==,1); // general error + g_assert_cmpuint(count_nl(res->standard_err), >, 1); +} + +static void +test_mu_find_links(void) +{ + TempDir temp_dir; + + { + auto res = run_command({MU_PROGRAM, "find", "--muhome", test_mu_home, + "--format", "links", "--linksdir", temp_dir.path(), + "mime:message/rfc822"}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code,==,0); + g_assert_cmpuint(count_nl(res->standard_out),==,0); + g_assert_cmpuint(count_nl(res->standard_err),==,0); + } + + + /* furthermore, two symlinks should be there */ + const auto f1{mu_format("{}/cur/rfc822.1", temp_dir)}; + const auto f2{mu_format("{}/cur/rfc822.2", temp_dir)}; + + g_assert_cmpuint(determine_dtype(f1.c_str(), true), ==, DT_LNK); + g_assert_cmpuint(determine_dtype(f2.c_str(), true), ==, DT_LNK); + + /* now we try again, we should get a line of error output, + * when we find the first target file already exists */ + { + auto res = run_command({MU_PROGRAM, "find", "--muhome", test_mu_home, + "--format", "links", "--linksdir", temp_dir.path(), + "mime:message/rfc822"}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code,==,1); + g_assert_cmpuint(count_nl(res->standard_out),==,0); + g_assert_cmpuint(count_nl(res->standard_err),==,1); + } + + /* now we try again with --clearlinks, and the we should be + * back to 0 errors */ + { + auto res = run_command({MU_PROGRAM, "find", "--muhome", test_mu_home, + "--format", "links", "--clearlinks", "--linksdir", temp_dir.path(), + "mime:message/rfc822"}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code,==,0); + g_assert_cmpuint(count_nl(res->standard_out),==,0); + g_assert_cmpuint(count_nl(res->standard_err),==,0); + } + + g_assert_cmpuint(determine_dtype(f1.c_str(), true), ==, DT_LNK); + g_assert_cmpuint(determine_dtype(f2.c_str(), true), ==, DT_LNK); +} + +/* some more tests */ + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + if (!set_en_us_utf8_locale()) + return 0; /* don't error out... */ + + TempDir temp_dir{}; + { + test_mu_home = temp_dir.path(); + + auto res1 = run_command({MU_PROGRAM, "--quiet", "init", + "--muhome", test_mu_home, "--maildir" , MU_TESTMAILDIR2}); + assert_valid_result(res1); + + auto res2 = run_command({MU_PROGRAM, "--quiet", "index", + "--muhome", test_mu_home}); + assert_valid_result(res2); + } + + g_test_add_func("/cmd/find/empty-query", test_mu_find_empty_query); + g_test_add_func("/cmd/find/01", test_mu_find_01); + g_test_add_func("/cmd/find/02", test_mu_find_02); + g_test_add_func("/cmd/find/file", test_mu_find_file); + g_test_add_func("/cmd/find/mime", test_mu_find_mime); + g_test_add_func("/cmd/find/links", test_mu_find_links); + g_test_add_func("/cmd/find/text-in-rfc822", test_mu_find_text_in_rfc822); + g_test_add_func("/cmd/find/wrong-muhome", test_mu_find_wrong_muhome); + g_test_add_func("/cmd/find/maildir-special", test_mu_find_maildir_special); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd-index.cc b/mu/mu-cmd-index.cc new file mode 100644 index 0000000..6b3b255 --- /dev/null +++ b/mu/mu-cmd-index.cc @@ -0,0 +1,201 @@ +/* +** Copyright (C) 2008-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-cmd.hh" + +#include <chrono> +#include <thread> +#include <atomic> + +#include <errno.h> +#include <string.h> +#include <cstdio> +#include <signal.h> +#include <unistd.h> + +#include "mu-store.hh" + +using namespace Mu; + +static std::atomic<bool> caught_signal; + +static void +sig_handler(int _sig) +{ + caught_signal = true; +} + +static void +install_sig_handler(void) +{ + struct sigaction action; + int i, sigs[] = {SIGINT, SIGHUP, SIGTERM}; + + sigemptyset(&action.sa_mask); + action.sa_flags = SA_RESETHAND; + action.sa_handler = sig_handler; + + for (i = 0; i != G_N_ELEMENTS(sigs); ++i) + if (sigaction(sigs[i], &action, NULL) != 0) + mu_critical("set sigaction for {} failed: {}", + sigs[i], g_strerror(errno)); +} + +static void +print_stats(const Indexer::Progress& stats, bool color) +{ + const char* kars = "-\\|/"; + static auto i = 0U; + + MaybeAnsi col{color}; + using Color = MaybeAnsi::Color; + + mu_print("{}{}{} indexing messages; " + "checked: {}{}{}; " + "updated/new: {}{}{}; " + "cleaned-up: {}{}{}", + col.fg(Color::Yellow), kars[++i % 4], col.reset(), + col.fg(Color::Green), static_cast<size_t>(stats.checked), col.reset(), + col.fg(Color::Green), static_cast<size_t>(stats.updated), col.reset(), + col.fg(Color::Green), static_cast<size_t>(stats.removed), col.reset()); +} + +Result<void> +Mu::mu_cmd_index(const Options& opts) +{ + auto store = std::invoke([&]{ + if (opts.index.reindex) + return Store::make(opts.runtime_path(RuntimePath::XapianDb), + Store::Options::ReInit|Store::Options::Writable); + else + return Store::make(opts.runtime_path(RuntimePath::XapianDb), + Store::Options::Writable); + }); + + if (!store) + return Err(store.error()); + + const auto mdir{store->root_maildir()}; + if (G_UNLIKELY(::access(mdir.c_str(), R_OK) != 0)) + return Err(Error::Code::File, "'{}' is not readable: {}", + mdir, g_strerror(errno)); + + MaybeAnsi col{!opts.nocolor}; + using Color = MaybeAnsi::Color; + if (!opts.quiet) { + if (opts.index.lazycheck) + mu_print("lazily "); + + mu_println("indexing maildir {}{}{} -> " + "store {}{}{}", + col.fg(Color::Green), store->root_maildir(), col.reset(), + col.fg(Color::Blue), store->path(), col.reset()); + } + + Mu::Indexer::Config conf{}; + conf.cleanup = !opts.index.nocleanup; + conf.lazy_check = opts.index.lazycheck; + // ignore .noupdate with an empty store. + conf.ignore_noupdate = store->empty(); + + install_sig_handler(); + + auto& indexer{store->indexer()}; + indexer.start(conf); + while (!caught_signal && indexer.is_running()) { + if (!opts.quiet) + print_stats(indexer.progress(), !opts.nocolor); + + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + + if (!opts.quiet) { + mu_print("\r"); + ::fflush({}); + } + } + + indexer.stop(); + + if (!opts.quiet) { + print_stats(indexer.progress(), !opts.nocolor); + mu_print("\n"); + ::fflush({}); + } + + return Ok(); +} + + +#ifdef BUILD_TESTS + +/* + * Tests. + * + */ +#include <config.h> +#include <mu-store.hh> +#include "utils/mu-test-utils.hh" + + +static void +test_mu_index(size_t batch_size=0) +{ + TempDir temp_dir{}; + + const auto mu_home{temp_dir.path()}; + + auto res1 = run_command({MU_PROGRAM, "--quiet", "init", "--batch-size", + mu_format("{}", batch_size == 0 ? 10000 : batch_size), + "--muhome", mu_home, "--maildir" , MU_TESTMAILDIR2}); + assert_valid_command(res1); + + auto res2 = run_command({MU_PROGRAM, "--quiet", "index", + "--muhome", mu_home}); + assert_valid_command(res2); + + auto&& store = unwrap(Store::make(join_paths(temp_dir.path(), "xapian"))); + g_assert_cmpuint(store.size(),==,14); +} + + +static void +test_mu_index_basic() +{ + test_mu_index(); +} + +static void +test_mu_index_batch() +{ + test_mu_index(2); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/cmd/index/basic", test_mu_index_basic); + g_test_add_func("/cmd/index/batch", test_mu_index_batch); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd-info.cc b/mu/mu-cmd-info.cc new file mode 100644 index 0000000..2e155fc --- /dev/null +++ b/mu/mu-cmd-info.cc @@ -0,0 +1,316 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include "mu-cmd.hh" +#include <message/mu-message.hh> +#include "utils/mu-utils.hh" + +#include <glib.h> +#include <gmime/gmime.h> + +#include <fmt/ostream.h> + +#include <thirdparty/tabulate.hpp> + +using namespace Mu; +using namespace tabulate; + +template <> struct fmt::formatter<Table> : ostream_formatter {}; + +static void +colorify(Table& table, const Options& opts) +{ + if (opts.nocolor || table.size() == 0) + return; + + for (auto&& c = 0U; c != table.row(0).size(); ++c) { + switch (c) { + case 0: + table.column(c).format() + .font_color(Color::green) + .font_style({FontStyle::bold}); + break; + case 1: + table.column(c).format() + .font_color(Color::blue); + break; + case 2: + table.column(c).format() + .font_color(Color::magenta); + break; + + case 3: + table.column(c).format() + .font_color(Color::yellow); + break; + case 4: + table.column(c).format() + .font_color(Color::green); + break; + case 5: + table.column(c).format() + .font_color(Color::blue); + break; + case 6: + table.column(c).format() + .font_color(Color::magenta); + break; + + case 7: + table.column(c).format() + .font_color(Color::yellow); + break; + default: + table.column(c).format() + .font_color(Color::grey); + break; + } + } + + for (auto&& c = 0U; c != table.row(0).size(); ++c) + table[0][c].format() + .font_color(Color::white) + .font_style({FontStyle::bold}); +} + + +static Result<void> +topic_fields(const Options& opts) +{ + using namespace std::string_literals; + + Table fields; + fields.add_row({"field-name", "alias", "short", "search", + "value", "sexp", "example query", "description"}); + + auto searchable=[&](const Field& field)->std::string { + if (field.is_boolean_term()) + return "boolean"; + if (field.is_phrasable_term()) + return "phrase"; + if (field.is_value()) + return "yes"; + if (field.is_contact()) + return "contact"; + if (field.is_range()) + return "range"; + return "no"; + }; + + size_t row{}; + field_for_each([&](auto&& field){ + if (field.is_internal()) + return; // skip. + + fields.add_row({mu_format("{}", field.name), + field.alias.empty() ? "" : mu_format("{}", field.alias), + field.shortcut ? mu_format("{}", field.shortcut) : ""s, + searchable(field), + field.is_value() ? "yes" : "no", + field.include_in_sexp() ? "yes" : "no", + field.example_query, + field.description}); + ++row; + }); + + colorify(fields, opts); + + std::cout << "# Message fields\n" << fields << '\n'; + + return Ok(); +} + +static Result<void> +topic_flags(const Options& opts) +{ + using namespace tabulate; + using namespace std::string_literals; + + Table flags; + flags.add_row({"flag", "shortcut", "category", "description"}); + + flag_infos_for_each([&](const MessageFlagInfo& info) { + + const auto catname = std::invoke( + [](MessageFlagCategory cat)->std::string { + switch(cat){ + case MessageFlagCategory::Mailfile: + return "file"; + case MessageFlagCategory::Maildir: + return "maildir"; + case MessageFlagCategory::Content: + return "content"; + case MessageFlagCategory::Pseudo: + return "pseudo"; + default: + return {}; + } + }, info.category); + + flags.add_row({mu_format("{}", info.name), + mu_format("{}", info.shortcut), + catname, + std::string{info.description}}); + }); + + colorify(flags, opts); + + std::cout << "# Message flags\n" << flags << '\n'; + + return Ok(); +} + +static Result<void> +topic_store(const Mu::Store& store, const Options& opts) +{ + auto tstamp = [](::time_t t)->std::string { + if (t == 0) + return "never"; + else + return mu_format("{:%c}", mu_time(t)); + }; + + Table info; + const auto conf{store.config()}; + info.add_row({"property", "value"}); + info.add_row({"maildir", store.root_maildir()}); + info.add_row({"database-path", store.path()}); + info.add_row({"schema-version", + mu_format("{}", conf.get<Config::Id::SchemaVersion>())}); + info.add_row({"max-message-size", mu_format("{}", conf.get<Config::Id::MaxMessageSize>())}); + info.add_row({"batch-size", mu_format("{}", conf.get<Config::Id::BatchSize>())}); + info.add_row({"created", tstamp(conf.get<Config::Id::Created>())}); + + for (auto&& c : conf.get<Config::Id::PersonalAddresses>()) + info.add_row({"personal-address", c}); + for (auto&& c : conf.get<Config::Id::IgnoredAddresses>()) + info.add_row({"ignored-address", c}); + + info.add_row({"messages in store", mu_format("{}", store.size())}); + info.add_row({"support-ngrams", conf.get<Config::Id::SupportNgrams>() ? "yes" : "no"}); + + info.add_row({"last-change", tstamp(store.statistics().last_change)}); + info.add_row({"last-index", tstamp(store.statistics().last_index)}); + + if (!opts.nocolor) + colorify(info, opts); + + std::cout << info << '\n'; + + return Ok(); +} + +static Result<void> +topic_maildirs(const Mu::Store& store, const Options& opts) +{ + for (auto&& mdir: store.maildirs()) + mu_println("{}", mdir); + + return Ok(); +} + +static Result<void> +topic_mu(const Options& opts) +{ + Table info; + + using namespace tabulate; + + info.add_row({"property", "value", "description"}); + info.add_row({"mu-version", std::string{VERSION}, "Mu runtime version"}); + info.add_row({"xapian-version", Xapian::version_string(), "Xapian runtime version"}); + info.add_row({"gmime-version", + mu_format("{}.{}.{}", gmime_major_version, gmime_minor_version, + gmime_micro_version), "GMime runtime version"}); + info.add_row({"glib-version", + mu_format("{}.{}.{}", glib_major_version, glib_minor_version, + glib_micro_version), "GLib runtime version"}); + info.add_row({"schema-version", mu_format("{}", MU_STORE_SCHEMA_VERSION), + "Version of mu's database schema"}); + + info.add_row({"cld2-support", +#if HAVE_CLD2 + "yes" +#else + "no" +#endif + , "Support searching by language-code?"}); + + info.add_row({"guile-support", +#if BUILD_GUILE + "yes" +#else + "no" +#endif + , "GNU Guile 3.x scripting support?"}); + info.add_row({"readline-support", +#if HAVE_LIBREADLINE + "yes" +#else + "no" +#endif + , "Better 'm server' REPL for debugging?"}); + + if (!opts.nocolor) + colorify(info, opts); + + std::cout << info << '\n'; + + return Ok(); +} + + +Result<void> +Mu::mu_cmd_info(const Mu::Store& store, const Options& opts) +{ + if (!locale_workaround()) + return Err(Error::Code::User, "failed to find a working locale"); + + const auto topic{opts.info.topic}; + if (topic == "store") + return topic_store(store, opts); + else if (topic == "maildirs") + return topic_maildirs(store, opts); + else if (topic == "fields") { + topic_fields(opts); + std::cout << std::endl; + return topic_flags(opts); + } else if (topic == "mu") { + return topic_mu(opts); + } else { + topic_mu(opts); + + MaybeAnsi col{!opts.nocolor}; + using Color = MaybeAnsi::Color; + + auto topic = [&](auto&& t, auto&& d)->std::string { + return mu_format("{}{:<10}{} - {:>12}", + col.fg(Color::Green), t, col.reset(), d); + }; + + mu_println("\nother info topics ('mu info <topic>'):\n{}\n{}\n{}", + topic("store", "information about the message store (database)"), + topic("maildirs", "list the maildirs under the store's root-maildir"), + topic("fields", "information about message fields")); + } + + return Ok(); +} diff --git a/mu/mu-cmd-init.cc b/mu/mu-cmd-init.cc new file mode 100644 index 0000000..26a9600 --- /dev/null +++ b/mu/mu-cmd-init.cc @@ -0,0 +1,144 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include "mu-cmd.hh" + +using namespace Mu; + +#ifndef BUILD_TESTS + +Result<void> +Mu::mu_cmd_init(const Options& opts) +{ + auto store = std::invoke([&]()->Result<Store> { + + /* + * reinit + */ + if (opts.init.reinit) + return Store::make(opts.runtime_path(RuntimePath::XapianDb), + Store::Options::ReInit|Store::Options::Writable); + /* + * full init + */ + + /* not provided, nor could we find a good default */ + if (opts.init.maildir.empty()) + return Err(Error::Code::InvalidArgument, + "missing --maildir parameter and could " + "not determine default"); + else if (!g_path_is_absolute(opts.init.maildir.c_str())) + return Err(Error{Error::Code::File, + "--maildir is not absolute"}); + + MemDb mdb; + Config conf{mdb}; + + if (opts.init.max_msg_size) + conf.set<Config::Id::MaxMessageSize>(*opts.init.max_msg_size); + if (opts.init.batch_size && *opts.init.batch_size != 0) + conf.set<Config::Id::BatchSize>(*opts.init.batch_size); + if (!opts.init.my_addresses.empty()) + conf.set<Config::Id::PersonalAddresses>(opts.init.my_addresses); + if (!opts.init.ignored_addresses.empty()) + conf.set<Config::Id::IgnoredAddresses>(opts.init.ignored_addresses); + if (opts.init.support_ngrams) + conf.set<Config::Id::SupportNgrams>(true); + + return Store::make_new(opts.runtime_path(RuntimePath::XapianDb), + opts.init.maildir, conf); + }); + + if (!store) + return Err(store.error()); + + if (!opts.quiet) { + + mu_println("mu has been {} with the following properties:", + opts.init.reinit ? "reinitialized" : "created"); + // mildly hacky + Options opts_copy{opts}; + opts_copy.info.topic = "store"; + mu_cmd_info(*store, opts_copy); + + mu_println("Database is empty. You can use 'mu index' to fill it."); + } + + return Ok(); +} + + + +#else /* BUILD_TESTS */ + +/* + * Tests. + * + */ +#include <config.h> +#include <mu-store.hh> +#include "utils/mu-test-utils.hh" + + +static void +test_mu_init_basic() +{ + TempDir temp_dir{}; + + const auto mu_home{temp_dir.path()}; + + auto res1 = run_command({MU_PROGRAM, "--quiet", "init", + "--muhome", mu_home, "--maildir" , MU_TESTMAILDIR2}); + assert_valid_command(res1); + + auto&& store = unwrap(Store::make(join_paths(temp_dir.path(), "xapian"))); + g_assert_true(store.empty()); +} + +static void +test_mu_init_maildir() +{ + TempDir temp_dir{}; + + const auto mu_home{temp_dir.path()}; + + g_setenv("MAILDIR", MU_TESTMAILDIR2, 1); + auto res1 = run_command({MU_PROGRAM, "--quiet", "init", + "--muhome", mu_home}); + assert_valid_command(res1); + + auto&& store = unwrap(Store::make(join_paths(temp_dir.path(), "xapian"))); + g_assert_true(store.empty()); + assert_equal(store.root_maildir(), MU_TESTMAILDIR2); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/cmd/init/basic", test_mu_init_basic); + g_test_add_func("/cmd/init/maildir", test_mu_init_maildir); + + return g_test_run(); +} + +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd-mkdir.cc b/mu/mu-cmd-mkdir.cc new file mode 100644 index 0000000..b91bdec --- /dev/null +++ b/mu/mu-cmd-mkdir.cc @@ -0,0 +1,99 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-cmd.hh" + +#include "mu-maildir.hh" + +using namespace Mu; + +Mu::Result<void> +Mu::mu_cmd_mkdir(const Options& opts) +{ + for (auto&& dir: opts.mkdir.dirs) { + if (auto&& res = + maildir_mkdir(dir, opts.mkdir.mode); !res) + return res; + } + + return Ok(); +} + + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +static void +test_mkdir_single() +{ + auto testroot{unwrap(make_temp_dir())}; + auto testdir1{join_paths(testroot, "testdir1")}; + + auto res = run_command({MU_PROGRAM, "mkdir", testdir1}); + assert_valid_command(res); + + g_assert_true(check_dir(join_paths(testdir1, "cur"), true, true)); + g_assert_true(check_dir(join_paths(testdir1, "new"), true, true)); + g_assert_true(check_dir(join_paths(testdir1, "tmp"), true, true)); +} + +static void +test_mkdir_multi() +{ + auto testroot{unwrap(make_temp_dir())}; + auto testdir2{join_paths(testroot, "testdir2")}; + auto testdir3{join_paths(testroot, "testdir3")}; + + auto res = run_command({MU_PROGRAM, "mkdir", testdir2, testdir3}); + assert_valid_command(res); + + g_assert_true(check_dir(join_paths(testdir2, "cur"), true, true)); + g_assert_true(check_dir(join_paths(testdir2, "new"), true, true)); + g_assert_true(check_dir(join_paths(testdir3, "tmp"), true, true)); + + g_assert_true(check_dir(join_paths(testdir3, "cur"), true, true)); + g_assert_true(check_dir(join_paths(testdir3, "new"), true, true)); + g_assert_true(check_dir(join_paths(testdir3, "tmp"), true, true)); +} + +int +main(int argc, char* argv[]) try { + + mu_test_init(&argc, &argv); + + g_test_add_func("/cmd/mkdir/single", test_mkdir_single); + g_test_add_func("/cmd/mkdir/multi", test_mkdir_multi); + + return g_test_run(); + +} catch (const Error& e) { + mu_printerrln("{}", e.what()); + return 1; +} catch (...) { + mu_printerrln("caught exception"); + return 1; +} +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd-move.cc b/mu/mu-cmd-move.cc new file mode 100644 index 0000000..7ad7a48 --- /dev/null +++ b/mu/mu-cmd-move.cc @@ -0,0 +1,273 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-cmd.hh" + +#include "mu-store.hh" +#include "mu-maildir.hh" +#include "message/mu-message-file.hh" + + #include <unistd.h> + +using namespace Mu; + + +Result<void> +Mu::mu_cmd_move(Mu::Store& store, const Options& opts) +{ + const auto& src{opts.move.src}; + if (::access(src.c_str(), R_OK) != 0 || determine_dtype(src) != DT_REG) + return Err(Error::Code::InvalidArgument, + "Source is not a readable file"); + + auto id{store.find_message_id(src)}; + if (!id) + return Err(Error{Error::Code::InvalidArgument, + "Source file is not present in database"} + .add_hint("Perhaps run mu index?")); + + std::string dest{opts.move.dest}; + Option<const std::string&> dest_path; + if (dest.empty() && opts.move.flags.empty()) + return Err(Error::Code::InvalidArgument, + "Must have at least one of destination and flags"); + else if (!dest.empty()) { + const auto mdirs{store.maildirs()}; + + if (!seq_some(mdirs, [&](auto &&d){ return d == dest;})) + return Err(Error{Error::Code::InvalidArgument, + "No maildir '{}' in store", dest} + .add_hint("Try 'mu mkdir'")); + else + dest_path = dest; + } + + auto old_flags{flags_from_path(src)}; + if (!old_flags) + return Err(Error::Code::InvalidArgument, "failed to determine old flags"); + + Flags new_flags{}; + if (!opts.move.flags.empty()) { + if (auto&& nflags{flags_from_expr(to_string_view(opts.move.flags), + *old_flags)}; !nflags) + return Err(Error::Code::InvalidArgument, "Invalid flags"); + else + new_flags = flags_maildir_file(*nflags); + + if (any_of(new_flags & Flags::New) && new_flags != Flags::New) + return Err(Error{Error::Code::File, + "the New flag cannot be combined with others"} + .add_hint("See the mu-move manpage")); + } + + Store::MoveOptions move_opts{}; + if (opts.move.change_name) + move_opts |= Store::MoveOptions::ChangeName; + if (opts.move.update_dups) + move_opts |= Store::MoveOptions::DupFlags; + if (opts.move.dry_run) + move_opts |= Store::MoveOptions::DryRun; + + auto id_paths = store.move_message(*id, dest_path, new_flags, move_opts); + if (!id_paths) + return Err(std::move(id_paths.error())); + + for (const auto&[_id, path]: *id_paths) + mu_println("{}", path); + + return Ok(); +} + + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +static void +test_move_dry_run() +{ + allow_warnings(); + + TempDir tdir; + const auto dbpath{runtime_path(RuntimePath::XapianDb, tdir.path())}; + + auto res = run_command0({CP_PROGRAM, "-r", MU_TESTMAILDIR, tdir.path()}); + assert_valid_command(res); + + const auto testpath{join_paths(tdir.path(), "testdir")}; + const auto src{join_paths(testpath, "cur", "1220863042.12663_1.mindcrime!2,S")}; + { + auto store = Store::make_new(dbpath, testpath, {}); + assert_valid_result(store); + g_assert_true(store->indexer().start({}, true/*block*/)); + } + + // make a message 'New' + { + auto res = run_command0({MU_PROGRAM, "move", "--muhome", tdir.path(), src, + "--flags", "N", "--dry-run"}); + assert_valid_command(res); + + auto dst{join_paths(testpath, "new", "1220863042.12663_1.mindcrime")}; + assert_equal(res->standard_out, dst + '\n'); + + g_assert_true(::access(dst.c_str(), F_OK) != 0); + g_assert_true(::access(src.c_str(), F_OK) == 0); + } + + // change some flags + { + auto res = run_command0({MU_PROGRAM, "move", "--muhome", tdir.path(), src, + "--flags", "FP", "--dry-run"}); + assert_valid_command(res); + + auto dst{join_paths(testpath, "cur", "1220863042.12663_1.mindcrime!2,FP")}; + assert_equal(res->standard_out, dst + '\n'); + } + + // change some relative flag + { + auto res = run_command0({MU_PROGRAM, "move", "--muhome", tdir.path(), src, + "--flags", "+F", "--dry-run"}); + assert_valid_command(res); + + auto dst{join_paths(testpath, "cur", "1220863042.12663_1.mindcrime!2,FS")}; + assert_equal(res->standard_out, dst + '\n'); + } + + { + auto res = run_command0({MU_PROGRAM, "move", "--muhome", tdir.path(), src, + "--flags", "-S+P+T", "--dry-run"}); + assert_valid_command(res); + + auto dst{join_paths(testpath, "cur", "1220863042.12663_1.mindcrime!2,PT")}; + assert_equal(res->standard_out, dst + '\n'); + } + + // change maildir + for (auto& o : {"o1", "o2"}) + assert_valid_result(maildir_mkdir(join_paths(tdir.path(), "testdir", o))); + + { + auto res = run_command0({MU_PROGRAM, "move", "--muhome", tdir.path(), src, + "/o1", "--flags", "-S+F", "--dry-run"}); + assert_valid_command(res); + assert_equal(res->standard_out, + join_paths(testpath, + "o1/cur", "1220863042.12663_1.mindcrime!2,F") + "\n"); + } + + // change-dups; first create some dups and index them. + assert_valid_result(run_command0({CP_PROGRAM, src, join_paths(testpath, "o1/cur")})); + assert_valid_result(run_command0({CP_PROGRAM, src, join_paths(testpath, "o2/cur")})); + { + auto store = Store::make(dbpath, Store::Options::Writable); + assert_valid_result(store); + g_assert_true(store->indexer().start({}, true/*block*/)); + } + + // change some flags + update dups + { + auto res = run_command0({MU_PROGRAM, "move", "--muhome", tdir.path(), src, + "--flags", "-S+S+T+R", "--update-dups", "--dry-run"}); + assert_valid_command(res); + + auto p{join_paths(testpath, "cur", "1220863042.12663_1.mindcrime!2,RST")}; + auto p1{join_paths(testpath, "o1", "cur", "1220863042.12663_1.mindcrime!2,RS")}; + auto p2{join_paths(testpath, "o2", "cur", "1220863042.12663_1.mindcrime!2,RS")}; + + assert_equal(res->standard_out, mu_format("{}\n{}\n{}\n", p, p1, p2)); + } +} + + +static void +test_move_real() +{ + allow_warnings(); + + TempDir tdir; + const auto dbpath{runtime_path(RuntimePath::XapianDb, tdir.path())}; + + auto res = run_command0({CP_PROGRAM, "-r", MU_TESTMAILDIR, tdir.path()}); + assert_valid_command(res); + + const auto testpath{join_paths(tdir.path(), "testdir")}; + const auto src{join_paths(testpath, "cur", "1220863042.12663_1.mindcrime!2,S")}; + { + auto store = Store::make_new(dbpath, testpath, {}); + assert_valid_result(res); + g_assert_true(store->indexer().start({}, true/*block*/)); + } + + { + auto res = run_command0({MU_PROGRAM, "move", "--muhome", tdir.path(), src, + "--flags", "N"}); + assert_valid_command(res); + auto dst{join_paths(testpath, "new", "1220863042.12663_1.mindcrime")}; + g_assert_true(::access(dst.c_str(), F_OK) == 0); + g_assert_true(::access(src.c_str(), F_OK) != 0); + } + + // change flags, maildir, update-dups + // change-dups; first create some dups and index them. + const auto src2{join_paths(testpath, "cur", "1305664394.2171_402.cthulhu!2,")}; + for (auto& o : {"o1", "o2", "o3"}) + assert_valid_result(maildir_mkdir(join_paths(tdir.path(), "testdir", o))); + assert_valid_result(run_command0({CP_PROGRAM, src2, join_paths(testpath, "o1/cur")})); + assert_valid_result(run_command0({CP_PROGRAM, src2, join_paths(testpath, "o2/new")})); + { + auto store = Store::make(dbpath, Store::Options::Writable); + assert_valid_result(store); + g_assert_true(store->indexer().start({}, true/*block*/)); + } + + auto res2 = run_command0({MU_PROGRAM, "move", "--muhome", tdir.path(), src2, "/o3", + "--flags", "-S+S+T+R", "--update-dups", "--change-name"}); + assert_valid_command(res2); + + auto store = Store::make(dbpath, Store::Options::Writable); + assert_valid_result(store); + g_assert_true(store->indexer().start({}, true/*block*/)); + + for (auto&& f: split(res2->standard_out, "\n")) { + //mu_println(">> {}", f); + if (f.length() > 2) + g_assert_true(::access(f.c_str(), F_OK) == 0); + } +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/cmd/move/dry-run", test_move_dry_run); + g_test_add_func("/cmd/move/real", test_move_real); + + return g_test_run(); + +} +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd-remove.cc b/mu/mu-cmd-remove.cc new file mode 100644 index 0000000..5eb96b8 --- /dev/null +++ b/mu/mu-cmd-remove.cc @@ -0,0 +1,100 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-cmd.hh" + +using namespace Mu; + +Result<void> +Mu::mu_cmd_remove(Mu::Store& store, const Options& opts) +{ + for (auto&& file: opts.remove.files) { + const auto res = store.remove_message(file); + if (!res) + return Err(Error::Code::File, "failed to remove {}", file.c_str()); + else + mu_debug("removed message @ {}", file); + } + + return Ok(); +} + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +static void +test_remove_ok() +{ + auto testhome{unwrap(make_temp_dir())}; + auto dbpath{runtime_path(RuntimePath::XapianDb, testhome)}; + + /* create a writable copy */ + const auto testmdir = join_paths(testhome, "test-maildir"); + const auto testmsg = join_paths(testmdir, "/cur/1220863042.12663_1.mindcrime!2,S"); + auto cres = run_command({CP_PROGRAM, "-r", MU_TESTMAILDIR, testmdir}); + assert_valid_command(cres); + + { + auto&& store = unwrap(Store::make_new(dbpath, testmdir)); + auto res = store.add_message(testmsg); + assert_valid_result(res); + g_assert_true(store.contains_message(testmsg)); + } + + { // remove the same + auto res = run_command({MU_PROGRAM, "remove", + mu_format("--muhome={}", testhome), + testmsg}); + assert_valid_command(res); + } + + { + auto&& store = unwrap(Store::make(dbpath)); + g_assert_false(!!store.contains_message(testmsg)); + g_assert_cmpuint(::access(testmsg.c_str(), F_OK), ==, 0); + } + + remove_directory(testhome); +} + + +int +main(int argc, char* argv[]) try { + + mu_test_init(&argc, &argv); + + g_test_add_func("/cmd/remove/ok", test_remove_ok); + + return g_test_run(); + +} catch (const Error& e) { + mu_printerrln("{}", e.what()); + return 1; +} catch (...) { + mu_printerrln("caught exception"); + return 1; +} +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd-script.cc b/mu/mu-cmd-script.cc new file mode 100644 index 0000000..2302dd7 --- /dev/null +++ b/mu/mu-cmd-script.cc @@ -0,0 +1,49 @@ +/* +** Copyright (C) 2012-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include "mu-cmd.hh" +#include "mu-script.hh" +#include "utils/mu-utils.hh" + +using namespace Mu; + +Result<void> +Mu::mu_cmd_script(const Options& opts) +{ + ScriptPaths paths = { MU_SCRIPTS_DIR }; + const auto&& scriptinfos{script_infos(paths)}; + auto script_it = Mu::seq_find_if(scriptinfos, [&](auto&& item) { + return item.name == opts.script.name; + }); + + if (script_it == scriptinfos.cend()) + return Err(Error::Code::InvalidArgument, + "cannot find script '{}'", opts.script.name); + + std::vector<std::string> params{opts.script.params}; + if (!opts.muhome.empty()) { + params.emplace_back("--muhome"); + params.emplace_back(opts.muhome); + } + + // won't return unless there's an error. + return run_script(script_it->path, opts.script.params); +} diff --git a/mu/mu-cmd-server.cc b/mu/mu-cmd-server.cc new file mode 100644 index 0000000..3a69456 --- /dev/null +++ b/mu/mu-cmd-server.cc @@ -0,0 +1,164 @@ +/* +** Copyright (C) 2020-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include <string> +#include <algorithm> +#include <atomic> +#include <cstdio> + +#include <unistd.h> + +#include "mu-cmd.hh" +#include "mu-server.hh" + +#include "utils/mu-utils.hh" +#include "utils/mu-readline.hh" + +using namespace Mu; + +static std::atomic<int> MuTerminate{0}; +static bool tty; + +static void +sig_handler(int sig) +{ + MuTerminate = sig; +} + +static void +install_sig_handler() +{ + MuTerminate = 0; + + struct sigaction action{}; + action.sa_handler = sig_handler; + sigemptyset(&action.sa_mask); + action.sa_flags = SA_RESETHAND; + + for (auto sig: {SIGINT, SIGHUP, SIGTERM, SIGPIPE}) + if (sigaction(sig, &action, NULL) != 0) + mu_critical("set sigaction for {} failed: {}", + sig, g_strerror(errno)); +} + +/* + * Markers for/after the length cookie that precedes the expression we write to + * output. We use octal 376, 377 (ie, 0xfe, 0xff) as they will never occur in + * utf8 + */ + +#define COOKIE_PRE "\376" +#define COOKIE_POST "\377" + +static void +cookie(size_t n) +{ + const auto num{static_cast<unsigned>(n)}; + + if (tty) // for testing. + ::printf("[%x]", num); + else + ::printf(COOKIE_PRE "%x" COOKIE_POST, num); +} + + + +static void +output_stdout(const std::string& str, Server::OutputFlags flags) +{ + cookie(str.size() + 1); + if (G_UNLIKELY(::puts(str.c_str()) < 0)) { + mu_critical("failed to write output '{}'", str); + ::raise(SIGTERM); /* terminate ourselves */ + } + if (any_of(flags & Server::OutputFlags::Flush)) + std::fflush(stdout); +} + + +static void +report_error(const Mu::Error& err) noexcept +{ + output_stdout(Sexp(":error"_sym, Error::error_number(err.code()), + ":message"_sym, err.what()).to_string(), + Server::OutputFlags::Flush); +} + +Result<void> +Mu::mu_cmd_server(const Mu::Options& opts) try { + + auto store = Store::make(opts.runtime_path(RuntimePath::XapianDb), + Store::Options::Writable); + if (!store) + return Err(store.error()); + + Server::Options sopts{}; + sopts.allow_temp_file = opts.server.allow_temp_file; + + Server server{*store, sopts, output_stdout}; + mu_message("created server with store @ {}; maildir @ {}; debug-mode {};" + "readline: {}", + store->path(), store->root_maildir(), + opts.debug ? "yes" : "no", + have_readline() ? "yes" : "no"); + + tty = ::isatty(::fileno(stdout)); + const auto eval = std::string{opts.server.commands ? "(help :full t)" : opts.server.eval}; + if (!eval.empty()) { + server.invoke(eval); + return Ok(); + } + + // Note, the readline stuff is inactive unless on a tty. + const auto histpath{opts.runtime_path(RuntimePath::Cache) + "/history"}; + setup_readline(histpath, 50); + + install_sig_handler(); + mu_println(";; Welcome to the " PACKAGE_STRING " command-server{}\n" + ";; Use (help) to get a list of commands, (quit) to quit.", + opts.debug ? " (debug-mode)" : ""); + + bool do_quit{}; + while (!MuTerminate && !do_quit) { + std::fflush(stdout); // Needed for Windows, see issue #1827. + const auto line{read_line(do_quit)}; + if (line.find_first_not_of(" \t") == std::string::npos) + continue; // skip whitespace-only lines + + do_quit = server.invoke(line) ? false : true; + save_line(line); + } + + if (MuTerminate != 0) + mu_message ("shutting down due to signal {}", MuTerminate.load()); + + shutdown_readline(); + + return Ok(); + +} catch (const Error& er) { /* note: user-level error, "OK" for mu */ + report_error(er); + mu_warning("server caught exception: {}", er.what()); + return Ok(); +} catch (...) { + mu_critical("server caught exception"); + return Err(Error::Code::Internal, "caught exception"); +} diff --git a/mu/mu-cmd-verify.cc b/mu/mu-cmd-verify.cc new file mode 100644 index 0000000..7fbb4b9 --- /dev/null +++ b/mu/mu-cmd-verify.cc @@ -0,0 +1,255 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" +#include "mu-cmd.hh" + +#include "message/mu-message.hh" +#include "message/mu-mime-object.hh" + +#include <iostream> +#include <iomanip> + +using namespace Mu; + +template <typename T> +static void +key_val(const Mu::MaybeAnsi& col, const std::string& key, T val) +{ + using Color = Mu::MaybeAnsi::Color; + + mu_println("{}{:<18}{}: {}{}{}", + col.fg(Color::BrightBlue), key, col.reset(), + col.fg(Color::Green), val, col.reset()); +} + +static void +print_signature(const Mu::MimeSignature& sig, const Options& opts) +{ + Mu::MaybeAnsi col{!opts.nocolor}; + + const auto created{sig.created()}; + key_val(col, "created", + created == 0 ? std::string{"unknown"} : + mu_format("{:%c}", mu_time(sig.created()))); + + const auto expires{sig.expires()}; + key_val(col, "expires", expires==0 ? std::string{"never"} : + mu_format("{:%c}", mu_time(sig.expires()))); + + const auto cert{sig.certificate()}; + key_val(col, "public-key algo", + to_string_view_opt(cert.pubkey_algo()).value_or("unknown")); + key_val(col, "digest algo", + to_string_view_opt(cert.digest_algo()).value_or("unknown")); + key_val(col, "id-validity", + to_string_view_opt(cert.id_validity()).value_or("unknown")); + key_val(col, "trust", + to_string_view_opt(cert.trust()).value_or("unknown")); + key_val(col, "issuer-serial", cert.issuer_serial().value_or("unknown")); + key_val(col, "issuer-name", cert.issuer_name().value_or("unknown")); + key_val(col, "finger-print", cert.fingerprint().value_or("unknown")); + key_val(col, "key-id", cert.key_id().value_or("unknown")); + key_val(col, "name", cert.name().value_or("unknown")); + key_val(col, "user-id", cert.user_id().value_or("unknown")); +} + + +static bool +verify(const MimeMultipartSigned& sigpart, const Options& opts) +{ + using VFlags = MimeMultipartSigned::VerifyFlags; + const auto vflags{opts.verify.auto_retrieve ? + VFlags::EnableKeyserverLookups: VFlags::None}; + + auto ctx{MimeCryptoContext::make_gpg()}; + if (!ctx) + return false; + + const auto sigs{sigpart.verify(*ctx, vflags)}; + Mu::MaybeAnsi col{!opts.nocolor}; + + if (!sigs || sigs->empty()) { + + if (!opts.quiet) + mu_println("cannot find signatures in part"); + + return true; + } + + bool valid{true}; + for (auto&& sig: *sigs) { + + const auto status{sig.status()}; + + if (!opts.quiet) + key_val(col, "status", to_string(status)); + + if (opts.verbose) + print_signature(sig, opts); + + if (none_of(sig.status() & MimeSignature::Status::Green)) + valid = false; + } + + return valid; +} + + +static bool +verify_message(const Message& message, const Options& opts, const std::string& name) +{ + if (none_of(message.flags() & Flags::Signed)) { + if (!opts.quiet) + mu_println("{}: no signed parts found", name); + return false; + } + + bool verified{true}; /* innocent until proven guilty */ + for(auto&& part: message.parts()) { + + if (!part.is_signed()) + continue; + + const auto& mobj{part.mime_object()}; + if (!mobj.is_multipart_signed()) + continue; + + if (!verify(MimeMultipartSigned(mobj), opts)) + verified = false; + } + + return verified; +} + + + +Mu::Result<void> +Mu::mu_cmd_verify(const Options& opts) +{ + bool all_ok{true}; + const auto mopts = message_options(opts.verify); + + for (auto&& file: opts.verify.files) { + + auto message{Message::make_from_path(file, mopts)}; + if (!message) + return Err(message.error()); + + if (!opts.quiet && opts.verify.files.size() > 1) + mu_println("verifying {}", file); + + if (!verify_message(*message, opts, file)) + all_ok = false; + } + + // when no messages provided, read from stdin + if (opts.verify.files.empty()) { + const auto msgtxt = read_from_stdin(); + if (!msgtxt) + return Err(msgtxt.error()); + auto message{Message::make_from_text(*msgtxt, {}, mopts)}; + if (!message) + return Err(message.error()); + + all_ok = verify_message(*message, opts, "<stdin>"); + } + + if (all_ok) + return Ok(); + else + return Err(Error::Code::UnverifiedSignature, + "failed to verify one or more signatures"); +} + + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include "utils/mu-test-utils.hh" + +/* we can only test 'verify' if gpg is installed, and has djcb@djcbsoftware's key in the keyring */ +static bool +verify_is_testable(void) +{ + auto gpg{program_in_path("gpg2")}; + if (!gpg) { + mu_message("cannot find gpg2 in path"); + return false; + } + + auto res{run_command({*gpg, "--list-keys", "DCC4A036"})}; /* djcb@djcbsoftware.nl's key */ + if (!res || res->exit_code != 0) { + mu_message("key DCC4A036 not found"); + return false; + } + + return true; +} + +static void +test_mu_verify_good(void) +{ + if (!verify_is_testable()) { + g_test_skip("cannot test verify"); + return; + } + + auto res = run_command({MU_PROGRAM, "verify", + join_paths(MU_TESTMAILDIR4, "signed!2,S")}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code ,==, 0); +} + +static void +test_mu_verify_bad(void) +{ + if (!verify_is_testable()) { + g_test_skip("cannot test verify"); + return; + } + + auto res = run_command({MU_PROGRAM, "verify", + join_paths(MU_TESTMAILDIR4, "signed-bad!2,S")}); + assert_valid_result(res); + g_assert_cmpuint(res->exit_code,==, 1); +} + +int +main(int argc, char* argv[]) try { + + mu_test_init(&argc, &argv); + + g_test_add_func("/cmd/verify/good", test_mu_verify_good); + g_test_add_func("/cmd/verify/bad", test_mu_verify_bad); + + return g_test_run(); + +} catch (const Error& e) { + mu_printerrln("{}", e.what()); + return 1; +} catch (...) { + mu_printerrln("caught exception"); + return 1; +} +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd-view.cc b/mu/mu-cmd-view.cc new file mode 100644 index 0000000..3ddd78e --- /dev/null +++ b/mu/mu-cmd-view.cc @@ -0,0 +1,424 @@ +/* +** Copyright (C) 2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ +#include <config.h> + +#include "mu-cmd.hh" + +#include "message/mu-message.hh" + +#include <iostream> +#include <iomanip> + +using namespace Mu; + + +#define VIEW_TERMINATOR '\f' /* form-feed */ + +using namespace Mu; + +static Mu::Result<void> +view_msg_sexp(const Message& message, const Options& opts) +{ + ::fputs(message.sexp().to_string().c_str(), stdout); + ::fputs("\n", stdout); + + return Ok(); +} + + +static std::string /* return comma-sep'd list of attachments */ +get_attach_str(const Message& message, const Options& opts) +{ + std::string str; + seq_for_each(message.parts(), [&](auto&& part) { + if (auto fname = part.raw_filename(); fname) { + if (str.empty()) + str = fname.value(); + else + str += ", " + fname.value(); + } + }); + + return str; +} + +#define color_maybe(C) \ + do { \ + if (color) \ + fputs((C), stdout); \ + } while (0) + +static void +print_field(const std::string& field, const std::string& val, bool color) +{ + if (val.empty()) + return; + + color_maybe(MU_COLOR_MAGENTA); + fputs_encoded(field, stdout); + color_maybe(MU_COLOR_DEFAULT); + fputs(": ", stdout); + + color_maybe(MU_COLOR_GREEN); + fputs_encoded(val, stdout); + + color_maybe(MU_COLOR_DEFAULT); + fputs("\n", stdout); +} + +/* a summary_len of 0 mean 'don't show summary, show body */ +static void +body_or_summary(const Message& message, const Options& opts) +{ + const auto color{!opts.nocolor}; + using Format = Options::View::Format; + + std::string body, btype; + switch (opts.view.format) { + case Format::Plain: + btype = "plain text"; + body = message.body_text().value_or(""); + break; + case Format::Html: + btype = "html"; + body = message.body_html().value_or(""); + break; + default: + throw std::range_error("unsupported format"); // bug + } + + if (body.empty()) { + if (any_of(message.flags() & Flags::Encrypted)) { + color_maybe(MU_COLOR_CYAN); + mu_println("[No {} body found; message does have encrypted parts]", + btype); + } else { + color_maybe(MU_COLOR_MAGENTA); + mu_println("[No {} body found]", btype); + } + color_maybe(MU_COLOR_DEFAULT); + return; + } + + if (opts.view.summary_len) { + const auto summ{summarize(body, *opts.view.summary_len)}; + print_field("Summary", summ, color); + } else { + mu_print_encoded("{}", body); + if (!g_str_has_suffix(body.c_str(), "\n")) + mu_println(""); + } +} + +/* we ignore fields for now */ +/* summary_len == 0 means "no summary */ +static Mu::Result<void> +view_msg_plain(const Message& message, const Options& opts) +{ + const auto color{!opts.nocolor}; + + print_field("From", to_string(message.from()), color); + print_field("To", to_string(message.to()), color); + print_field("Cc", to_string(message.cc()), color); + print_field("Bcc", to_string(message.bcc()), color); + print_field("Subject", message.subject(), color); + + if (auto&& date = message.date(); date != 0) + print_field("Date", mu_format("{:%c}", mu_time(date)), color); + + print_field("Tags", join(message.tags(), ", "), color); + + print_field("Attachments",get_attach_str(message, opts), color); + + mu_println(""); + body_or_summary(message, opts); + + return Ok(); +} + +static Mu::Result<void> +handle_msg(const Message& message, const Options& opts) +{ + using Format = Options::View::Format; + + switch (opts.view.format) { + case Format::Plain: + case Format::Html: + return view_msg_plain(message, opts); + + case Format::Sexp: + return view_msg_sexp(message, opts); + default: + mu_critical("bug: should not be reached"); + return Err(Error::Code::Internal, "error"); + } +} + +Mu::Result<void> +Mu::mu_cmd_view(const Options& opts) +{ + for (auto&& file: opts.view.files) { + auto message{Message::make_from_path( + file, message_options(opts.view))}; + if (!message) + return Err(message.error()); + + if (auto res = handle_msg(*message, opts); !res) + return res; + /* add a separator between two messages? */ + if (opts.view.terminate) + mu_print("{}", VIEW_TERMINATOR); + } + + // no files? read from stding + if (opts.view.files.empty()) { + const auto msgtxt = read_from_stdin(); + if (!msgtxt) + return Err(msgtxt.error()); + auto message = Message::make_from_text(*msgtxt,{}, + message_options(opts.view)); + if (!message) + return Err(message.error()); + else + return handle_msg(*message, opts); + } + return Ok(); +} + + +#ifdef BUILD_TESTS +/* + * Tests. + * + */ + +#include <fcntl.h> /* Definition of AT_* constants */ +#include <sys/stat.h> +#include <fstream> +#include <utils/mu-regex.hh> +#include "utils/mu-test-utils.hh" + +static constexpr std::string_view test_msg = +R"(From: Test <test@example.com> +To: abc@example.com +Date: Mon, 23 May 2011 10:53:45 +0200 +Subject: vla +MIME-Version: 1.0 +Content-Type: multipart/alternative; + boundary="_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d" +Message-ID: <10374608.109906.11909.20115aabbccdd.MSGID@mailinglijst.nl> + +--_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d +Content-Type: text/plain; charset="iso-8859-15" +Content-Transfer-Encoding: quoted-printable + +text + +--_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d +Content-Type: text/html; charset="iso-8859-15" +Content-Transfer-Encoding: quoted-printable + +html + +--_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d-- +)"; + + +static std::string msgpath; + +static void +test_view_plain() +{ + auto res = run_command({MU_PROGRAM, "view", msgpath}); + assert_valid_command(res); + auto output{*res}; + + // silly hack to avoid locale diffs + auto rx = unwrap(Regex::make("^Date:.*", G_REGEX_MULTILINE)); + output.standard_out = unwrap(rx.replace(output.standard_out, "Date: xxx")); + + g_assert_true(output.standard_err.empty()); + assert_equal(output.standard_out, +R"(From: Test <test@example.com> +To: abc@example.com +Subject: vla +Date: xxx + +text +)"); + +} + +static void +test_view_html() +{ + auto res = run_command({MU_PROGRAM, "view", "--format=html", msgpath}); + assert_valid_command(res); + auto output{*res}; + + auto rx = unwrap(Regex::make("^Date:.*", G_REGEX_MULTILINE)); + output.standard_out = unwrap(rx.replace(output.standard_out, "Date: xxx")); + + g_assert_true(output.standard_err.empty()); + assert_equal(output.standard_out, +R"(From: Test <test@example.com> +To: abc@example.com +Subject: vla +Date: xxx + +html +)"); + +} + +static void +test_view_sexp() +{ + TempTz tz("Europe/Amsterdam"); + if (!tz.available()) { + g_test_skip("timezone not available"); + return; + } + + auto res = run_command({MU_PROGRAM, "view", "--format=sexp", msgpath}); + assert_valid_command(res); + auto output{*res}; + + g_assert_true(output.standard_err.empty()); + + // Note: :path, :changed (file ctime) change per run. + struct stat statbuf{}; + g_assert_true(::stat(msgpath.c_str(), &statbuf) == 0); + + const auto expected = mu_format( + R"((:path "{}" :size 638 :changed ({} {} 0) :date (19930 8345 0) :flags (unread) :from ((:email "test@example.com" :name "Test")) :message-id "10374608.109906.11909.20115aabbccdd.MSGID@mailinglijst.nl" :priority normal :subject "vla" :to ((:email "abc@example.com"))) +)", + msgpath, + statbuf.st_ctime >> 16, + statbuf.st_ctime & 0xffff); + + assert_equal(output.standard_out, expected); +} + +static void +test_mu_view_01(void) +{ + TempDir temp_dir{}; + + if (!set_en_us_utf8_locale()) { + g_test_skip("failed to switch to en_US/utf8"); + return; + } + + auto res = run_command({MU_PROGRAM, "view", + join_paths(MU_TESTMAILDIR2, "bar", "cur", "mail4")}); + assert_valid_result(res); + g_assert_true(res->standard_err.empty()); + + g_assert_cmpuint(res->standard_out.size(), ==, 364); +} + +static void +test_mu_view_multi(void) +{ + TempDir temp_dir{}; + + if (!set_en_us_utf8_locale()) { + g_test_skip("failed to switch to en_US/utf8"); + return; + } + + auto res = run_command({MU_PROGRAM, "view", + join_paths(MU_TESTMAILDIR2, "bar", "cur", "mail5"), + join_paths(MU_TESTMAILDIR2, "bar", "cur", "mail5")}); + assert_valid_result(res); + g_assert_true(res->standard_err.empty()); + + g_assert_cmpuint(res->standard_out.size(), ==, 162); +} + +static void +test_mu_view_multi_separate(void) +{ + TempDir temp_dir{}; + + if (!set_en_us_utf8_locale()) { + g_test_skip("failed to switch to en_US/utf8"); + return; + } + + auto res = run_command({MU_PROGRAM, "view", "--terminate", + join_paths(MU_TESTMAILDIR2, "bar", "cur", "mail5"), + join_paths(MU_TESTMAILDIR2, "bar", "cur", "mail5")}); + assert_valid_result(res); + g_assert_true(res->standard_err.empty()); + + g_assert_cmpuint(res->standard_out.size(), ==, 164); +} + +static void +test_mu_view_attach(void) +{ + TempDir temp_dir{}; + + if (!set_en_us_utf8_locale()) { + g_test_skip("failed to switch to en_US/utf8"); + return; + } + + auto res = run_command({MU_PROGRAM, "view", "--terminate", + join_paths(MU_TESTMAILDIR2, "Foo", "cur", "mail5")}); + assert_valid_result(res); + g_assert_true(res->standard_err.empty()); + + g_assert_cmpuint(res->standard_out.size(), ==, 164); +} + + + +int +main(int argc, char* argv[]) try { + + TempDir tmpdir{}; + msgpath = join_paths(tmpdir.path(), "test-message.txt"); + std::ofstream strm{msgpath}; + strm.write(test_msg.data(), test_msg.size()); + strm.close(); + g_assert_true(strm.good()); + + mu_test_init(&argc, &argv); + + g_test_add_func("/cmd/view/01", test_mu_view_01); + g_test_add_func("/cmd/view/multi", test_mu_view_multi); + g_test_add_func("/cmd/view/multi-separate", test_mu_view_multi_separate); + g_test_add_func("/cmd/view/attach", test_mu_view_attach); + g_test_add_func("/cmd/view/plain", test_view_plain); + g_test_add_func("/cmd/view/html", test_view_html); + g_test_add_func("/cmd/view/sexp", test_view_sexp); + + return g_test_run(); + +} catch (const Error& e) { + mu_printerrln("{}", e.what()); + return 1; +} catch (...) { + mu_printerrln("caught exception"); + return 1; +} +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-cmd.cc b/mu/mu-cmd.cc new file mode 100644 index 0000000..bcdca98 --- /dev/null +++ b/mu/mu-cmd.cc @@ -0,0 +1,169 @@ +/* +** Copyright (C) 2010-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include <iostream> +#include <iomanip> + +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include <unistd.h> +#include <errno.h> + +#include "mu-options.hh" +#include "mu-cmd.hh" +#include "mu-maildir.hh" +#include "mu-contacts-cache.hh" +#include "message/mu-message.hh" +#include "message/mu-mime-object.hh" + +#include "utils/mu-error.hh" +#include "utils/mu-utils-file.hh" +#include "utils/mu-utils.hh" + +#include <thirdparty/tabulate.hpp> + +using namespace Mu; + + +static Result<void> +cmd_fields(const Options& opts) +{ + mu_printerrln("the 'mu fields' command has been superseded by 'mu info'; try:\n" + " mu info fields\n"); + return Ok(); +} + + +static Result<void> +cmd_find(const Options& opts) +{ + auto store{Store::make(opts.runtime_path(RuntimePath::XapianDb))}; + if (!store) + return Err(store.error()); + else + return mu_cmd_find(*store, opts); +} + + +static void +show_usage(void) +{ + mu_println("usage: mu command [options] [parameters]"); + mu_println("where command is one of index, find, cfind, view, mkdir, " + "extract, add, remove, script, verify or server"); + mu_println("see the mu, mu-<command> or mu-easy manpages for " + "more information"); +} + + +using ReadOnlyStoreFunc = std::function<Result<void>(const Store&, const Options&)>; +using WritableStoreFunc = std::function<Result<void>(Store&, const Options&)>; + +static Result<void> +with_readonly_store(const ReadOnlyStoreFunc& func, const Options& opts) +{ + auto store{Store::make(opts.runtime_path(RuntimePath::XapianDb))}; + if (!store) + return Err(store.error()); + + return func(store.value(), opts); +} + +static Result<void> +with_writable_store(const WritableStoreFunc func, const Options& opts) +{ + auto store{Store::make(opts.runtime_path(RuntimePath::XapianDb), + Store::Options::Writable)}; + if (!store) + return Err(store.error()); + + return func(store.value(), opts); +} + +Result<void> +Mu::mu_cmd_execute(const Options& opts) try { + + if (!opts.sub_command) + return Err(Error::Code::Internal, "missing subcommand"); + + switch (*opts.sub_command) { + case Options::SubCommand::Help: + return Ok(); /* already handled in mu-options.cc */ + /* + * no store needed + */ + case Options::SubCommand::Fields: + return cmd_fields(opts); + case Options::SubCommand::Mkdir: + return mu_cmd_mkdir(opts); + case Options::SubCommand::Script: + return mu_cmd_script(opts); + case Options::SubCommand::View: + return mu_cmd_view(opts); + case Options::SubCommand::Verify: + return mu_cmd_verify(opts); + case Options::SubCommand::Extract: + return mu_cmd_extract(opts); + /* + * read-only store + */ + + case Options::SubCommand::Cfind: + return with_readonly_store(mu_cmd_cfind, opts); + case Options::SubCommand::Find: + return cmd_find(opts); + case Options::SubCommand::Info: + return with_readonly_store(mu_cmd_info, opts); + + /* writable store */ + + case Options::SubCommand::Add: + return with_writable_store(mu_cmd_add, opts); + case Options::SubCommand::Remove: + return with_writable_store(mu_cmd_remove, opts); + case Options::SubCommand::Move: + return with_writable_store(mu_cmd_move, opts); + + /* + * commands instantiate store themselves + */ + case Options::SubCommand::Index: + return mu_cmd_index(opts); + case Options::SubCommand::Init: + return mu_cmd_init(opts); + case Options::SubCommand::Server: + return mu_cmd_server(opts); + + default: + show_usage(); + return Ok(); + } + +} catch (const Mu::Error& er) { + return Err(er); +} catch (const std::runtime_error& re) { + return Err(Error::Code::Internal, "runtime-error: {}", re.what()); +} catch (const std::exception& ex) { + return Err(Error::Code::Internal, "error: {}", ex.what()); +} catch (...) { + return Err(Error::Code::Internal, "caught exception"); +} diff --git a/mu/mu-cmd.hh b/mu/mu-cmd.hh new file mode 100644 index 0000000..7b591f8 --- /dev/null +++ b/mu/mu-cmd.hh @@ -0,0 +1,198 @@ +/* +** Copyright (C) 2008-2022-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_CMD_HH__ +#define MU_CMD_HH__ + +#include <glib.h> +#include <mu-store.hh> +#include <utils/mu-result.hh> + +#include "mu-options.hh" + +namespace Mu { + + +/** + * Get message options from (sub)command options + * + * @param cmdopts (sub) command options + * + * @return message options + */ +template<typename CmdOpts> +constexpr Message::Options +message_options(const CmdOpts& cmdopts) +{ + Message::Options mopts{Message::Options::AllowRelativePath}; + + if (cmdopts.decrypt) + mopts |= Message::Options::Decrypt; + if (cmdopts.auto_retrieve) + mopts |= Message::Options::RetrieveKeys; + + return mopts; +} + +/** + * execute the 'add' command + * + * @param store store object to use + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_add(Store& store, const Options& opts); + +/** + * execute the 'cfind' command + * + * @param store store object to use + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_cfind(const Store& store, const Options& opts); + +/** + * execute the 'extract' command + * + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_extract(const Options& opts); + +/** + * execute the 'find' command + * + * @param store store object to use + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_find(const Store& store, const Options& opts); + +/** + * execute the 'index' command + * + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_index(const Options& opt); + +/** + * execute the 'info' command + * + * @param store message store object. + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_info(const Mu::Store& store, const Options& opts); + +/** + * execute the 'init' command + * + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_init(const Options& opts); + +/** + * execute the 'mkdir' command + * + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_mkdir(const Options& opts); + +/** + * execute the 'move' command + * + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_move(Store& store, const Options& opts); + +/** + * execute the 'remove' command + * + * @param store store object to use + * @param opts configuration options + * + * @return Ok() or some error + */ +Result<void> mu_cmd_remove(Store& store, const Options& opt); + +/** + * execute the 'script' command + * + * @param opts configuration options + * @param err receives error information, or NULL + * + * @return Ok() or some error + */ +Result<void> mu_cmd_script(const Options& opts); + + +/** + * execute the server command + * @param opts configuration options + * @param err receives error information, or NULL + * + * @return Ok() or some error + */ +Result<void> mu_cmd_server(const Options& opts); + +/** + * execute the 'verify' command + * + * @param opts configuration options + * + * @return Ok() or some error + */ +Mu::Result<void> mu_cmd_verify(const Options& opts); + +/** + * execute the 'view' command + * + * @param opts configuration options + * + * @return Ok() or some error + */ +Mu::Result<void> mu_cmd_view(const Options& opts); + +/** + * execute some mu command, based on 'opts' + * + * @param opts configuration option + * @param err receives error information, or NULL + * + * @return Ok() or some error + */ +Result<void> mu_cmd_execute(const Options& opts); + +} // namespace Mu + +#endif /*__MU_CMD_H__*/ diff --git a/mu/mu-memcheck.in b/mu/mu-memcheck.in new file mode 100644 index 0000000..73a2329 --- /dev/null +++ b/mu/mu-memcheck.in @@ -0,0 +1,6 @@ +#!/bin/sh + +export G_SLICE=always-malloc +export G_DEBUG=gc-friendly + +libtool --mode=execute valgrind --tool=memcheck --leak-check=full --show-possibly-lost=no --leak-resolution=med --track-origins=yes --num-callers=20 --log-file='@abs_top_builddir@/mu-%p.vgdump' @abs_top_builddir@/mu/mu $@ diff --git a/mu/mu-options.cc b/mu/mu-options.cc new file mode 100644 index 0000000..9533e9c --- /dev/null +++ b/mu/mu-options.cc @@ -0,0 +1,956 @@ +/* +** Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +/** + * @brief Command-line handling + * + * Here we implement mu's command-line parsing based on the CLI11 library. At + * the time of writing, that library seems to be the best based on the criteria + * that it supports the features we need and is available as a header-only + * include. + * + * CLI11 can do quite a bit, and we're only scratching the surface here, + * plan is to slowly improve things. + * + * - we do quite a bit of sanity-checking, but the errors are a rather terse + * - the docs could be improved, e.g., `mu find --help` and --format/--sortfield + * + */ + + +#include <config.h> +#include <stdexcept> +#include <array> +#include <unordered_map> +#include <iostream> +#include <string_view> +#include <unistd.h> + +#include <utils/mu-utils.hh> +#include <utils/mu-utils-file.hh> +#include <utils/mu-error.hh> +#include "utils/mu-test-utils.hh" +#include "mu-options.hh" +#include "mu-script.hh" + +#include <thirdparty/CLI11.hpp> + +using namespace Mu; + + +/* + * helpers + */ + + + +/** + * array of associated pair elements -- like an alist + * but based on std::array and thus can be constexpr + */ +template<typename T1, typename T2, std::size_t N> + using AssocPairs = std::array<std::pair<T1, T2>, N>; + + +/** + * Get the first value of the pair where the second element is @param s. + * + * @param p AssocPairs + * @param s some second pair value + * + * @return the matching first pair value, or Nothing if not found. + */ +template<typename P> +constexpr Option<typename P::value_type::first_type> +to_first(const P& p, typename P::value_type::second_type s) +{ + for (const auto& item: p) + if (item.second == s) + return item.first; + return Nothing; +} + +/** + * Get the second value of the pair where the first element is @param f. + * + * @param p AssocPairs + * @param f some first pair value + * + * @return the matching second pair value, or Nothing if not found. + */ +template<typename P> +constexpr Option<typename P::value_type::second_type> +to_second(const P& p, typename P::value_type::first_type f) +{ + for (const auto& item: p) + if (item.first == f) + return item.second; + return Nothing; +} + + +/** + * Options-specific array-bases type that maps some enum to a <name, description> pair + */ +template<typename T, std::size_t N> +using InfoEnum = AssocPairs<T, std::pair<std::string_view, std::string_view>, N>; + +/** + * Get the name (shortname) for some InfoEnum, based on the enum + * + * @param ie an InfoEnum + * @param e an enum value + * + * @return the name if found, or Nothing + */ +template<typename IE> +static constexpr Option<std::string_view> +to_name(const IE& ie, typename IE::value_type::first_type e) { + if (auto&& s{to_second(ie, e)}; s) + return s->first; + else + return Nothing; +} + +/** + * Get the enum value for some InfoEnum, based on the name + * + * @param ie an InfoEnum + * @param name some name (shortname) + * + * @return the name if found, or Nothing + */ +template<typename IE> +static constexpr Option<typename IE::value_type::first_type> +to_enum(const IE& ie, std::string_view name) { + for(auto&& item: ie) + if (item.second.first == name) + return item.first; + else + return Nothing; +} + +/** + * List help options for as a string, with the default marked with '(*)' + * + * @param ie infoenum + * @param default_opt default option + * + * @return a help string + */ +template<typename IE> +static std::string +options_help(const IE& ie, typename IE::value_type::first_type default_opt) +{ + std::string s; + for(auto&& item: ie) { + if (!s.empty()) + s += ", "; + s += std::string{item.second.first}; + if (item.first == default_opt) + s += "(*)"; /* default option */ + } + return s; +} + + +/** + * Get map from string->type + */ +template<typename IE> +static std::unordered_map<std::string, typename IE::value_type::first_type> +options_map(const IE& ie) +{ + std::unordered_map<std::string, typename IE::value_type::first_type> map; + for (auto&& item : ie) + map.emplace(std::string{item.second.first}, item.first); + + return map; +} + +// transformers + +// Expand the path using wordexp +static const std::function ExpandPath = [](std::string filepath)->std::string { + if (auto&& res{expand_path(filepath)}; !res) + throw CLI::ValidationError{res.error().what()}; + else + return res.value(); +}; + +// Canonicalize path +static const std::function CanonicalizePath = [](std::string filepath)->std::string { + return filepath = canonicalize_filename(filepath); +}; + +/* + * common + */ + +template<typename T> +static void +sub_crypto(CLI::App& sub, T& opts) +{ + sub.add_flag("--auto-retrieve,-r", opts.auto_retrieve, + "Attempt to automatically retrieve online keys"); + sub.add_flag("--decrypt", opts.decrypt, + "Attempt to decrypt"); +} + +/* + * subcommands + */ + +static void +sub_add(CLI::App& sub, Options& opts) +{ + sub.add_option("files", opts.add.files, + "Path(s) to message files(s)") + ->required(); +} + +static void +sub_cfind(CLI::App& sub, Options& opts) +{ + using Format = Options::Cfind::Format; + static constexpr InfoEnum<Format, 8> FormatInfos = {{ + { Format::Plain, {"plain", "Plain output"} }, + { Format::MuttAlias, {"mutt-alias", "Mutt alias"} }, + { Format::MuttAddressBook, {"mutt-ab", "Mutt address book"}}, + { Format::Wanderlust, {"wl", "Wanderlust"}}, + { Format::OrgContact, {"org-contact", "org-contact"}}, + { Format::Bbdb, {"bbdb", "Emacs BBDB"}}, + { Format::Csv, {"csv", "comma-separated values"}}, + { Format::Json, {"json", "format as json array"}}, + }}; + + const auto fhelp = options_help(FormatInfos, Format::Plain); + const auto fmap = options_map(FormatInfos); + + sub.add_option("--format,-o", opts.cfind.format, + "Output format; one of " + fhelp) + ->type_name("<format>") + ->default_str("plain") + ->default_val(Format::Plain) + ->transform(CLI::CheckedTransformer(fmap)); + + sub.add_option("pattern", opts.cfind.rx_pattern, + "Regular expression pattern to match"); + sub.add_flag("--personal,-p", opts.cfind.personal, + "Only show 'personal' contacts"); + sub.add_option("--after", opts.cfind.after, + "Only show results after some timestamps") + ->type_name("<time_t>") + ->check(CLI::PositiveNumber); + sub.add_option("--maxnum,-n", opts.cfind.maxnum, + "Maximum number of results") + ->type_name("<number>") + ->check(CLI::PositiveNumber); +} + + + +static void +sub_extract(CLI::App& sub, Options& opts) +{ + sub_crypto(sub, opts.extract); + + sub.add_flag("--save-attachments,-a", opts.extract.save_attachments, + "Save all attachments"); + sub.add_flag("--save-all", opts.extract.save_all, "Save all MIME parts") + ->excludes("--save-attachments"); + sub.add_flag("--overwrite", opts.extract.overwrite, + "Overwrite existing files"); + sub.add_flag("--play", opts.extract.play, + "Attempt to open the extracted parts"); + sub.add_option("--parts", opts.extract.parts, + "Save specific parts (comma-sep'd list)") + ->type_name("<parts>")->delimiter(','); + sub.add_option("--target-dir", opts.extract.targetdir, + "Target directory for saving") + ->type_name("<dir>") + ->transform(ExpandPath, "expand target path") + ->default_str("<current>") + ->default_val("."); + sub.add_flag("--uncooked,-u", opts.extract.uncooked, + "Avoid massaging extracted file-names"); + // optional; otherwise use standard-input + sub.add_option("message-path", opts.extract.message, + "Path to message file") + ->type_name("<message-path>"); + + sub.add_option("--matches", opts.extract.filename_rx, + "Regular expression for files to save") + ->type_name("<filename-rx>") + ->excludes("--parts") + ->excludes("--save-attachments") + ->excludes("--save-all"); + + // backward compat: filename-rx as non-option + sub.add_option("filename-rx", opts.extract.filename_rx, + "Regular expression for files to save") + ->type_name("<filename-rx>") + ->excludes("--parts") + ->excludes("--save-attachments") + ->excludes("--matches") + ->excludes("--save-all"); +} + +static void +sub_fields(CLI::App& sub, Options& opts) +{ + // nothing to do. +} + + +static void +sub_find(CLI::App& sub, Options& opts) +{ + using Format = Options::Find::Format; + static constexpr InfoEnum<Format, 7> FormatInfos = {{ + { Format::Plain, + {"plain", "Plain output"} + }, + { Format::Links, + {"links", "Maildir with symbolic links"} + }, + { Format::Xml, + {"xml", "XML"} + }, + { Format::Sexp, + {"sexp", "S-expressions"} + }, + { Format::Json, + {"json", "JSON"} + }, + }}; + + sub.add_flag("--threads,-t", opts.find.threads, + "Show message threads"); + sub.add_flag("--skip-dups,-u", opts.find.skip_dups, + "Show only one of messages with same message-id"); + sub.add_flag("--include-related,-r", opts.find.include_related, + "Include related messages in results"); + sub.add_flag("--analyze,-a", opts.find.analyze, + "Analyze the query"); + + const auto fhelp = options_help(FormatInfos, Format::Plain); + const auto fmap = options_map(FormatInfos); + + sub.add_option("--format,-o", opts.find.format, + "Output format; one of " + fhelp) + ->type_name("<format>") + ->default_str("plain") + ->default_val(Format::Plain) + ->transform(CLI::CheckedTransformer(fmap)); + + sub.add_option("--maxnum,-n", opts.find.maxnum, + "Maximum number of results") + ->type_name("<number>") + ->check(CLI::PositiveNumber); + + sub.add_option("--fields,-f", opts.find.fields, + "Fields to display") + ->default_val("d f s"); + + std::unordered_map<std::string, Field::Id> smap; + std::string sopts; + field_for_each([&](auto&& field){ + if (field.is_sortable()) { + smap.emplace(std::string(field.name), field.id); + smap.emplace(std::string(1, field.shortcut), field.id); + if (!sopts.empty()) + sopts += ", "; + sopts += mu_format("{}|{}", field.name, field.shortcut); + } + }); + sub.add_option("--sortfield,-s", opts.find.sortfield, + "Field to sort the results by; one of " + sopts) + ->type_name("<field>") + ->default_str("date") + ->default_val(Field::Id::Date) + ->transform(CLI::CheckedTransformer(smap)); + + sub.add_flag("--reverse,-z", opts.find.reverse, + "Sort in descending order"); + + sub.add_option("--bookmark,-b", opts.find.bookmark, + "Use bookmarked query") + ->type_name("<bookmark>"); + + sub.add_flag("--clearlinks", opts.find.clearlinks, + "Clear old links first"); + sub.add_option("--linksdir", opts.find.linksdir, + "Use bookmarked query") + ->type_name("<dir>") + ->transform(ExpandPath, "expand linksdir path"); + + sub.add_option("--summary-len", opts.find.summary_len, + "Use up to so many lines for the summary") + ->type_name("<lines>") + ->check(CLI::PositiveNumber); + + sub.add_option("--exec", opts.find.exec, + "Command to execute on message file") + ->type_name("<command>"); + + sub.add_option("query", opts.find.query, + "Search query pattern(s)") + ->type_name("<query>"); +} + +static void +sub_help(CLI::App& sub, Options& opts) +{ + sub.add_option("command", opts.help.command, + "Command to request help for") + ->type_name("<command>"); +} + +static void +sub_index(CLI::App& sub, Options& opts) +{ + sub.add_flag("--lazy-check", opts.index.lazycheck, + "Skip based on dir-timestamps"); + sub.add_flag("--nocleanup", opts.index.nocleanup, + "Don't clean up database after indexing"); + sub.add_flag("--reindex", opts.index.reindex, + "Perform a complete reindexing"); +} + + +static void +sub_info(CLI::App& sub, Options& opts) +{ + sub.add_option("topic", opts.info.topic, + "Information topic") + ->type_name("<topic>") ; +} + +static void +sub_init(CLI::App& sub, Options& opts) +{ + const auto default_mdir = std::invoke([]()->std::string { + if (const auto mdir_env{::getenv("MAILDIR")}; mdir_env) + return mdir_env; + else if (const auto mdir_home = ::join_paths(g_get_home_dir(), "Maildir"); + check_dir(mdir_home)) + return mdir_home; + else + return {}; + }); + + sub.add_option("--maildir,-m", opts.init.maildir, "Top of the maildir") + ->type_name("<maildir>") + ->default_val(default_mdir) + ->transform(ExpandPath, "expand maildir path"); + sub.add_option("--my-address", opts.init.my_addresses, + "Personal e-mail address or regexp") + ->type_name("<address>"); + sub.add_option("--ignored-address", opts.init.ignored_addresses, + "Ignored e-mail address or regexp") + ->type_name("<address>"); + + sub.add_option("--max-message-size", opts.init.max_msg_size, + "Maximum allowed message size in bytes"); + sub.add_option("--batch-size", opts.init.batch_size, + "Maximum size of database transaction"); + sub.add_option("--support-ngrams", opts.init.support_ngrams, + "Support CJK n-grams if for querying/indexing"); + sub.add_flag("--reinit", opts.init.reinit, + "Re-initialize database with current settings") + ->excludes("--maildir") + ->excludes("--my-address") + ->excludes("--ignored-address") + ->excludes("--max-message-size") + ->excludes("--batch-size") + ->excludes("--support-ngrams"); +} + +static void +sub_mkdir(CLI::App& sub, Options& opts) +{ + sub.add_option("--mode", opts.mkdir.mode, "Set the access mode (octal)") + ->default_val(0755) + ->type_name("<mode>"); + + sub.add_option("dirs", opts.mkdir.dirs, "Path to directory/ies") + ->type_name("<dir>") + ->required(); +} + + +static void +sub_move(CLI::App& sub, Options& opts) +{ + sub.add_flag("--change-name", opts.move.change_name, + "Change name of target file"); + sub.add_flag("--update-dups", opts.move.update_dups, + "Update duplicate messages too"); + sub.add_flag("--dry-run,-n", opts.move.dry_run, + "Print target name, but do not change anything"); + + sub.add_option("--flags", opts.move.flags, "Target flags") + ->type_name("<flags>"); + + sub.add_option("source", opts.move.src, "Message file to move") + ->type_name("<message-path>") + ->transform(ExpandPath, "expand source path") + ->transform(CanonicalizePath, "canonicalize source path") + ->required(); + sub.add_option("destination", opts.move.dest, + "Destination maildir") + ->type_name("<maildir>"); +} + + +static void +sub_remove(CLI::App& sub, Options& opts) +{ + sub.add_option("files", opts.remove.files, + "Paths to message files to remove") + ->type_name("<files>"); +} + +static void +sub_server(CLI::App& sub, Options& opts) +{ + sub.add_flag("--commands", opts.server.commands, + "List available commands"); + sub.add_option("--eval", opts.server.eval, + "Evaluate mu server expression") + ->excludes("--commands"); + sub.add_flag("--allow-temp-file", opts.server.allow_temp_file, + "Allow for the temp-file optimization") + ->excludes("--commands"); + +} + +static void +sub_verify(CLI::App& sub, Options& opts) +{ + sub_crypto(sub, opts.verify); + + // optional; otherwise use standard-input + sub.add_option("message-paths", opts.verify.files, + "Message files to verify") + ->type_name("<message-path>"); +} + +static void +sub_view(CLI::App& sub, Options& opts) +{ + using Format = Options::View::Format; + static constexpr InfoEnum<Format, 3> FormatInfos = {{ + { Format::Plain, + {"plain", "Plain output"} + }, + { Format::Html, + {"html", "Plain output with HTML body"} + }, + { Format::Sexp, + {"sexp", "S-expressions"} + }, + }}; + + const auto fhelp = options_help(FormatInfos, Format::Plain); + const auto fmap = options_map(FormatInfos); + + sub.add_option("--format,-o", opts.view.format, + "Output format; one of " + fhelp) + ->type_name("<format>") + ->default_str("plain") + ->default_val(Format::Plain) + ->transform(CLI::CheckedTransformer(fmap)); + + sub_crypto(sub, opts.view); + + sub.add_option("--summary-len", opts.view.summary_len, + "Use up to so many lines for the summary") + ->type_name("<lines>") + ->check(CLI::PositiveNumber); + + sub.add_flag("--terminate", opts.view.terminate, + "Insert form-feed after each message"); + + // optional; otherwise use standard-input + sub.add_option("message-paths", opts.view.files, + "Message files to view") + ->type_name("<message-path>"); +} + + +using SubCommand = Options::SubCommand; +using Category = Options::Category; + +struct CommandInfo { + Category category; + std::string_view name; + std::string_view help; + + // std::function is not constexp-friendly + typedef void(*setup_func_t)(CLI::App&, Options&); + setup_func_t setup_func{}; +}; + +static constexpr +AssocPairs<SubCommand, CommandInfo, Options::SubCommandNum> SubCommandInfos= {{ + { SubCommand::Add, + { Category::NeedsWritableStore, + "add", "Add message(s) to the database", sub_add} + }, + { SubCommand::Cfind, + { Category::NeedsReadOnlyStore, + "cfind", "Find contacts matching pattern", sub_cfind} + }, + { SubCommand::Extract, + {Category::None, + "extract", "Extract MIME-parts from messages", sub_extract} + }, + { SubCommand::Fields, + {Category::None, + "fields", "Superseded by 'mu info'", sub_fields} + }, + { SubCommand::Find, + {Category::NeedsReadOnlyStore, + "find", "Find messages matching query", sub_find } + }, + { SubCommand::Help, + {Category::None, + "help", "Show help information", sub_help } + }, + { SubCommand::Index, + {Category::NeedsWritableStore, + "index", "Store message information in the database", sub_index } + }, + { SubCommand::Info, + {Category::NeedsReadOnlyStore, + "info", "Show information", sub_info } + }, + { SubCommand::Init, + {Category::NeedsWritableStore, + "init", "Initialize the database", sub_init } + }, + { SubCommand::Mkdir, + {Category::None, + "mkdir", "Create a new Maildir", sub_mkdir } + }, + { SubCommand::Move, + {Category::NeedsWritableStore, + "move", "Move a message or change flags", sub_move } + }, + { SubCommand::Remove, + {Category::NeedsWritableStore, + "remove", "Remove message from file-system and database", sub_remove } + }, + { SubCommand::Script, + // Note: SubCommand::Script is special; there's no literal + // "script" subcommand, there subcommands for all the scripts. + {Category::None, + "script", "Invoke a script", {}} + }, + { SubCommand::Server, + {Category::NeedsWritableStore, + "server", "Start a mu server (for mu4e)", sub_server} + }, + { SubCommand::Verify, + {Category::None, + "verify", "Verify cryptographic signatures", sub_verify} + }, + { SubCommand::View, + {Category::None, + "view", "View specific messages", sub_view} + }, + }}; + + + +static ScriptInfos +add_scripts(CLI::App& app, Options& opts) +{ +#ifndef BUILD_GUILE + return {}; +#else + ScriptPaths paths = { MU_SCRIPTS_DIR }; + auto scriptinfos{script_infos(paths)}; + for (auto&& script: scriptinfos) { + auto&& sub = app.add_subcommand(script.name)->group("Scripts") + ->description(script.oneline); + sub->add_option("params", opts.script.params, + "Parameter to script") + ->type_name("<params>"); + } + + return scriptinfos; +#endif /*BUILD_GUILE*/ +} + + +static Result<Options> +show_manpage(Options& opts, const std::string& name) +{ + char *path = g_find_program_in_path("man"); + if (!path) + return Err(Error::Code::Command, + "cannot find 'man' program"); + + GError* err{}; + auto cmd{to_string_gchar(std::move(path)) + " " + name}; + auto res = g_spawn_command_line_sync(cmd.c_str(), {}, {}, {}, &err); + if (!res) + return Err(Error::Code::Command, &err, + "error running man command"); + + return Ok(std::move(opts)); +} + + +static Result<Options> +cmd_help(const CLI::App& app, Options& opts) +{ + if (opts.help.command.empty()) { + mu_println("{}", app.help()); + return Ok(std::move(opts)); + } + + for (auto&& item: SubCommandInfos) { + if (item.second.name == opts.help.command) + return show_manpage(opts, "mu-" + opts.help.command); + } + + for (auto&& item: {"query", "easy"}) + if (item == opts.help.command) + return show_manpage(opts, "mu-" + opts.help.command); + + return Err(Error::Code::Command, + "no help available for '{}'", opts.help.command); +} + +bool +Options::default_no_color() +{ + static const auto no_color = + !::isatty(::fileno(stdout)) || + !::isatty(::fileno(stderr)) || + ::getenv("NO_COLOR") != NULL; + + return no_color; +} + +static void +add_global_options(CLI::App& cli, Options& opts) +{ + opts.nocolor = Options::default_no_color(); + errno = 0; + + cli.add_flag("-q,--quiet", opts.quiet, "Hide non-essential output"); + cli.add_flag("-v,--verbose", opts.verbose, "Show verbose output"); + cli.add_flag("--log-stderr", opts.log_stderr, "Log to stderr") + ->group(""/*always hide*/); + cli.add_flag("--nocolor", opts.nocolor, "Don't show ANSI colors") + ->default_val(Options::default_no_color()) + ->default_str(Options::default_no_color() ? "<true>" : "<false>"); + cli.add_flag("-d,--debug", opts.debug, "Run in debug mode") + ->group(""/*always hide*/); +} + +Result<Options> +Options::make(int argc, char *argv[]) +{ + Options opts{}; + CLI::App app{"mu mail indexer/searcher", "mu"}; + + app.description(R"(mu mail indexer/searcher +Copyright (C) 2008-2023 Dirk-Jan C. Binnema + +License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>. +This is free software: you are free to change and redistribute it. +There is NO WARRANTY, to the extent permitted by law. +)"); + app.set_version_flag("-V,--version", PACKAGE_VERSION); + app.set_help_flag("-h,--help", "Show help informmation"); + app.set_help_all_flag("--help-all"); + app.require_subcommand(0, 1); + + add_global_options(app, opts); + + /* + * subcommands + * + * we keep around a map of the subcommand pointers, so we can + * easily find the chosen one (if any) later. + */ + for (auto&& cmdinfo: SubCommandInfos) { + //const auto cmdtype = cmdinfo.first; + const auto name{std::string{cmdinfo.second.name}}; + const auto help{std::string{cmdinfo.second.help}}; + const auto setup{cmdinfo.second.setup_func}; + const auto cat{category(cmdinfo.first)}; + + if (!setup) + continue; + + auto sub = app.add_subcommand(name, help); + setup(*sub, opts); + + /* allow global options _after_ subcommand as well; + * this is for backward compat with the older + * command-line parsing */ + sub->fallthrough(true); + + /* store commands get the '--muhome' parameter as well */ + if (cat == Category::NeedsReadOnlyStore || + cat == Category::NeedsWritableStore) + sub->add_option("--muhome", + opts.muhome, "Specify alternative mu directory") + ->envname("MUHOME") + ->type_name("<dir>") + ->transform(ExpandPath, "expand muhome path"); + } + + /* add scripts (if supported) as semi-subcommands as well */ + const auto scripts = add_scripts(app, opts); + + try { + app.parse(argc, argv); + + // find the chosen sub command, if any. + for (auto&& cmdinfo: SubCommandInfos) { + if (cmdinfo.first == SubCommand::Script) + continue; // not a _real_ subcommand. + const auto name{std::string{cmdinfo.second.name}}; + if (app.got_subcommand(name)) { + opts.sub_command = cmdinfo.first; + } + } + + // otherwise, perhaps it's a script? + if (!opts.sub_command) { + for (auto&& info: scripts) { // find the chosen script, if any. + if (app.got_subcommand(info.name)) { + opts.sub_command = SubCommand::Script; + opts.script.name = info.name; + } + } + } + + // if nothing else, try "help" + if (opts.sub_command.value_or(SubCommand::Help) == SubCommand::Help) + return cmd_help(app, opts); + + } catch (const CLI::CallForHelp& cfh) { + mu_println("{}", app.help()); + } catch (const CLI::CallForAllHelp& cfah) { + mu_println("{}", app.help("", CLI::AppFormatMode::All)); + } catch (const CLI::CallForVersion&) { + mu_println("version {}", PACKAGE_VERSION); + } catch (const CLI::ParseError& pe) { + return Err(Error::Code::InvalidArgument, "{}", pe.what()); + } catch (...) { + return Err(Error::Code::Internal, "error parsing arguments"); + } + + return Ok(std::move(opts)); +} + +Category +Options::category(Options::SubCommand sub) +{ + for (auto&& item: SubCommandInfos) + if (item.first == sub) + return item.second.category; + + return Category::None; +} + +/* + * trust but verify + */ + +static constexpr bool +validate_subcommand_ids() +{ + size_t val{}; + for (auto& cmd: Options::SubCommands) + if (static_cast<size_t>(cmd) != val++) + return false; + + for (auto u = 0U; u != SubCommandInfos.size(); ++u) + if (static_cast<size_t>(SubCommandInfos.at(u).first) != u) + return false; + return true; +} + + +/* + * tests... also build as runtime-tests, so we can get coverage info + */ +#ifdef BUILD_TESTS +#define static_assert g_assert_true +#endif /*BUILD_TESTS*/ + + +[[maybe_unused]] +static void +test_ids() +{ + static_assert(validate_subcommand_ids()); +} + +#ifdef BUILD_TESTS + +enum struct TestEnum { A, B, C }; +constexpr AssocPairs<TestEnum, std::string_view, 3> +test_epairs = {{ + {TestEnum::A, "a"}, + {TestEnum::B, "b"}, + {TestEnum::C, "c"}, +}}; + +static constexpr Option<std::string_view> +to_name(TestEnum te) +{ + return to_second(test_epairs, te); +} + +static constexpr Option<TestEnum> +to_type(std::string_view name) +{ + return to_first(test_epairs, name); + +} + +static void +test_enum_pairs(void) +{ + assert_equal(to_name(TestEnum::A).value(), "a"); + g_assert_true(to_type("c").value() == TestEnum::C); +} + +int +main(int argc, char* argv[]) +{ + mu_test_init(&argc, &argv); + + g_test_add_func("/options/ids", test_ids); + g_test_add_func("/option/enum-pairs", test_enum_pairs); + + return g_test_run(); +} +#endif /*BUILD_TESTS*/ diff --git a/mu/mu-options.hh b/mu/mu-options.hh new file mode 100644 index 0000000..fa440bf --- /dev/null +++ b/mu/mu-options.hh @@ -0,0 +1,310 @@ +/* +** Copyright (C) 2022-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#ifndef MU_OPTIONS_HH__ +#define MU_OPTIONS_HH__ + +#include <sstream> +#include <string> +#include <vector> +#include <utils/mu-option.hh> +#include <utils/mu-result.hh> +#include <utils/mu-utils.hh> +#include <utils/mu-utils-file.hh> + +#include <message/mu-fields.hh> +#include <mu-script.hh> +#include <ctime> +#include <sys/stat.h> + +/* command-line options for Mu */ +namespace Mu { +struct Options { + using OptSize = Option<std::size_t>; + using SizeVec = std::vector<std::size_t>; + using OptTStamp = Option<std::time_t>; + using OptFieldId = Option<Field::Id>; + using StringVec = std::vector<std::string>; + + /* + * general options + */ + bool quiet; /**< don't give any output */ + bool debug; /**< log debug-level info */ + bool version; /**< request mu version */ + bool log_stderr; /**< log to stderr */ + bool nocolor; /**< don't use use ansi-colors */ + bool verbose; /**< verbose output */ + std::string muhome; /**< alternative mu dir */ + + /** + * Whether by default, we should show color + * + * @return true or false + */ + static bool default_no_color(); + + enum struct SubCommand { + Add, Cfind, Extract, Fields, Find, Help, Index,Info, Init, Mkdir, + Move, Remove, Script, Server, Verify, View, + // <private> + __count__ + }; + static constexpr auto SubCommandNum = static_cast<size_t>(SubCommand::__count__); + static constexpr std::array<SubCommand, SubCommandNum> SubCommands = {{ + SubCommand::Add, + SubCommand::Cfind, + SubCommand::Extract, + SubCommand::Fields, + SubCommand::Find, + SubCommand::Help, + SubCommand::Index, + SubCommand::Info, + SubCommand::Init, + SubCommand::Mkdir, + SubCommand::Move, + SubCommand::Remove, + SubCommand::Script, + SubCommand::Server, + SubCommand::Verify, + SubCommand::View + }}; + + Option<SubCommand> sub_command; /**< The chosen sub-command, if any. */ + + /* + * Add + */ + struct Add { + StringVec files; /**< field to add */ + } add; + + /* + * Cfind + */ + struct Cfind { + enum struct Format { Plain, MuttAlias, MuttAddressBook, + Wanderlust, OrgContact, Bbdb, Csv, Json }; + Format format; /**< Output format */ + bool personal; /**< only show personal contacts */ + OptTStamp after; /**< only last seen after tstamp */ + OptSize maxnum; /**< maximum number of results */ + std::string rx_pattern; /**< contact regexp to match */ + } cfind; + + + struct Crypto { + bool auto_retrieve; /**< auto-retrieve keys */ + bool decrypt; /**< decrypt */ + }; + + /* + * Extract + */ + struct Extract: public Crypto { + std::string message; /**< path to message file */ + bool save_all; /**< extract all parts */ + bool save_attachments; /**< extract all attachment parts */ + SizeVec parts; /**< parts to save / open */ + std::string targetdir{}; /**< where to save attachments */ + bool overwrite; /**< overwrite same-named files */ + bool play; /**< try to 'play' attachment */ + std::string filename_rx; /**< Filename rx to save */ + bool uncooked{}; /**< Whether to avoid massaging + * the output filename */ + } extract; + + /* + * Fields + */ + + /* + * Find + */ + struct Find { + std::string fields; /**< fields to show in output */ + Field::Id sortfield; /**< field to sort by */ + OptSize maxnum; /**< max # of entries to print */ + bool reverse; /**< sort in revers order (z->a) */ + bool threads; /**< show message threads */ + bool clearlinks; /**< clear linksdir first */ + std::string linksdir; /**< directory for links */ + OptSize summary_len; /**< max # of lines for summary */ + std::string bookmark; /**< use bookmark */ + bool analyze; /**< analyze query */ + + enum struct Format { Plain, Links, Xml, Json, Sexp, Exec }; + Format format; /**< Output format */ + std::string exec; /**< cmd to execute on matches */ + bool skip_dups; /**< show only first with msg id */ + bool include_related; /**< included related messages */ + /**< for find and cind */ + OptTStamp after; /**< only last seen after T */ + bool auto_retrieve; /**< assume we're online */ + bool decrypt; /**< try to decrypt the body */ + + StringVec query; /**< search query */ + } find; + + struct Help { + std::string command; /**< Help parameter */ + } help; + + /* + * Index + */ + struct Index { + bool nocleanup; /**< don't cleanup del'd mails */ + bool lazycheck; /**< don't check uptodate dirs */ + bool reindex; /**< do a full re-index */ + } index; + + + /* + * Info + */ + struct Info { + std::string topic; /**< what to get info about? */ + } info; + + /* + * Init + */ + struct Init { + std::string maildir; /**< where the mails are */ + StringVec my_addresses; /**< personal e-mail addresses */ + StringVec ignored_addresses; /**< addresses to be ignored for + * the contacts-cache */ + OptSize max_msg_size; /**< max size for message files */ + OptSize batch_size; /**< db transaction batch size */ + bool reinit; /**< re-initialize */ + bool support_ngrams; /**< support CJK etc. ngrams */ + + } init; + + /* + * Mkdir + */ + struct Mkdir { + StringVec dirs; /**< Dir(s) to create */ + mode_t mode; /**< Mode for the maildir */ + } mkdir; + + /* + * Move + */ + struct Move { + std::string src; /**< Source file */ + std::string dest; /**< Destination dir */ + std::string flags; /**< Flags for destination */ + bool change_name; /**< Change basename for destination */ + bool update_dups; /**< Update duplicate messages too */ + bool dry_run; /**< Just print the result path, + but do not change anything */ + } move; + + /* + * Remove + */ + struct Remove { + StringVec files; /**< Files to remove */ + } remove; + + /* + * Scripts (i.e., finding scriot) + */ + struct Script { + std::string name; /**< name of script */ + StringVec params; /**< script params */ + } script; + + /* + * Server + */ + struct Server { + bool commands; /**< dump docs for commands */ + std::string eval; /**< command to evaluate */ + bool allow_temp_file; /**< temp-file optimization allowed? */ + } server; + + /* + * Verify + */ + struct Verify: public Crypto { + StringVec files; /**< message files to verify */ + } verify; + /* + * View + */ + struct View: public Crypto { + bool terminate; /**< add \f between msgs in view */ + OptSize summary_len; /**< max # of lines for summary */ + + enum struct Format { Plain, Sexp, Html }; + Format format; /**< output format*/ + + StringVec files; /**< Message file(s) */ + } view; + + + /** + * Create an Options structure fo the given command-line arguments. + * + * @param argc argc + * @param argv argc + * + * @return Options, or an Error + */ + static Result<Options> make(int argc, char *argv[]); + + + /** + * Different commands need different things + * + */ + enum struct Category { + None, + NeedsReadOnlyStore, + NeedsWritableStore, + }; + + /** + * Get the category for some subcommand + * + * @param sub subcommand + * + * @return the category + */ + static Category category(SubCommand sub); + + /** + * Get some well-known Path + * + * @param path the Path to find + * + * @return the path name + */ + std::string runtime_path(RuntimePath path) const { + return Mu::runtime_path(path, muhome); + } +}; + +} // namepace Mu + +#endif /* MU_OPTIONS_HH__ */ diff --git a/mu/mu.cc b/mu/mu.cc new file mode 100644 index 0000000..69d3c48 --- /dev/null +++ b/mu/mu.cc @@ -0,0 +1,136 @@ +/* +** Copyright (C) 2008-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include <config.h> +#include <functional> + +#include <glib.h> +#include <glib-object.h> +#include <locale.h> + +#include "mu-cmd.hh" +#include "mu-options.hh" +#include "utils/mu-utils.hh" +#include "utils/mu-logger.hh" + +#include "mu-cmd.hh" + +using namespace Mu; + + +static void +output_error(const std::string& what, bool use_color) +{ + using Color = MaybeAnsi::Color; + MaybeAnsi col{use_color}; + + mu_printerrln("{}error{}: {}{}{}", + col.fg(Color::Red), col.reset(), + col.fg(Color::BrightYellow), what, col.reset()); +} + +static int +handle_result(const Result<void>& res, const Mu::Options& opts) +{ + if (res) + return 0; + + using Color = MaybeAnsi::Color; + MaybeAnsi col{!opts.nocolor}; + + // show the error and some help, but not if it's only a softerror. + if (!res.error().is_soft_error()) + output_error(res.error().what(), !opts.nocolor); + else + mu_printerrln("{}{}{}", + col.fg(Color::BrightBlue), res.error().what(), col.reset()); + + // perhaps give some useful hint on how to solve it. + if (!res.error().hint().empty()) + mu_printerrln("{}hint{}: {}{}{}", + col.fg(Color::Blue), col.reset(), + col.fg(Color::Green), res.error().hint(), col.reset()); + + if (res.error().exit_code() != 0 && !res.error().is_soft_error()) { + mu_warning("mu finishing with error: {}", + format_as(res.error())); + if (const auto& hint = res.error().hint(); !hint.empty()) + mu_info("hint: {}", hint); + } + + return res.error().exit_code(); +} + +int +main(int argc, char* argv[]) try +{ + /* + * We handle this through explicit options + */ + g_unsetenv("XAPIAN_CJK_NGRAM"); + + /* + * set up locale + */ + ::setlocale(LC_ALL, ""); + + /* + * read command-line options + */ + const auto opts{Options::make(argc, argv)}; + if (!opts) { + output_error(opts.error().what(), !Options::default_no_color()); + return opts.error().exit_code(); + } else if (!opts->sub_command) { + // nothing more to do. + return 0; + } + + // setup logging + Logger::Options lopts{Logger::Options::None}; + if (opts->log_stderr) + lopts |= Logger::Options::StdOutErr; + if (opts->debug) + lopts |= Logger::Options::Debug; + if (!!g_getenv("MU_TEST")) + lopts |= Logger::Options::File; + + const auto logger{Logger::make(opts->runtime_path(RuntimePath::LogFile), lopts)}; + if (!logger) { + output_error(logger.error().what(), !opts->nocolor); + return logger.error().exit_code(); + } + + /* + * handle sub command + */ + return handle_result(mu_cmd_execute(*opts), *opts); + + // exceptions should have been handled earlier, but catch them here, + // just in case... +} catch (const std::logic_error& le) { + mu_printerrln("caught logic-error: {}", le.what()); + return 97; +} catch (const std::runtime_error& re) { + mu_printerrln("caught runtime-error: {}", re.what()); + return 98; +} catch (...) { + mu_printerrln("caught exception"); + return 99; +} diff --git a/mu/tests/gmime-test.c b/mu/tests/gmime-test.c new file mode 100644 index 0000000..f269ecb --- /dev/null +++ b/mu/tests/gmime-test.c @@ -0,0 +1,264 @@ +/* +** Copyright (C) 2011-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#define _POSIX_C_SOURCE 1 + +#include <gmime/gmime.h> +#include <stdio.h> +#include <errno.h> +#include <string.h> +#include <locale.h> + +static gchar* +get_recip(GMimeMessage* msg, GMimeAddressType atype) +{ + char* recep; + InternetAddressList* receps; + + receps = g_mime_message_get_addresses(msg, atype); + recep = (char*)internet_address_list_to_string(receps, NULL, FALSE); + + if (!recep || !*recep) { + g_free(recep); + return NULL; + } + + return recep; +} + +static gchar* +get_refs_str(GMimeMessage* msg) +{ + const gchar* str; + GMimeReferences* mime_refs; + int i, refs_len; + gchar* rv; + + str = g_mime_object_get_header(GMIME_OBJECT(msg), "References"); + if (!str) + return NULL; + + mime_refs = g_mime_references_parse(NULL, str); + refs_len = g_mime_references_length(mime_refs); + for (rv = NULL, i = 0; i < refs_len; ++i) { + const char* msgid; + char *tmp; + + msgid = g_mime_references_get_message_id(mime_refs, i); + tmp = rv; + rv = g_strdup_printf("%s%s%s", rv ? rv : "", rv ? "," : "", msgid); + g_free(tmp); + } + g_mime_references_free(mime_refs); + + return rv; +} + +static void +print_date(GMimeMessage* msg) +{ + GDateTime* dt; + gchar* buf; + + dt = g_mime_message_get_date(msg); + if (!dt) + return; + + dt = g_date_time_to_local(dt); + buf = g_date_time_format(dt, "%c"); + g_date_time_unref(dt); + + if (buf) { + g_print("Date : %s\n", buf); + g_free(buf); + } +} + +static void +print_body(GMimeMessage* msg) +{ + GMimeObject* body; + GMimeDataWrapper* wrapper; + GMimeStream* stream; + + body = g_mime_message_get_body(msg); + + if (GMIME_IS_MULTIPART(body)) + body = g_mime_multipart_get_part(GMIME_MULTIPART(body), 0); + if (!GMIME_IS_PART(body)) + return; + + wrapper = g_mime_part_get_content(GMIME_PART(body)); + if (!GMIME_IS_DATA_WRAPPER(wrapper)) + return; + + stream = g_mime_data_wrapper_get_stream(wrapper); + if (!GMIME_IS_STREAM(stream)) + return; + + do { + char buf[512]; + ssize_t len; + + len = g_mime_stream_read(stream, buf, sizeof(buf)); + if (len == -1) + break; + + if (write(fileno(stdout), buf, len) == -1) + break; + + if (len < (int)sizeof(buf)) + break; + + } while (1); +} + +static gboolean +test_message(GMimeMessage* msg) +{ + gchar* val; + const gchar* str; + + val = get_recip(msg, GMIME_ADDRESS_TYPE_FROM); + g_print("From : %s\n", val ? val : "<none>"); + g_free(val); + + val = get_recip(msg, GMIME_ADDRESS_TYPE_TO); + g_print("To : %s\n", val ? val : "<none>"); + g_free(val); + + val = get_recip(msg, GMIME_ADDRESS_TYPE_CC); + g_print("Cc : %s\n", val ? val : "<none>"); + g_free(val); + + val = get_recip(msg, GMIME_ADDRESS_TYPE_BCC); + g_print("Bcc : %s\n", val ? val : "<none>"); + g_free(val); + + str = g_mime_message_get_subject(msg); + g_print("Subject: %s\n", str ? str : "<none>"); + + print_date(msg); + + str = g_mime_message_get_message_id(msg); + g_print("Msg-id : %s\n", str ? str : "<none>"); + + { + gchar* refsstr; + refsstr = get_refs_str(msg); + g_print("Refs : %s\n", refsstr ? refsstr : "<none>"); + g_free(refsstr); + } + + print_body(msg); + + return TRUE; +} + +static gboolean +test_stream(GMimeStream* stream) +{ + GMimeParser* parser; + GMimeMessage* msg; + gboolean rv; + + parser = NULL; + msg = NULL; + + parser = g_mime_parser_new_with_stream(stream); + if (!parser) { + g_warning("failed to create parser"); + rv = FALSE; + goto leave; + } + + msg = g_mime_parser_construct_message(parser, NULL); + if (!msg) { + g_warning("failed to construct message"); + rv = FALSE; + goto leave; + } + + rv = test_message(msg); + +leave: + if (parser) + g_object_unref(parser); + + if (msg) + g_object_unref(msg); + + return rv; +} + +static gboolean +test_file(const char* path) +{ + FILE* file; + GMimeStream* stream; + gboolean rv; + + stream = NULL; + file = NULL; + + file = fopen(path, "r"); + if (!file) { + g_warning("cannot open file '%s': %s", path, g_strerror(errno)); + rv = FALSE; + goto leave; + } + + stream = g_mime_stream_file_new(file); + if (!stream) { + g_warning("cannot open stream for '%s'", path); + rv = FALSE; + goto leave; + } + + rv = test_stream(stream); + g_object_unref(stream); + return rv; + +leave: + if (file) + fclose(file); + + return rv; +} + +int +main(int argc, char* argv[]) +{ + gboolean rv; + + if (argc != 2) { + g_printerr("usage: %s <msg-file>\n", argv[0]); + return 1; + } + + setlocale(LC_ALL, ""); + + g_mime_init(); + + rv = test_file(argv[1]); + + g_mime_shutdown(); + + return rv ? 0 : 1; +} diff --git a/mu/tests/meson.build b/mu/tests/meson.build new file mode 100644 index 0000000..cc8a342 --- /dev/null +++ b/mu/tests/meson.build @@ -0,0 +1,110 @@ +## Copyright (C) 2022-2024 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# +# tests +# + + +test('test-cmd-add', + executable('test-cmd-add', + '../mu-cmd-add.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-cfind', + executable('test-cmd-cfind', + '../mu-cmd-cfind.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-extract', + executable('test-cmd-extract', + '../mu-cmd-extract.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-find', + executable('test-cmd-find', + '../mu-cmd-find.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-index', + executable('test-cmd-index', + '../mu-cmd-index.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-init', + executable('test-cmd-init', + '../mu-cmd-init.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-mkdir', + executable('test-cmd-mkdir', + '../mu-cmd-mkdir.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-move', + executable('test-cmd-move', + '../mu-cmd-move.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-remove', + executable('test-cmd-remove', + '../mu-cmd-remove.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-verify', + executable('test-cmd-verify', + '../mu-cmd-verify.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-view', + executable('test-cmd-view', + '../mu-cmd-view.cc', + install: false, + cpp_args: ['-DBUILD_TESTS'], + dependencies: [glib_dep, lib_mu_dep])) + +test('test-cmd-query', + executable('test-cmd-query', + 'test-mu-query.cc', + install: false, + dependencies: [glib_dep, config_h_dep, lib_mu_dep])) + +gmime_test = executable( + 'gmime-test', [ + 'gmime-test.c' +], + dependencies: [ glib_dep, gmime_dep ], + install: false) diff --git a/mu/tests/test-mu-query.cc b/mu/tests/test-mu-query.cc new file mode 100644 index 0000000..09c0cde --- /dev/null +++ b/mu/tests/test-mu-query.cc @@ -0,0 +1,612 @@ +/* +** Copyright (C) 2008-2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +** +** This program is free software; you can redistribute it and/or modify it +** under the terms of the GNU General Public License as published by the +** Free Software Foundation; either version 3, or (at your option) any +** later version. +** +** This program is distributed in the hope that it will be useful, +** but WITHOUT ANY WARRANTY; without even the implied warranty of +** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +** GNU General Public License for more details. +** +** You should have received a copy of the GNU General Public License +** along with this program; if not, write to the Free Software Foundation, +** Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +** +*/ + +#include "config.h" + +#include <unordered_set> +#include <string> + +#include <glib.h> +#include <glib/gstdio.h> + +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <locale.h> + +#include "utils/mu-test-utils.hh" +#include "mu-query.hh" +#include "utils/mu-result.hh" +#include "utils/mu-utils.hh" +#include "utils/mu-utils-file.hh" +#include "mu-store.hh" + +using namespace Mu; + +static std::string DB_PATH1; +static std::string DB_PATH2; + +static std::string +make_database(const std::string& dbdir, const std::string& testdir) +{ + /* use the env var rather than `--muhome` */ + g_setenv("MUHOME", dbdir.c_str(), 1); + const auto cmdline{mu_format( + "/bin/sh -c '" + "{} --quiet init --maildir={} ; " + "{} --quiet index'", + MU_PROGRAM, testdir, MU_PROGRAM)}; + + if (g_test_verbose()) + mu_printerrln("\n{}", cmdline); + + g_assert(g_spawn_command_line_sync(cmdline.c_str(), NULL, NULL, NULL, NULL)); + auto xpath = join_paths(dbdir, "xapian"); + /* ensure MUHOME worked */ + g_assert_cmpuint(::access(xpath.c_str(), F_OK), ==, 0); + + return xpath; +} + +static void +assert_no_dups(const QueryResults& qres) +{ + std::unordered_set<std::string> msgid_set, path_set; + + for (auto&& mi : qres) { + g_assert_true(msgid_set.find(mi.message_id().value()) == msgid_set.end()); + g_assert_true(path_set.find(mi.path().value()) == path_set.end()); + + path_set.emplace(*mi.path()); + msgid_set.emplace(*mi.message_id()); + + g_assert_false(msgid_set.find(mi.message_id().value()) == msgid_set.end()); + g_assert_false(path_set.find(mi.path().value()) == path_set.end()); + } +} + +/* note: this also *moves the iter* */ +static size_t +run_and_count_matches(const std::string& xpath, + const std::string& expr, + Mu::QueryFlags flags = Mu::QueryFlags::None) +{ + auto store{Store::make(xpath)}; + assert_valid_result(store); + + // if (g_test_verbose()) { + // std::cout << "==> mquery: " << store.parse_query(expr, false) << "\n"; + // std::cout << "==> xquery: " << store.parse_query(expr, true) << "\n"; + // } + + Mu::allow_warnings(); + + auto qres{store->run_query(expr, {}, flags)}; + g_assert_true(!!qres); + assert_no_dups(*qres); + + if (g_test_verbose()) + mu_println("'{}' => {}\n", expr, qres->size()); + + return qres->size(); +} + +typedef struct { + const char* query; + size_t count; /* expected number of matches */ +} QResults; + +static void +test_mu_query_01(void) +{ + int i; + QResults queries[] = { + {"basic", 3}, + {"question", 5}, + {"thanks", 2}, + {"html", 4}, + {"subject:exception", 1}, + {"exception", 1}, + {"subject:A&B", 1}, + {"A&B", 1}, + {"subject:elisp", 1}, + {"html AND contains", 1}, + {"html and contains", 1}, + {"from:pepernoot", 0}, + {"foo:pepernoot", 0}, + {"funky", 1}, + {"fünkÿ", 1}, + { "", 19 }, + {"msgid:abcd$efgh@example.com", 1}, + {"i:abcd$efgh@example.com", 1}, +#ifdef HAVE_CLD2 +{ "lang:en", 14}, +#endif /*HAVE_CLD2*/ + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH1, queries[i].query), + ==, queries[i].count); +} + +static void +test_mu_query_02(void) +{ + const char* q; + q = "i:f7ccd24b0808061357t453f5962w8b61f9a453b684d0@mail.gmail.com"; + g_assert_cmpuint(run_and_count_matches(DB_PATH1, q), ==, 1); +} + +static void +test_mu_query_03(void) +{ + int i; + QResults queries[] = {{"ploughed", 1}, + {"i:3BE9E6535E3029448670913581E7A1A20D852173@" + "emss35m06.us.lmco.com", + 1}, + {"i:!&!AAAAAAAAAYAAAAAAAAAOH1+8mkk+lLn7Gg5fke7" + "FbCgAAAEAAAAJ7eBDgcactKhXL6r8cEnJ8BAAAAAA==@" + "example.com", + 1}, + + /* subsets of the words in the subject should match */ + {"s:gcc include search order", 1}, + {"s:gcc include search", 1}, + {"s:search order", 1}, + {"s:include", 1}, + + {"s:lisp", 1}, + {"s:LISP", 1}, + + // { "s:\"Re: Learning LISP; Scheme vs elisp.\"", 1}, + // { "subject:Re: Learning LISP; Scheme vs elisp.", 1}, + // { "subject:\"Re: Learning LISP; Scheme vs elisp.\"", 1}, + {"to:help-gnu-emacs@gnu.org", 4}, + //{"t:help-gnu-emacs", 4}, + {"flag:flagged", 1}}; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH1, queries[i].query), + ==, queries[i].count); +} + +static void +test_mu_query_04(void) +{ + int i; + + QResults queries[] = { + {"frodo@example.com", 1}, + {"f:frodo@example.com", 1}, + {"f:Frodo Baggins", 1}, + {"bilbo@anotherexample.com", 1}, + {"t:bilbo@anotherexample.com", 1}, + {"t:bilbo", 1}, + {"f:bilbo", 0}, + {"baggins", 1}, + {"prio:h", 1}, + {"prio:high", 1}, + {"prio:normal", 11}, + {"prio:l", 7}, + {"not prio:l", 12}, + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH1, queries[i].query), + ==, queries[i].count); +} + +static void +test_mu_query_logic(void) +{ + int i; + QResults queries[] = {{"subject:gcc", 1}, + {"subject:lisp", 1}, + {"subject:gcc OR subject:lisp", 2}, + {"subject:gcc or subject:lisp", 2}, + {"subject:gcc AND subject:lisp", 0}, + {"subject:gcc OR (subject:scheme AND subject:elisp)", 2}, + {"(subject:gcc OR subject:scheme) AND subject:elisp", 1}}; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH1, queries[i].query), + ==, queries[i].count); +} + +static void +test_mu_query_accented_chars_01(void) +{ + auto store = Store::make(DB_PATH1); + assert_valid_result(store); + + auto qres{store->run_query("fünkÿ")}; + g_assert_true(!!qres); + g_assert_false(qres->empty()); + + const auto msg{qres->begin().message()}; + if (!msg) { + mu_warning("error getting message"); + g_assert_not_reached(); + } + + assert_equal(msg->subject(), "Greetings from Lothlórien"); +} + +static void +test_mu_query_accented_chars_02(void) +{ + int i; + + QResults queries[] = {{"f:mü", 1}, + { "s:motörhead", 1}, + {"t:Helmut", 1}, + {"t:Kröger", 1}, + {"s:MotorHeäD", 1}, + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) { + auto count = run_and_count_matches(DB_PATH1, queries[i].query); + if (count != queries[i].count) + mu_warning("query '{}'; expected {} but got {}", + queries[i].query, queries[i].count, count); + g_assert_cmpuint(run_and_count_matches(DB_PATH1, queries[i].query), + ==, queries[i].count); + } +} + +static void +test_mu_query_accented_chars_fraiche(void) +{ + int i; + + QResults queries[] = {{"crème fraîche", 1}, + {"creme fraiche", 1}, + {"fraîche crème", 1}, + {"будланула", 1}, + {"БУДЛАНУЛА", 1}, + {"CRÈME FRAÎCHE", 1}, + {"CREME FRAICHE", 1}}; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) { + if (g_test_verbose()) + mu_println("{}", queries[i].query); + + g_assert_cmpuint(run_and_count_matches(DB_PATH2, queries[i].query), + ==, queries[i].count); + } +} + +static void +test_mu_query_wildcards(void) +{ + int i; + + QResults queries[] = { + {"f:mü", 1}, + {"s:mo*", 1}, + {"t:Helm*", 1}, + {"queensryche", 1}, + {"Queen*", 1}, + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH1, queries[i].query), + ==, queries[i].count); +} + +static void +test_mu_query_dates_helsinki(void) +{ + const auto hki = "Europe/Helsinki"; + if (!timezone_available(hki)) { + g_test_skip("timezone not available"); + return; + } + + int i; + const char* old_tz; + + QResults queries[] = {{"date:20080731..20080804", 5}, + {"date:20080731..20080804 s:gcc", 1}, + {"date:200808110803..now", 7}, + {"date:200808110803..today", 7}, + {"date:200808110801..now", 7}}; + + old_tz = set_tz(hki); + TempDir tdir; + const auto xpath{make_database(tdir.path(), MU_TESTMAILDIR)}; + g_assert_false(xpath.empty()); + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(xpath, queries[i].query), + ==, queries[i].count); + + set_tz(old_tz); +} + +static void +test_mu_query_dates_sydney(void) +{ + const auto syd = "Australia/Sydney"; + if (!timezone_available(syd)) { + g_test_skip("timezone not available"); + return; + } + + int i; + const char* old_tz; + QResults queries[] = {{"date:20080731..20080804", 5}, + {"date:20080731..20080804 s:gcc", 1}, + {"date:200808110803..now", 7}, + {"date:200808110803..today", 7}, + {"date:200808110801..now", 7}}; + old_tz = set_tz(syd); + + TempDir tdir; + const auto xpath{make_database(tdir.path(), MU_TESTMAILDIR)}; + g_assert_false(xpath.empty()); + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(xpath, queries[i].query), + ==, queries[i].count); + set_tz(old_tz); +} + +static void +test_mu_query_dates_la(void) +{ + const auto la = "America/Los_Angeles"; + if (!timezone_available(la)) { + g_test_skip("timezone not available"); + return; + } + + int i; + const char* old_tz; + + QResults queries[] = {{"date:20080731..20080804", 5}, + {"date:2008-07-31..2008-08-04", 5}, + {"date:20080804..20080731", 5}, + {"date:20080731..20080804 s:gcc", 1}, + {"date:200808110803..now", 6}, + {"date:200808110803..today", 6}, + {"date:200808110801..now", 6}}; + old_tz = set_tz(la); + + TempDir tdir; + const auto xpath{make_database(tdir.path(), MU_TESTMAILDIR)}; + g_assert_false(xpath.empty()); + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) { + /* g_print ("%s\n", queries[i].query); */ + g_assert_cmpuint(run_and_count_matches(xpath, queries[i].query), + ==, queries[i].count); + } + + set_tz(old_tz); +} + +static void +test_mu_query_sizes(void) +{ + int i; + QResults queries[] = { + {"size:0b..2m", 19}, + {"size:3b..2m", 19}, + {"size:2k..4k", 4}, + + {"size:0b..2m", 19}, + {"size:2m..0b", 19}, + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH1, queries[i].query), + ==, queries[i].count); +} + +static void +test_mu_query_attach(void) +{ + int i; + QResults queries[] = {{"j:sittingbull.jpg", 1}, {"file:custer", 0}, {"file:custer.jpg", 1}}; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) { + if (g_test_verbose()) + mu_println("query: {}", queries[i].query); + g_assert_cmpuint(run_and_count_matches(DB_PATH2, queries[i].query), + ==, queries[i].count); + } +} + +static void +test_mu_query_msgid(void) +{ + int i; + QResults queries[] = { + {"i:CAHSaMxZ9rk5ASjqsbXizjTQuSk583=M6TORHz" + "=bfogtmbGGs5A@mail.gmail.com", + 1}, + {"msgid:CAHSaMxZ9rk5ASjqsbXizjTQuSk583=M6TORHz=" + "bfogtmbGGs5A@mail.gmail.com", + 1}, + + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) { + if (g_test_verbose()) + mu_println("query: {}", queries[i].query); + g_assert_cmpuint(run_and_count_matches(DB_PATH2, queries[i].query), + ==, queries[i].count); + } +} + +static void +test_mu_query_tags(void) +{ + int i; + QResults queries[] = { + {"x:paradise", 1}, + {"tag:lost", 1}, + {"tag:lost tag:paradise", 1}, + {"tag:lost tag:horizon", 0}, + {"tag:lost OR tag:horizon", 1}, + {"tag:queensryche", 1}, + {"tag:Queensrÿche", 1}, + {"x:paradise,lost", 0}, + {"x:paradise AND x:lost", 1}, + {"x:\\\\backslash", 1}, + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH2, queries[i].query), + ==, queries[i].count); +} + +static void +test_mu_query_wom_bat(void) +{ + int i; + QResults queries[] = { + {"maildir:/wom_bat", 3}, + //{ "\"maildir:/wom bat\"", 3}, + // as expected, no longer works with new parser + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH2, queries[i].query), + ==, queries[i].count); +} + +static void +test_mu_query_signed_encrypted(void) +{ + int i; + QResults queries[] = { + {"flag:encrypted", 2}, + {"flag:signed", 2}, + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH1, queries[i].query), + ==, + queries[i].count); +} + +static void +test_mu_query_multi_to_cc(void) +{ + int i; + QResults queries[] = { + {"to:a@example.com", 1}, + {"cc:d@example.com", 1}, + {"to:b@example.com", 1}, + {"cc:e@example.com", 1}, + {"cc:e@example.com AND cc:d@example.com", 1}, + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) + g_assert_cmpuint(run_and_count_matches(DB_PATH1, queries[i].query), + ==, queries[i].count); +} + +static void +test_mu_query_tags_02(void) +{ + int i; + QResults queries[] = { + {"x:paradise", 1}, + {"tag:@NextActions", 1}, + {"x:queensrÿche", 1}, + {"tag:lost OR tag:operation*", 2}, + }; + + for (i = 0; i != G_N_ELEMENTS(queries); ++i) { + g_assert_cmpuint(run_and_count_matches(DB_PATH2, queries[i].query), + ==, queries[i].count); + } +} + +/* Tests for https://github.com/djcb/mu/issues/380 + + On certain platforms, something goes wrong during compilation and the + --related option doesn't work. +*/ +static void +test_mu_query_threads_compilation_error(void) +{ + TempDir tdir; + const auto xpath = make_database(tdir.path(), MU_TESTMAILDIR); + + g_assert_cmpuint(run_and_count_matches(xpath, "msgid:uwsireh25.fsf@one.dot.net"), ==, 1); + + g_assert_cmpuint(run_and_count_matches(xpath, + "msgid:uwsireh25.fsf@one.dot.net", + QueryFlags::IncludeRelated), ==, 3); +} + +int +main(int argc, char* argv[]) +{ + TempDir td1; + TempDir td2; + + mu_test_init(&argc, &argv); + DB_PATH1 = make_database(td1.path(), MU_TESTMAILDIR); + g_assert_false(DB_PATH1.empty()); + + DB_PATH2 = make_database(td2.path(), MU_TESTMAILDIR2); + g_assert_false(DB_PATH2.empty()); + + g_test_add_func("/mu-query/test-mu-query-01", test_mu_query_01); + g_test_add_func("/mu-query/test-mu-query-02", test_mu_query_02); + g_test_add_func("/mu-query/test-mu-query-03", test_mu_query_03); + g_test_add_func("/mu-query/test-mu-query-04", test_mu_query_04); + + g_test_add_func("/mu-query/test-mu-query-signed-encrypted", test_mu_query_signed_encrypted); + g_test_add_func("/mu-query/test-mu-query-multi-to-cc", test_mu_query_multi_to_cc); + g_test_add_func("/mu-query/test-mu-query-logic", test_mu_query_logic); + + g_test_add_func("/mu-query/test-mu-query-accented-chars-1", + test_mu_query_accented_chars_01); + g_test_add_func("/mu-query/test-mu-query-accented-chars-2", + test_mu_query_accented_chars_02); + g_test_add_func("/mu-query/test-mu-query-accented-chars-fraiche", + test_mu_query_accented_chars_fraiche); + + g_test_add_func("/mu-query/test-mu-query-msgid", test_mu_query_msgid); + + g_test_add_func("/mu-query/test-mu-query-wom-bat", test_mu_query_wom_bat); + + g_test_add_func("/mu-query/test-mu-query-wildcards", test_mu_query_wildcards); + g_test_add_func("/mu-query/test-mu-query-sizes", test_mu_query_sizes); + + g_test_add_func("/mu-query/test-mu-query-dates-helsinki", test_mu_query_dates_helsinki); + g_test_add_func("/mu-query/test-mu-query-dates-sydney", test_mu_query_dates_sydney); + g_test_add_func("/mu-query/test-mu-query-dates-la", test_mu_query_dates_la); + + g_test_add_func("/mu-query/test-mu-query-attach", test_mu_query_attach); + g_test_add_func("/mu-query/test-mu-query-tags", test_mu_query_tags); + g_test_add_func("/mu-query/test-mu-query-tags_02", test_mu_query_tags_02); + + g_test_add_func("/mu-query/test-mu-query-threads-compilation-error", + test_mu_query_threads_compilation_error); + + return g_test_run(); +} diff --git a/mu4e/fdl.texi b/mu4e/fdl.texi new file mode 100644 index 0000000..96ce74e --- /dev/null +++ b/mu4e/fdl.texi @@ -0,0 +1,451 @@ +@c The GNU Free Documentation License. +@center Version 1.2, November 2002 + +@c This file is intended to be included within another document, +@c hence no sectioning command or @node. + +@display +Copyright @copyright{} 2000,2001,2002 Free Software Foundation, Inc. +51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA + +Everyone is permitted to copy and distribute verbatim copies +of this license document, but changing it is not allowed. +@end display + +@enumerate 0 +@item +PREAMBLE + +The purpose of this License is to make a manual, textbook, or other +functional and useful document @dfn{free} in the sense of freedom: to +assure everyone the effective freedom to copy and redistribute it, +with or without modifying it, either commercially or noncommercially. +Secondarily, this License preserves for the author and publisher a way +to get credit for their work, while not being considered responsible +for modifications made by others. + +This License is a kind of ``copyleft'', which means that derivative +works of the document must themselves be free in the same sense. It +complements the GNU General Public License, which is a copyleft +license designed for free software. + +We have designed this License in order to use it for manuals for free +software, because free software needs free documentation: a free +program should come with manuals providing the same freedoms that the +software does. But this License is not limited to software manuals; +it can be used for any textual work, regardless of subject matter or +whether it is published as a printed book. We recommend this License +principally for works whose purpose is instruction or reference. + +@item +APPLICABILITY AND DEFINITIONS + +This License applies to any manual or other work, in any medium, that +contains a notice placed by the copyright holder saying it can be +distributed under the terms of this License. Such a notice grants a +world-wide, royalty-free license, unlimited in duration, to use that +work under the conditions stated herein. The ``Document'', below, +refers to any such manual or work. Any member of the public is a +licensee, and is addressed as ``you''. You accept the license if you +copy, modify or distribute the work in a way requiring permission +under copyright law. + +A ``Modified Version'' of the Document means any work containing the +Document or a portion of it, either copied verbatim, or with +modifications and/or translated into another language. + +A ``Secondary Section'' is a named appendix or a front-matter section +of the Document that deals exclusively with the relationship of the +publishers or authors of the Document to the Document's overall +subject (or to related matters) and contains nothing that could fall +directly within that overall subject. (Thus, if the Document is in +part a textbook of mathematics, a Secondary Section may not explain +any mathematics.) The relationship could be a matter of historical +connection with the subject or with related matters, or of legal, +commercial, philosophical, ethical or political position regarding +them. + +The ``Invariant Sections'' are certain Secondary Sections whose titles +are designated, as being those of Invariant Sections, in the notice +that says that the Document is released under this License. If a +section does not fit the above definition of Secondary then it is not +allowed to be designated as Invariant. The Document may contain zero +Invariant Sections. If the Document does not identify any Invariant +Sections then there are none. + +The ``Cover Texts'' are certain short passages of text that are listed, +as Front-Cover Texts or Back-Cover Texts, in the notice that says that +the Document is released under this License. A Front-Cover Text may +be at most 5 words, and a Back-Cover Text may be at most 25 words. + +A ``Transparent'' copy of the Document means a machine-readable copy, +represented in a format whose specification is available to the +general public, that is suitable for revising the document +straightforwardly with generic text editors or (for images composed of +pixels) generic paint programs or (for drawings) some widely available +drawing editor, and that is suitable for input to text formatters or +for automatic translation to a variety of formats suitable for input +to text formatters. A copy made in an otherwise Transparent file +format whose markup, or absence of markup, has been arranged to thwart +or discourage subsequent modification by readers is not Transparent. +An image format is not Transparent if used for any substantial amount +of text. A copy that is not ``Transparent'' is called ``Opaque''. + +Examples of suitable formats for Transparent copies include plain +@sc{ascii} without markup, Texinfo input format, La@TeX{} input +format, @acronym{SGML} or @acronym{XML} using a publicly available +@acronym{DTD}, and standard-conforming simple @acronym{HTML}, +PostScript or @acronym{PDF} designed for human modification. Examples +of transparent image formats include @acronym{PNG}, @acronym{XCF} and +@acronym{JPG}. Opaque formats include proprietary formats that can be +read and edited only by proprietary word processors, @acronym{SGML} or +@acronym{XML} for which the @acronym{DTD} and/or processing tools are +not generally available, and the machine-generated @acronym{HTML}, +PostScript or @acronym{PDF} produced by some word processors for +output purposes only. + +The ``Title Page'' means, for a printed book, the title page itself, +plus such following pages as are needed to hold, legibly, the material +this License requires to appear in the title page. For works in +formats which do not have any title page as such, ``Title Page'' means +the text near the most prominent appearance of the work's title, +preceding the beginning of the body of the text. + +A section ``Entitled XYZ'' means a named subunit of the Document whose +title either is precisely XYZ or contains XYZ in parentheses following +text that translates XYZ in another language. (Here XYZ stands for a +specific section name mentioned below, such as ``Acknowledgements'', +``Dedications'', ``Endorsements'', or ``History''.) To ``Preserve the Title'' +of such a section when you modify the Document means that it remains a +section ``Entitled XYZ'' according to this definition. + +The Document may include Warranty Disclaimers next to the notice which +states that this License applies to the Document. These Warranty +Disclaimers are considered to be included by reference in this +License, but only as regards disclaiming warranties: any other +implication that these Warranty Disclaimers may have is void and has +no effect on the meaning of this License. + +@item +VERBATIM COPYING + +You may copy and distribute the Document in any medium, either +commercially or noncommercially, provided that this License, the +copyright notices, and the license notice saying this License applies +to the Document are reproduced in all copies, and that you add no other +conditions whatsoever to those of this License. You may not use +technical measures to obstruct or control the reading or further +copying of the copies you make or distribute. However, you may accept +compensation in exchange for copies. If you distribute a large enough +number of copies you must also follow the conditions in section 3. + +You may also lend copies, under the same conditions stated above, and +you may publicly display copies. + +@item +COPYING IN QUANTITY + +If you publish printed copies (or copies in media that commonly have +printed covers) of the Document, numbering more than 100, and the +Document's license notice requires Cover Texts, you must enclose the +copies in covers that carry, clearly and legibly, all these Cover +Texts: Front-Cover Texts on the front cover, and Back-Cover Texts on +the back cover. Both covers must also clearly and legibly identify +you as the publisher of these copies. The front cover must present +the full title with all words of the title equally prominent and +visible. You may add other material on the covers in addition. +Copying with changes limited to the covers, as long as they preserve +the title of the Document and satisfy these conditions, can be treated +as verbatim copying in other respects. + +If the required texts for either cover are too voluminous to fit +legibly, you should put the first ones listed (as many as fit +reasonably) on the actual cover, and continue the rest onto adjacent +pages. + +If you publish or distribute Opaque copies of the Document numbering +more than 100, you must either include a machine-readable Transparent +copy along with each Opaque copy, or state in or with each Opaque copy +a computer-network location from which the general network-using +public has access to download using public-standard network protocols +a complete Transparent copy of the Document, free of added material. +If you use the latter option, you must take reasonably prudent steps, +when you begin distribution of Opaque copies in quantity, to ensure +that this Transparent copy will remain thus accessible at the stated +location until at least one year after the last time you distribute an +Opaque copy (directly or through your agents or retailers) of that +edition to the public. + +It is requested, but not required, that you contact the authors of the +Document well before redistributing any large number of copies, to give +them a chance to provide you with an updated version of the Document. + +@item +MODIFICATIONS + +You may copy and distribute a Modified Version of the Document under +the conditions of sections 2 and 3 above, provided that you release +the Modified Version under precisely this License, with the Modified +Version filling the role of the Document, thus licensing distribution +and modification of the Modified Version to whoever possesses a copy +of it. In addition, you must do these things in the Modified Version: + +@enumerate A +@item +Use in the Title Page (and on the covers, if any) a title distinct +from that of the Document, and from those of previous versions +(which should, if there were any, be listed in the History section +of the Document). You may use the same title as a previous version +if the original publisher of that version gives permission. + +@item +List on the Title Page, as authors, one or more persons or entities +responsible for authorship of the modifications in the Modified +Version, together with at least five of the principal authors of the +Document (all of its principal authors, if it has fewer than five), +unless they release you from this requirement. + +@item +State on the Title page the name of the publisher of the +Modified Version, as the publisher. + +@item +Preserve all the copyright notices of the Document. + +@item +Add an appropriate copyright notice for your modifications +adjacent to the other copyright notices. + +@item +Include, immediately after the copyright notices, a license notice +giving the public permission to use the Modified Version under the +terms of this License, in the form shown in the Addendum below. + +@item +Preserve in that license notice the full lists of Invariant Sections +and required Cover Texts given in the Document's license notice. + +@item +Include an unaltered copy of this License. + +@item +Preserve the section Entitled ``History'', Preserve its Title, and add +to it an item stating at least the title, year, new authors, and +publisher of the Modified Version as given on the Title Page. If +there is no section Entitled ``History'' in the Document, create one +stating the title, year, authors, and publisher of the Document as +given on its Title Page, then add an item describing the Modified +Version as stated in the previous sentence. + +@item +Preserve the network location, if any, given in the Document for +public access to a Transparent copy of the Document, and likewise +the network locations given in the Document for previous versions +it was based on. These may be placed in the ``History'' section. +You may omit a network location for a work that was published at +least four years before the Document itself, or if the original +publisher of the version it refers to gives permission. + +@item +For any section Entitled ``Acknowledgements'' or ``Dedications'', Preserve +the Title of the section, and preserve in the section all the +substance and tone of each of the contributor acknowledgements and/or +dedications given therein. + +@item +Preserve all the Invariant Sections of the Document, +unaltered in their text and in their titles. Section numbers +or the equivalent are not considered part of the section titles. + +@item +Delete any section Entitled ``Endorsements''. Such a section +may not be included in the Modified Version. + +@item +Do not retitle any existing section to be Entitled ``Endorsements'' or +to conflict in title with any Invariant Section. + +@item +Preserve any Warranty Disclaimers. +@end enumerate + +If the Modified Version includes new front-matter sections or +appendices that qualify as Secondary Sections and contain no material +copied from the Document, you may at your option designate some or all +of these sections as invariant. To do this, add their titles to the +list of Invariant Sections in the Modified Version's license notice. +These titles must be distinct from any other section titles. + +You may add a section Entitled ``Endorsements'', provided it contains +nothing but endorsements of your Modified Version by various +parties---for example, statements of peer review or that the text has +been approved by an organization as the authoritative definition of a +standard. + +You may add a passage of up to five words as a Front-Cover Text, and a +passage of up to 25 words as a Back-Cover Text, to the end of the list +of Cover Texts in the Modified Version. Only one passage of +Front-Cover Text and one of Back-Cover Text may be added by (or +through arrangements made by) any one entity. If the Document already +includes a cover text for the same cover, previously added by you or +by arrangement made by the same entity you are acting on behalf of, +you may not add another; but you may replace the old one, on explicit +permission from the previous publisher that added the old one. + +The author(s) and publisher(s) of the Document do not by this License +give permission to use their names for publicity for or to assert or +imply endorsement of any Modified Version. + +@item +COMBINING DOCUMENTS + +You may combine the Document with other documents released under this +License, under the terms defined in section 4 above for modified +versions, provided that you include in the combination all of the +Invariant Sections of all of the original documents, unmodified, and +list them all as Invariant Sections of your combined work in its +license notice, and that you preserve all their Warranty Disclaimers. + +The combined work need only contain one copy of this License, and +multiple identical Invariant Sections may be replaced with a single +copy. If there are multiple Invariant Sections with the same name but +different contents, make the title of each such section unique by +adding at the end of it, in parentheses, the name of the original +author or publisher of that section if known, or else a unique number. +Make the same adjustment to the section titles in the list of +Invariant Sections in the license notice of the combined work. + +In the combination, you must combine any sections Entitled ``History'' +in the various original documents, forming one section Entitled +``History''; likewise combine any sections Entitled ``Acknowledgements'', +and any sections Entitled ``Dedications''. You must delete all +sections Entitled ``Endorsements.'' + +@item +COLLECTIONS OF DOCUMENTS + +You may make a collection consisting of the Document and other documents +released under this License, and replace the individual copies of this +License in the various documents with a single copy that is included in +the collection, provided that you follow the rules of this License for +verbatim copying of each of the documents in all other respects. + +You may extract a single document from such a collection, and distribute +it individually under this License, provided you insert a copy of this +License into the extracted document, and follow this License in all +other respects regarding verbatim copying of that document. + +@item +AGGREGATION WITH INDEPENDENT WORKS + +A compilation of the Document or its derivatives with other separate +and independent documents or works, in or on a volume of a storage or +distribution medium, is called an ``aggregate'' if the copyright +resulting from the compilation is not used to limit the legal rights +of the compilation's users beyond what the individual works permit. +When the Document is included in an aggregate, this License does not +apply to the other works in the aggregate which are not themselves +derivative works of the Document. + +If the Cover Text requirement of section 3 is applicable to these +copies of the Document, then if the Document is less than one half of +the entire aggregate, the Document's Cover Texts may be placed on +covers that bracket the Document within the aggregate, or the +electronic equivalent of covers if the Document is in electronic form. +Otherwise they must appear on printed covers that bracket the whole +aggregate. + +@item +TRANSLATION + +Translation is considered a kind of modification, so you may +distribute translations of the Document under the terms of section 4. +Replacing Invariant Sections with translations requires special +permission from their copyright holders, but you may include +translations of some or all Invariant Sections in addition to the +original versions of these Invariant Sections. You may include a +translation of this License, and all the license notices in the +Document, and any Warranty Disclaimers, provided that you also include +the original English version of this License and the original versions +of those notices and disclaimers. In case of a disagreement between +the translation and the original version of this License or a notice +or disclaimer, the original version will prevail. + +If a section in the Document is Entitled ``Acknowledgements'', +``Dedications'', or ``History'', the requirement (section 4) to Preserve +its Title (section 1) will typically require changing the actual +title. + +@item +TERMINATION + +You may not copy, modify, sublicense, or distribute the Document except +as expressly provided for under this License. Any other attempt to +copy, modify, sublicense or distribute the Document is void, and will +automatically terminate your rights under this License. However, +parties who have received copies, or rights, from you under this +License will not have their licenses terminated so long as such +parties remain in full compliance. + +@item +FUTURE REVISIONS OF THIS LICENSE + +The Free Software Foundation may publish new, revised versions +of the GNU Free Documentation License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. See +@uref{http://www.gnu.org/copyleft/}. + +Each version of the License is given a distinguishing version number. +If the Document specifies that a particular numbered version of this +License ``or any later version'' applies to it, you have the option of +following the terms and conditions either of that specified version or +of any later version that has been published (not as a draft) by the +Free Software Foundation. If the Document does not specify a version +number of this License, you may choose any version ever published (not +as a draft) by the Free Software Foundation. +@end enumerate + +@page +@heading ADDENDUM: How to use this License for your documents + +To use this License in a document you have written, include a copy of +the License in the document and put the following copyright and +license notices just after the title page: + +@smallexample +@group + Copyright (C) @var{year} @var{your name}. + Permission is granted to copy, distribute and/or modify this document + under the terms of the GNU Free Documentation License, Version 1.2 + or any later version published by the Free Software Foundation; + with no Invariant Sections, no Front-Cover Texts, and no Back-Cover + Texts. A copy of the license is included in the section entitled ``GNU + Free Documentation License''. +@end group +@end smallexample + +If you have Invariant Sections, Front-Cover Texts and Back-Cover Texts, +replace the ``with@dots{}Texts.'' line with this: + +@smallexample +@group + with the Invariant Sections being @var{list their titles}, with + the Front-Cover Texts being @var{list}, and with the Back-Cover Texts + being @var{list}. +@end group +@end smallexample + +If you have Invariant Sections without Cover Texts, or some other +combination of the three, merge those two alternatives to suit the +situation. + +If your document contains nontrivial examples of program code, we +recommend releasing these examples in parallel under your choice of +free software license, such as the GNU General Public License, +to permit their use in free software. + +@c Local Variables: +@c ispell-local-pdict: "ispell-dict" +@c End: + diff --git a/mu4e/htmlxref.cnf b/mu4e/htmlxref.cnf new file mode 100644 index 0000000..1af587b --- /dev/null +++ b/mu4e/htmlxref.cnf @@ -0,0 +1,788 @@ +# htmlxref.cnf - reference file for free Texinfo manuals on the web. + +htmlxrefversion=2023-04-02.12; # UTC + +# Copyright 2010-2023 Free Software Foundation, Inc. +# +# Copying and distribution of this file, with or without modification, +# are permitted in any medium without royalty provided the copyright +# notice and this notice are preserved. +# +# The latest version of this file is available at +# http://ftpmirror.gnu.org/texinfo/htmlxref.cnf. +# Email corrections or additions to bug-texinfo@gnu.org. +# The primary goal is to list all relevant GNU manuals; +# other free manuals are also welcome. +# +# To be included in this list, a manual must: +# +# - have a generic url, e.g., no version numbers; +# - have a unique file name (e.g., manual identifier), i.e., be related to the +# package name. Things like "refman" or "tutorial" don't work. +# - follow the naming convention for nodes described at +# http://www.gnu.org/software/texinfo/manual/texinfo/html_node/HTML-Xref.html +# This is what makeinfo and texi2html implement. +# +# Unless the above criteria are met, it's not possible to generate +# reliable cross-manual references. +# +# For information on automatically generating all the useful formats for +# a manual to put on the web, see +# http://www.gnu.org/prep/maintain/html_node/Manuals-on-Web-Pages.html. + +# For people editing this file: when a manual named foo is related to a +# package named bar, the url should contain a variable reference ${BAR}. +# Otherwise, the gnumaint scripts have no way of knowing they are +# associated, and thus gnu.org/manual can't include them. + +# shorten references to manuals on www.gnu.org. +G = https://www.gnu.org +GS = ${G}/software + +3dldf mono ${GS}/3dldf/manual/user_ref/3DLDF.html +3dldf node ${GS}/3dldf/manual/user_ref/ + +alive mono ${GS}/alive/manual/alive.html +alive node ${GS}/alive/manual/html_node/ + +anubis mono ${GS}/anubis/manual/anubis.html +anubis chapter ${GS}/anubis/manual/html_chapter/ +anubis section ${GS}/anubis/manual/html_section/ +anubis node ${GS}/anubis/manual/html_node/ + +artanis mono ${GS}/artanis/manual/artanis.html +artanis node ${GS}/artanis/manual/html_node/ + +aspell section http://aspell.net/man-html/index.html + +auctex mono ${GS}/auctex/manual/auctex.html +auctex node ${GS}/auctex/manual/auctex/ + +autoconf mono ${GS}/autoconf/manual/autoconf.html +autoconf node ${GS}/autoconf/manual/html_node/ + +autogen mono ${GS}/autogen/manual/autogen.html +autogen chapter ${GS}/autogen/manual/html_chapter/ +autogen node ${GS}/autoconf/manual/html_node/ + +automake mono ${GS}/automake/manual/automake.html +automake node ${GS}/automake/manual/html_node/ + +avl node http://adtinfo.org/libavl.html/ + +bash mono ${GS}/bash/manual/bash.html +bash node ${GS}/bash/manual/html_node/ + +BINUTILS = https://sourceware.org/binutils/docs +binutils mono ${BINUTILS}/binutils.html +binutils node ${BINUTILS}/binutils/ + # + as mono ${BINUTILS}/as.html + as node ${BINUTILS}/as/ + # + bfd mono ${BINUTILS}/bfd.html + bfd node ${BINUTILS}/bfd/ + # + gprof mono ${BINUTILS}/gprof.html + gprof node ${BINUTILS}/gprof/ + # + ld mono ${BINUTILS}/ld.html + ld node ${BINUTILS}/ld/ + +bison mono ${GS}/bison/manual/bison.html +bison node ${GS}/bison/manual/html_node/ + +bpel2owfn mono ${GS}/bpel2owfn/manual/2.0.x/bpel2owfn.html + +ccd2cue mono ${GS}/ccd2cue/manual/ccd2cue.html +ccd2cue node ${GS}/ccd2cue/manual/html_node/ + +cflow mono ${GS}/cflow/manual/cflow.html +cflow node ${GS}/cflow/manual/html_node/ + +chess mono ${GS}/chess/manual/gnuchess.html +chess node ${GS}/chess/manual/html_node/ + +combine mono ${GS}/combine/manual/combine.html +combine chapter ${GS}/combine/manual/html_chapter/ +combine section ${GS}/combine/manual/html_section/ +combine node ${GS}/combine/manual/html_node/ + +complexity mono ${GS}/complexity/manual/complexity.html +complexity node ${GS}/complexity/manual/html_node/ + +coreutils mono ${GS}/coreutils/manual/coreutils.html +coreutils node ${GS}/coreutils/manual/html_node/ + +cpio mono ${GS}/cpio/manual/cpio.html +cpio node ${GS}/cpio/manual/html_node/ + +cssc node ${GS}/cssc/manual/ + +CVS = ${GS}/trans-coord/manual +cvs mono ${CVS}/cvs/cvs.html +cvs node ${CVS}/cvs/html_node/ + +ddd mono ${GS}/ddd/manual/html_mono/ddd.html + +ddrescue mono ${GS}/ddrescue/manual/ddrescue_manual.html + +dejagnu node ${GS}/dejagnu/manual/ + +DICO = https://www.gnu.org.ua/software/dico/manual +dico mono ${DICO}/dico.html +dico chapter ${DICO}/html_chapter/ +dico section ${DICO}/html_section/ +dico node ${DICO}/html_node/ + +diffutils mono ${GS}/diffutils/manual/diffutils.html +diffutils node ${GS}/diffutils/manual/html_node/ + +ed mono ${GS}/ed/manual/ed_manual.html + +EMACS = ${GS}/emacs/manual +emacs mono ${EMACS}/html_mono/emacs.html +emacs node ${EMACS}/html_node/emacs/ + # + auth mono ${EMACS}/html_mono/auth.html + auth node ${EMACS}/html_node/auth/ + # + autotype mono ${EMACS}/html_mono/autotype.html + autotype node ${EMACS}/html_node/autotype/ + # + calc mono ${EMACS}/html_mono/calc.html + calc node ${EMACS}/html_node/calc/ + # + ccmode mono ${EMACS}/html_mono/ccmode.html + ccmode node ${EMACS}/html_node/ccmode/ + # + cl mono ${EMACS}/html_mono/cl.html + cl node ${EMACS}/html_node/cl/ + # + dbus mono ${EMACS}/html_mono/dbus.html + dbus node ${EMACS}/html_node/dbus/ + # + ebrowse mono ${EMACS}/html_mono/ebrowse.html + ebrowse node ${EMACS}/html_node/ebrowse/ + # + ede mono ${EMACS}/html_mono/ede.html + ede node ${EMACS}/html_node/ede/ + # + edt mono ${EMACS}/html_mono/edt.html + edt node ${EMACS}/html_node/edt/ + # + ediff mono ${EMACS}/html_mono/ediff.html + ediff node ${EMACS}/html_node/ediff/ + # + eieio mono ${EMACS}/html_mono/eieio.html + eieio node ${EMACS}/html_node/eieio/ + # + elisp mono ${EMACS}/html_mono/elisp.html + elisp node ${EMACS}/html_node/elisp/ + # + emacs-gnutls mono ${EMACS}/html_mono/emacs-gnutls.html + emacs-gnutls node ${EMACS}/html_node/emacs-gnutls/ + # + emacs-mime mono ${EMACS}/html_mono/emacs-mime.html + emacs-mime node ${EMACS}/html_node/emacs-mime/ + # + epa mono ${EMACS}/html_mono/epa.html + epa node ${EMACS}/html_node/epa/ + # + erc mono ${EMACS}/html_mono/erc.html + erc node ${EMACS}/html_node/erc/ + # + dired-x mono ${EMACS}/html_mono/dired-x.html + dired-x node ${EMACS}/html_node/dired-x/ + # + ert mono ${EMACS}/html_mono/ert.html + ert node ${EMACS}/html_node/ert/ + # + eshell mono ${EMACS}/html_mono/eshell.html + eshell node ${EMACS}/html_node/eshell/ + # + eudc mono ${EMACS}/html_mono/eudc.html + eudc node ${EMACS}/html_node/eudc/ + # + eww mono ${EMACS}/html_mono/eww.html + eww node ${EMACS}/html_node/eww/ + # + forms mono ${EMACS}/html_mono/forms.html + forms node ${EMACS}/html_node/forms/ + # + flymake mono ${EMACS}/html_mono/flymake.html + flymake node ${EMACS}/html_node/flymake/ + # + gnus mono ${EMACS}/html_mono/gnus.html + gnus node ${EMACS}/html_node/gnus/ + # + htmlfontify mono ${EMACS}/html_mono/htmlfontify.html + htmlfontify node ${EMACS}/html_node/htmlfontify/ + # + idlwave mono ${EMACS}/html_mono/idlwave.html + idlwave node ${EMACS}/html_node/idlwave/ + # + ido mono ${EMACS}/html_mono/ido.html + ido node ${EMACS}/html_node/ido/ + # + info mono ${EMACS}/html_mono/info.html + info node ${EMACS}/html_node/info/ + # + mairix-el mono ${EMACS}/html_mono/mairix-el.html + mairix-el node ${EMACS}/html_node/mairix-el/ + # + message mono ${EMACS}/html_mono/message.html + message node ${EMACS}/html_node/message/ + # + mh-e mono ${EMACS}/html_mono/mh-e.html + mh-e node ${EMACS}/html_node/mh-e/ + # + newsticker mono ${EMACS}/html_mono/newsticker.html + newsticker node ${EMACS}/html_node/newsticker/ + # + nxml-mode mono ${EMACS}/html_mono/nxml-mode.html + nxml-mode node ${EMACS}/html_node/nxml-mode/ + # + octave-mode mono ${EMACS}/html_mono/octave-mode.html + octave-mode node ${EMACS}/html_node/octave-mode/ + # + org mono ${EMACS}/html_mono/org.html + org node ${EMACS}/html_node/org/ + # + pcl-cvs mono ${EMACS}/html_mono/pcl-cvs.html + pcl-cvs node ${EMACS}/html_node/pcl-cvs/ + # + pgg mono ${EMACS}/html_mono/pgg.html + pgg node ${EMACS}/html_node/pgg/ + # + rcirc mono ${EMACS}/html_mono/rcirc.html + rcirc node ${EMACS}/html_node/rcirc/ + # + reftex mono ${EMACS}/html_mono/reftex.html + reftex node ${EMACS}/html_node/reftex/ + # + remember mono ${EMACS}/html_mono/remember.html + remember node ${EMACS}/html_node/remember/ + # + sasl mono ${EMACS}/html_mono/sasl.html + sasl node ${EMACS}/html_node/sasl/ + # + semantic mono ${EMACS}/html_mono/semantic.html + semantic node ${EMACS}/html_node/semantic/ + # + bovine mono ${EMACS}/html_mono/bovine.html + bovine node ${EMACS}/html_node/bovine/ + # + srecode mono ${EMACS}/html_mono/srecode.html + srecode node ${EMACS}/html_node/srecode/ + # + ses mono ${EMACS}/html_mono/ses.html + ses node ${EMACS}/html_node/ses/ + # + sieve mono ${EMACS}/html_mono/sieve.html + sieve node ${EMACS}/html_node/sieve/ + # + smtp mono ${EMACS}/html_mono/smtpmail.html + smtp node ${EMACS}/html_node/smtpmail/ + # + speedbar mono ${EMACS}/html_mono/speedbar.html + speedbar node ${EMACS}/html_node/speedbar/ + # + sc mono ${EMACS}/html_mono/sc.html + sc node ${EMACS}/html_node/sc/ + # + todo-mode mono ${EMACS}/html_mono/todo-mode.html + todo-mode node ${EMACS}/html_node/todo-mode/ + # + tramp mono ${EMACS}/html_mono/tramp.html + tramp node ${EMACS}/html_node/tramp/ + # + url mono ${EMACS}/html_mono/url.html + url node ${EMACS}/html_node/url/ + # + vhdl-mode mono ${EMACS}/html_mono/vhdl-mode.html + vhdl-mode node ${EMACS}/html_node/vhdl-mode/ + # + vip mono ${EMACS}/html_mono/vip.html + vip node ${EMACS}/html_node/vip/ + # + viper mono ${EMACS}/html_mono/viper.html + viper node ${EMACS}/html_node/viper/ + # + widget mono ${EMACS}/html_mono/widget.html + widget node ${EMACS}/html_node/widget/ + # + wisent mono ${EMACS}/html_mono/wisent.html + wisent node ${EMACS}/html_node/wisent/ + # + woman mono ${EMACS}/html_mono/woman.html + woman node ${EMACS}/html_node/woman/ + # (end emacs manuals in EMACS) + +easejs mono ${GS}/easejs/manual/easejs.html +easejs node ${GS}/easejs/manual/ + +emacs-muse mono ${GS}/emacs-muse/manual/muse.html +emacs-muse node ${GS}/emacs-muse/manual/html_node/ + +emms node ${GS}/emms/manual/ + +ada-mode mono https://elpa.gnu.org/packages/ada-mode.html + +gpr-mode mono https://elpa.gnu.org/packages/doc/gpr-mode.html + +findutils mono ${GS}/findutils/manual/html_mono/find.html +findutils node ${GS}/findutils/manual/html_node/find_html + +flex node https://westes.github.io/flex/manual/ + +gama mono ${GS}/gama/manual/gama.html +gama node ${GS}/gama/manual/html_node/ + +GAWK = ${GS}/gawk/manual +gawk mono ${GAWK}/gawk.html +gawk node ${GAWK}/html_node/ + gawkinet mono ${GAWK}/gawkinet/gawkinet.html + gawkinet node ${GAWK}/gawkinet/html_node/ + +gcal mono ${GS}/gcal/manual/gcal.html +gcal node ${GS}/gcal/manual/html_node/ + +GCC = https://gcc.gnu.org/onlinedocs +gcc node ${GCC}/gcc/ + cpp node ${GCC}/cpp/ + gfortran node ${GCC}/gfortran/ + gnat_rm node ${GCC}/gnat_rm/ + gnat_ugn node ${GCC}/gnat_ugn/ + libgomp node ${GCC}/libgomp/ + libstdc++ node ${GCC}/libstdc++/ + # + gccint node ${GCC}/gccint/ + cppinternals node ${GCC}/cppinternals/ + gfc-internals node ${GCC}/gfc-internals/ + gnat-style node ${GCC}/gnat-style/ + libiberty node ${GCC}/libiberty/ + +GDB = https://sourceware.org/gdb/current/onlinedocs +gdb node ${GDB}/gdb.html/ + stabs node ${GDB}/stabs.html/ + +GDBM = http://www.gnu.org.ua/software/gdbm/manual +gdbm node ${GDBM}/ + +gettext mono ${GS}/gettext/manual/gettext.html +gettext node ${GS}/gettext/manual/html_node/ + +gforth node https://www.complang.tuwien.ac.at/forth/gforth/Docs-html/ + +global mono ${GS}/global/manual/global.html + +gmediaserver node ${GS}/gmediaserver/manual/ + +gmp node https://www.gmplib.org/manual/ + +gnu-arch node ${GS}/gnu-arch/tutorial/ + +gnu-c-manual mono ${GS}/gnu-c-manual/gnu-c-manual.html + +gnu-crypto node ${GS}/gnu-crypto/manual/ + +gnubg mono ${GS}/gnubg/manual/gnubg.html +gnubg node ${GS}/gnubg/manual/html_node/ + +GNUCOBOL = https://gnucobol.sourceforge.io/HTML +gnucobpg mono ${GNUCOBOL}/gnucobpg.html + gnucobqr mono ${GNUCOBOL}/gnucobqr.html + gnucobsp mono ${GNUCOBOL}/gnucobsp.html + +gnubik mono ${GS}/gnubik/manual/gnubik.html +gnubik node ${GS}/gnubik/manual/html_node/ + +gnulib mono ${GS}/gnulib/manual/gnulib.html +gnulib node ${GS}/gnulib/manual/html_node/ + +GNUN = ${GS}/trans-coord/manual +gnun mono ${GNUN}/gnun/gnun.html +gnun node ${GNUN}/gnun/html_node/ + web-trans mono ${GNUN}/web-trans/web-trans.html + web-trans node ${GNUN}/web-trans/html_node/ + +GNUPG = https://www.gnupg.org/documentation/manuals +gnupg node ${GNUPG}/gnupg/ + dirmngr node ${GNUPG}/dirmngr/ + gcrypt node ${GNUPG}/gcrypt/ + libgcrypt node ${GNUPG}/gcrypt/ + ksba node ${GNUPG}/ksba/ + assuan node ${GNUPG}/assuan/ + gpgme node ${GNUPG}/gpgme/ + +gnuprologjava node ${GS}/gnuprologjava/manual/ + +gnuschool mono ${GS}/gnuschool/gnuschool.html + +GNUSTANDARDS = ${G}/prep + maintain mono ${GNUSTANDARDS}/maintain/maintain.html + maintain node ${GNUSTANDARDS}/maintain/html_node/ + # + standards mono ${GNUSTANDARDS}/standards/standards.html + standards node ${GNUSTANDARDS}/standards/html_node/ + +# following url is a redirect, which cannot be used for links within the manual +#gnutls mono ${GS}/gnutls/manual/gnutls.html +# empty directory +#gnutls node ${GS}/gnutls/manual/html_node/ +GNUTLS = http://www.gnutls.org/manual +gnutls mono ${GNUTLS}/gnutls.html +gnutls node ${GNUTLS}/html_node/ + +gperf mono ${GS}/gperf/manual/gperf.html +gperf node ${GS}/gperf/manual/html_node/ + +grep mono ${GS}/grep/manual/grep.html +grep node ${GS}/grep/manual/html_node/ + +groff node ${GS}/groff/manual/html_node/ + +GRUB = ${GS}/grub/manual/ + grub mono ${GRUB}/grub/grub.html + grub node ${GRUB}/grub/html_node/ + # + multiboot mono ${GRUB}/multiboot/multiboot.html + multiboot node ${GRUB}/multiboot/html_node/ + # + grub-dev mono ${GRUB}/grub-dev/grub-dev.html + grub-dev node ${GRUB}/grub-dev/html_node/ + +gsasl mono ${GS}/gsasl/manual/gsasl.html +gsasl node ${GS}/gsasl/manual/html_node/ + +gsl node ${GS}/gsl/manual/html_node/ + +gsrc mono ${GS}/gsrc/manual/gsrc.html +gsrc node ${GS}/gsrc/manual/html_node/ + +gss mono ${GS}/gss/manual/gss.html +gss node ${GS}/gss/manual/html_node/ + +gtypist mono ${GS}/gtypist/doc/gtypist.html + +guile mono ${GS}/guile/manual/guile.html +guile node ${GS}/guile/manual/html_node/ + +GUILE_GNOME = ${GS}/guile-gnome/docs + gobject node ${GUILE_GNOME}/gobject/html/ + glib node ${GUILE_GNOME}/glib/html/ + atk node ${GUILE_GNOME}/atk/html/ + pango node ${GUILE_GNOME}/pango/html/ + pangocairo node ${GUILE_GNOME}/pangocairo/html/ + gdk node ${GUILE_GNOME}/gdk/html/ + gtk node ${GUILE_GNOME}/gtk/html/ + libglade node ${GUILE_GNOME}/libglade/html/ + gnome-vfs node ${GUILE_GNOME}/gnome-vfs/html/ + libgnomecanvas node ${GUILE_GNOME}/libgnomecanvas/html/ + gconf node ${GUILE_GNOME}/gconf/html/ + libgnome node ${GUILE_GNOME}/libgnome/html/ + libgnomeui node ${GUILE_GNOME}/libgnomeui/html/ + corba node ${GUILE_GNOME}/corba/html/ + clutter node ${GUILE_GNOME}/clutter/html/ + clutter-glx node ${GUILE_GNOME}/clutter-glx/html/ + +guile-gtk node ${GS}/guile-gtk/docs/guile-gtk/ + +guile-rpc mono ${GS}/guile-rpc/manual/guile-rpc.html +guile-rpc node ${GS}/guile-rpc/manual/html_node/ + +guix mono ${GS}/guix/manual/guix.html +guix node ${GS}/guix/manual/html_node/ + +gv mono ${GS}/gv/manual/gv.html +gv node ${GS}/gv/manual/html_node/ + +gzip mono ${GS}/gzip/manual/gzip.html +gzip node ${GS}/gzip/manual/html_node/ + +hello mono ${GS}/hello/manual/hello.html +hello node ${GS}/hello/manual/html_node/ + +help2man mono ${GS}/help2man/help2man.html + +idutils mono ${GS}/idutils/manual/idutils.html +idutils node ${GS}/idutils/manual/html_node/ + +inetutils mono ${GS}/inetutils/manual/inetutils.html +inetutils node ${GS}/inetutils/manual/html_node/ + +# No manual, redirects to git sources +#jwhois mono ${GS}/jwhois/manual/jwhois.html +# 404 Not Found +#jwhois node ${GS}/jwhois/manual/html_node/ + +libc mono ${GS}/libc/manual/html_mono/libc.html +libc node ${GS}/libc/manual/html_node/ + +LIBCDIO = ${GS}/libcdio + libcdio mono ${LIBCDIO}/libcdio.html + cd-text mono ${LIBCDIO}/cd-text-format.html + +libextractor mono ${GS}/libextractor/manual/libextractor.html +libextractor node ${GS}/libextractor/manual/html_node/ + +libidn mono ${GS}/libidn/manual/libidn.html +libidn node ${GS}/libidn/manual/html_node/ + +libidn2 mono ${GS}/libidn/libidn2/manual/libidn2.html +libidn2 node ${GS}/libidn/libidn2/manual/html_node/ + +librejs mono ${GS}/librejs/manual/librejs.html +librejs node ${GS}/librejs/manual/html_node/ + +libmatheval mono ${GS}/libmatheval/manual/libmatheval.html + +LIBMICROHTTPD = ${GS}/libmicrohttpd +libmicrohttpd mono ${LIBMICROHTTPD}/manual/libmicrohttpd.html +libmicrohttpd node ${LIBMICROHTTPD}/manual/html_node/ + # The manual name is based on the Texinfo file name in the code, + # not on the file name for the tutorial which is too generic. + microhttpd-tutorial mono ${LIBMICROHTTPD}/tutorial.html + +libtasn1 mono ${GS}/libtasn1/manual/libtasn1.html +libtasn1 node ${GS}/libtasn1/manual/html_node/ + +libtool mono ${GS}/libtool/manual/libtool.html +libtool node ${GS}/libtool/manual/html_node/ + +lightning mono ${GS}/lightning/manual/lightning.html +lightning node ${GS}/lightning/manual/html_node/ + +# The stable/ url redirects immediately, but that's ok. +# The .html extension is omitted on their web site, but it works if given. +LILYPOND = http://lilypond.org/doc/stable/Documentation + lilypond-internals node ${LILYPOND}/internals/ + lilypond-learning node ${LILYPOND}/learning/ + lilypond-notation node ${LILYPOND}/notation/ + lilypond-snippets node ${LILYPOND}/snippets/ + lilypond-usage node ${LILYPOND}/usage/ + lilypond-web node ${LILYPOND}/web/ + music-glossary node ${LILYPOND}/music-glossary/ + +liquidwar6 mono ${GS}/liquidwar6/manual/liquidwar6.html +liquidwar6 node ${GS}/liquidwar6/manual/html_node/ + +lispintro mono ${GS}/emacs/emacs-lisp-intro/html_mono/emacs-lisp-intro.html +lispintro node ${GS}/emacs/emacs-lisp-intro/html_node/index.html + +LSH = http://www.lysator.liu.se/~nisse/lsh + lsh mono ${LSH}/lsh.html + +m4 mono ${GS}/m4/manual/m4.html +m4 node ${GS}/m4/manual/html_node/ + +MITGNUSCHEME = ${GS}/mit-scheme/documentation/stable +mit-scheme-user mono ${MITGNUSCHEME}/mit-scheme-user.html +mit-scheme-user node ${MITGNUSCHEME}/mit-scheme-user/ + # + mit-scheme-ref mono ${MITGNUSCHEME}/mit-scheme-ref.html + mit-scheme-ref node ${MITGNUSCHEME}/mit-scheme-ref/ + # + mit-scheme-ffi mono ${MITGNUSCHEME}/mit-scheme-ffi.html + mit-scheme-ffi node ${MITGNUSCHEME}/mit-scheme-ffi/ + # + mit-scheme-sos mono ${MITGNUSCHEME}/mit-scheme-sos.html + mit-scheme-sos node ${MITGNUSCHEME}/mit-scheme-sos/ + # + mit-scheme-imail mono ${MITGNUSCHEME}/mit-scheme-imail.html + # + mit-scheme-blowfish mono ${MITGNUSCHEME}/mit-scheme-blowfish.html + # + mit-scheme-gdbm mono ${MITGNUSCHEME}/mit-scheme-gdbm.html + +mailutils mono ${GS}/mailutils/manual/mailutils.html +mailutils chapter ${GS}/mailutils/manual/html_chapter/ +mailutils section ${GS}/mailutils/manual/html_section/ +mailutils node ${GS}/mailutils/manual/html_node/ + +make mono ${GS}/make/manual/make.html +make node ${GS}/make/manual/html_node/ + +mdk mono ${GS}/mdk/manual/mdk.html +mdk node ${GS}/mdk/manual/html_node/ + +METAEXCHANGE = https://ftp.gwdg.de/pub/gnu2/iwfmdh/doc/texinfo + iwf_mh node ${METAEXCHANGE}/iwf_mh.html + scantest node ${METAEXCHANGE}/scantest.html + +MIT_SCHEME = ${GS}/mit-scheme/documentation/stable + mit-scheme-ref node ${MIT_SCHEME}/mit-scheme-ref/ + mit-scheme-user node ${MIT_SCHEME}/mit-scheme-user/ + sos node ${MIT_SCHEME}/mit-scheme-sos/ + mit-scheme-imail mono ${MIT_SCHEME}/mit-scheme-imail.html + +moe mono ${GS}/moe/manual/moe_manual.html + +motti node ${GS}/motti/manual/ + +# only PDF is available as documentation to download +#mpc node http://www.multiprecision.org/index.php?prog=mpc&page=html + +mpfr mono https://www.mpfr.org/mpfr-current/mpfr.html + +mtools mono ${GS}/mtools/manual/mtools.html + +nano mono https://www.nano-editor.org/dist/latest/nano.html + +nettle mono https://www.lysator.liu.se/~nisse/nettle/nettle.html + +ocrad mono ${GS}/ocrad/manual/ocrad_manual.html + +parted mono ${GS}/parted/manual/parted.html +parted node ${GS}/parted/manual/html_node/ + +pascal node https://www.gnu-pascal.de/gpc/ + +# can't use pcb since url's contain dates --30nov10 + +PIES = http://www.gnu.org.ua/software/pies/manual +pies node ${PIES}/ + +plotutils mono ${GS}/plotutils/manual/en/plotutils.html +plotutils node ${GS}/plotutils/manual/en/html_node/ + +proxyknife mono ${GS}/proxyknife/manual/proxyknife.html +proxyknife node ${GS}/proxyknife/manual/html_node/ + +pspp mono ${GS}/pspp/manual/pspp.html +pspp node ${GS}/pspp/manual/html_node/ + +pyconfigure mono ${GS}/pyconfigure/manual/pyconfigure.html +pyconfigure node ${GS}/pyconfigure/manual/html_node/ + +R = https://cran.r-project.org/doc/manuals + R-intro mono ${R}/R-intro.html + R-lang mono ${R}/R-lang.html + R-exts mono ${R}/R-exts.html + R-data mono ${R}/R-data.html + R-admin mono ${R}/R-admin.html + R-ints mono ${R}/R-ints.html + +rcs mono ${GS}/rcs/manual/rcs.html +rcs node ${GS}/rcs/manual/html_node/ + +READLINE = https://tiswww.cwru.edu/php/chet/readline +readline mono ${READLINE}/readline.html + rluserman mono ${READLINE}/rluserman.html + history mono ${READLINE}/history.html + +# no manual for Recode found. Most recent fork seems to be at +# https://github.com/rrthomas/recode/ + +recutils mono ${GS}/recutils/manual/recutils.html +recutils node ${GS}/recutils/manual/html_node/ + +remotecontrol mono ${GS}/remotecontrol/manual/remotecontrol.html +remotecontrol node ${GS}/remotecontrol/manual/html_node/ + +rottlog mono ${GS}/rottlog/manual/rottlog.html +rottlog node ${GS}/rottlog/manual/html_node/ + +RUSH = http://www.gnu.org.ua/software/rush/manual +rush mono ${RUSH}/rush.html +rush chapter ${RUSH}/html_chapter/ +rush section ${RUSH}/html_section/ +rush node ${RUSH}/html_node/ + +screen mono ${GS}/screen/manual/screen.html +screen node ${GS}/screen/manual/html_node/ + +sed mono ${GS}/sed/manual/sed.html +sed node ${GS}/sed/manual/html_node/ + +sharutils mono ${GS}/sharutils/manual/sharutils.html +sharutils chapter ${GS}/sharutils/manual/html_chapter/ +sharutils node ${GS}/sharutils/manual/html_node/ + +# replaces dmd +shepherd mono ${GS}/shepherd/manual/shepherd.html +shepherd node ${GS}/shepherd/manual/html_node/ + +SMALLTALK = ${GS}/smalltalk +gst mono ${SMALLTALK}/manual/gst.html +gst node ${SMALLTALK}/manual/html_node/ + # + gst-base mono ${SMALLTALK}/manual-base/gst-base.html + gst-base node ${SMALLTALK}/manual-base/html_node/ + # + gst-libs mono ${SMALLTALK}/manual-libs/gst-libs.html + gst-libs node ${SMALLTALK}/manual-libs/html_node/ + +sourceinstall mono ${GS}/sourceinstall/manual/sourceinstall.html +sourceinstall node ${GS}/sourceinstall/manual/html_node/ + +sqltutor mono ${GS}/sqltutor/manual/sqltutor.html +sqltutor node ${GS}/sqltutor/manual/html_node/ + +src-highlite mono ${GS}/src-highlite/source-highlight.html + +swbis mono ${GS}/swbis/manual.html + +tar mono ${GS}/tar/manual/tar.html +tar chapter ${GS}/tar/manual/html_chapter/ +tar section ${GS}/tar/manual/html_section/ +tar node ${GS}/tar/manual/html_node/ + +teseq mono ${GS}/teseq/manual/teseq.html +teseq node ${GS}/teseq/manual/html_node/ + +TEXINFO = ${GS}/texinfo/manual +texinfo mono ${TEXINFO}/texinfo/texinfo.html +texinfo node ${TEXINFO}/texinfo/html_node/ + # + info-stnd mono ${TEXINFO}/info-stnd/info-stnd.html + info-stnd node ${TEXINFO}/info-stnd/html_node/ + # + texi2any_api mono ${TEXINFO}/texi2any_api/texi2any_api.html + texi2any_api node ${TEXINFO}/texi2any_api/html_node/ + # + texi2any_internals mono ${TEXINFO}/texi2any_internals/texi2any_internals.html + texi2any_internals chapter ${TEXINFO}/texi2any_internals/html_chapter/ + +thales node ${GS}/thales/manual/ + +units mono ${GS}/units/manual/units.html +units node ${GS}/units/manual/html_node/ + +vc-dwim mono ${GS}/vc-dwim/manual/vc-dwim.html +vc-dwim node ${GS}/vc-dwim/manual/html_node/ + +wdiff mono ${GS}/wdiff/manual/wdiff.html +wdiff node ${GS}/wdiff/manual/html_node/ + +websocket4j mono ${GS}/websocket4j/manual/websocket4j.html +websocket4j node ${GS}/websocket4j/manual/html_node/ + +wget mono ${GS}/wget/manual/wget.html +wget node ${GS}/wget/manual/html_node/ + +xboard mono ${GS}/xboard/manual/xboard.html +xboard node ${GS}/xboard/manual/html_node/ + +# emacs-page +# Free TeX-related Texinfo manuals on tug.org. + +T = https://tug.org/texinfohtml + +dvipng mono ${T}/dvipng.html +dvips mono ${T}/dvips.html +eplain mono ${T}/eplain.html +kpathsea mono ${T}/kpathsea.html +latex2e mono ${T}/latex2e.html +tlbuild mono ${T}/tlbuild.html +web2c mono ${T}/web2c.html + + +# Local Variables: +# eval: (add-hook 'write-file-hooks 'time-stamp) +# time-stamp-start: "htmlxrefversion=" +# time-stamp-format: "%:y-%02m-%02d.%02H" +# time-stamp-time-zone: "UTC" +# time-stamp-end: "; # UTC" +# End: diff --git a/mu4e/meson.build b/mu4e/meson.build new file mode 100644 index 0000000..a2a22bb --- /dev/null +++ b/mu4e/meson.build @@ -0,0 +1,142 @@ +## Copyright (C) 2022 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +## +## This program is free software; you can redistribute it and/or modify +## it under the terms of the GNU General Public License as published by +## the Free Software Foundation; either version 3 of the License, or +## (at your option) any later version. +## +## This program is distributed in the hope that it will be useful, +## but WITHOUT ANY WARRANTY; without even the implied warranty of +## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +## GNU General Public License for more details. +## +## You should have received a copy of the GNU General Public License +## along with this program; if not, write to the Free Software Foundation, +## Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + + +# generate some build data for use in mu4e +mu4e_meta = configure_file( + input: 'mu4e-config.el.in', + output: 'mu4e-config.el', + install: true, + install_dir: mu4e_lispdir, + configuration: { + 'VERSION' : meson.project_version(), + 'MU_DOC_DIR' : join_paths(datadir, 'doc', 'mu'), + }) + +mu4e_pkg_desc = configure_file( + input: 'mu4e-pkg.el.in', + output: 'mu4e-pkg.el', + install: true, + install_dir: mu4e_lispdir, + configuration: { + 'VERSION' : meson.project_version(), + 'EMACS_MIN_VERSION' : emacs_min_version, + }) + +mu4e_srcs=[ + 'mu4e-actions.el', + 'mu4e-bookmarks.el', + 'mu4e-compose.el', + 'mu4e-contacts.el', + 'mu4e-context.el', + 'mu4e-contrib.el', + 'mu4e-draft.el', + 'mu4e-folders.el', + 'mu4e.el', + 'mu4e-headers.el', + 'mu4e-helpers.el', + 'mu4e-icalendar.el', + 'mu4e-lists.el', + 'mu4e-main.el', + 'mu4e-mark.el', + 'mu4e-message.el', + 'mu4e-mime-parts.el', + 'mu4e-modeline.el', + 'mu4e-notification.el', + 'mu4e-obsolete.el', + 'mu4e-org.el', + 'mu4e-query-items.el', + 'mu4e-search.el', + 'mu4e-server.el', + 'mu4e-speedbar.el', + 'mu4e-thread.el', + 'mu4e-update.el', + 'mu4e-vars.el', + 'mu4e-view.el', + 'mu4e-window.el' +] + +# note, we cannot compile mu4e-config.el without incurring +# WARNING: Source item +# '[...]/build/mu4e/mu4e-meta.el' cannot be converted to File object, because +# it is a generated file. This will become a hard error in the future. +# +#... so let's not do that! + +foreach src : mu4e_srcs + target_name= '@BASENAME@.elc' + target_path = join_paths(meson.current_build_dir(), target_name) + target_func = '(setq byte-compile-dest-file-function(lambda(_) "' + target_path + '"))' + + # hack-around for native compile issue: copy sources to builddir. + # see: https://debbugs.gnu.org/db/47/47987.html + configure_file(input: src, output:'@BASENAME@.el', copy:true, + install_mode: 'r--r--r--') + + custom_target(src.underscorify() + '_el', + build_by_default: true, + input: src, + output: target_name, + install_dir: mu4e_lispdir, + install: true, + # rebuild all if any changed. + depend_files: mu4e_srcs, + command: [emacs, + '--no-init-file', + '--batch', + '--directory', meson.current_source_dir(), + '--directory', meson.current_build_dir(), + # we don't need warnings for items that have become + # obsolete _after_ our last supported emacs release. + '--eval', '(setq byte-compile-warnings \'(not obsolete))', + '--eval', target_func, + '--funcall', 'batch-byte-compile', '@INPUT@']) + +endforeach + +# this depends on the above hack: all mu4e elisp files needs to be in builddir +mu4e_autoloads = configure_file( + output: 'mu4e-autoloads.el', + install: true, + install_dir: mu4e_lispdir, + command: [emacs, + '--no-init-file', + '--batch', + '--load', 'package', + '--eval', '(package-generate-autoloads "mu4e" "' + + meson.current_build_dir() + '" )']) + +# also install the sources and the config +install_data(mu4e_srcs, install_dir: mu4e_lispdir) + +# install mu4e-about.org +install_data('mu4e-about.org', install_dir : join_paths(datadir, 'doc', 'mu')) + +if makeinfo.found() + custom_target('mu4e_info', + input: 'mu4e.texi', + output: 'mu4e.info', + install_dir: infodir, + install: true, + command: [makeinfo, + '-o', join_paths(meson.current_build_dir(), 'mu4e.info'), + join_paths(meson.current_source_dir(), 'mu4e.texi'), + '-I', join_paths(meson.current_build_dir(), '..')]) + if install_info.found() + infodir = join_paths(get_option('prefix') / get_option('infodir')) + meson.add_install_script(install_info_script, infodir, 'mu4e.info') + endif +endif diff --git a/mu4e/mu4e-about.org b/mu4e/mu4e-about.org new file mode 100644 index 0000000..5c015b3 --- /dev/null +++ b/mu4e/mu4e-about.org @@ -0,0 +1,15 @@ +#+STARTUP:showall +* About mu4e + + *mu4e* is an emacs e-mail client based on the [[http://djcbsoftware.nl/code/mu][mu]] email search engine. It was + written & designed by /Dirk-Jan C. Binnema/, with contributions from others. + + *mu4e* and *mu* are free software, licensed under the terms of the [[http://www.gnu.org/licenses/gpl-3.0.html][GNU GPLv3]]. + + You can get the code from [[https://github.com/djcb/mu][the git repository]]; there, you can also + [[https://github.com/djcb/mu/issues][file bugs and feature requests]]. + + *mu4e* has its own [[info:mu4e][manual]], which includes an [[info:mu4e#FAQ%20-%20Frequently%20Anticipated%20Questions][FAQ]]. If that is not enough, + there's also the [[http://groups.google.com/group/mu-discuss][mu mailing list]]. + + [Press *q* to quit this buffer] diff --git a/mu4e/mu4e-actions.el b/mu4e/mu4e-actions.el new file mode 100644 index 0000000..543ebac --- /dev/null +++ b/mu4e/mu4e-actions.el @@ -0,0 +1,275 @@ +;;; mu4e-actions.el --- Actions for messages/attachments -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Example actions for messages, attachments (see chapter 'Actions' in the +;; manual) + +;;; Code: + +(require 'ido) +(require 'browse-url) + +(require 'mu4e-helpers) +(require 'mu4e-message) +(require 'mu4e-search) +(require 'mu4e-contacts) +(require 'mu4e-lists) + +;;; Count lines + +(defun mu4e-action-count-lines (msg) + "Count the number of lines in the e-mail MSG. +Works for headers view and message-view." + (message "Number of lines: %s" + (shell-command-to-string + (concat "wc -l < " + (shell-quote-argument (mu4e-message-field msg :path)))))) + +;;; Org Helpers + +(defvar mu4e-captured-message nil + "The most recently captured message.") + +(defun mu4e-action-capture-message (msg) + "Remember MSG. +Later, we can create an attachment based on this message with +`mu4e-compose-attach-captured-message'." + (setq mu4e-captured-message msg) + (message "Message has been captured")) + + +(defun mu4e-action-copy-message-file-path (msg) + "Save the full path for the current MSG to the kill ring." + (kill-new (mu4e-message-field msg :path))) + +(defvar mu4e-org-contacts-file nil + "File to store contact information for org-contacts. +Needed by `mu4e-action-add-org-contact'.") + +(eval-when-compile ;; silence compiler warning about free variable + (unless (require 'org-capture nil 'noerror) + (defvar org-capture-templates nil))) + +(defun mu4e-action-add-org-contact (msg) + "Add an org-contact based on the sender ddress of the current MSG. +You need to set `mu4e-org-contacts-file' to the full path to the +file where you store your org-contacts." + (unless (require 'org-capture nil 'noerror) + (mu4e-error "Feature org-capture is not available")) + (unless mu4e-org-contacts-file + (mu4e-error "Variable `mu4e-org-contacts-file' is nil")) + (let* ((sender (car-safe (mu4e-message-field msg :from))) + (name (mu4e-contact-name sender)) + (email (mu4e-contact-email sender)) + (blurb + (format + (concat + "* %%?%s\n" + ":PROPERTIES:\n" + ":EMAIL: %s\n" + ":NICK:\n" + ":BIRTHDAY:\n" + ":END:\n\n") + (or name email "") + (or email ""))) + (key "mu4e-add-org-contact-key") + (org-capture-templates + (append org-capture-templates + (list (list key "contacts" 'entry + (list 'file mu4e-org-contacts-file) blurb))))) + (when (fboundp 'org-capture) + (org-capture nil key)))) + +;;; Patches + +(defvar mu4e--patch-directory-history nil + "History of directories we have applied patches to.") + +;; This essentially works around the fact that read-directory-name +;; can't have custom history. +(defun mu4e--read-patch-directory (&optional prompt) + "Read a `PROMPT'ed directory name via `completing-read' with history." + (unless prompt + (setq prompt "Target directory:")) + (file-truename + (completing-read prompt 'read-file-name-internal #'file-directory-p + nil nil 'mu4e--patch-directory-history))) + +(defun mu4e-action-git-apply-patch (msg) + "Apply `MSG' as a git patch." + (let ((path (mu4e--read-patch-directory "Target directory: "))) + (let ((default-directory path)) + (shell-command + (format "git apply %s" + (shell-quote-argument (mu4e-message-field msg :path))))))) + +(defun mu4e-action-git-apply-mbox (msg &optional signoff) + "Apply `MSG' a git patch with optional `SIGNOFF'. + +If the `default-directory' matches the most recent history entry don't +bother asking for the git tree again (useful for bulk actions)." + + (let ((cwd (substring-no-properties + (or (car mu4e--patch-directory-history) + "not-a-dir")))) + (unless (and (stringp cwd) (string= default-directory cwd)) + (setq cwd (mu4e--read-patch-directory "Target directory: "))) + (let ((default-directory cwd)) + (shell-command + (format "git am %s %s" + (if signoff "--signoff" "") + (shell-quote-argument (mu4e-message-field msg :path))))))) + +;;; Tagging + +(defvar mu4e-action-tags-header "X-Keywords" + "Header where tags are stored. +Used by `mu4e-action-retag-message'. Make sure it is one of the +headers mu recognizes for storing tags: X-Keywords, X-Label, +Keywords. Also note that changing this setting on already tagged +messages can lead to messages with multiple tags headers.") + +(defvar mu4e-action-tags-completion-list '() + "List of tags for completion in `mu4e-action-retag-message'.") + +(defun mu4e--contains-line-matching (regexp path) + "Return non-nil if the file at PATH contain a line matching REGEXP. +Otherwise return nil." + (with-temp-buffer + (insert-file-contents path) + (save-excursion + (goto-char (point-min)) + (re-search-forward regexp nil t)))) + +(defun mu4e--replace-first-line-matching (regexp to-string path) + "Replace first line matching REGEXP in PATH with TO-STRING." + (with-temp-file path + (insert-file-contents path) + (save-excursion + (goto-char (point-min)) + (if (re-search-forward regexp nil t) + (replace-match to-string t nil))))) + +(declare-function mu4e--server-add "mu4e-server") +(defun mu4e--refresh-message (path) + "Re-parse message at PATH. +if this works, we will +receive (:info add :path <path> :docid <docid>) as well as (:update +<msg-sexp>)." + (mu4e--server-add path)) + +(defun mu4e-action-retag-message (msg &optional retag-arg) + "Change tags of MSG with RETAG-ARG. + +RETAG-ARG is a comma-separated list of additions and removals. + +Example: +tag,+long tag,-oldtag +would add \"tag\" and \"long tag\", and remove \"oldtag\"." + (let* ( + (path (mu4e-message-field msg :path)) + (oldtags (mu4e-message-field msg :tags)) + (tags-completion + (append + mu4e-action-tags-completion-list + (mapcar (lambda (tag) (format "+%s" tag)) + mu4e-action-tags-completion-list) + (mapcar (lambda (tag) (format "-%s" tag)) + oldtags))) + (retag (if retag-arg + (split-string retag-arg ",") + (completing-read-multiple "Tags: " tags-completion))) + (header mu4e-action-tags-header) + (sep (cond ((string= header "Keywords") ", ") + ((string= header "X-Label") " ") + ((string= header "X-Keywords") ", ") + (t ", "))) + (taglist (if oldtags (copy-sequence oldtags) '())) + tagstr) + (dolist (tag retag taglist) + (cond + ((string-match "^\\+\\(.+\\)" tag) + (setq taglist (push (match-string 1 tag) taglist))) + ((string-match "^\\-\\(.+\\)" tag) + (setq taglist (delete (match-string 1 tag) taglist))) + (t + (setq taglist (push tag taglist))))) + + (setq taglist (sort (delete-dups taglist) 'string<)) + (setq tagstr (mapconcat 'identity taglist sep)) + + (setq tagstr (replace-regexp-in-string "[\\&]" "\\\\\\&" tagstr)) + (setq tagstr (replace-regexp-in-string "[/]" "\\&" tagstr)) + + (if (not (mu4e--contains-line-matching (concat header ":.*") path)) + ;; Add tags header just before the content + (mu4e--replace-first-line-matching + "^$" (concat header ": " tagstr "\n") path) + + ;; replaces keywords, restricted to the header + (mu4e--replace-first-line-matching + (concat header ":.*") + (concat header ": " tagstr) + path)) + + (mu4e-message (concat "tagging: " (mapconcat 'identity taglist ", "))) + (mu4e--refresh-message path))) + +(defun mu4e-action-show-thread (msg) + "Show thread for message at point with point remaining on MSG. +I.e., point remains on the message with the message-id where the +action was invoked. If invoked in view mode, continue to display +the message." + (let ((msgid (mu4e-message-field msg :message-id))) + (when msgid + (let ((mu4e-search-threads t) + (mu4e-search-include-related t)) + (mu4e-search + (format "msgid:%s" msgid) + nil nil nil + msgid (and (eq major-mode 'mu4e-view-mode) + (not (eq mu4e-split-view 'single-window)))))))) + + +;;; Mailing list URLS + +(defun mu4e-action-browse-list-archive (msg) + "Browse the archive for a mailing list message MSG. +See `mu4e-mailing-list-archive-url'." + (interactive (list (mu4e-message-at-point))) + (if-let ((url (mu4e-mailing-list-archive-url msg))) + (browse-url url) + (mu4e-warn "No archive available for this message"))) + +(defun mu4e-action-copy-list-archive-url (msg) + "Copy the archive url for a mailing list message MSG. +See `mu4e-mailing-list-archive-url'." + (interactive (list (mu4e-message-at-point))) + (let ((url (mu4e-mailing-list-archive-url msg))) + (if (stringp url) + (kill-new url) + (mu4e-warn "Cannot get archive URL for this message")))) + +;;; +(provide 'mu4e-actions) +;;; mu4e-actions.el ends here diff --git a/mu4e/mu4e-bookmarks.el b/mu4e/mu4e-bookmarks.el new file mode 100644 index 0000000..452169b --- /dev/null +++ b/mu4e/mu4e-bookmarks.el @@ -0,0 +1,195 @@ +;;; mu4e-bookmarks.el --- Bookmarks handling -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;;; Code: +(require 'mu4e-helpers) +(require 'mu4e-modeline) +(require 'mu4e-folders) +(require 'mu4e-query-items) + + +;;; Configuration + +(defgroup mu4e-bookmarks nil + "Settings for bookmarks." + :group 'mu4e) + +(defcustom mu4e-bookmarks + '(( :name "Unread messages" + :query "flag:unread AND NOT flag:trashed" + :key ?u) + ( :name "Today's messages" + :query "date:today..now" + :key ?t) + ( :name "Last 7 days" + :query "date:7d..now" + :hide-unread t + :key ?w) + ( :name "Messages with images" + :query "mime:image/*" + :key ?p)) + "List of pre-defined queries that are shown on the main screen. + +Each of the list elements is a plist with at least: +`:name' - the name of the query +`:query' - the query expression string or function +`:key' - the shortcut key (single character) + +Optionally, you can add the following: + +- `:favorite' - if t, monitor the results of this query, and make +it eligible for showing its status in the modeline. At most +one bookmark should have this set to t (otherwise the _first_ +bookmark is the implicit favorite). The query for the `:favorite' +item must be unique among `mu4e-bookmarks' and +`mu4e-maildir-shortcuts'. +- `:hide' - if t, the bookmark is hidden from the main-view and +speedbar. +- `:hide-unread' - do not show the counts of +unread/total number of matches for the query in the main-view. +This can be useful if a bookmark uses a very slow query. + +`:hide-unread' is implied from `:hide'. + +Note: for efficiency, queries used to determine the unread/all +counts do not discard duplicate or unreadable messages. Thus, the +numbers shown may differ from the number you get from a normal +query." + :type '(repeat (plist)) + :group 'mu4e-bookmarks) + + +(defun mu4e-ask-bookmark (prompt) + "Ask user for bookmark using PROMPT. +Return the corresponding query. The bookmark are as defined in +`mu4e-bookmarks'." + (unless (mu4e-bookmarks) (mu4e-error "No bookmarks defined")) + (let* ((bmarks (seq-map (lambda (bm) + (cons (format "%c%s" + (plist-get bm :key) + (plist-get bm :name)) + (plist-get bm :query))) + (mu4e-filter-single-key (mu4e-bookmarks))))) + (mu4e-read-option prompt bmarks))) + +(defun mu4e-get-bookmark-query (kar) + "Get the corresponding bookmarked query for shortcut KAR. +Raise an error if none is found." + (let ((chosen-bm + (or (seq-find + (lambda (bm) + (= kar (plist-get bm :key))) + (mu4e-bookmarks)) + (mu4e-warn "Unknown shortcut '%c'" kar)))) + (mu4e--bookmark-query chosen-bm))) + +(defun mu4e-bookmark-define (query name key) + "Define a bookmark for QUERY with NAME and shortcut KEY. +Append it to `mu4e-bookmarks'. Replaces any existing bookmark +with KEY." + (setq mu4e-bookmarks + (seq-remove + (lambda (bm) + (= (plist-get bm :key) key)) + (mu4e-bookmarks))) + (cl-pushnew `(:name ,name + :query ,query + :key ,key) + mu4e-bookmarks :test 'equal)) + +(defun mu4e-bookmarks () + "Get `mu4e-bookmarks' in the (new) format. +Convert from the old format if needed." + (seq-map (lambda (item) + (if (and (listp item) (= (length item) 3)) + `(:name ,(nth 1 item) :query ,(nth 0 item) + :key ,(nth 2 item)) + item)) + mu4e-bookmarks)) + +(defun mu4e-bookmark-favorite () + "Find the favorite bookmark." + ;; note, use query-items, which will have picked a favorite + ;; even if user did not provide one explictly + (seq-find + (lambda (item) + (plist-get item :favorite)) + (mu4e-query-items 'bookmarks))) + +;; for Zero-Inbox afficionados +(defvar mu4e-modeline-all-clear '("C:" . "🌀") + "No more messages at all for this query.") +(defvar mu4e-modeline-all-read '("R:" . "✅") + "No unread messages left.") +(defvar mu4e-modeline-unread-items '("U:" . "📫") + "There are some unread items.") +(defvar mu4e-modeline-new-items '("N:" . "🔥") + "There are some new items after the baseline. +I.e., very new messages.") + +(declare-function mu4e-search-bookmark "mu4e-search") +(defun mu4e-jump-to-favorite () + "Jump to to the favorite bookmark, if any." + (interactive) + (when-let ((fav (mu4e--bookmark-query (mu4e-bookmark-favorite)))) + (mu4e-search-bookmark fav))) + +(defun mu4e--bookmarks-modeline-item () + "Modeline item showing message counts for the favorite bookmark. + +This uses the one special ':favorite' bookmark, and if there is +one, creates a propertized string for display in the modeline." + (when-let ((fav ;; any results for the favorite bookmark item? + (seq-find (lambda (bm) (plist-get bm :favorite)) + (mu4e-query-items 'bookmarks)))) + (cl-destructuring-bind (&key unread count delta-unread + &allow-other-keys) fav + (propertize + (format "%s%s " + (funcall (if mu4e-use-fancy-chars 'cdr 'car) + (cond + ((> delta-unread 0) mu4e-modeline-new-items) + ((> unread 0) mu4e-modeline-unread-items) + ((> count 0) mu4e-modeline-all-read) + (t mu4e-modeline-all-clear))) + (mu4e--query-item-display-counts fav)) + 'help-echo + (format + (concat + "mu4e favorite bookmark '%s':\n" + "\t%s\n\n" + "number of matches: %d\n" + "unread messages: %d\n" + "changes since baseline: %+d\n") + (plist-get fav :name) + (mu4e--bookmark-query fav) + count unread delta-unread) + 'mouse-face 'mode-line-highlight + 'keymap '(mode-line keymap + (mouse-1 . mu4e-jump-to-favorite) + (mouse-2 . mu4e-jump-to-favorite) + (mouse-3 . mu4e-jump-to-favorite)))))) + +(provide 'mu4e-bookmarks) +;;; mu4e-bookmarks.el ends here diff --git a/mu4e/mu4e-compose.el b/mu4e/mu4e-compose.el new file mode 100644 index 0000000..6135ef5 --- /dev/null +++ b/mu4e/mu4e-compose.el @@ -0,0 +1,521 @@ +;;; mu4e-compose.el --- Compose and send messages -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2024 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Implements mu4e-compose-mode, which is a `message-mode' derivative. There's +;; quite a bit of trickery involved to make the message-mode functions work in +;; this context; see mu4e-draft for details. + + +;;; Code: +(require 'message) +(require 'sendmail) +(require 'gnus-msg) +(require 'nnheader) ;; for make-full-mail-header + +(require 'mu4e-obsolete) +(require 'mu4e-server) +(require 'mu4e-message) +(require 'mu4e-context) +(require 'mu4e-folders) + +(require 'mu4e-draft) + + +;;; User configuration for compose-mode +(defgroup mu4e-compose nil + "Customization for composing/sending messages." + :group 'mu4e) + +(defcustom mu4e-compose-format-flowed nil + "Whether to compose messages to be sent as format=flowed. +\(Or with long lines if variable `use-hard-newlines' is set to +nil). The variable `fill-flowed-encode-column' lets you customize +the width beyond which format=flowed lines are wrapped." + :type 'boolean + :safe 'booleanp + :group 'mu4e-compose) + +(defcustom mu4e-compose-pre-hook nil + "Hook run just *before* message composition starts. + +If the compose-type is a symbol, either `reply' or `forward', the +variable `mu4e-compose-parent-message' is the message replied to +/ being forwarded / edited, and `mu4e-compose-type' contains the +type of message to be composed. + +Note that there is no draft message yet when this hook runs, it +is meant for influencing the how mu4e constructs the draft +message. If you want to do something with the draft messages +after it has been constructed, `mu4e-compose-mode-hook' would be +the place to do that." + :type 'hook + :group 'mu4e-compose) + +(defcustom mu4e-compose-post-hook + (list + ;; kill compose frames + #'mu4e-compose-post-kill-frame + ;; attempt to restore the old configuration. + #'mu4e-compose-post-restore-window-configuration) + "Hook run *after* message composition is over. + +This is hook is run when closing the composition buffer, either +by sending, postponing, exiting or killing it. + +This multiplexes the `message-mode' hooks `message-send-actions', +`message-postpone-actions', `message-exit-actions' and +`message-kill-actions', and the hook is run with a variable +`mu4e-compose-post-trigger' set correspondingly to a symbol, +`send', `postpone', `exit' or `kill'." + :type 'hook + :group 'mu4e-compose) + + + +(defvar mu4e-captured-message) +(defun mu4e-compose-attach-captured-message () + "Insert the last captured message file as an attachment. +Messages are captured with `mu4e-action-capture-message'." + (interactive) + (if-let* ((msg mu4e-captured-message) + (path (plist-get msg :path)) + (path (and (file-exists-p path) path))) + (mml-attach-file + path + "message/rfc822" + (or (plist-get msg :subject) "No subject") + "attachment") + (mu4e-warn "No valid message has been captured"))) + +;; Go to bottom / top + +(defun mu4e-compose-goto-top (&optional arg) + "Go to the beginning of the message or buffer. +Go to the beginning of the message or, if already there, go to +the beginning of the buffer. + +Push mark at previous position, unless either a +\\[universal-argument] prefix ARG is supplied, or Transient Mark mode +is enabled and the mark is active." + (interactive "P") + (or arg + (region-active-p) + (push-mark)) + (let ((old-position (point))) + (message-goto-body) + (when (equal (point) old-position) + (goto-char (point-min))))) + +(defun mu4e-compose-goto-bottom (&optional arg) + "Go to the end of the message or buffer. +Go to the end of the message (before signature) or, if already +there, go to the end of the buffer. + +Push mark at previous position, unless either a +\\[universal-argument] prefix ARG is supplied, or Transient Mark mode +is enabled and the mark is active." + (interactive "P") + (or arg + (region-active-p) + (push-mark)) + (let ((old-position (point)) + (message-position (save-excursion (message-goto-body) (point)))) + (goto-char (point-max)) + (when (re-search-backward message-signature-separator message-position t) + (forward-line -1)) + (when (equal (point) old-position) + (goto-char (point-max))))) + +(defun mu4e-compose-context-switch (&optional force name) + "Change the context for the current draft message. + +With NAME, switch to the context with NAME, and with FORCE non-nil, +switch even if the switch is to the same context. + +Like `mu4e-context-switch' but with some changes after switching: +1. Update the From and Organization headers as per the new context +2. Update the `message-signature' as per the new context. + +Unlike some earlier version of this function, does _not_ update +the draft folder for the messages, as that would require changing +the file under our feet, which is a bit fragile." + (interactive "P") + + (unless (derived-mode-p 'mu4e-compose-mode) + (mu4e-error "Only available in mu4e compose buffers")) + + (let ((old-context (mu4e-context-current))) + (unless (and name (not force) (eq old-context name)) + (unless (and (not force) + (eq old-context (mu4e-context-switch nil name))) + (save-excursion + ;; Change From / Organization if needed. + (message-replace-header "Organization" + (or (message-make-organization) "") + '("Subject")) ;; keep in same place + (message-replace-header "From" + (or (message-make-from) "")) + ;; Update signature. + (when (message-goto-signature) ;; delete old signature. + (if message-signature-insert-empty-line + (forward-line -2) (forward-line -1)) + (delete-region (point) (point-max))) + (when message-signature + (save-excursion (message-insert-signature)))))))) + + +;;; address completion + +;; inspired by org-contacts.el and +;; https://github.com/nordlow/elisp/blob/master/mine/completion-styles-cycle.el + +(defun mu4e--compose-complete-handler (str pred action) + "Complete address STR with predication PRED for ACTION." + (cond + ((eq action nil) + (try-completion str mu4e--contacts-set pred)) + ((eq action t) + (all-completions str mu4e--contacts-set pred)) + ((eq action 'metadata) + ;; our contacts are already sorted - just need to tell the completion + ;; machinery not to try to undo that... + '(metadata + (display-sort-function . identity) + (cycle-sort-function . identity))))) + +(defun mu4e-complete-contact () + "Attempt to complete the text at point with a contact. +I.e., either \"name <email>\" or \"email\". Return nil if not found. + +This function can be used for `completion-at-point-functions', to +complete addresses. This can be used from outside mu4e, but mu4e +must be active (running) for this to work." + (let* ((end (point)) + (start (save-excursion + (re-search-backward "\\(\\`\\|[\n:,]\\)[ \t]*") + (goto-char (match-end 0)) + (point)))) + (list start end #'mu4e--compose-complete-handler))) + +(defun mu4e--compose-complete-contact-field () + "Attempt to complete a contact when in a contact field. + +This is like `mu4e-compose-complete-contact', but limited to the +contact fields." + (let ((mail-abbrev-mode-regexp + "^\\(To\\|B?Cc\\|Reply-To\\|From\\|Sender\\):") + (mail-header-separator mu4e--header-separator)) + (when (mail-abbrev-in-expansion-header-p) + (mu4e-complete-contact)))) + +(defun mu4e--compose-setup-completion () + "Maybe enable auto-completion of addresses. +Do this when `mu4e-compose-complete-addresses' is non-nil. + +When enabled, this attempts to put mu4e's completions at the +start of the buffer-local `completion-at-point-functions'. Other +completion functions still apply." + (when mu4e-compose-complete-addresses + (set (make-local-variable 'completion-ignore-case) t) + (set (make-local-variable 'completion-cycle-threshold) 7) + (add-to-list (make-local-variable 'completion-styles) 'substring) + (add-hook 'completion-at-point-functions + #'mu4e--compose-complete-contact-field -10 t))) + + ;;; mu4e-compose-mode +(defun mu4e--compose-remap-faces () + "Remap `message-mode' faces to mu4e ones. + +Our parent `message-mode' uses font-locking for the compose +buffers; lets remap its faces so it uses the ones for mu4e." + ;; normal headers + (face-remap-add-relative 'message-header-name 'mu4e-header-field-face) + (face-remap-add-relative 'message-header-other 'mu4e-header-value-face) + ;; special headers + (face-remap-add-relative 'message-header-from 'mu4e-contact-face) + (face-remap-add-relative 'message-header-to 'mu4e-contact-face) + (face-remap-add-relative 'message-header-cc 'mu4e-contact-face) + (face-remap-add-relative 'message-header-bcc 'mu4e-contact-face) + (face-remap-add-relative 'message-header-subject + 'mu4e-special-header-value-face)) + +(defvar mu4e-compose-mode-map + (let ((map (make-sparse-keymap))) + (set-keymap-parent map message-mode-map) + (define-key map (kbd "C-S-u") #'mu4e-update-mail-and-index) + (define-key map (kbd "C-c C-u") #'mu4e-update-mail-and-index) + (define-key map (kbd "C-c ;") #'mu4e-compose-context-switch) + + ;; emacs 29 + ;;(keymap-set map "<remap> <beginning-of-buffer>" #'mu4e-compose-goto-top) + ;;(keymap-set map "<remap> <end-of-buffer>" #'mu4e-compose-goto-bottom) + (define-key map (vector 'remap #'beginning-of-buffer) + #'mu4e-compose-goto-top) + (define-key map (vector 'remap #'end-of-buffer) + #'mu4e-compose-goto-bottom) + + ;; remove some unsupported commands... [remap ..] does not work here + ;; XXX remove from menu, too. + (define-key map (kbd "C-c C-f C-n") nil) ;; message-goto-newsgroups + (define-key map (kbd "C-c C-n") nil) ;; message-insert-newsgroups + (define-key map (kbd "C-c C-j") nil) ;; gnus-delay-article + map) + "The keymap for mu4e-compose buffers.") + +(defun mu4e--compose-unsupported (&rest _args) + "Advise wrapper for Gnus unsupported functions in mu4e." + (when (eq major-mode 'mu4e-compose-mode) + (mu4e-warn "Not available in mu4e"))) + +(defun mu4e--neutralize-undesirables () + "Beware Gnus commands that do not work with mu4e." + ;; the Field menu contains many items that don't apply. + (advice-add 'gnus-delay-article + :before #'mu4e--compose-unsupported) ;; # XXX does not work?! + (advice-add 'message-goto-newsgroups :before #'mu4e--compose-unsupported) + (advice-add 'message-insert-newsgroups :before #'mu4e--compose-unsupported)) + +(define-derived-mode mu4e-compose-mode message-mode "mu4e:compose" + "Major mode for the mu4e message composition, derived from `message-mode'. +\\{mu4e-compose-mode-map}." + (progn + (use-local-map mu4e-compose-mode-map) + (mu4e-context-minor-mode) + (mu4e--neutralize-undesirables) + (mu4e--compose-remap-faces) + (setq-local nobreak-char-display nil) + ;; set this to allow mu4e to work when gnus-agent is unplugged in gnus + (set (make-local-variable 'message-send-mail-real-function) nil) + ;; Set to nil to enable `electric-quote-local-mode' to work: + (set (make-local-variable 'comment-use-syntax) nil) + (mu4e--compose-setup-completion) ;; maybe offer address completion + (if mu4e-compose-format-flowed ;; format-flowed + (progn + (turn-off-auto-fill) + (setq truncate-lines nil + word-wrap t + mml-enable-flowed t + use-hard-newlines t) + (visual-line-mode t)) + (setq mml-enable-flowed nil)))) + +(declare-function mu4e-view-message-text "mu4e-view") + +(defun mu4e-message-cite-nothing () + "Function for `message-cite-function' that cites _nothing_." + (save-excursion + (message-cite-original-without-signature) + (delete-region (point-min) (point-max)))) + +(defun mu4e--compose-cite (msg) + "Return a cited version of the ORIG message MSG (a string). +This function uses `message-cite-function', and its settings apply." + (with-temp-buffer + (insert (mu4e-view-message-text msg)) + (goto-char (point-min)) + (push-mark (point-max)) + (let ((message-signature-separator "^-- *$") + (message-signature-insert-empty-line t)) + (funcall message-cite-function)) + (pop-mark) + (goto-char (point-min)) + (buffer-string))) + + +;;;###autoload +(defalias 'mu4e-compose-mail #'mu4e-compose-new) + +;;;###autoload +(defun mu4e-compose-new (&optional to subject other-headers continue + _switch-function yank-action send-actions + return-action &rest _) + "Mu4e's implementation of `compose-mail'. +TO, SUBJECT, OTHER-HEADERS, CONTINUE, YANK-ACTION SEND-ACTIONS +RETURN-ACTION are as described in `compose-mail', and to the +extend that they do not conflict with mu4e's inner workings. +SWITCH-FUNCTION is ignored." + (interactive) + (mu4e--draft + 'new + (lambda () (mu4e--message-call + #'message-mail to subject other-headers continue + nil ;; switch-function -> we handle it ourselves. + yank-action send-actions return-action)))) + +;;;###autoload +(defun mu4e-compose-reply-to (&optional to wide) + "Reply to the message at point. +Optional TO can be the To: address for the message. If WIDE is +non-nil, make it a \"wide\" reply (a.k.a. \"reply-to-all\")." + (interactive) + (let ((parent (mu4e-message-at-point))) + (mu4e--draft-with-parent + 'reply parent + (lambda () + (with-current-buffer (mu4e--message-call #'message-reply to wide) + (message-goto-body) + (insert (mu4e--compose-cite parent)) + (current-buffer)))))) + +;;;###autoload +(defun mu4e-compose-reply (&optional wide) + "Reply to the message at point. If WIDE is +non-nil, make it a \"wide\" reply (a.k.a. \"reply-to-all\")." + (interactive "P") + (mu4e-compose-reply-to nil wide)) + +;;;###autoload +(defun mu4e-compose-wide-reply () + "Wide reply to the message at point. +(a.k.a. \"reply-to-all\")." + (interactive) + (mu4e-compose-reply-to nil t))1 + +;;;###autoload +(defun mu4e-compose-supersede () + "Supersede the message at point. + +That is, send the message again, with all the same recipients; +this can be useful to follow-up on a sent message. The message +must originate from the current user, as determined through +`mu4e-personal-or-alternative-address-p'." + (interactive) + (let ((parent (mu4e-message-at-point))) + (mu4e--draft-with-parent + 'reply ;; it's a special kind of reply. + parent + (lambda () + (with-current-buffer (mu4e--message-call #'message-supersede)))))) + +(defun mu4e-compose-forward () + "Forward the message at point. +To influence the way a message is forwarded, you can use the +variables ‘message-forward-as-mime’ and +‘message-forward-show-mml’." + (interactive) + (let ((parent (mu4e-message-at-point))) + (mu4e--draft-with-parent + 'forward parent + (lambda () + (setq + message-reply-headers (make-full-mail-header + 0 + (or (message-field-value "Subject") "none") + (or (message-field-value "From") "nobody") + (message-field-value "Date") + (message-field-value "Message-Id" t) + (message-field-value "References") + 0 0 "")) + ;; a bit of a hack; mu4e--draft-with-parent will insert the decoded + ;; version of the message, but that's not good enough for + ;; message-forward, which needs the raw message instead; see #2662. + (erase-buffer) + (insert-file-contents-literally (mu4e-message-readable-path parent)) + (with-current-buffer (mu4e--message-call #'message-forward) + (current-buffer)))))) + +;;;###autoload +(defun mu4e-compose-edit() + "Edit an existing draft message." + (interactive) + (let* ((msg (mu4e-message-at-point))) + (unless (member 'draft (mu4e-message-field msg :flags)) + (mu4e-warn "Cannot edit non-draft messages")) + (mu4e--draft + 'edit + (lambda () + (with-current-buffer + (find-file-noselect (mu4e-message-readable-path msg)) + (mu4e--delimit-headers) + (current-buffer)))))) + +;;;###autoload +(defun mu4e-compose-resend (address) + "Re-send the message at point to ADDRESS. +The message is resent as-is, without any editing. See +`message-resend' for details." + (interactive + (list (completing-read + "Resend message to address: " mu4e--contacts-set))) + (let ((msg (mu4e-message-at-point))) + (with-temp-buffer + (mu4e--prepare-draft msg) + (insert-file-contents (mu4e-message-readable-path msg)) + (message-resend address)))) + +;;; Compose Mail + +(declare-function mu4e "mu4e") + +;;;###autoload +(define-mail-user-agent 'mu4e-user-agent + #'mu4e-compose-mail + #'message-send-and-exit + #'message-kill-buffer + 'message-send-hook) + +;; Without this, `mail-user-agent' cannot be set to `mu4e-user-agent' +;; through customize, as the custom type expects a function. Not +;; sure whether this function is actually ever used; if it is then +;; returning the symbol is probably the correct thing to do, as other +;; such functions suggest. +(defun mu4e-user-agent () + "Return the `mu4e-user-agent' symbol." + 'mu4e-user-agent) + +;;; minor mode for use in other modes. +(defvar mu4e-compose-minor-mode-map + (let ((map (make-sparse-keymap))) + (define-key map "R" #'mu4e-compose-reply) + (define-key map "W" #'mu4e-compose-wide-reply) + (define-key map "F" #'mu4e-compose-forward) + (define-key map "E" #'mu4e-compose-edit) + (define-key map "C" #'mu4e-compose-new) + map) + "Keymap for compose minor-mode.") + +(define-minor-mode mu4e-compose-minor-mode + "Mode for searching for messages." + :global nil + :init-value nil ;; disabled by default + :group 'mu4e + :lighter "" + :keymap mu4e-compose-minor-mode-map) + +(defvar mu4e--compose-menu-items + '("--" + ["Compose new" mu4e-compose-new + :help "Compose new message"] + ["Reply" mu4e-compose-reply + :help "Reply to message"] + ["Reply to all" mu4e-compose-wide-reply + :help "Reply to all-recipients"] + ["Forward" mu4e-compose-forward + :help "Forward message"] + ["Resend" mu4e-compose-resend + :help "Re-send message"]) + "Easy menu items for message composition.") + ;;; +(provide 'mu4e-compose) +;;; mu4e-compose.el ends here diff --git a/mu4e/mu4e-config.el.in b/mu4e/mu4e-config.el.in new file mode 100644 index 0000000..5f99db4 --- /dev/null +++ b/mu4e/mu4e-config.el.in @@ -0,0 +1,9 @@ +;; auto-generated + +(defconst mu4e-mu-version "@VERSION@" + "Required mu binary version; mu4e's version must agree with this.") + +(defconst mu4e-doc-dir "@MU_DOC_DIR@" + "Mu4e's data-dir.") + +(provide 'mu4e-config) diff --git a/mu4e/mu4e-contacts.el b/mu4e/mu4e-contacts.el new file mode 100644 index 0000000..ab6079c --- /dev/null +++ b/mu4e/mu4e-contacts.el @@ -0,0 +1,308 @@ +;;; mu4e-contacts.el --- Dealing with contacts -*- lexical-binding: t -*- + +;; Copyright (C) 2022-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Utility functions used in the mu4e + +;;; Code: +(require 'cl-lib) +(require 'message) +(require 'mu4e-helpers) +(require 'mu4e-update) + + +;;; Configuration +(defcustom mu4e-compose-complete-addresses t + "Whether to do auto-completion of e-mail addresses." + :type 'boolean + :group 'mu4e-compose) + +(defcustom mu4e-compose-complete-only-personal nil + "Whether to consider only \"personal\" e-mail addresses for completion. +That is, addresses from messages where user was explicitly in one +of the address fields (this excludes mailing list messages). +These addresses are the ones specified with \"mu init\"." + :type 'boolean + :group 'mu4e-compose) + +(defcustom mu4e-compose-complete-only-after "2018-01-01" + "Consider only contacts last seen after this date. + +Date must be a string of the form YYYY-MM-DD. + +This is useful for limiting a potentially enormous set of +contacts for auto-completion to just those that are present in +the e-mail corpus in recent times. Set to nil to not have any +time-based restriction." + :type 'string + :group 'mu4e-compose) + +(defcustom mu4e-compose-complete-max nil + "Limit the amount of contacts for completion, nil for no limits. +After considering the other constraints +\(`mu4e-compose-complete-addresses' and +`mu4e-compose-complete-only-after'), pick only the highest-ranked +<n>. + +Lowering this variable reduces start-up time and memory usage." + :type '(choice natnum (const :tag "No limits" nil)) + :group 'mu4e-compose) + +;; names and mail-addresses can be mapped onto their canonical +;; counterpart. use the customizeable function +;; mu4e-canonical-contact-function to do that. below the identity +;; function for mapping a contact onto the canonical one. +(defun mu4e-contact-identity (contact) + "Return the name and the mail-address of a CONTACT. +It is used as the identity function for converting contacts to +their canonical counterpart; useful as an example." + (let ((name (plist-get contact :name)) + (mail (plist-get contact :mail))) + (list :name name :mail mail))) + +(defcustom mu4e-contact-process-function + (lambda(addr) + (cond + ((string-match-p "reply" addr) + ;; no-reply adresses are not useful of course, but neither are are + ;; reply-xxxx addresses since they're autogenerated only useful for direct + ;; replies. + nil) + (t addr))) + "Function for processing contact information for use in auto-completion. + +The function receives the contact as a string, e.g \"Foo Bar + <foo.bar@example.com>\" \"cuux@example.com\" + +The function should return either: +- nil: do not use this contact for completion +- the (possibly rewritten) address, which must be +an RFC-2822-compatible e-mail address." + :type 'function + :group 'mu4e-compose) + +(defcustom mu4e-compose-reply-ignore-address + '("no-?reply") + "Addresses to prune when doing wide replies. + +This can be a regexp matching the address, a list of regexps or a +predicate function. A value of nil keeps all the addresses." + :type '(choice + (const nil) + function + string + (repeat string)) + :group 'mu4e-compose) + + +;;; Internal variables +(defvar mu4e--contacts-tstamp "0" + "Timestamp for the most recent contacts update." ) + +(defvar mu4e--contacts-set nil + "Set with the full contact addresses for autocompletion.") + +;;; user mail address +(defun mu4e-personal-addresses (&optional no-regexp) + "Get the list user's personal addresses, as passed to \"mu init\". + +The address are either plain e-mail addresses or regexps (strings + wrapped / /). When NO-REGEXP is non-nil, do not include regexp + address patterns (if any)." + (seq-remove + (lambda (addr) (and no-regexp (string-match-p "^/.*/" addr))) + (when-let ((props (mu4e-server-properties))) + (plist-get props :personal-addresses)))) + +(defun mu4e-personal-address-p (addr) + "Is ADDR a personal address? +Evaluate to nil if ADDR does not match any of the personal +addresses. Uses \\=(mu4e-personal-addresses) for the addresses +with both the plain addresses and /regular expressions/." + (when addr + (seq-find + (lambda (m) + (if (string-match "/\\(.*\\)/" m) + (let ((rx (match-string 1 m)) + (case-fold-search t)) + (string-match rx addr)) + (eq t (compare-strings addr nil nil m nil nil 'case-insensitive)))) + (mu4e-personal-addresses)))) + + +(defun mu4e-personal-or-alternative-address-p (addr) + "Is ADDR either a personal or an alternative address? + +That is, does it match either `mu4e-personal-address-p' or +`message-alternative-emails'. + +Note that this expanded definition of user-addresses is only used +in emacs, not in `mu' (e.g. when indexing). + +Also see `mu4e-personal-or-alternative-address-or-empty-p'." + (let ((alts message-alternative-emails)) + (or (mu4e-personal-address-p addr) + (cond + ((functionp alts) (funcall alts addr)) + ((stringp alts) (string-match alts addr)) + (t nil))))) + +(defun mu4e-personal-or-alternative-address-or-empty-p (addr) + "Is ADDR either a personal, alternative address or nil? + +This is like `mu4e-personal-or-alternative-address-p' but also +return t for _empty_ ADDR. This can be useful for use with +`message-dont-reply-to-names' since it can receive empty strings; +those can be filtered-out by returning t here. + +See #2680 for further details. " + (or (and addr (string= addr "")) + (mu4e-personal-or-alternative-address-p addr))) + + +;; Helpers + +;;; RFC2822 handling of phrases in mail-addresses +;; +;; The optional display-name contains a phrase, it sits before the +;; angle-addr as specified in RFC2822 for email-addresses in header +;; fields. Contributed by jhelberg. + +(defun mu4e--rfc822-phrase-type (ph) + "Return an atom or quoted-string for the phrase PH. +This checks for empty string first. Then quotes around the phrase +\(returning symbol `rfc822-quoted-string'). Then whether there is +a quote inside the phrase (returning symbol +`rfc822-containing-quote'). + +The reverse of the RFC atext definition is then tested. If it +matches, nil is returned, if not, it returns a symbol +`rfc822-atom'." + (cond + ((= (length ph) 0) 'rfc822-empty) + ((= (aref ph 0) ?\") + (if (string-match "\"\\([^\"\\\n]\\|\\\\.\\|\\\\\n\\)*\"" ph) + 'rfc822-quoted-string + 'rfc822-containing-quote)) ; starts with quote, but doesn't end with one + ((string-match-p "[\"]" ph) 'rfc822-containing-quote) + ((string-match-p "[\000-\037()\*<>@,;:\\\.]+" ph) nil) + (t 'rfc822-atom))) + +(defun mu4e--rfc822-quote-phrase (ph) + "Quote an RFC822 phrase PH only if necessary. +Atoms and quoted strings don't need quotes. The rest do. In +case a phrase contains a quote, it will be escaped." + (let ((type (mu4e--rfc822-phrase-type ph))) + (cond + ((eq type 'rfc822-atom) ph) + ((eq type 'rfc822-quoted-string) ph) + ((eq type 'rfc822-containing-quote) + (format "\"%s\"" + (replace-regexp-in-string "\"" "\\\\\"" ph))) + (t (format "\"%s\"" ph))))) + +(defsubst mu4e-contact-name (contact) + "Get the name of this CONTACT, or nil." + (plist-get contact :name)) + +(defsubst mu4e-contact-email (contact) + "Get the name of this CONTACT, or nil." + (plist-get contact :email)) + +(defsubst mu4e-contact-cons (contact) + "Convert a CONTACT plist into a old-style (name . email)." + (cons + (mu4e-contact-name contact) + (mu4e-contact-email contact))) + +(defsubst mu4e-contact-make (name email) + "Create a contact plist from NAME and EMAIL." + `(:name ,name :email ,email)) + +(defun mu4e-contact-full (contact) + "Get the full combination of name and email address from CONTACT." + (let* ((email (mu4e-contact-email contact)) + (name (mu4e-contact-name contact))) + (if (and name (> (length name) 0)) + (format "%s <%s>" (mu4e--rfc822-quote-phrase name) email) + email))) + + +(defun mu4e--update-contacts (contacts &optional tstamp) + "Receive a sorted list of CONTACTS newer than TSTAMP. +Update an internal set with it. + +This is used by the completion function in mu4e-compose." + (let ((n 0)) + (unless mu4e--contacts-set + (setq mu4e--contacts-set (make-hash-table :test 'equal :weakness nil + :size (length contacts)))) + (dolist (contact contacts) + (cl-incf n) + (when (functionp mu4e-contact-process-function) + (setq contact (funcall mu4e-contact-process-function contact))) + (when contact ;; note the explicit deccode; the strings we get are + ;; utf-8, but emacs doesn't know yet. + (puthash (decode-coding-string contact 'utf-8) t mu4e--contacts-set))) + (setq mu4e--contacts-tstamp (or tstamp "0")) + (unless (zerop n) + (mu4e-index-message "Contacts updated: %d; total %d" + n (hash-table-count mu4e--contacts-set))))) + +(defun mu4e-contacts-info () + "Display information about the contacts-cache. +For testing/debugging." + (interactive) + (with-current-buffer (get-buffer-create "*mu4e-contacts-info*") + (erase-buffer) + (insert (format "complete addresses: %s\n" + (if mu4e-compose-complete-addresses "yes" "no"))) + (insert (format "only personal addresses: %s\n" + (if mu4e-compose-complete-only-personal "yes" "no"))) + (insert (format "only addresses seen after: %s\n" + (or mu4e-compose-complete-only-after "no restrictions"))) + + (when mu4e--contacts-set + (insert (format "number of contacts cached: %d\n\n" + (hash-table-count mu4e--contacts-set))) + (maphash (lambda (contact _) + (insert (format "%s\n" contact))) mu4e--contacts-set)) + (pop-to-buffer "*mu4e-contacts-info*"))) + +(declare-function mu4e--server-contacts "mu4e-server") + +(defun mu4e--request-contacts-maybe () + "Maybe update the set of contacts for autocompletion. + +If `mu4e-compose-complete-addresses' is non-nil, get/update the +list of contacts we use for autocompletion; otherwise, do +nothing." + (when mu4e-compose-complete-addresses + (mu4e--server-contacts + mu4e-compose-complete-only-personal + mu4e-compose-complete-only-after + mu4e-compose-complete-max + mu4e--contacts-tstamp))) + +(provide 'mu4e-contacts) +;;; mu4e-contacts.el ends here diff --git a/mu4e/mu4e-context.el b/mu4e/mu4e-context.el new file mode 100644 index 0000000..98cfedc --- /dev/null +++ b/mu4e/mu4e-context.el @@ -0,0 +1,243 @@ +;;; mu4e-context.el --- Switching between settings -*- lexical-binding: t -*- + +;; Copyright (C) 2015-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; A mu4e 'context' is a set of variable-settings and functions, which can be +;; used e.g. to switch between accounts. + +;;; Code: + +(require 'mu4e-helpers) +(require 'mu4e-modeline) +(require 'mu4e-query-items) + + +;;; Configuration +(defcustom mu4e-context-policy 'ask-if-none + "The policy to determine the context when entering the mu4e main view. + +If the value is `always-ask', ask the user unconditionally. + +In all other cases, if any context matches (using its match +function), this context is used. Otherwise, if none of the +contexts match, we have the following choices: + +- `pick-first': pick the first of the contexts available (ie. the default) +- `ask': ask the user `ask-if-none': ask if there is no context yet, + otherwise leave it as it is +- nil: return nil; eaves the current context as is. + +Also see `mu4e-compose-context-policy'." + :type '(choice + (const :tag "Always ask what context to use, even if one matches" + always-ask) + (const :tag "Ask if none of the contexts match" ask) + (const :tag "Ask when there's no context yet" ask-if-none) + (const :tag "Pick the first context if none match" pick-first) + (const :tag "Don't change the context when none match" nil)) + :group 'mu4e) + + +(defvar mu4e-contexts nil + "The list of `mu4e-context' objects describing mu4e's contexts.") + +(defvar mu4e-context-changed-hook nil + "Hook run just *after* the context changed.") + +(defface mu4e-context-face + '((t :inherit mu4e-title-face :weight bold)) + "Face for displaying the context in the modeline." + :group 'mu4e-faces) + +(defvar mu4e--context-current nil + "The current context. +Internal; use `mu4e-context-switch' to change it.") + +(defun mu4e-context-current (&optional output) + "Get the currently active context, or nil if there is none. +When OUTPUT is non-nil, echo the name of the current context or +none." + (interactive "p") + (let ((ctx mu4e--context-current)) + (when output + (mu4e-message "Current context: %s" + (if ctx (mu4e-context-name ctx) "<none>"))) + ctx)) + +(cl-defstruct mu4e-context + "A mu4e context object with the following members: +- `name': the name of the context, eg. \"Work\" or \"Private\". +- `enter-func': a parameterless function invoked when entering + this context, or nil +- `leave-func':a parameterless function invoked when leaving this + context, or nil +- `match-func': a function called when composing a new message, + that takes a message plist for the message replied to or + forwarded, and nil otherwise. Before composing a new message, + `mu4e' switches to the first context for which `match-func' + returns t. +- `vars': variables to set when entering context." + name ;; name of the context, e.g. "work" + (enter-func nil) ;; function invoked when entering the context + (leave-func nil) ;; function invoked when leaving the context + (match-func nil) ;; function that takes a msg-proplist, and return t + ;; if it matches, nil otherwise + vars) ;; alist of variables. + + +(defun mu4e--context-ask-user (prompt) + "Let user choose some context based on its name with PROMPT." + (when mu4e-contexts + (let* ((names (seq-map (lambda (context) + (cons (mu4e-context-name context) context)) + mu4e-contexts)) + (context (mu4e-read-option prompt names))) + (or context (mu4e-error "No such context"))))) + +(defun mu4e-context-switch (&optional force name) + "Switch to a context with NAME. +Context must be part of `mu4e-contexts'; if NAME is nil, query user. + +If the new context is the same as the current context, only +switch (run associated functions) when prefix argument FORCE is +non-nil." + (interactive "P") + (unless mu4e-contexts + (mu4e-error "No contexts defined")) + (let* ((names (seq-map (lambda (context) + (cons (mu4e-context-name context) context)) + mu4e-contexts)) + (old-context mu4e--context-current) ; i.e., context before switch + (context + (if name + (cdr-safe (assoc name names)) + (mu4e--context-ask-user "Switch to context: ")))) + (unless context (mu4e-error "No such context")) + ;; if new context is same as old one, only switch with FORCE + (when (or force (not (eq context (mu4e-context-current)))) + (when (and (mu4e-context-current) + (mu4e-context-leave-func mu4e--context-current)) + (funcall (mu4e-context-leave-func mu4e--context-current))) + ;; enter the new context + (when (mu4e-context-enter-func context) + (funcall (mu4e-context-enter-func context))) + (when (mu4e-context-vars context) + (mapc (lambda (cell) + (set (car cell) (cdr cell))) + (mu4e-context-vars context))) + (setq mu4e--context-current context) + (run-hooks 'mu4e-context-changed-hook) + ;; refresh the cached query items if there was a context before; we have + ;; have different bookmarks/maildirs now. + (when old-context + (mu4e--query-items-refresh 'reset-baseline)) + (mu4e-message "Switched context to %s" + (mu4e-context-name context))) + context)) + +(defun mu4e--context-autoswitch (&optional msg policy) + "Automatically switch to some context. + +When contexts are defined but there is no context yet, switch to +the first whose :match-func return non-nil. If none of them +match, return the first. For MSG and POLICY, see +`mu4e-context-determine'." + (when mu4e-contexts + (let ((context (mu4e-context-determine msg policy))) + (when context (mu4e-context-switch + nil (mu4e-context-name context)))))) + +(defun mu4e-context-determine (msg &optional policy) + "Return the first context where match-func evaluate to non-nil. + +MSG points to the plist for the message replied to or forwarded, +or nil if there is no such MSG; similar to what +`mu4e-compose-pre-hook' does. + +POLICY specifies how to do the determination. If POLICY is +`always-ask', we ask the user unconditionally. + +In all other cases, if any context matches (using its match +function), this context is returned. If none of the contexts +match, POLICY determines what to do: + +- `pick-first': pick the first of the contexts available +- `ask': ask the user +- `ask-if-none': ask if there is no context yet +- otherwise, return nil. Effectively, this leaves the current context +as it is." + (when mu4e-contexts + (if (eq policy 'always-ask) + (mu4e--context-ask-user "Select context: ") + (or ;; is there a matching one? + (seq-find (lambda (context) + (when (mu4e-context-match-func context) + (funcall (mu4e-context-match-func context) msg))) + mu4e-contexts) + ;; no context found yet; consult policy + (pcase policy + ('pick-first (car mu4e-contexts)) + ('ask (mu4e--context-ask-user "Select context: ")) + ('ask-if-none (or (mu4e-context-current) + (mu4e--context-ask-user "Select context: "))) + (_ nil)))))) + +(defmacro with-mu4e-context-vars (context &rest body) + "Evaluate BODY, with variables let-bound for CONTEXT (if any). +`funcall'." + (declare (indent 2)) + `(let* ((vars (and ,context (mu4e-context-vars ,context)))) + (cl-progv ;; XXX: perhaps use eval's lexical environment instead of progv? + (mapcar (lambda(cell) (car cell)) vars) + (mapcar (lambda(cell) (cdr cell)) vars) + (eval ,@body)))) + +(defun mu4e--context-modeline-item () + "Propertized string with the current context or nil." + (when-let* ((ctx (mu4e-context-current)) + (name (and ctx (mu4e-context-name ctx)))) + (concat + "<" + (propertize + name + 'face 'mu4e-context-face + 'help-echo (format "mu4e context: %s" name)) + ">"))) + +(define-minor-mode mu4e-context-minor-mode + "Mode for switching the mu4e context." + :global nil + :init-value nil ;; disabled by default + :group 'mu4e + :lighter "" + (mu4e--modeline-register #'mu4e--context-modeline-item)) + +(defvar mu4e--context-menu-items + '("--" + ["Switch-context" mu4e-context-switch + :help "Switch the mu4e context"]) + "Easy menu items for mu4e-context.") + +;;; +(provide 'mu4e-context) +;;; mu4e-context.el ends here diff --git a/mu4e/mu4e-contrib.el b/mu4e/mu4e-contrib.el new file mode 100644 index 0000000..1da3c9b --- /dev/null +++ b/mu4e/mu4e-contrib.el @@ -0,0 +1,201 @@ +;;; mu4e-contrib.el --- User-contributed functions -*- lexical-binding: t -*- + +;; Copyright (C) 2013-2023 Dirk-Jan C. Binnema + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Some user-contributed functions for mu4e + +;;; Code: + +(require 'mu4e-headers) +(require 'mu4e-view) +(require 'bookmark) +(require 'eshell) + + +;;; Various simple commands +(defun mu4e-headers-mark-all-unread-read () + "Put a ! \(read) mark on all visible unread messages." + (interactive) + (mu4e-headers-mark-for-each-if + (cons 'read nil) + (lambda (msg _param) + (memq 'unread (mu4e-msg-field msg :flags))))) + +(defun mu4e-headers-flag-all-read () + "Flag all visible messages as \"read\"." + (interactive) + (mu4e-headers-mark-all-unread-read) + (mu4e-mark-execute-all t)) + +(defun mu4e-headers-mark-all () + "Mark all headers for some action. +Ask user what action to execute." + (interactive) + (mu4e-headers-mark-for-each-if + (cons 'something nil) + (lambda (_msg _param) t)) + (mu4e-mark-execute-all)) + + + +;;; Bogofilter/SpamAssassin +;; +;; Support for handling spam with Bogofilter with the possibility +;; to define it for SpamAssassin, contributed by Gour. +;; +;; To add the actions to the menu, you can use something like: +;; +;; (add-to-list 'mu4e-headers-actions +;; '("sMark as spam" . mu4e-register-msg-as-spam) t) +;; (add-to-list 'mu4e-headers-actions +;; '("hMark as ham" . mu4e-register-msg-as-ham) t) + +(defvar mu4e-register-as-spam-cmd nil + "Command for invoking spam processor to register message as spam. +For example for bogofilter, use \"/usr/bin/bogofilter -Ns < %s\"") + +(defvar mu4e-register-as-ham-cmd nil + "Command for invoking spam processor to register message as ham. +For example for bogofile, use \"/usr/bin/bogofilter -Sn < %s\"") + +(defun mu4e-register-msg-as-spam (msg) + "Register MSG as spam." + (interactive) + (let* ((path (shell-quote-argument (mu4e-message-field msg :path))) + (command (format mu4e-register-as-spam-cmd path))) + (shell-command command)) + (mu4e-mark-at-point 'delete nil)) + +(defun mu4e-register-msg-as-ham (msg) + "Register MSG as ham." + (interactive) + (let* ((path (shell-quote-argument(mu4e-message-field msg :path))) + (command (format mu4e-register-as-ham-cmd path))) + (shell-command command)) + (mu4e-mark-at-point 'something nil)) + +;; (add-to-list 'mu4e-view-actions +;; '("sMark as spam" . mu4e-view-register-msg-as-spam) t) +;; (add-to-list 'mu4e-view-actions +;; '("hMark as ham" . mu4e-view-register-msg-as-ham) t) + +(defun mu4e-view-register-msg-as-spam (msg) + "Register MSG as spam (view mode)." + (interactive) + (let* ((path (shell-quote-argument (mu4e-message-field msg :path))) + (command (format mu4e-register-as-spam-cmd path))) + (shell-command command)) + (mu4e-view-mark-for-delete)) + +(defun mu4e-view-register-msg-as-ham (msg) + "Mark MSG as ham (view mode)." + (interactive) + (let* ((path (shell-quote-argument(mu4e-message-field msg :path))) + (command (format mu4e-register-as-ham-cmd path))) + (shell-command command)) + (mu4e-view-mark-for-something)) + + +;;; Eshell functions +;; +;; Code for `gnus-dired-attached' modified to run from eshell, +;; allowing files to be attached to an email via mu4e using the +;; eshell. Does not depend on gnus. + + +(defun mu4e--active-composition-buffers () + "Return all active mu4e composition buffers." + (let (buffers) + (save-excursion + (dolist (buffer (buffer-list t)) + (set-buffer buffer) + (when (eq major-mode 'mu4e-compose-mode) + (push (buffer-name buffer) buffers)))) + (nreverse buffers))) + + + +;; backward compat until 27.1 is univeral. +(defalias 'mu4e--flatten-list + (if (fboundp 'flatten-list) + #'flatten-list + (with-no-warnings + #'eshell-flatten-list))) + +;; backward compat ntil 28.1 is universal. +(defalias 'mu4e--mm-default-file-type + (if (fboundp 'mm-default-file-type) + #'mm-default-file-type + (with-no-warnings + #'mm-default-file-encoding))) + +(defun eshell/mu4e-attach (&rest args) + "Attach files to a mu4e message using eshell with ARGS. +If no mu4e buffers found, compose a new message and then attach +the file." + (let ((destination nil) + (files-str nil) + (bufs nil) + ;; Remove directories from the list + (files-to-attach + (delq nil (mapcar + (lambda (f) (if (or (not (file-exists-p f)) + (file-directory-p f)) + nil + (expand-file-name f))) + (mu4e--flatten-list (reverse args)))))) + ;; warn if user tries to attach without any files marked + (if (null files-to-attach) + (error "No files to attach") + (setq files-str + (mapconcat + (lambda (f) (file-name-nondirectory f)) + files-to-attach ", ")) + (setq bufs (mu4e--active-composition-buffers)) + ;; set up destination mail composition buffer + (if (and bufs + (y-or-n-p "Attach files to existing mail composition buffer? ")) + (setq destination + (if (= (length bufs) 1) + (get-buffer (car bufs)) + (let ((prompt (mu4e-format "%s" "Attach to buffer"))) + (substring-no-properties + (funcall mu4e-completing-read-function prompt + bufs))))) + ;; setup a new mail composition buffer + (if (y-or-n-p "Compose new mail and attach this file? ") + (progn (mu4e-compose-new) + (setq destination (current-buffer))))) + ;; if buffer was found, set buffer to destination buffer, and attach files + (if (not (eq destination 'nil)) + (progn (set-buffer destination) + (goto-char (point-max)) ; attach at end of buffer + (while files-to-attach + (mml-attach-file (car files-to-attach) + (or (mu4e--mm-default-file-type + (car files-to-attach)) + "application/octet-stream") nil) + (setq files-to-attach (cdr files-to-attach))) + (message "Attached file(s) %s" files-str)) + (message "No buffer to attach file to."))))) + +;;; _ +(provide 'mu4e-contrib) +;;; mu4e-contrib.el ends here diff --git a/mu4e/mu4e-draft.el b/mu4e/mu4e-draft.el new file mode 100644 index 0000000..3b94cdd --- /dev/null +++ b/mu4e/mu4e-draft.el @@ -0,0 +1,746 @@ +;;; mu4e-draft.el --- Helpers for m4e-compose -*- lexical-binding: t -*- + +;; Copyright (C) 2024 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Implements various helper functions for mu4e-compose. This all +;; look a little convoluted since we need to subvert the gnus/message +;; functions a bit to work with mu4e. + +(require 'message) +(require 'mu4e-config) +(require 'mu4e-helpers) +(require 'mu4e-contacts) +(require 'mu4e-folders) +(require 'mu4e-message) +(require 'mu4e-context) +(require 'mu4e-window) + +;;; Code: + +(declare-function mu4e-compose-mode "mu4e-compose") +(declare-function mu4e "mu4e") + +(defcustom mu4e-compose-crypto-policy + '(encrypt-encrypted-replies sign-encrypted-replies) + "Policy to control when messages will be signed/encrypted. + +The value is a list which influence the way draft messages are +created. Specifically, it might contain: + +- `sign-all-messages': Always add a signature. +- `sign-new-messages': Add a signature to new message, ie. + messages that aren't responses to another message. +- `sign-forwarded-messages': Add a signature when forwarding + a message +- `sign-edited-messages': Add a signature to drafts +- `sign-all-replies': Add a signature when responding to + another message. +- `sign-plain-replies': Add a signature when responding to + non-encrypted messages. +- `sign-encrypted-replies': Add a signature when responding + to encrypted messages. + +It should be noted that certain symbols have priorities over one +another. So `sign-all-messages' implies `sign-all-replies', which +in turn implies `sign-plain-replies'. Adding both to the set, is +not a contradiction, but a redundant configuration. + +All `sign-*' options have a `encrypt-*' analogue." + :type '(set :greedy t + (const :tag "Sign all messages" sign-all-messages) + (const :tag "Encrypt all messages" encrypt-all-messages) + (const :tag "Sign new messages" sign-new-messages) + (const :tag "Encrypt new messages" encrypt-new-messages) + (const :tag "Sign forwarded messages" sign-forwarded-messages) + (const :tag "Encrypt forwarded messages" + encrypt-forwarded-messages) + (const :tag "Sign edited messages" sign-edited-messages) + (const :tag "Encrypt edited messages" edited-forwarded-messages) + (const :tag "Sign all replies" sign-all-replies) + (const :tag "Encrypt all replies" encrypt-all-replies) + (const :tag "Sign replies to plain messages" sign-plain-replies) + (const :tag "Encrypt replies to plain messages" + encrypt-plain-replies) + (const :tag "Sign replies to encrypted messages" + sign-encrypted-replies) + (const :tag "Encrypt replies to encrypted messages" + encrypt-encrypted-replies)) + :group 'mu4e-compose) + +;;; Crypto +(defun mu4e--prepare-crypto (parent compose-type) + "Possibly encrypt or sign a message based on PARENT and COMPOSE-TYPE. +See `mu4e-compose-crypto-policy' for more details." + (let* ((encrypted-p + (and parent (memq 'encrypted (mu4e-message-field parent :flags)))) + (encrypt + (or (memq 'encrypt-all-messages mu4e-compose-crypto-policy) + (and (memq 'encrypt-new-messages mu4e-compose-crypto-policy) + (eq compose-type 'new)) ;; new messages + (and (eq compose-type 'forward) ;; forwarded + (memq 'encrypt-forwarded-messages mu4e-compose-crypto-policy)) + (and (eq compose-type 'edit) ;; edit + (memq 'encrypt-edited-messages mu4e-compose-crypto-policy)) + (and (eq compose-type 'reply) ;; all replies + (memq 'encrypt-all-replies mu4e-compose-crypto-policy)) + (and (eq compose-type 'reply) (not encrypted-p) ;; plain replies + (memq 'encrypt-plain-replies mu4e-compose-crypto-policy)) + (and (eq compose-type 'reply) encrypted-p + (memq 'encrypt-encrypted-replies + mu4e-compose-crypto-policy)))) ;; encrypted replies + (sign + (or (memq 'sign-all-messages mu4e-compose-crypto-policy) + (and (eq compose-type 'new) ;; new messages + (memq 'sign-new-messages mu4e-compose-crypto-policy)) + (and (eq compose-type 'forward) ;; forwarded messages + (memq 'sign-forwarded-messages mu4e-compose-crypto-policy)) + (and (eq compose-type 'edit) ;; edited messages + (memq 'sign-edited-messages mu4e-compose-crypto-policy)) + (and (eq compose-type 'reply) ;; all replies + (memq 'sign-all-replies mu4e-compose-crypto-policy)) + (and (eq compose-type 'reply) (not encrypted-p) ;; plain replies + (memq 'sign-plain-replies mu4e-compose-crypto-policy)) + (and (eq compose-type 'reply) encrypted-p ;; encrypted replies + (memq 'sign-encrypted-replies mu4e-compose-crypto-policy))))) + (cond ((and sign encrypt) (mml-secure-message-sign-encrypt)) + (sign (mml-secure-message-sign)) + (encrypt (mml-secure-message-encrypt))))) + +(defcustom mu4e-sent-messages-behavior 'sent + "Determines what mu4e does with sent messages. + +This is one of the symbols: +* `sent' move the sent message to the Sent-folder (`mu4e-sent-folder') +* `trash' move the sent message to the Trash-folder (`mu4e-trash-folder') +* `delete' delete the sent message. + +Note, when using GMail/IMAP, you should set this to either +`trash' or `delete', since GMail already takes care of keeping +copies in the sent folder. + +Alternatively, `mu4e-sent-messages-behavior' can be a function +which takes no arguments, and which should return one of the mentioned +symbols, for example: + + (setq mu4e-sent-messages-behavior (lambda () + (if (string= (message-sendmail-envelope-from) \"foo@example.com\") + \\='delete \\='sent))) + +The various `message-' functions from `message-mode' are available +for querying the message information." + :type '(choice (const :tag "move message to mu4e-sent-folder" sent) + (const :tag "move message to mu4e-trash-folder" trash) + (const :tag "delete message" delete)) + :group 'mu4e-compose) + +(defcustom mu4e-compose-context-policy 'ask + "Policy for determining the context when composing a new message. + +If the value is `always-ask', ask the user unconditionally. + +In all other cases, if any context matches (using its match +function), this context is used. Otherwise, if none of the +contexts match, we have the following choices: + +- `pick-first': pick the first of the contexts available (ie. the default) +- `ask': ask the user +- `ask-if-none': ask if there is no context yet, otherwise leave it as it is +- nil: return nil; leaves the current context as is. + +Also see `mu4e-context-policy'." + :type '(choice + (const :tag "Always ask what context to use" always-ask) + (const :tag "Ask if none of the contexts match" ask) + (const :tag "Ask when there's no context yet" ask-if-none) + (const :tag "Pick the first context if none match" pick-first) + (const :tag "Don't change the context when none match" nil)) + :safe 'symbolp + :group 'mu4e-compose) + +;; +;; display the ready-to-go display buffer in the desired way. +;; +(defun mu4e--display-draft-buffer (cbuf) + "Display the message composition buffer CBUF. +Display is influenced by `mu4e-compose-switch'." + (let ((func + (pcase mu4e-compose-switch + ('nil #'switch-to-buffer) + ('window #'switch-to-buffer-other-window) + ((or 'frame 't) #'switch-to-buffer-other-frame) + ('display-buffer #'display-buffer) + (_ (mu4e-error "Invalid mu4e-compose-switch"))))) + (funcall func cbuf))) + +(defvar mu4e-user-agent-string + (format "mu4e %s; emacs %s" mu4e-mu-version emacs-version) + "The User-Agent string for mu4e, or nil.") + +;;; Runtime variables; useful for user-hooks etc. +;; mu4e-compose-parent-message & mu4e-compose-type are buffer-local and +;; permanent-local so they'll survive the mode change to mu4e-compose-mode and +;; we can use them in the corresponding mode-hook. +(defvar-local mu4e-compose-parent-message nil + "The parent message plist. +This is the message being replied to, forwarded or edited; used +in `mu4e-compose-pre-hook'. For new (non-reply, forward etc.) +messages, it is nil.") +(put 'mu4e-compose-parent-message 'permanent-local t) + +(defvar-local mu4e-compose-type nil + "The compose-type for the current message.") +(put 'mu4e-compose-type 'permanent-local t) + +;;; Filenames +(defun mu4e--draft-basename() + "Construct a randomized filename for a message with flags FLAGSTR. +It looks something like + <time>-<random>.<hostname> + +This filename is used for the draft message and the sent message, +depending on `mu4e-sent-messages-behavior'." + (let* ((sysname (if (fboundp 'system-name) + (system-name) (with-no-warnings system-name))) + (sysname (if (string= sysname "") "localhost" sysname)) + (hostname (downcase + (save-match-data + (substring sysname + (string-match "^[^.]+" sysname) + (match-end 0)))))) + (format "%s.%04x%04x%04x%04x.%s" + (format-time-string "%s" (current-time)) + (random 65535) (random 65535) (random 65535) (random 65535) + hostname))) + +(defun mu4e--draft-message-path (base-name &optional parent) + "Construct a draft message path, based on PARENT if provided. + +PARENT is either nil or the original message (being replied + to/forwarded etc.), and is used to determine the draft folder. +BASE-NAME is the base filename without any Maildir decoration." + (let ((draft-dir (mu4e-get-drafts-folder parent))) + (mu4e-join-paths + (mu4e-root-maildir) draft-dir "cur" + (format "%s%s2,DS" base-name mu4e-maildir-info-delimiter)))) + +(defun mu4e--fcc-path (base-name &optional parent) + "Construct a Fcc: path, based on PARENT and `mu4e-sent-messages-behavior'. + +PARENT is either nil or the original message (being replied +to/forwarded etc.), and is used to determine the sent folder, +together with `mu4e-sent-messages-behavior'. BASE-NAME is the +base filename without any Maildir decoration. + +Returns the path for the sent message, either in the sent or +trash folder, or nil if the message should be removed after +sending." + (let* ((behavior + (if (and (functionp mu4e-sent-messages-behavior) + ;; don't interpret 'delete as a function... + (not (eq mu4e-sent-messages-behavior 'delete))) + (funcall mu4e-sent-messages-behavior) + mu4e-sent-messages-behavior)) + (sent-dir + (pcase behavior + ('delete nil) + ('trash (mu4e-get-trash-folder parent)) + ('sent (mu4e-get-sent-folder parent)) + (_ (mu4e-error "Error in `mu4e-sent-messages-behavior'"))))) + (when sent-dir + (mu4e-join-paths + (mu4e-root-maildir) sent-dir "cur" + (format "%s%s2,S" base-name mu4e-maildir-info-delimiter))))) + + +(defconst mu4e--header-separator + ;; XX properties don't show... why not? + (propertize "--text follows this line--" 'read-only t 'intangible t) + "Line used to separate headers from text in messages being composed.") + +(defun mu4e--delimit-headers (&optional undelimit) + "Delimit or undelimit (with UNDELIMIT) headers." + (let ((mail-header-separator (substring-no-properties mu4e--header-separator)) + (inhibit-read-only t)) + (save-excursion (mail-sendmail-undelimit-header)) ;; clear first + (unless undelimit (save-excursion (mail-sendmail-delimit-header))))) + +(defun mu4e--decoded-message (msg &optional headers-only) + "Get the message MSG, decoded as a string. +With HEADERS-ONLY non-nil, only include the headers part." + (with-temp-buffer + (setq-local gnus-article-decode-hook + '(article-decode-charset + article-decode-encoded-words + article-decode-idna-rhs + article-treat-non-ascii + article-remove-cr + article-de-base64-unreadable + article-de-quoted-unreadable) + gnus-inhibit-mime-unbuttonizing nil + gnus-unbuttonized-mime-types '(".*/.*") + gnus-original-article-buffer (current-buffer)) + (insert-file-contents-literally + (mu4e-message-readable-path msg) nil nil nil t) + ;; remove the body / attachments and what not. + (when headers-only + (rfc822-goto-eoh) + (delete-region (point) (point-max))) + ;; in rare (broken) case, if a message-id is missing use the generated one + ;; from mu. + (mu4e--delimit-headers) + (unless (message-field-value "Message-Id") + (goto-char (point-min)) + (insert (format "Message-Id: <%s>\n" (plist-get msg :message-id)))) + (mu4e--delimit-headers 'undelimit) + (ignore-errors (run-hooks 'gnus-article-decode-hook)) + (buffer-substring-no-properties (point-min) (point-max)))) + +(defvar mu4e--draft-buffer-max-name-length 48) +(defun mu4e--draft-set-friendly-buffer-name () + "Use some friendly name for this draft buffer." + (let* ((subj (message-field-value "subject")) + (subj (if (or (not subj) (string-match "^[:blank:]*$" subj)) + "No subject" subj))) + (rename-buffer (generate-new-buffer-name + (format "\"%s\"" + (truncate-string-to-width subj + mu4e--draft-buffer-max-name-length + 0 nil t))) + (buffer-name)))) + +;; hook impls + +(defun mu4e--fcc-handler (msgpath) + "Handle Fcc: for MSGPATH. +This ensures that a copy of a sent messages ends up in the +appropriate sent-messages folder. If MSGPATH is nil, do nothing." + (when msgpath + (let* ((target-dir (file-name-directory msgpath)) + (target-mdir (file-name-directory target-dir))) + ;; create maildir if needed + (unless (file-exists-p target-mdir) + (make-directory + (mu4e-join-paths target-mdir "cur" 'parents)) + (make-directory + (mu4e-join-paths target-mdir "new" 'parents))) + (write-file msgpath) + (mu4e--server-add msgpath)))) + +;; save / send hooks + +(defvar-local mu4e--compose-undo nil + "Remember the undo-state.") + +(defun mu4e--compose-before-save () + "Function called just before the draft buffer is saved." + ;; This does 3 things: + ;; - set the Message-Id if not already + ;; - set the Date if not already + ;; - (temporarily) remove the mail-header separator + (setq mu4e--compose-undo buffer-undo-list) + (save-excursion + (unless (message-field-value "Message-ID") + (message-generate-headers '(Message-ID))) + ;; older Emacsen (<= 28 perhaps?) won't update the Date + ;; if there already is one; so make sure it's gone. + (message-remove-header "Date") + (message-generate-headers '(Date Subject From)) + (mu4e--delimit-headers 'undelimit))) ;; remove separator + +(defun mu4e--set-parent-flags (path) + "Set flags for replied-to and forwarded for the message at PATH. +That is, set the `replied' \"R\" flag on messages we replied to, +and the `passed' \"F\" flag on message we have forwarded. + +If a message has an \"In-Reply-To\" header, it is considered a +reply to the message with the corresponding message id. +Otherwise, if it does not have an \"In-Reply-To\" header, but +does have a \"References:\" header, it is considered to be a +forward message for the message corresponding with the /last/ +message-id in the references header. + +If the message has been determined to be either a forwarded +message or a reply, we instruct the server to update that message +with resp. the \"P\" (passed) flag for a forwarded message, or +the \"R\" flag for a replied message. The original messages are +also marked as Seen. + +Function assumes that it is executed in the context of the +message buffer." + (when-let ((buf (find-file-noselect path))) + (with-current-buffer buf + (let ((in-reply-to (message-field-value "in-reply-to")) + (forwarded-from) + (references (message-field-value "references"))) + (unless in-reply-to + (when references + (with-temp-buffer ;; inspired by `message-shorten-references'. + (insert references) + (goto-char (point-min)) + (let ((refs)) + (while (re-search-forward "<[^ <]+@[^ <]+>" nil t) + (push (match-string 0) refs)) + ;; the last will be the first + (setq forwarded-from (car refs)))))) + ;; remove the <> and update the flags on the server-side. + (when (and in-reply-to (string-match "<\\(.*\\)>" in-reply-to)) + (mu4e--server-move (match-string 1 in-reply-to) nil "+R-N")) + (when (and forwarded-from (string-match "<\\(.*\\)>" forwarded-from)) + (mu4e--server-move (match-string 1 forwarded-from) nil "+P-N")))))) + +(defun mu4e--compose-after-save() + "Function called immediately after the draft buffer is saved." + ;; This does 3 things: + ;; - restore the mail-header-separator (see mu4e--compose-before-save) + ;; - update the buffer name (based on the message subject + ;; - tell the mu server about the updated draft message + (mu4e--delimit-headers) + (mu4e--draft-set-friendly-buffer-name) + ;; tell the server + (mu4e--server-add (buffer-file-name)) + ;; restore history. + (set-buffer-modified-p nil) + (setq buffer-undo-list mu4e--compose-undo)) + +(defun mu4e-sent-handler (docid path) + "Handler called with DOCID and PATH for the just-sent message. +For Forwarded ('Passed') and Replied messages, try to set the +appropriate flag at the message forwarded or replied-to." + ;; XXX we don't need this function anymore here, but we have an external + ;; caller in mu4e-icalendar... we should update that. + (mu4e--set-parent-flags path) + ;; if the draft file exists, remove it now. + (when (file-exists-p path) + (mu4e--server-remove docid))) + +(defun mu4e--send-harden-newlines () + "Set the hard property to all newlines." + (save-excursion + (goto-char (point-min)) + (while (search-forward "\n" nil t) + (put-text-property (1- (point)) (point) 'hard t)))) + +(defun mu4e--compose-before-send () + "Function called just before sending a message." + ;; Remove References: if In-Reply-To: is missing. + ;; This allows the user to effectively start a new message-thread by + ;; removing the In-Reply-To header. + (when (eq mu4e-compose-type 'reply) + (unless (message-field-value "In-Reply-To") + (message-remove-header "References"))) + (when use-hard-newlines + (mu4e--send-harden-newlines)) + ;; now handle what happens _after_ sending; typically, draft is gone and + ;; the sent message appears in sent. Update flags for related messages, + ;; i.e. for Forwarded ('Passed') and Replied messages, try to set the + ;; appropriate flag at the message forwarded or replied-to. + (add-hook 'message-sent-hook + (lambda () + (when-let ((fcc-path (message-field-value "Fcc"))) + (mu4e--set-parent-flags fcc-path) + ;; we end up with a ((buried) buffer here, visiting the + ;; fcc-path; not quite sure why. But let's get rid of it (#2681) + (when-let ((buf (find-buffer-visiting fcc-path))) + (kill-buffer buf)) + ;; remove draft + (when-let ((draft (buffer-file-name))) + (mu4e--server-remove draft)))) + nil t)) + +;; overrides for message-* functions +;; +;; mostly some magic because the message-reply/-forward/... functions want to +;; create and switch to buffer by themselves; but mu4e wants to control +;; when/where the buffers are shown so we subvert the message-functions and get +;; the buffer without display it. + +(defvar mu4e--message-buf nil + "The message buffer created by (overridden) message-* functions.") + +(defun mu4e--message-pop-to-buffer (name &optional _switch) + "Mu4e override for `message-pop-to-buffer'. +Creates a buffer NAME and returns it." + (set-buffer (get-buffer-create name)) + (erase-buffer) + (setq mu4e--message-buf (current-buffer))) + +(defun mu4e--message-is-yours-p () + "Mu4e's override for `message-is-yours-p'." + (seq-some (lambda (field) + (if-let ((recip (message-field-value field))) + (mu4e-personal-or-alternative-address-p + (car (mail-header-parse-address recip))))) + '("From" "Sender"))) + +(defmacro mu4e--validate-hidden-buffer (&rest body) + "Macro to evaluate BODY and asserts that it yields a valid buffer. +Where valid means that it is a live an non-active buffer. +Returns said buffer." + `(let ((buf (progn ,@body))) + (cl-assert (buffer-live-p buf)) + (cl-assert (not (eq buf (window-buffer (selected-window))))) + buf)) + +(defun mu4e--message-call (func &rest params) + "Call message/gnus functions from a mu4e-context. +E.g., functions such as `message-reply' or `message-forward', but +manipulate such that they do *not* switch to the created buffer, +but merely return it. + +FUNC is the function to call and PARAMS are its parameters. + +For replying/forwarding, this functions expects to be called +while in a buffer with the to-be-forwarded/replied-to message." + (let* ((message-this-is-mail t) + (message-generate-headers-first nil) + (message-newsreader mu4e-user-agent-string) + (message-mail-user-agent nil)) + (cl-letf + ;; `message-pop-to-buffer' attempts switching the visible buffer; + ;; instead, we manipulate it to _return_ the buffer. + (((symbol-function #'message-pop-to-buffer) + #'mu4e--message-pop-to-buffer) + ;; teach `message-is-yours-p' about how mu4e defines that + ((symbol-function #'message-is-yours-p) + #'mu4e--message-is-yours-p)) + ;; also turn off all the gnus crypto handling, we do that ourselves.. + (setq-local gnus-message-replysign nil + gnus-message-replyencrypt nil + gnus-message-replysignencrypted nil) + (setq mu4e--message-buf nil) + (apply func params)) + (mu4e--validate-hidden-buffer mu4e--message-buf))) +;; +;; make the draft buffer ready for use. +;; + +(defun mu4e--jump-to-a-reasonable-place () + "Jump to a reasonable place for writing an email." + (if (not (message-field-value "To")) + (message-goto-to) + (if (not (message-field-value "Subject")) + (message-goto-subject) + (pcase message-cite-reply-position + ((or 'above 'traditional) (message-goto-body)) + (_ (when (message-goto-signature) (forward-line -2))))))) + +(defvar mu4e-draft-hidden-headers + (append message-hidden-headers '("^User-agent:" "^Fcc:")) + "Message headers to hide when composing. +This is mu4e's version of `message-hidden-headers'.") + +(defun mu4e--prepare-draft (&optional parent) + "Get ready for message composition. +PARENT is the parent message, if any." + (unless (mu4e-running-p) (mu4e 'background)) ;; start if needed + (mu4e--context-autoswitch parent mu4e-compose-context-policy)) + +(defun mu4e--prepare-draft-headers (compose-type) + "Add extra headers for message based on COMPOSE-TYPE." + (message-generate-headers + (seq-filter #'identity ;; ensure needed headers are generated. + `(From Subject Date Message-ID + ,(when (memq compose-type '(reply forward)) 'References) + ,(when (eq compose-type 'reply) 'In-Reply-To) + ,(when message-newsreader 'User-Agent) + ,(when message-user-organization 'Organization))))) + +(defun mu4e--prepare-draft-buffer (compose-type parent) + "Prepare the current buffer as a draft-buffer. +COMPOSE-TYPE and PARENT are as in `mu4e--draft'." + (cl-assert (member compose-type '(reply forward edit new))) + (cl-assert (eq (if parent t nil) + (if (member compose-type '(reply forward)) t nil))) + ;; remember some variables, e.g for user hooks. These are permanent-local + ;; hence survive the mode-switch below (we do this so these useful vars are + ;; available in mode-hooks. + (setq-local + mu4e-compose-parent-message parent + mu4e-compose-type compose-type) + + ;; draft path + (unless (eq compose-type 'edit) + (set-visited-file-name ;; make it a draft file + (mu4e--draft-message-path (mu4e--draft-basename) parent))) + ;; fcc + (when-let ((fcc-path (mu4e--fcc-path (mu4e--draft-basename) parent))) + (message-add-header (concat "Fcc: " fcc-path "\n"))) + + (mu4e--prepare-draft-headers compose-type) + (mu4e--prepare-crypto parent compose-type) + ;; set the attachment dir to something more reasonable than the draft + ;; directory. + (setq default-directory (mu4e-determine-attachment-dir)) + (mu4e--draft-set-friendly-buffer-name) + + ;; now, switch to compose mode + (mu4e-compose-mode) + + ;; hide some internal headers + (let ((message-hidden-headers mu4e-draft-hidden-headers)) + (message-hide-headers)) + + ;; hooks + (add-hook 'before-save-hook #'mu4e--compose-before-save nil t) + (add-hook 'after-save-hook #'mu4e--compose-after-save nil t) + (add-hook 'message-send-hook #'mu4e--compose-before-send nil t) + (setq-local message-fcc-handler-function #'mu4e--fcc-handler) + + (mu4e--jump-to-a-reasonable-place) + + (set-buffer-modified-p nil) + (undo-boundary)) + +;; +;; mu4e-compose-pos-hook helpers + +(defvar mu4e--before-draft-window-config nil + "The window configuration just before creating the draft.") + +(defun mu4e-compose-post-restore-window-configuration() + "Function that perhaps restores the window configuration. +I.e. the configuration just before the draft buffer appeared. +This is for use in `mu4e-compose-post-hook'. +See `set-window-configuration' for further details." + (when mu4e--before-draft-window-config + ;;(message "RESTORE to %s" mu4e--before-draft-window-config) + (set-window-configuration mu4e--before-draft-window-config) + (setq mu4e--before-draft-window-config nil))) + +(defvar mu4e--draft-activation-frame nil + "Frame from which composition was activated. +Used internally for mu4e-compose-post-kill-frame.") + +(defun mu4e-compose-post-kill-frame () + "Function that perhaps kills the composition frame. +This is for use in `mu4e-compose-post-hook'." + (let ((msgframe (selected-frame))) + ;;(message "kill frame? %s %s" mu4e--draft-activation-frame msgframe) + (when (and (frame-live-p msgframe) + (not (eq mu4e--draft-activation-frame msgframe))) + (delete-frame msgframe)))) + +(defvar mu4e-message-post-action nil + "Runtime variable for use with `mu4e-compose-post-hook'. +It contains a symbol denoting the action that triggered the hook, +either `send', `exit', `kill' or `postpone'.") + +(defvar mu4e-compose-post-hook) +(defun mu4e--message-post-actions (trigger) + "Invoked after we're done with a message. + +I.e. this multiplexes the `message-(send|exit|kill|postpone)-actions'; +with the mu4e-message-post-action set accordingly." + (setq mu4e-message-post-action trigger) + (run-hooks 'mu4e-compose-post-hook)) + +(defun mu4e--prepare-post (&optional oldframe oldwindconf) + "Prepare the `mu4e-compose-post-hook` handling. + +Set up some message actions. In particular, handle closing frames +when we created it. OLDFRAME is the frame from which the +message-composition was triggered. OLDWINDCONF is the current +window configuration." + ;; remember current frame & window conf + (setq mu4e--draft-activation-frame oldframe + mu4e--before-draft-window-config oldwindconf) + + ;; make message's "post" hooks local, and multiplex them + (make-local-variable 'message-send-actions) + (make-local-variable 'message-postpone-actions) + (make-local-variable 'message-exit-actions) + (make-local-variable 'message-kill-actions) + + (push (lambda () (mu4e--message-post-actions 'send)) + message-send-actions) + (push (lambda () (mu4e--message-post-actions 'postpone)) + message-postpone-actions) + (push (lambda () (mu4e--message-post-actions 'exit)) + message-exit-actions) + (push (lambda () (mu4e--message-post-actions 'kill)) + message-kill-actions)) + +;; +;; creating drafts +;; + +(defun mu4e--draft (compose-type compose-func &optional parent) + "Create a new message draft. + +This is the central access point for creating new mail buffers; +when there's a parent message, use `mu4e--compose-with-parent'. + +COMPOSE-TYPE is the type of message to create. COMPOSE-FUNC is a +function that must return a buffer that satisfies +`mu4e--validate-hidden-buffer'. + +Optionally, PARENT is the message parent or nil. For compose-type +`reply' and `forward' we require a PARENT; for the other compose +it must be nil. + +After this, user is presented with a message composition buffer. + +Returns the new buffer." + (mu4e--prepare-draft parent) + ;; evaluate BODY; this must yield a hidden, live buffer. This is evaluated in + ;; a temp buffer with contains the parent-message, if any. if there's a + ;; PARENT, load the corresponding message into a temp-buffer before calling + ;; compose-func + (let ((draft-buffer) + (oldframe (selected-frame)) + (oldwinconf (current-window-configuration))) + (with-temp-buffer + ;; provide a temp buffer so the compose-func can do its thing + (setq draft-buffer (mu4e--validate-hidden-buffer (funcall compose-func))) + (with-current-buffer draft-buffer + ;; we have our basic buffer; turn it into a full mu4e composition + ;; buffer. + (mu4e--prepare-draft-buffer compose-type parent))) + ;; we're ready for composition; let's display it in the way user configured + ;; things: directly through display buffer (via pop-t or otherwise through + ;; mu4e-window. + (if (eq mu4e-compose-switch 'display-buffer) + (pop-to-buffer draft-buffer) + (mu4e-display-buffer draft-buffer 'do-select)) + ;; prepare possible message actions (such as cleaning-up) + (mu4e--prepare-post oldframe oldwinconf) + draft-buffer)) + +(defun mu4e--draft-with-parent (compose-type parent compose-func) + "Draft a message based on some parent message. +COMPOSE-TYPE, COMPOSE-FUNC and PARENT are as in `mu4e--draft', +but note the different order." + (mu4e--draft + compose-type + (lambda () + (let ( ;; only needed for Fwd. Gnus has a bad default. + (message-make-forward-subject-function + (list #'message-forward-subject-fwd))) + (insert (mu4e--decoded-message parent)) + ;; let's make sure we don't use message-reply-headers from + ;; some unrelated message. + (setq message-reply-headers nil) + (funcall compose-func))) + parent)) + +(provide 'mu4e-draft) diff --git a/mu4e/mu4e-folders.el b/mu4e/mu4e-folders.el new file mode 100644 index 0000000..330264a --- /dev/null +++ b/mu4e/mu4e-folders.el @@ -0,0 +1,302 @@ +;;; mu4e-folders.el --- Dealing with maildirs & folders -*- lexical-binding: t -*- + +;; Copyright (C) 2021-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Dealing with maildirs & folders + +;;; Code: +(require 'mu4e-helpers) +(require 'mu4e-context) +(require 'mu4e-server) + +;;; Customization +(defgroup mu4e-folders nil + "Special folders." + :group 'mu4e) + +(defcustom mu4e-drafts-folder "/drafts" + "Folder for draft messages, relative to the root maildir. +For instance, \"/drafts\". Instead of a string, may also be a +function that takes a message (a msg plist, see +`mu4e-message-field'), and returns a folder. Note, the message +parameter refers to the original message being replied to / being +forwarded / re-edited and is nil otherwise. `mu4e-drafts-folder' +is only evaluated once." + :type '(choice + (string :tag "Folder name") + (function :tag "Function return folder name")) + :group 'mu4e-folders) + +(defcustom mu4e-refile-folder "/archive" + "Folder for refiling messages, relative to the root maildir. +For instance \"/Archive\". Instead of a string, may also be a +function that takes a message (a msg plist, see +`mu4e-message-field'), and returns a folder. Note that the +message parameter refers to the message-at-point." + :type '(choice + (string :tag "Folder name") + (function :tag "Function return folder name")) + :group 'mu4e-folders) + +(defcustom mu4e-sent-folder "/sent" + "Folder for sent messages, relative to the root maildir. +For instance, \"/Sent Items\". Instead of a string, may also be a +function that takes a message (a msg plist, see +`mu4e-message-field'), and returns a folder. Note that the +message parameter refers to the original message being replied to +/ being forwarded / re-edited, and is nil otherwise." + :type '(choice + (string :tag "Folder name") + (function :tag "Function return folder name")) + :group 'mu4e-folders) + +(defcustom mu4e-trash-folder "/trash" + "Folder for trashed messages, relative to the root maildir. +For instance, \"/trash\". Instead of a string, may also be a +function that takes a message (a msg plist, see +`mu4e-message-field'), and returns a folder. When using +`mu4e-trash-folder' in the headers view (when marking messages +for trash). Note that the message parameter refers to the +message-at-point. When using it when composing a message (see +`mu4e-sent-messages-behavior'), this refers to the original +message being replied to / being forwarded / re-edited, and is +nil otherwise." + :type '(choice + (string :tag "Folder name") + (function :tag "Function return folder name")) + :group 'mu4e-folders) + +(defcustom mu4e-maildir-shortcuts nil + "A list of maildir shortcuts. +This makes it possible to quickly go to a particular +maildir (folder), or quickly moving messages to them (e.g., for +archiving or refiling). + +Each of the list elements is a plist with at least: +`:maildir' - the maildir for the shortcut (e.g. \"/archive\") +`:key' - the shortcut key. + +Optionally, you can add the following: +`:name' - name of the maildir to be displayed in main-view. +`:hide' - if t, the shortcut is hidden from the main-view and +speedbar. +`:hide-unread' - do not show the counts of unread/total number + of matches for the maildir in the main-view, and is implied +from `:hide'. + +For backward compatibility, an older form is recognized as well: + + (maildir . key), where MAILDIR is a maildir (such as +\"/archive/\"), and key is a single character. + +You can use these shortcuts in the headers and view buffers, for +example with `mu4e-mark-for-move-quick' (or \"m\", by default) or +`mu4e-jump-to-maildir' (or \"j\", by default), followed by the +designated shortcut character for the maildir. + +Unlike in search queries, folder names with spaces in them must +NOT be quoted, since mu4e does this for you." + :type '(choice + (alist :key-type (string :tag "Maildir") + :value-type character + :tag "Alist (old format)") + (repeat (plist + :key-type (choice (const :tag "Maildir" :maildir) + (const :tag "Shortcut" :key) + (const :tag "Name of maildir" :name) + (const :tag "Hide from main view" :hide) + (const :tag "Do not count" :hide-unread)) + :tag "Plist (new format)"))) + :version "1.3.9" + :group 'mu4e-folders) + +(defcustom mu4e-maildir-initial-input "/" + "Initial input for `mu4e-completing-completing-read' function." + :type 'string + :group 'mu4e-folders) + +(defcustom mu4e-maildir-info-delimiter + (if (member system-type '(ms-dos windows-nt cygwin)) + ";" ":") + "Separator character between message identifier and flags. +It defaults to ':' on most platforms, except on Windows, where it +is not allowed and we use ';' for compatibility with mbsync, +offlineimap and other programs." + :type 'string + :group 'mu4e-folders) + +(defcustom mu4e-attachment-dir (expand-file-name "~/") + "Default directory for attaching and saving attachments. + +This can be either a string (a file system path), or a function +that takes a filename and the mime-type as arguments, and returns +the attachment dir. See Info node `(mu4e) Attachments' for +details. + +When this called for composing a message, both filename and +mime-type are nil." + :type 'directory + :group 'mu4e-folders + :safe 'stringp) + +(defvar mu4e-maildir-list nil + "Cached list of maildirs.") + + +(defun mu4e-maildir-shortcuts () + "Get `mu4e-maildir-shortcuts' in the (new) format. +Converts from the old format if needed." + (seq-map (lambda (item) ;; convert from old format? + (if (and (consp item) (not (consp (cdr item)))) + `(:maildir ,(car item) :key ,(cdr item)) + item)) + mu4e-maildir-shortcuts)) + +;; the standard folders can be functions too +(defun mu4e--get-folder (foldervar msg) + "Within the mu-context of MSG, get message folder FOLDERVAR. +If FOLDER is a string, return it, if it is a function, evaluate +this function with MSG as parameter which may be nil, and return +the result." + (unless (member foldervar + '(mu4e-sent-folder mu4e-drafts-folder + mu4e-trash-folder mu4e-refile-folder)) + (mu4e-error "Folder must be one of mu4e-(sent|drafts|trash|refile)-folder")) + ;; get the value with the vars for the relevants context let-bound + (with-mu4e-context-vars (mu4e-context-determine msg nil) + (let* ((folder (symbol-value foldervar)) + (val + (cond + ((stringp folder) folder) + ((functionp folder) (funcall folder msg)) + (t (mu4e-error "Unsupported type for %S" folder))))) + (or val (mu4e-error "%S evaluates to nil" foldervar))))) + +(defun mu4e-get-drafts-folder (&optional msg) + "Get the drafts folder, optionally based on MSG. +See `mu4e-drafts-folder'." (mu4e--get-folder 'mu4e-drafts-folder msg)) + +(defun mu4e-get-refile-folder (&optional msg) + "Get the folder for refiling, optionally based on MSG. +See `mu4e-refile-folder'." (mu4e--get-folder 'mu4e-refile-folder msg)) + +(defun mu4e-get-sent-folder (&optional msg) + "Get the sent folder, optionally based on MSG. +See `mu4e-sent-folder'." (mu4e--get-folder 'mu4e-sent-folder msg)) + +(defun mu4e-get-trash-folder (&optional msg) + "Get the trash folder, optionally based on MSG. +See `mu4e-trash-folder'." (mu4e--get-folder 'mu4e-trash-folder msg)) + +;;; Maildirs +(defun mu4e--guess-maildir (path) + "Guess the maildir for PATH, or nil if cannot find it." + (let ((idx (string-match (mu4e-root-maildir) path))) + (when (and idx (zerop idx)) + (replace-regexp-in-string + (mu4e-root-maildir) + "" + (expand-file-name + (mu4e-join-paths path ".." "..")))))) + +(defun mu4e-create-maildir-maybe (dir) + "Offer to create maildir DIR if it does not exist yet. +Return t if it already exists or (after asking) an attempt has been +to create it; otherwise return nil." + (let ((seems-to-exist (file-directory-p dir))) + (when (or seems-to-exist + (yes-or-no-p (mu4e-format "%s does not exist yet. Create now?" dir))) + ;; even when the maildir already seems to exist, call mkdir for a deeper + ;; check. However only get an update when the maildir is totally new. + (mu4e--server-mkdir dir (not seems-to-exist)) + t))) + +(defun mu4e-get-maildirs () + "Get maildirs under `mu4e-maildir'." + mu4e-maildir-list) + +(defun mu4e-ask-maildir (prompt) + "Ask the user for a maildir (using PROMPT). + +If the special shortcut \"o\" (for _o_ther) is used, or +if (mu4e-maildir-shortcuts) evaluates to nil, let user choose +from all maildirs under `mu4e-maildir'." + (let* ((options + (seq-map (lambda (md) + (cons + (format "%c%s" (plist-get md :key) + (or (plist-get md :name) + (plist-get md :maildir))) + (plist-get md :maildir))) + (mu4e-filter-single-key (mu4e-maildir-shortcuts)))) + (response + (if (not options) + 'other + (mu4e-read-option prompt + (append options + '(("oOther..." . other))))))) + (substring-no-properties + (if (eq response 'other) + (progn + (funcall mu4e-completing-read-function prompt + (mu4e-get-maildirs) nil nil + mu4e-maildir-initial-input)) + response)))) + +(defun mu4e-ask-maildir-check-exists (prompt) + "Like `mu4e-ask-maildir', PROMPT for existence of the maildir. +Offer to create it if it does not exist yet." + (let* ((mdir (mu4e-ask-maildir prompt)) + (fullpath (mu4e-join-paths (mu4e-root-maildir) mdir))) + (unless (file-directory-p fullpath) + (and (yes-or-no-p + (mu4e-format "%s does not exist. Create now?" fullpath)) + (mu4e--server-mkdir fullpath))) + mdir)) + +;; mu4e-attachment-dir is either a string or a function that takes a +;; filename and the mime-type as argument, either (or both) which can +;; be nil + +(defun mu4e-determine-attachment-dir (&optional fname mimetype) + "Get the target-directory for attachments. + +This is based on the variable `mu4e-attachment-dir', which is either: +- if is a string, used it as-is +- a function taking two string parameters, both of which can be nil: + (1) a filename or a URL + (2) a mime-type (such as \"text/plain\"." + (let ((dir + (cond + ((stringp mu4e-attachment-dir) + mu4e-attachment-dir) + ((functionp mu4e-attachment-dir) + (funcall mu4e-attachment-dir fname mimetype)) + (t + (mu4e-error "Unsupported type for mu4e-attachment-dir" ))))) + (if dir + (expand-file-name dir) + (mu4e-error "Mu4e-attachment-dir evaluates to nil")))) + +(provide 'mu4e-folders) +;;; mu4e-folders.el ends here diff --git a/mu4e/mu4e-headers.el b/mu4e/mu4e-headers.el new file mode 100644 index 0000000..11d1dea --- /dev/null +++ b/mu4e/mu4e-headers.el @@ -0,0 +1,1648 @@ +;;; mu4e-headers.el --- Message headers -*- lexical-binding: t; coding:utf-8 -*- + +;; Copyright (C) 2011-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; In this file are function related mu4e-headers-mode, to creating the list of +;; one-line descriptions of emails, aka 'headers' (not to be confused with +;; headers like 'To:' or 'Subject:') + +;;; Code: + +(require 'cl-lib) +(require 'fringe) +(require 'hl-line) +(require 'mailcap) +(require 'mule-util) ;; seems _some_ people need this for + ;; truncate-string-ellipsis + +(require 'mu4e-update) + + ;; utility functions +(require 'mu4e-server) +(require 'mu4e-vars) +(require 'mu4e-mark) +(require 'mu4e-context) +(require 'mu4e-contacts) +(require 'mu4e-search) +(require 'mu4e-compose) +(require 'mu4e-actions) +(require 'mu4e-message) +(require 'mu4e-lists) +(require 'mu4e-update) +(require 'mu4e-folders) +(require 'mu4e-thread) + +(declare-function mu4e-view "mu4e-view") +(declare-function mu4e--main-view "mu4e-main") + + +;;; Configuration + +(defgroup mu4e-headers nil + "Settings for the headers view." + :group 'mu4e) + +(defcustom mu4e-headers-fields + '( (:human-date . 12) + (:flags . 6) + (:mailing-list . 10) + (:from . 22) + (:subject . nil)) + "A list of header fields to show in the headers buffer. +Each element has the form (HEADER . WIDTH), where HEADER is one of +the available headers (see `mu4e-header-info') and WIDTH is the +respective width in characters. + +A width of nil means \"unrestricted\", and this is best reserved +for the rightmost (last) field. Note that emacs may become very +slow with excessively long lines (1000s of characters), so if you +regularly get such messages, you want to avoid fields with nil +altogether." + :type `(repeat (cons (choice + ,@(mapcar (lambda (h) + (list 'const :tag + (plist-get (cdr h) :help) + (car h))) + mu4e-header-info) + (restricted-sexp + :tag "User-specified header" + :match-alternatives (mu4e--headers-header-p))) + (choice (integer :tag "width") + (const :tag "unrestricted width" nil)))) + :group 'mu4e-headers) + +(defun mu4e--headers-header-p (symbol) + "Is symbol a valid mu4e header? +This means its either one of the build-in or user-specified headers." + (assoc symbol (append mu4e-header-info mu4e-header-info-custom))) + +(defcustom mu4e-headers-date-format "%x" + "Date format to use in the headers view. +In the format of `format-time-string'." + :type 'string + :group 'mu4e-headers) + +(defcustom mu4e-headers-time-format "%X" + "Time format to use in the headers view. +In the format of `format-time-string'." + :type 'string + :group 'mu4e-headers) + +(defcustom mu4e-headers-long-date-format "%c" + "Date format to use in the headers view tooltip. +In the format of `format-time-string'." + :type 'string + :group 'mu4e-headers) + +(defcustom mu4e-headers-precise-alignment nil + "When set, use precise (but relatively slow) alignment for columns. +By default, do it in a slightly inaccurate but faster way. To get +an idea about the difference, In some tests, the rendering time +was around 5.8 ms per messages for precise alignment, versus 3.3 +for non-precise aligment (for 445 messages)." + :type 'boolean + :group 'mu4e-headers) + +(defcustom mu4e-headers-auto-update t + "Whether to automatically update the current headers buffer if an +indexing operation showed changes." + :type 'boolean + :group 'mu4e-headers) + +(defcustom mu4e-headers-advance-after-mark t + "With this option set to non-nil, automatically advance to the +next mail after marking a message in header view." + :type 'boolean + :group 'mu4e-headers) + + +(defcustom mu4e-headers-visible-flags + '(draft flagged new passed replied trashed attach encrypted signed + list personal) + "An ordered list of flags to show in the headers buffer. +Each element is a symbol in the list. + +By default, we leave out `unread' and `seen', since those are +mostly covered by `new', and the display gets cluttered otherwise." + :type '(set + (const :tag "Draft" draft) + (const :tag "Flagged" flagged) + (const :tag "New" new) + (const :tag "Passed" passed) + (const :tag "Replied" replied) + (const :tag "Seen" seen) + (const :tag "Trashed" trashed) + (const :tag "Attach" attach) + (const :tag "Encrypted" encrypted) + (const :tag "Signed" signed) + (const :tag "List" list) + (const :tag "Personal" personal) + (const :tag "Calendar" calendar)) + :group 'mu4e-headers) + +(defcustom mu4e-headers-found-hook nil + "Hook run just *after* all of the headers for the last search +query have been received and are displayed." + :type 'hook + :group 'mu4e-headers) + +;;; Public variables +(defcustom mu4e-headers-from-or-to-prefix '("" . "To ") + "Prefix for the :from-or-to field when it is showing, + respectively, From: or To:. It is a cons cell with the car + element being the From: prefix, the cdr element the To: prefix." + :type '(cons string string) + :group 'mu4e-headers) + +;;;; Fancy marks + +;; marks for headers of the form; each is a cons-cell (basic . fancy) +;; each of which is basic ascii char and something fancy, respectively +;; by default, we some conservative marks, even when 'fancy' +;; so they're less likely to break if people don't have certain fonts. +;; However, if you want to be really 'fancy', you could use something like +;; the following; esp. with a newer Emacs with color-icon support. +;; (setq +;; mu4e-headers-draft-mark '("D" . "💈") +;; mu4e-headers-flagged-mark '("F" . "📍") +;; mu4e-headers-new-mark '("N" . "🔥") +;; mu4e-headers-passed-mark '("P" . "❯") +;; mu4e-headers-replied-mark '("R" . "❮") +;; mu4e-headers-seen-mark '("S" . "☑") +;; mu4e-headers-trashed-mark '("T" . "💀") +;; mu4e-headers-attach-mark '("a" . "📎") +;; mu4e-headers-encrypted-mark '("x" . "🔒") +;; mu4e-headers-signed-mark '("s" . "🔑") +;; mu4e-headers-unread-mark '("u" . "⎕") +;; mu4e-headers-list-mark '("l" . "🔈") +;; mu4e-headers-personal-mark '("p" . "👨") +;; mu4e-headers-calendar-mark '("c" . "📅")) + + +(defvar mu4e-headers-draft-mark '("D" . "⚒") "Draft.") +(defvar mu4e-headers-flagged-mark '("F" . "✚") "Flagged.") +(defvar mu4e-headers-new-mark '("N" . "✱") "New.") +(defvar mu4e-headers-passed-mark '("P" . "❯") "Passed (fwd).") +(defvar mu4e-headers-replied-mark '("R" . "❮") "Replied.") +(defvar mu4e-headers-seen-mark '("S" . "✔") "Seen.") +(defvar mu4e-headers-trashed-mark '("T" . "⏚") "Trashed.") +(defvar mu4e-headers-attach-mark '("a" . "⚓") "W/ attachments.") +(defvar mu4e-headers-encrypted-mark '("x" . "⚴") "Encrypted.") +(defvar mu4e-headers-signed-mark '("s" . "☡") "Signed.") +(defvar mu4e-headers-unread-mark '("u" . "⎕") "Unread.") +(defvar mu4e-headers-list-mark '("l" . "Ⓛ") "Mailing list.") +(defvar mu4e-headers-personal-mark '("p" . "Ⓟ") "Personal.") +(defvar mu4e-headers-calendar-mark '("c" . "Ⓒ") "Calendar invitation.") + + +;;;; Graph drawing + +(defvar mu4e-headers-thread-mark-as-orphan 'first + "Define which messages should be prefixed with the orphan mark. +`all' marks all the messages without a parent as orphan, `first' +only marks the first message in the thread.") + +(defvar mu4e-headers-thread-root-prefix '("* " . "□ ") + "Prefix for root messages.") +(defvar mu4e-headers-thread-child-prefix '("|>" . "│ ") + "Prefix for messages in sub threads that do have a following sibling.") +(defvar mu4e-headers-thread-first-child-prefix '("o " . "⚬ ") + "Prefix for the first child messages in a sub thread.") +(defvar mu4e-headers-thread-last-child-prefix '("L" . "└ ") + "Prefix for messages in sub threads that do not have a following sibling.") +(defvar mu4e-headers-thread-connection-prefix '("|" . "│ ") + "Prefix to connect sibling messages that do not follow each other. +Must have the same length as `mu4e-headers-thread-blank-prefix'.") +(defvar mu4e-headers-thread-blank-prefix '(" " . " ") + "Prefix to separate non connected messages. +Must have the same length as `mu4e-headers-thread-connection-prefix'.") +(defvar mu4e-headers-thread-orphan-prefix '("<>" . "♢ ") + "Prefix for orphan messages with siblings.") +(defvar mu4e-headers-thread-single-orphan-prefix '("<>" . "♢ ") + "Prefix for orphan messages with no siblings.") +(defvar mu4e-headers-thread-duplicate-prefix '("=" . "≡ ") + "Prefix for duplicate messages.") + +;;;; Various + +(defcustom mu4e-headers-actions + '( ("capture message" . mu4e-action-capture-message) + ("browse online archive" . mu4e-action-browse-list-archive) + ("show this thread" . mu4e-action-show-thread)) + "List of actions to perform on messages in the headers list. +The actions are cons-cells of the form (NAME . FUNC) where: +* NAME is the name of the action (e.g. \"Count lines\") +* FUNC is a function which receives a message plist as an argument. + +The first character of NAME is used as the shortcut." + :group 'mu4e-headers + :type '(alist :key-type string :value-type function)) + +(defvar mu4e-headers-custom-markers + '(("Older than" + (lambda (msg date) (time-less-p (mu4e-msg-field msg :date) date)) + (lambda () (mu4e-get-time-date "Match messages before: "))) + ("Newer than" + (lambda (msg date) (time-less-p date (mu4e-msg-field msg :date))) + (lambda () (mu4e-get-time-date "Match messages after: "))) + ("Bigger than" + (lambda (msg bytes) (> (mu4e-msg-field msg :size) (* 1024 bytes))) + (lambda () (read-number "Match messages bigger than (Kbytes): ")))) + "List of custom markers -- functions to mark message that match +some custom function. Each of the list members has the following +format: + (NAME PREDICATE-FUNC PARAM-FUNC) +* NAME is the name of the predicate function, and the first +character is the shortcut (so keep those unique). +* PREDICATE-FUNC is a function that takes two parameters, MSG +and (optionally) PARAM, and should return non-nil when there's a +match. +* PARAM-FUNC is function that is evaluated once, and its value is +then passed to PREDICATE-FUNC as PARAM. This is useful for +getting user-input.") +;;; Internal variables/constants + +;; docid cookies +(defconst mu4e~headers-docid-pre "\376" + "Each header starts (invisibly) with the `mu4e~headers-docid-pre', +followed by the docid, followed by `mu4e~headers-docid-post'.") +(defconst mu4e~headers-docid-post "\377" + "Each header starts (invisibly) with the `mu4e~headers-docid-pre', +followed by the docid, followed by `mu4e~headers-docid-post'.") + + +(defvar mu4e~headers-search-start nil) +(defvar mu4e~headers-render-start nil) +(defvar mu4e~headers-render-time nil) + +(defvar mu4e-headers-report-render-time nil + "If non-nil, report on the time it took to render the messages. +This is mostly useful for profiling.") + +(defvar mu4e~headers-hidden 0 + "Number of headers hidden due to `mu4e-headers-hide-predicate'.") + + +;;; Clear + +(defun mu4e~headers-clear (&optional text) + "Clear the headers buffer and related data structures. +Optionally, show TEXT." + (when (buffer-live-p (mu4e-get-headers-buffer)) + (setq mu4e~headers-render-start (float-time) + mu4e~headers-hidden 0) + (let ((inhibit-read-only t)) + (with-current-buffer (mu4e-get-headers-buffer) + (mu4e--mark-clear) + (remove-overlays) + (erase-buffer) + (when text + (goto-char (point-min)) + (insert (propertize text 'face 'mu4e-system-face 'intangible t))))))) + +;;; Misc + +(defun mu4e~headers-contact-str (contacts) + "Turn the list of contacts CONTACTS (with elements (NAME . EMAIL) +into a string." + (mapconcat + (lambda (contact) + (let ((name (mu4e-contact-name contact)) + (email (mu4e-contact-email contact))) + (or name email "?"))) contacts ", ")) + +(defun mu4e~headers-thread-prefix-map (type) + "Return the thread prefix based on the symbol TYPE." + (let ((get-prefix + (lambda (cell) + (if mu4e-use-fancy-chars (cdr cell) (car cell))))) + (propertize + (cl-case type + (child (funcall get-prefix mu4e-headers-thread-child-prefix)) + (first-child (funcall get-prefix mu4e-headers-thread-first-child-prefix)) + (last-child (funcall get-prefix mu4e-headers-thread-last-child-prefix)) + (connection (funcall get-prefix mu4e-headers-thread-connection-prefix)) + (blank (funcall get-prefix mu4e-headers-thread-blank-prefix)) + (orphan (funcall get-prefix mu4e-headers-thread-orphan-prefix)) + (single-orphan (funcall get-prefix + mu4e-headers-thread-single-orphan-prefix)) + (duplicate (funcall get-prefix mu4e-headers-thread-duplicate-prefix)) + (t "?")) + 'face 'mu4e-thread-fold-face))) + + +;; headers in the buffer are prefixed by an invisible string with the docid +;; followed by an EOT ('end-of-transmission', \004, ^D) non-printable ascii +;; character. this string also has a text-property with the docid. the former +;; is used for quickly finding a certain header, the latter for retrieving the +;; docid at point without string matching etc. + +(defun mu4e~headers-docid-pos (docid) + "Return the pos of the beginning of the line with the header with +docid DOCID, or nil if it cannot be found." + (let ((pos)) + (save-excursion + (setq pos (mu4e~headers-goto-docid docid))) + pos)) + +(defun mu4e~headers-docid-cookie (docid) + "Create an invisible string containing DOCID; this is to be used +at the beginning of lines to identify headers." + (propertize (format "%s%d%s" + mu4e~headers-docid-pre docid mu4e~headers-docid-post) + 'docid docid 'invisible t));; + +(defun mu4e~headers-docid-at-point (&optional point) + "Get the docid for the header at POINT, or at current (point) if +nil. Returns the docid, or nil if there is none." + (save-excursion + (when point + (goto-char point)) + (get-text-property (line-beginning-position) 'docid))) + + + +(defun mu4e~headers-goto-docid (docid &optional to-mark) + "Go to the beginning of the line with the header with docid +DOCID, or nil if it cannot be found. If the optional TO-MARK is +non-nil, go to the point directly *after* the docid-cookie instead +of the beginning of the line." + (let ((oldpoint (point)) (newpoint)) + (goto-char (point-min)) + (setq newpoint + (search-forward (mu4e~headers-docid-cookie docid) nil t)) + (unless to-mark + (if (null newpoint) + (goto-char oldpoint) ;; not found; restore old pos + (progn + (beginning-of-line) ;; found, move to beginning of line + (setq newpoint (point))))) + newpoint)) ;; return the point, or nil if not found + +(defun mu4e~headers-field-for-docid (docid field) + "Get FIELD (a symbol, see `mu4e-headers-names') for the message +with DOCID which must be present in the headers buffer." + (save-excursion + (when (mu4e~headers-goto-docid docid) + (mu4e-message-field (mu4e-message-at-point) field)))) + + +;; In order to print a thread tree with all the message connections, +;; it's necessary to keep track of all sub levels that still have +;; following messages. For each level, mu4e~headers-thread-state keeps +;; the value t for a connection or nil otherwise. +(defvar-local mu4e~headers-thread-state '()) + +(defun mu4e~headers-thread-prefix (thread) + "Calculate the thread prefix based on thread info THREAD." + (when thread + (let* ((prefix "") + (level (plist-get thread :level)) + (has-child (plist-get thread :has-child)) + (first-child (plist-get thread :first-child)) + (last-child (plist-get thread :last-child)) + (orphan (plist-get thread :orphan)) + (single-orphan(and orphan first-child last-child)) + (duplicate (plist-get thread :duplicate))) + ;; Do not prefix root messages. + (if (= level 0) + (setq mu4e~headers-thread-state '())) + (if (> level 0) + (let* ((length (length mu4e~headers-thread-state)) + (padding (make-list (max 0 (- level length)) nil))) + ;; Trim and pad the state to ensure a message will + ;; always be shown with the correct indentation, even if + ;; a broken thread is returned. It's trimmed to level-1 + ;; because the current level has always an connection + ;; and it used a special formatting. + (setq mu4e~headers-thread-state + (cl-subseq (append mu4e~headers-thread-state padding) + 0 (- level 1))) + ;; Prepare the thread prefix. + (setq prefix + (concat + ;; Current mu4e~headers-thread-state, composed by + ;; connections or blanks. + (mapconcat + (lambda (s) + (mu4e~headers-thread-prefix-map + (if s 'connection 'blank))) + mu4e~headers-thread-state "") + ;; Current entry. + (mu4e~headers-thread-prefix-map + (if single-orphan 'single-orphan + (if (and orphan + (or first-child + (not (eq mu4e-headers-thread-mark-as-orphan + 'first)))) + 'orphan + (if last-child 'last-child + (if first-child 'first-child + 'child))))))))) + ;; If a new sub-thread will follow (has-child) and the current + ;; one is still not done (not last-child), then a new + ;; connection needs to be added to the tree-state. It's not + ;; necessary to a blank (nil), because padding will handle + ;; that. + (if (and has-child (not last-child)) + (setq mu4e~headers-thread-state + (append mu4e~headers-thread-state '(t)))) + ;; Return the thread prefix. + (format "%s%s" + prefix + (if duplicate + (mu4e~headers-thread-prefix-map 'duplicate) ""))))) + +(defun mu4e~headers-flags-str (flags) + "Get a display string for FLAGS. +Note that `mu4e-flags-to-string' is for internal use only; this +function is for display. (This difference is significant, since +internally, the Maildir spec determines what the flags look like, +while our display may be different)." + (or (mapconcat + (lambda (flag) + (when (member flag mu4e-headers-visible-flags) + (if-let* ((mark (intern-soft + (format "mu4e-headers-%s-mark" (symbol-name flag)))) + (cell (symbol-value mark))) + (if mu4e-use-fancy-chars (cdr cell) (car cell)) + ""))) + flags "") + "")) + +;;; Special headers + +(defun mu4e~headers-from-or-to (msg) + "Get the From: address from MSG if not one of user's; otherwise get To:. +When the from address for message MSG is one of the the user's +addresses, (as per `mu4e-personal-address-p'), show the To +address. Otherwise, show the From address, prefixed with the +appropriate `mu4e-headers-from-or-to-prefix'." + (let* ((from1 (car-safe (mu4e-message-field msg :from))) + (from1-addr (and from1 (mu4e-contact-email from1))) + (is-user (and from1-addr (mu4e-personal-address-p from1-addr)))) + (if is-user + (concat (cdr mu4e-headers-from-or-to-prefix) + (mu4e~headers-contact-str (mu4e-message-field msg :to))) + (concat (car mu4e-headers-from-or-to-prefix) + (mu4e~headers-contact-str (mu4e-message-field msg :from)))))) + +(defun mu4e~headers-human-date (msg) + "Show a \"human\" date for MSG. +If the date is today, show the time, otherwise, show the date. +The formats used for date and time are `mu4e-headers-date-format' +and `mu4e-headers-time-format'." + (let ((date (mu4e-msg-field msg :date))) + (if (equal date '(0 0 0)) + "None" + (let ((day1 (decode-time date)) + (day2 (decode-time (current-time)))) + (if (and + (eq (nth 3 day1) (nth 3 day2)) ;; day + (eq (nth 4 day1) (nth 4 day2)) ;; month + (eq (nth 5 day1) (nth 5 day2))) ;; year + (format-time-string mu4e-headers-time-format date) + (format-time-string mu4e-headers-date-format date)))))) + +(defun mu4e~headers-thread-subject (msg) + "Get the subject for MSG if it is the first one in a thread. +Otherwise, return the thread-prefix without the subject-text. In +other words, show the subject of a thread only once, similar to +e.g. \"mutt\"." + (let* ((tinfo (mu4e-message-field msg :meta)) + (subj (mu4e-msg-field msg :subject))) + (concat ;; prefix subject with a thread indicator + (mu4e~headers-thread-prefix tinfo) + (if (plist-get tinfo :thread-subject) + (truncate-string-to-width subj 600) "")))) + +(defun mu4e~headers-mailing-list (list) + "Get some identifier for the mailing list." + (if list + (propertize (mu4e-get-mailing-list-shortname list) 'help-echo list) + "")) + +(defsubst mu4e~headers-custom-field-value (msg field) + "Show some custom header field, or raise an error if it is not +found." + (let* ((item (or (assoc field mu4e-header-info-custom) + (mu4e-error "field %S not found" field))) + (func (or (plist-get (cdr-safe item) :function) + (mu4e-error "no :function defined for field %S %S" + field (cdr item))))) + (funcall func msg))) + +(defun mu4e~headers-field-value (msg field) + (let ((val (mu4e-message-field msg field))) + (cl-case field + (:subject + (concat ;; prefix subject with a thread indicator + (mu4e~headers-thread-prefix (mu4e-message-field msg :meta)) + ;; "["(plist-get (mu4e-message-field msg :meta) :path) "] " + ;; work-around: emacs' display gets really slow when lines are too long; + ;; so limit subject length to 600 + (truncate-string-to-width val 600))) + (:thread-subject ;; if not searching threads, fall back to :subject + (if mu4e-search-threads + (mu4e~headers-thread-subject msg) + (mu4e~headers-field-value msg :subject))) + ((:maildir :path :message-id) val) + ((:to :from :cc :bcc) (mu4e~headers-contact-str val)) + ;; if we (ie. `user-mail-address' is the 'From', show + ;; 'To', otherwise show From + (:from-or-to (mu4e~headers-from-or-to msg)) + (:date (format-time-string mu4e-headers-date-format val)) + (:list (or val "")) + (:mailing-list (mu4e~headers-mailing-list (mu4e-msg-field msg :list))) + (:human-date (propertize (mu4e~headers-human-date msg) + 'help-echo (format-time-string + mu4e-headers-long-date-format + (mu4e-msg-field msg :date)))) + (:flags (propertize (mu4e~headers-flags-str val) + 'help-echo (format "%S" val))) + (:tags (propertize (mapconcat 'identity val ", "))) + (:size (mu4e-display-size val)) + (t (mu4e~headers-custom-field-value msg field))))) + +(defsubst mu4e~headers-truncate-field-fast (val width) + "Truncate VAL to WIDTH. Fast and somewhat inaccurate." + (if width + (truncate-string-to-width val width 0 ?\s truncate-string-ellipsis) + val)) + +(defun mu4e~headers-truncate-field-precise (field val width) + "Return VAL truncated to one less than WIDTH, with a trailing +space propertized with a `display' text property which expands to + the correct column for display." + (when width + (let ((end-col (cl-loop for (f . w) in mu4e-headers-fields + sum w + until (equal f field)))) + (setq val (string-trim-right val)) + (if (> width (length val)) + (setq val (concat val " ")) + (setq val + (concat + (truncate-string-to-width val (1- width) 0 ?\s t) + " "))) + (put-text-property (1- (length val)) + (length val) + 'display + `(space . (:align-to ,end-col)) + val))) + val) + +(defsubst mu4e~headers-truncate-field (field val width) + "Truncate VAL to WIDTH." + (if mu4e-headers-precise-alignment + (mu4e~headers-truncate-field-precise field val width) + (mu4e~headers-truncate-field-fast val width))) + +(defsubst mu4e~headers-field-handler (f-w msg) + "Create a description of the field of MSG described by F-W." + (let* ((field (car f-w)) + (width (cdr f-w)) + (val (mu4e~headers-field-value msg field)) + (val (and val + (if width + (mu4e~headers-truncate-field field val width) + val)))) + val)) + +(defsubst mu4e~headers-apply-flags (msg fieldval) + "Adjust FIELDVAL's face property based on flags in MSG." + (let* ((flags (plist-get msg :flags)) + (meta (plist-get msg :meta)) + (face (cond + ((memq 'trashed flags) 'mu4e-trashed-face) + ((memq 'draft flags) 'mu4e-draft-face) + ((or (memq 'unread flags) (memq 'new flags)) + 'mu4e-unread-face) + ((memq 'flagged flags) 'mu4e-flagged-face) + ((plist-get meta :related) 'mu4e-related-face) + ((memq 'replied flags) 'mu4e-replied-face) + ((memq 'passed flags) 'mu4e-forwarded-face) + (t 'mu4e-header-face)))) + (add-face-text-property 0 (length fieldval) face t fieldval) + fieldval)) + +(defsubst mu4e~message-header-line (msg) + "Return a propertized description of message MSG suitable for +displaying in the header view." + (if (and mu4e-search-hide-enabled mu4e-search-hide-predicate + (funcall mu4e-search-hide-predicate msg)) + (progn + (cl-incf mu4e~headers-hidden) + nil) + (progn + (mu4e~headers-apply-flags + msg + (mapconcat (lambda (f-w) (mu4e~headers-field-handler f-w msg)) + mu4e-headers-fields " "))))) + +(defsubst mu4e~headers-insert-header (msg pos) + "Insert a header for MSG at point POS." + (when-let ((line (mu4e~message-header-line msg)) + (docid (plist-get msg :docid))) + (goto-char pos) + (insert + (propertize + (concat + (mu4e~headers-docid-cookie docid) + mu4e--mark-fringe line "\n") + 'docid docid 'msg msg)))) + +(defun mu4e~headers-remove-header (docid &optional ignore-missing) + "Remove header with DOCID at point. +When IGNORE-MISSING is non-nill, don't raise an error when the +docid is not found." + (with-current-buffer (mu4e-get-headers-buffer) + (if (mu4e~headers-goto-docid docid) + (let ((inhibit-read-only t)) + (delete-region (line-beginning-position) (line-beginning-position 2))) + (unless ignore-missing + (mu4e-error "Cannot find message with docid %S" docid))))) + + +;;; Handler functions + +;; next are a bunch of handler functions; those will be called from mu4e~proc in +;; response to output from the server process + +(defun mu4e~headers-view-handler (msg) + "Handler function for displaying a message." + (mu4e-view msg)) + +(defun mu4e~headers-view-this-message-p (docid) + "Is DOCID currently being viewed?" + (mu4e-get-view-buffers + (lambda (_) (eq docid (plist-get mu4e--view-message :docid))))) + +;; note: this function is very performance-sensitive +(defun mu4e~headers-append-handler (msglst) + "Append one-line descriptions of messages in MSGLIST. +Do this at the end of the headers-buffer." + (when (buffer-live-p (mu4e-get-headers-buffer)) + (with-current-buffer (mu4e-get-headers-buffer) + (save-excursion + (let ((inhibit-read-only t)) + (seq-do + (lambda (msg) + (mu4e~headers-insert-header msg (point-max))) + msglst)))))) + + +(defun mu4e~headers-update-handler (msg is-move maybe-view) + "Update handler, will be called when a message has been updated +in the database. This function will update the current list of +headers." + (when (buffer-live-p (mu4e-get-headers-buffer)) + (with-current-buffer (mu4e-get-headers-buffer) + (let* ((docid (mu4e-message-field msg :docid)) + (initial-message-at-point (mu4e~headers-docid-at-point)) + (initial-column (current-column)) + (inhibit-read-only t) + (point (mu4e~headers-docid-pos docid)) + (markinfo (gethash docid mu4e--mark-map))) + (when point ;; is the message present in this list? + + ;; if it's marked, unmark it now + (when (mu4e-mark-docid-marked-p docid) + (mu4e-mark-set 'unmark)) + + ;; re-use the thread info from the old one; this is needed because + ;; *update* messages don't have thread info by themselves (unlike + ;; search results) + ;; since we still have the search results, re-use + ;; those + (plist-put msg :meta + (mu4e~headers-field-for-docid docid :meta)) + + ;; first, remove the old one (otherwise, we'd have two headers with + ;; the same docid... + (mu4e~headers-remove-header docid t) + + ;; if we're actually viewing this message (in mu4e-view mode), we + ;; update it; that way, the flags can be updated, as well as the path + ;; (which is useful for viewing the raw message) + (when (and maybe-view (mu4e~headers-view-this-message-p docid)) + (save-excursion (mu4e-view msg))) + ;; now, if this update was about *moving* a message, we don't show it + ;; anymore (of course, we cannot be sure if the message really no + ;; longer matches the query, but this seem a good heuristic. if it + ;; was only a flag-change, show the message with its updated flags. + (unless is-move + (save-excursion + (mu4e~headers-insert-header msg point))) + + ;; restore the mark, if any. See #2076. + (when (and markinfo (mu4e~headers-goto-docid docid)) + (mu4e-mark-at-point (car markinfo) (cdr markinfo))) + + (if (and initial-message-at-point + (mu4e~headers-goto-docid initial-message-at-point)) + (progn + (move-to-column initial-column) + (mu4e~headers-highlight initial-message-at-point)) + ;; attempt to highlight the corresponding line and make it visible + (mu4e~headers-highlight docid)) + (run-hooks 'mu4e-message-changed-hook)))))) + +(defun mu4e~headers-remove-handler (docid) + "Remove handler, will be called when a message with DOCID has +been removed from the database. This function will hide the removed +message from the current list of headers. If the message is not +present, don't do anything." + (when (buffer-live-p (mu4e-get-headers-buffer)) + (mu4e~headers-remove-header docid t)) + ;; if we were viewing this message, close it now. + (when (and (mu4e~headers-view-this-message-p docid) + (buffer-live-p (mu4e-get-view-buffer))) + (let ((buf (mu4e-get-view-buffer))) + (mapc #'delete-window (get-buffer-window-list + buf nil t)) + (kill-buffer buf)))) + + + +;;; Performing queries (internal) +(defconst mu4e~search-message "Searching...") +(defconst mu4e~no-matches "No matching messages found") +(defconst mu4e~end-of-results "End of search results") + +(defvar mu4e--search-background nil + "Is this a background search? + If so, do not attempt to switch buffers. This variable is to be let-bound +to t before \"automatic\" searches.") + +(defun mu4e--search-execute (expr ignore-history) + "Search for query EXPR. + +Switch to the output buffer for the results. If IGNORE-HISTORY is +true, do *not* update the query history stack." + (let* ((buf (mu4e-get-headers-buffer nil t)) + (view-window mu4e~headers-view-win) + (inhibit-read-only t) + (rewritten-expr (funcall mu4e-query-rewrite-function expr)) + (maxnum (unless mu4e-search-full mu4e-search-results-limit))) + (with-current-buffer buf + ;; NOTE: this resets all buffer-local variables, including + ;; `mu4e~headers-view-win', which may have a live window if the + ;; headers buffer already exists when `mu4e-get-headers-buffer' + ;; is called. + (mu4e-headers-mode) + (setq mu4e~headers-view-win view-window) + (unless ignore-history + ;; save the old present query to the history list + (when mu4e--search-last-query + (mu4e--search-push-query mu4e--search-last-query 'past))) + (setq mu4e--search-last-query rewritten-expr) + (setq list-buffers-directory rewritten-expr) + (mu4e--modeline-update)) + + ;; when the buffer is already visible, select it; otherwise, + ;; switch to it. + (unless (get-buffer-window buf (if mu4e--search-background 0 nil)) + (mu4e-display-buffer buf t)) + (run-hook-with-args 'mu4e-search-hook expr) + (mu4e~headers-clear mu4e~search-message) + (setq mu4e~headers-search-start (float-time)) + (mu4e--server-find + rewritten-expr + mu4e-search-threads + mu4e-search-sort-field + mu4e-search-sort-direction + maxnum + mu4e-search-skip-duplicates + mu4e-search-include-related))) + +(defun mu4e~headers-benchmark-message (count) + "Get some report message for messaging search and rendering speed." + (if (and mu4e-headers-report-render-time + mu4e~headers-search-start + mu4e~headers-render-start + (> count 0)) + (let ((render-time-ms (* 1000(- (float-time) mu4e~headers-render-start))) + (search-time-ms (* 1000(- (float-time) mu4e~headers-search-start)))) + (format (concat + "; search: %0.1f ms (%0.2f ms/msg)" + "; render: %0.1f ms (%0.2f ms/msg)") + search-time-ms (/ search-time-ms count) + render-time-ms (/ render-time-ms count))) + "")) + +(defun mu4e~headers-found-handler (count) + "Create a one line description of the number of headers found +after the end of the search results." + (when (buffer-live-p (mu4e-get-headers-buffer)) + (with-current-buffer (mu4e-get-headers-buffer) + (save-excursion + (goto-char (point-max)) + (let ((inhibit-read-only t) + (str (if (zerop count) mu4e~no-matches mu4e~end-of-results)) + (msg (format "Found %d matching message%s; %d hidden%s" + count (if (= 1 count) "" "s") + mu4e~headers-hidden + (mu4e~headers-benchmark-message count)))) + + (insert (propertize str 'face 'mu4e-system-face 'intangible t)) + (unless (zerop count) + (mu4e-message "%s" msg)))) + + ;; if we need to jump to some specific message, do so now + (goto-char (point-min)) + (when mu4e--search-msgid-target + (if (eq (current-buffer) (window-buffer)) + (mu4e-headers-goto-message-id mu4e--search-msgid-target) + (let* ((pos (mu4e-headers-goto-message-id + mu4e--search-msgid-target))) + (when pos + (set-window-point (get-buffer-window nil t) pos))))) + (when (and mu4e--search-view-target (mu4e-message-at-point 'noerror)) + ;; view the message at point when there is one. + (mu4e-headers-view-message)) + (setq mu4e--search-view-target nil + mu4e--search-msgid-target nil) + (when (mu4e~headers-docid-at-point) + (mu4e~headers-highlight (mu4e~headers-docid-at-point))) + ;; maybe enable thread folding + (when mu4e-search-threads + (mu4e-thread-mode)))) + ;; run-hooks + (run-hooks 'mu4e-headers-found-hook)) + + +;;; Marking + +(defmacro mu4e~headers-defun-mark-for (mark) + "Define a function mu4e~headers-mark-MARK." + (let ((funcname (intern (format "mu4e-headers-mark-for-%s" mark))) + (docstring (format "Mark header at point with %s." mark))) + `(progn + (defun ,funcname () ,docstring + (interactive) + (mu4e-headers-mark-and-next ',mark)) + (put ',funcname 'definition-name ',mark)))) + +(mu4e~headers-defun-mark-for refile) +(mu4e~headers-defun-mark-for something) +(mu4e~headers-defun-mark-for delete) +(mu4e~headers-defun-mark-for trash) +(mu4e~headers-defun-mark-for flag) +(mu4e~headers-defun-mark-for move) +(mu4e~headers-defun-mark-for read) +(mu4e~headers-defun-mark-for unflag) +(mu4e~headers-defun-mark-for untrash) +(mu4e~headers-defun-mark-for unmark) +(mu4e~headers-defun-mark-for unread) +(mu4e~headers-defun-mark-for action) + +(declare-function mu4e-view-pipe "mu4e-view") + +(defvar mu4e-headers-mode-map + (let ((map (make-sparse-keymap))) + + (define-key map "q" #'mu4e~headers-quit-buffer) + (define-key map "g" #'mu4e-search-rerun) ;; for compatibility + + + (define-key map "%" #'mu4e-headers-mark-pattern) + (define-key map "t" #'mu4e-headers-mark-subthread) + (define-key map "T" #'mu4e-headers-mark-thread) + + (define-key map "," #'mu4e-sexp-at-point) + (define-key map ";" #'mu4e-context-switch) + + ;; navigation between messages + (define-key map "p" #'mu4e-headers-prev) + (define-key map "n" #'mu4e-headers-next) + (define-key map (kbd "<M-up>") #'mu4e-headers-prev) + (define-key map (kbd "<M-down>") #'mu4e-headers-next) + + (define-key map (kbd "[") #'mu4e-headers-prev-unread) + (define-key map (kbd "]") #'mu4e-headers-next-unread) + + (define-key map (kbd "{") #'mu4e-headers-prev-thread) + (define-key map (kbd "}") #'mu4e-headers-next-thread) + + ;; change the number of headers + (define-key map (kbd "C-+") #'mu4e-headers-split-view-grow) + (define-key map (kbd "C--") #'mu4e-headers-split-view-shrink) + (define-key map (kbd "<C-kp-add>") 'mu4e-headers-split-view-grow) + (define-key map (kbd "<C-kp-subtract>") + #'mu4e-headers-split-view-shrink) + + ;; switching to view mode (if it's visible) + (define-key map "y" #'mu4e-select-other-view) + + ;; marking/unmarking ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; + (define-key map (kbd "<backspace>") #'mu4e-headers-mark-for-trash) + (define-key map (kbd "d") #'mu4e-headers-mark-for-trash) + (define-key map (kbd "<delete>") #'mu4e-headers-mark-for-delete) + (define-key map (kbd "<deletechar>") #'mu4e-headers-mark-for-delete) + (define-key map (kbd "D") #'mu4e-headers-mark-for-delete) + (define-key map (kbd "m") #'mu4e-headers-mark-for-move) + (define-key map (kbd "r") #'mu4e-headers-mark-for-refile) + + (define-key map (kbd "?") #'mu4e-headers-mark-for-unread) + (define-key map (kbd "!") #'mu4e-headers-mark-for-read) + (define-key map (kbd "A") #'mu4e-headers-mark-for-action) + + (define-key map (kbd "u") #'mu4e-headers-mark-for-unmark) + (define-key map (kbd "+") #'mu4e-headers-mark-for-flag) + (define-key map (kbd "-") #'mu4e-headers-mark-for-unflag) + (define-key map (kbd "=") #'mu4e-headers-mark-for-untrash) + (define-key map (kbd "&") #'mu4e-headers-mark-custom) + + (define-key map (kbd "*") + #'mu4e-headers-mark-for-something) + (define-key map (kbd "<kp-multiply>") + #'mu4e-headers-mark-for-something) + (define-key map (kbd "<insertchar>") + #'mu4e-headers-mark-for-something) + (define-key map (kbd "<insert>") + #'mu4e-headers-mark-for-something) + + (define-key map (kbd "#") #'mu4e-mark-resolve-deferred-marks) + + (define-key map "U" #'mu4e-mark-unmark-all) + (define-key map "x" #'mu4e-mark-execute-all) + + (define-key map "a" #'mu4e-headers-action) + + ;; message composition + + (define-key map (kbd "RET") #'mu4e-headers-view-message) + (define-key map [mouse-2] #'mu4e-headers-view-message) + + (define-key map "$" #'mu4e-show-log) + (define-key map "H" #'mu4e-display-manual) + + (define-key map "|" #'mu4e-view-pipe) + map) + "Keymap for mu4e's headers mode.") + +(easy-menu-define mu4e-headers-mode-menu + mu4e-headers-mode-map "Menu for mu4e's headers-mode." + (append + '("Headers" ;;:visible mu4e-headers-mode + "--" + ["Previous" mu4e-headers-prev + :help "Move to previous header"] + ["Next" mu4e-headers-prev + :help "Move to next header"] + "--" + ["Mark for move" mu4e-headers-mark-for-move + :help "Mark message for move" + ]) + mu4e--compose-menu-items + mu4e--search-menu-items + mu4e--context-menu-items + '( + "--" + ["Quit" mu4e~headers-quit-buffer + :help "Quit the headers"] + ))) + +;;; Headers-mode and mode-map + +(defun mu4e~header-line-format () + "Get the format for the header line." + (let ((uparrow (if mu4e-use-fancy-chars " ▲" " ^")) + (downarrow (if mu4e-use-fancy-chars " ▼" " V"))) + (cons + (make-string + (+ mu4e--mark-fringe-len (floor (fringe-columns 'left t))) ?\s) + (mapcar + (lambda (item) + (let* (;; with threading enabled, we're necessarily sorting by date. + (sort-field (if mu4e-search-threads + :date mu4e-search-sort-field)) + (field (car item)) (width (cdr item)) + (info (cdr (assoc field + (append mu4e-header-info + mu4e-header-info-custom)))) + (sortable (plist-get info :sortable)) + ;; if sortable, it is either t (when field is sortable itself) + ;; or a symbol (if another field is used for sorting) + (this-field (when sortable (if (booleanp sortable) + field + sortable))) + (help (plist-get info :help)) + ;; triangle to mark the sorted-by column + (arrow + (when (and sortable (eq this-field sort-field)) + (if (eq mu4e-search-sort-direction 'descending) + downarrow + uparrow))) + (name (concat (plist-get info :shortname) arrow)) + (map (make-sparse-keymap))) + (when sortable + (define-key map [header-line mouse-1] + (lambda (&optional e) + ;; getting the field, inspired by + ;; `tabulated-list-col-sort' + (interactive "e") + (let* ((obj (posn-object (event-start e))) + (field + (and obj + (get-text-property 0 'field (car obj))))) + ;; "t": if we're already sorted by field, the + ;; sort-order is changed + (mu4e-search-change-sorting field t))))) + (concat + (propertize + (if width + (truncate-string-to-width + name width 0 ?\s truncate-string-ellipsis) + name) + 'face (when arrow 'bold) + 'help-echo help + 'mouse-face (when sortable 'highlight) + 'keymap (when sortable map) + 'field field) " "))) + mu4e-headers-fields)))) + +(defun mu4e~headers-maybe-auto-update () + "Update the current headers buffer after indexing has brought +some changes, `mu4e-headers-auto-update' is non-nil and there is +no user-interaction ongoing." + (when (and mu4e-headers-auto-update ;; must be set + mu4e-index-update-status + (not (mu4e-get-view-buffer)) ;; not when viewing a message + (not (zerop (plist-get mu4e-index-update-status :updated))) + ;; NOTE: `mu4e-mark-marks-num' can return nil. Is that intended? + (zerop (or (mu4e-mark-marks-num) 0)) ;; non active marks + (not (active-minibuffer-window))) ;; no user input only + ;; rerun search if there's a live window with search results; + ;; otherwise we'd trigger a headers view from out of nowhere. + (when (and (buffer-live-p (mu4e-get-headers-buffer)) + (window-live-p (get-buffer-window (mu4e-get-headers-buffer) t))) + (let ((mu4e--search-background t)) + (mu4e-search-rerun))))) + +(defcustom mu4e-headers-eldoc-format "“%s” from %f on %d" + "Format for the `eldoc' string for the current message in the headers buffer. +The following specs are supported: +- %s: the message Subject +- %f: the message From +- %t: the message To +- %c: the message Cc +- %d: the message Date +- %p: the message priority +- %m: the maildir containing the message +- %F: the message’s flags +- %M: the Message-Id" + :type 'string + :group 'mu4e-headers) + +(defun mu4e-headers-eldoc-function (&rest _args) + (let ((msg (get-text-property (point) 'msg))) + (when msg + (format-spec + mu4e-headers-eldoc-format + `((?s . ,(mu4e-message-field msg :subject)) + (?f . ,(mu4e~headers-contact-str (mu4e-message-field msg :from))) + (?t . ,(mu4e~headers-contact-str (mu4e-message-field msg :to))) + (?c . ,(mu4e~headers-contact-str (mu4e-message-field msg :cc))) + (?d . ,(mu4e~headers-human-date msg)) + (?p . ,(mu4e-message-field msg :priority)) + (?m . ,(mu4e-message-field msg :maildir)) + (?F . ,(mu4e-message-field msg :flags)) + (?M . ,(mu4e-message-field msg :message-id))))))) + +(define-derived-mode mu4e-headers-mode special-mode + "mu4e:headers" + "Major mode for displaying mu4e search results. +\\{mu4e-headers-mode-map}." + (use-local-map mu4e-headers-mode-map) + (make-local-variable 'mu4e~headers-proc) + (make-local-variable 'mu4e~highlighted-docid) + (set (make-local-variable 'hl-line-face) 'mu4e-header-highlight-face) + + ;; Eldoc support + (when (and (featurep 'eldoc) mu4e-eldoc-support) + (if (boundp 'eldoc-documentation-functions) + ;; Emacs 28 or newer + (add-hook 'eldoc-documentation-functions + #'mu4e-headers-eldoc-function nil t) + ;; Emacs 27 or older + (add-function :before-until (local 'eldoc-documentation-function) + #'mu4e-headers-eldoc-function))) + + ;; support bookmarks. + (set (make-local-variable 'bookmark-make-record-function) + 'mu4e--make-bookmark-record) + ;; maybe update the current headers upon indexing changes + (add-hook 'mu4e-index-updated-hook #'mu4e~headers-maybe-auto-update) + (setq + truncate-lines t + buffer-undo-list t ;; don't record undo information + overwrite-mode nil + header-line-format (mu4e~header-line-format)) + + (mu4e--mark-initialize) ;; initialize the marking subsystem + (mu4e-context-minor-mode) + (mu4e-update-minor-mode) + (mu4e-search-minor-mode) + (mu4e-compose-minor-mode) + (hl-line-mode 1) + + (mu4e--modeline-register #'mu4e--search-modeline-item) + (mu4e--modeline-update)) + +;;; Highlighting + +(defvar mu4e~highlighted-docid nil + "The highlighted docid") + +(defun mu4e~headers-highlight (docid) + "Highlight the header with DOCID, or do nothing if it's not found. +Also, unhighlight any previously highlighted headers." + (with-current-buffer (mu4e-get-headers-buffer) + (save-excursion + ;; first, unhighlight the previously highlighted docid, if any + (when (and docid mu4e~highlighted-docid + (mu4e~headers-goto-docid mu4e~highlighted-docid)) + (hl-line-unhighlight)) + ;; now, highlight the new one + (when (mu4e~headers-goto-docid docid) + (hl-line-highlight))) + (setq mu4e~highlighted-docid docid))) + +;;; Misc 2 + +(defun mu4e~headers-select-window () + "When there is a visible window for the headers buffer, make sure +to select it. This is needed when adding new headers, otherwise +adding a lot of new headers looks really choppy." + (let ((win (get-buffer-window (mu4e-get-headers-buffer)))) + (when win (select-window win)))) + +(defun mu4e-headers-goto-message-id (msgid) + "Go to the next message with message-id MSGID. Return the +message plist, or nil if not found." + (mu4e-headers-find-if + (lambda (msg) + (let ((this-msgid (mu4e-message-field msg :message-id))) + (when (and this-msgid (string= msgid this-msgid)) + msg))))) + +;;; Marking 2 + +(defun mu4e~headers-mark (docid mark) + "(Visually) mark the header for DOCID with character MARK." + (with-current-buffer (mu4e-get-headers-buffer) + (let ((inhibit-read-only t) (oldpoint (point))) + (unless (mu4e~headers-goto-docid docid) + (mu4e-error "Cannot find message with docid %S" docid)) + ;; now, we're at the beginning of the header, looking at + ;; <docid>\004 + ;; (which is invisible). jump past that… + (unless (re-search-forward mu4e~headers-docid-post nil t) + (mu4e-error "Cannot find the `mu4e~headers-docid-post' separator")) + + ;; clear old marks, and add the new ones. + (let ((msg (get-text-property (point) 'msg))) + (delete-char mu4e--mark-fringe-len) + (insert (propertize + (format mu4e--mark-fringe-format mark) + 'face 'mu4e-header-marks-face + 'docid docid + 'msg msg))) + (goto-char oldpoint)))) + + +;;; Queries & searching + +;;; Search-based marking + +(defun mu4e-headers-for-each (func) + "Call FUNC for each header, moving point to the header. +FUNC receives one argument, the message s-expression for the +corresponding header." + (save-excursion + (goto-char (point-min)) + (while (search-forward mu4e~headers-docid-pre nil t) + ;; not really sure why we need to jump to bol; we do need to, otherwise we + ;; miss lines sometimes... + (let ((msg (get-text-property (line-beginning-position) 'msg))) + (when msg + (funcall func msg)))))) + + +(defun mu4e-headers-find-if (func &optional backward) + "Move to the header for which FUNC returns non-`nil'. +if BACKWARD is non-nil, search backwards. + + FUNC receives one argument, the message s-expression for the +corresponding header. If BACKWARD is non-`nil', search backwards. +Returns the new position, or `nil' if nothing was found. If you +want to exclude matches for the current message, you can use +`mu4e-headers-find-if-next'. + +Return the found position or nil if not found." + (let ((pos) + (search-func (if backward 'search-backward 'search-forward))) + (save-excursion + (while (and (null pos) + (funcall search-func mu4e~headers-docid-pre nil t)) + ;; not really sure why we need to jump to bol; we do need to, otherwise + ;; we miss lines sometimes... + (let ((msg (get-text-property (line-beginning-position) 'msg))) + (when (and msg (funcall func msg)) + (setq pos (point)))))) + (when pos + (goto-char pos)) + pos)) + +(defun mu4e-headers-find-if-next (func &optional backwards) + "Like `mu4e-headers-find-if', but do not match the current header. +Move to the next or (if BACKWARDS is non-`nil') header for which FUNC +returns non-`nil', starting from the current position." + (let ((pos)) + (save-excursion + (if backwards (beginning-of-line) (end-of-line)) + (setq pos (mu4e-headers-find-if func backwards))) + (when pos (goto-char pos)))) + +(defvar mu4e~headers-regexp-hist nil + "History list of regexps used.") + +(defun mu4e-headers-mark-for-each-if (markpair mark-pred &optional param) + "Mark all headers for which predicate function MARK-PRED returns +non-nil with MARKPAIR. MARK-PRED is function that receives two +arguments, MSG (the message at point) and PARAM (a user-specified +parameter). MARKPAIR is a cell (MARK . TARGET); see +`mu4e-mark-at-point' for details about marks." + (mu4e-headers-for-each + (lambda (msg) + (when (funcall mark-pred msg param) + (mu4e-mark-at-point (car markpair) (cdr markpair)))))) + +(defun mu4e-headers-mark-pattern () + "Ask user for a kind of mark (move, delete etc.), a field to +match and a regular expression to match with. Then, mark all +matching messages with that mark." + (interactive) + (let ((markpair (mu4e--mark-get-markpair "Mark matched messages with: " t)) + (field (mu4e-read-option "Field to match: " + '( ("subject" . :subject) + ("from" . :from) + ("to" . :to) + ("cc" . :cc) + ("bcc" . :bcc) + ("list" . :list)))) + (pattern (read-string + (mu4e-format "Regexp:") + nil 'mu4e~headers-regexp-hist))) + (mu4e-headers-mark-for-each-if + markpair + (lambda (msg _param) + (let* ((value (mu4e-msg-field msg field))) + (if (member field '(:to :from :cc :bcc :reply-to)) + (cl-find-if (lambda (contact) + (let ((name (mu4e-contact-name contact)) + (email (mu4e-contact-email contact))) + (or (and name (string-match pattern name)) + (and email (string-match pattern email))))) + value) + (string-match pattern (or value "")))))))) + +(defun mu4e-headers-mark-custom () + "Mark messages based on a user-provided predicate function." + (interactive) + (let* ((pred (mu4e-read-option "Match function: " + mu4e-headers-custom-markers)) + (param (when (cdr pred) (eval (cdr pred)))) + (markpair (mu4e--mark-get-markpair "Mark matched messages with: " t))) + (mu4e-headers-mark-for-each-if markpair (car pred) param))) + +(defun mu4e~headers-get-thread-info (msg what) + "Get WHAT (a symbol, either path or thread-id) for MSG." + (let* ((meta (or (mu4e-message-field msg :meta) + (mu4e-error "No thread info found"))) + (path (or (plist-get meta :path) + (mu4e-error "No threadpath found")))) + (cl-case what + (path path) + (thread-id + (save-match-data + ;; the thread id is the first segment of the thread path + (when (string-match "^\\([[:xdigit:]]+\\):?" path) + (match-string 1 path)))) + (otherwise (mu4e-error "Not supported"))))) + +(defun mu4e-headers-mark-thread-using-markpair (markpair &optional subthread) + "Mark the thread at point using the given markpair. If SUBTHREAD is +non-nil, marking is limited to the message at point and its +descendants." + (let* ((mark (car markpair)) + (allowed-marks (mapcar 'car mu4e-marks))) + (unless (memq mark allowed-marks) + (mu4e-error "The mark (%s) has to be one of: %s" + mark allowed-marks))) + ;; note: the thread id is shared by all messages in a thread + (let* ((msg (mu4e-message-at-point)) + (thread-id (mu4e~headers-get-thread-info msg 'thread-id)) + (path (mu4e~headers-get-thread-info msg 'path)) + ;; the thread path may have a ':z' suffix for sorting; + ;; remove it for subthread matching. + (match-path (replace-regexp-in-string ":z$" "" path)) + (last-marked-point)) + (mu4e-headers-for-each + (lambda (cur-msg) + (let ((cur-thread-id (mu4e~headers-get-thread-info cur-msg 'thread-id)) + (cur-thread-path (mu4e~headers-get-thread-info cur-msg 'path))) + (if subthread + ;; subthread matching; mymsg's thread path should have path as its + ;; prefix + (when (string-match (concat "^" match-path) cur-thread-path) + (mu4e-mark-at-point (car markpair) (cdr markpair)) + (setq last-marked-point (point))) + ;; nope; not looking for the subthread; looking for the whole thread + (when (string= thread-id cur-thread-id) + (mu4e-mark-at-point (car markpair) (cdr markpair)) + (setq last-marked-point (point))))))) + (when last-marked-point + (goto-char last-marked-point) + (mu4e-headers-next)))) + +(defun mu4e-headers-mark-thread (&optional subthread markpair) + "Like `mu4e-headers-mark-thread-using-markpair' but prompt for the markpair." + (interactive + (let* ((subthread current-prefix-arg)) + (list current-prefix-arg + ;; FIXME: e.g., for refiling we should evaluate this + ;; for each line separately + (mu4e--mark-get-markpair + (if subthread "Mark subthread with: " "Mark whole thread with: ") + t)))) + (mu4e-headers-mark-thread-using-markpair markpair subthread)) + +(defun mu4e-headers-mark-subthread (&optional markpair) + "Like `mu4e-mark-thread', but only for a sub-thread." + (interactive) + (if markpair (mu4e-headers-mark-thread t markpair) + (let ((current-prefix-arg t)) + (call-interactively 'mu4e-headers-mark-thread)))) + + + +(defun mu4e-headers-view-message () + "View message at point." + (interactive) + (unless (eq major-mode 'mu4e-headers-mode) + (mu4e-error "Must be in mu4e-headers-mode (%S)" major-mode)) + (let* ((msg (mu4e-message-at-point)) + (path (mu4e-message-field msg :path)) + (_exists (or (file-readable-p path) + (mu4e-warn "No message at %s" path))) + (docid (or (mu4e-message-field msg :docid) + (mu4e-warn "No message at point"))) + (mark-as-read + (if (functionp mu4e-view-auto-mark-as-read) + (funcall mu4e-view-auto-mark-as-read msg) + mu4e-view-auto-mark-as-read))) + (when-let ((buf (mu4e-get-view-buffer (current-buffer) nil))) + (with-current-buffer buf + (mu4e-loading-mode 1))) + (mu4e--server-view docid mark-as-read))) + +(defvar-local mu4e-headers-open-after-move t + "If set to non-nil, open message after `mu4e-headers-next' and +`mu4e-headers-prev' if pointing at a message after the move +and there is a live message view. + +This variable is for let-binding when scripting.") + +(defun mu4e~headers-move (lines) + "Move point LINES lines. +Move forward if LINES is positive or backwards if LINES is +negative. If this succeeds, return the new docid. Otherwise, +return nil. + +If pointing at a message after the move and there is a +view-window, open the message unless +`mu4e-headers-open-after-move' is non-nil." + (cl-assert (eq major-mode 'mu4e-headers-mode)) + (when (ignore-errors + (let (line-move-visual) + (line-move lines) + t)) + (let* ((docid (mu4e~headers-docid-at-point)) + (folded (and docid (mu4e-thread-message-folded-p)))) + (if folded + (mu4e~headers-move (if (< lines 0) -1 1)) ;; skip folded + (when docid + ;; Skip invisible text at BOL possibly hidden by + ;; the end of another invisible overlay covering + ;; previous EOL. + (move-to-column 2) + ;; update all windows showing the headers buffer + (walk-windows + (lambda (win) + (when (eq (window-buffer win) + (mu4e-get-headers-buffer (buffer-name))) + (set-window-point win (point)))) + nil t) + ;; If the assigned (and buffer-local) `mu4e~headers-view-win' + ;; is not live then that is indicates the user does not want + ;; to pop up the view when they navigate in the headers + ;; buffer. + (when (and mu4e-headers-open-after-move + (window-live-p mu4e~headers-view-win)) + (mu4e-headers-view-message)) + ;; attempt to highlight the new line, display the message + (mu4e~headers-highlight docid) + docid))))) + +(defun mu4e-headers-next (&optional n) + "Move point to the next message header. +If this succeeds, return the new docid. Otherwise, return nil. +Optionally, takes an integer N (prefix argument), to the Nth next +header. + +If pointing at a message after the move and there is a +view-window, open the message unless +`mu4e-headers-open-after-move' is non-nil." + (interactive "P") + (mu4e~headers-move (or n 1))) + +(defun mu4e-headers-prev (&optional n) + "Move point to the previous message header. +If this succeeds, return the new docid. Otherwise, return nil. +Optionally, takes an integer N (prefix argument), to the Nth +previous header. + +If pointing at a message after the move and there is a +view-window, open the message unless +`mu4e-headers-open-after-move' is non-nil." + (interactive "P") + (mu4e~headers-move (- (or n 1)))) + +(defun mu4e~headers-prev-or-next-unread (backwards) + "Move point to the next message that is unread (and +untrashed). If BACKWARDS is non-`nil', move backwards." + (interactive "P") + (or (mu4e-headers-find-if-next + (lambda (msg) + (let ((flags (mu4e-message-field msg :flags))) + (and (member 'unread flags) (not (member 'trashed flags))))) + backwards) + (mu4e-message (format "No %s unread message found" + (if backwards "previous" "next"))))) + +(defun mu4e-headers-prev-unread () + "Move point to the previous message that is unread (and +untrashed)." + (interactive) + (mu4e~headers-prev-or-next-unread t)) + +(defun mu4e-headers-next-unread () + "Move point to the next message that is unread (and +untrashed)." + (interactive) + (mu4e~headers-prev-or-next-unread nil)) + +(defun mu4e~headers-thread-root-p (&optional msg) + "Is MSG at the root of a thread? +If MSG is nil, use message at point." + (when-let* ((msg (or msg (get-text-property (point) 'msg))) + (meta (mu4e-message-field msg :meta))) + (let* ((orphan (plist-get meta :orphan)) + (first-child (plist-get meta :first-child)) + (root (plist-get meta :root))) + (or root (and orphan first-child))))) + +(defun mu4e~headers-prev-or-next-thread (backwards) + "Move point to the top of the next thread. +If BACKWARDS is non-`nil', move backwards." + (when (mu4e-headers-find-if-next #'mu4e~headers-thread-root-p backwards) + (point))) + +(defun mu4e-headers-prev-thread () + "Move point to the previous thread." + (interactive) (mu4e~headers-prev-or-next-thread t)) + +(defun mu4e-headers-next-thread () + "Move point to the previous thread." + (interactive) (mu4e~headers-prev-or-next-thread nil)) + +(defun mu4e-headers-split-view-grow (&optional n) + "In split-view, grow the headers window. +In horizontal split-view, increase the number of lines shown by N. +In vertical split-view, increase the number of columns shown by N. +If N is negative shrink the headers window. When not in split-view +do nothing." + (interactive "P") + (let ((n (or n 1)) + (hwin (get-buffer-window (mu4e-get-headers-buffer)))) + (when (and (buffer-live-p (mu4e-get-view-buffer)) (window-live-p hwin)) + (let ((n (or n 1))) + (cl-case mu4e-split-view + ;; emacs has weird ideas about what horizontal, vertical means... + (horizontal + (window-resize hwin n nil) + (cl-incf mu4e-headers-visible-lines n)) + (vertical + (window-resize hwin n t) + (cl-incf mu4e-headers-visible-columns n))))))) + +(defun mu4e-headers-split-view-shrink (&optional n) + "In split-view, shrink the headers window. +In horizontal split-view, decrease the number of lines shown by N. +In vertical split-view, decrease the number of columns shown by N. +If N is negative grow the headers window. When not in split-view +do nothing." + (interactive "P") + (mu4e-headers-split-view-grow (- (or n 1)))) + +(defun mu4e-headers-action (&optional actionfunc) + "Ask user what to do with message-at-point, then do it. +The actions are specified in `mu4e-headers-actions'. Optionally, +pass ACTIONFUNC, which is a function that takes a msg-plist +argument." + (interactive) + (let ((msg (mu4e-message-at-point)) + (afunc (or actionfunc + (mu4e-read-option "Action: " mu4e-headers-actions)))) + (funcall afunc msg))) + +(defun mu4e-headers-mark-and-next (mark) + "Set mark MARK on the message at point or on all messages in the +region if there is a region, then move to the next message." + (interactive) + (when (mu4e-thread-message-folded-p) + (mu4e-warn "Cannot mark folded messages")) + (mu4e-mark-set mark) + (when mu4e-headers-advance-after-mark (mu4e-headers-next))) + +(defun mu4e~headers-quit-buffer () + "Quit the mu4e-headers buffer and go back to the main view." + (interactive) + (mu4e-mark-handle-when-leaving) + (quit-window t) + ;; clear the decks before going to the main-view + (mu4e--query-items-refresh 'reset-baseline) + (mu4e--main-view)) + + +;;; Loading messages +;; + + +(defvar-local mu4e--loading-overlay-bg nil + "Internal variable that holds the loading overlay for the background.") + +(defvar-local mu4e--loading-overlay-text nil + "Internal variable that holds the loading overlay for the text.") + +(define-minor-mode mu4e-loading-mode + "Minor mode for buffers awaiting data from mu" + :init-value nil :lighter " Loading" :keymap nil + (if mu4e-loading-mode + (progn + (when mu4e-dim-when-loading + (setq mu4e--loading-overlay-bg + (let ((overlay (make-overlay (point-min) (point-max)))) + (overlay-put overlay 'face + `(:foreground "gray22" :background + ,(face-attribute 'default + :background))) + (overlay-put overlay 'priority 9998) + overlay)) + (setq mu4e--loading-overlay-text + (let ((overlay (make-overlay (point-min) (point-min)))) + (overlay-put overlay 'priority 9999) + (overlay-put overlay 'before-string + (propertize "Loading…\n" + 'face 'mu4e-header-title-face)) + overlay)))) + (when mu4e--loading-overlay-bg + (delete-overlay mu4e--loading-overlay-bg)) + (when mu4e--loading-overlay-text + (delete-overlay mu4e--loading-overlay-text)))) + +(provide 'mu4e-headers) +;;; mu4e-headers.el ends here diff --git a/mu4e/mu4e-helpers.el b/mu4e/mu4e-helpers.el new file mode 100644 index 0000000..e2718bb --- /dev/null +++ b/mu4e/mu4e-helpers.el @@ -0,0 +1,604 @@ +;;; mu4e-helpers.el --- Helper functions -*- lexical-binding: t -*- + +;; Copyright (C) 2022-2024 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Helper functions used in the mu4e. This is slowly usurp all the code from +;; mu4e-utils.el that does not depend on other parts of mu4e. + +;;; Code: + +(require 'seq) +(require 'ido) +(require 'cl-lib) +(require 'bookmark) + +(require 'mu4e-window) +(require 'mu4e-config) + +;;; Customization + +(defcustom mu4e-debug nil + "When set to non-nil, log debug information to the mu4e log buffer." + :type 'boolean + :group 'mu4e) + +(defcustom mu4e-completing-read-function #'ido-completing-read + "Function to be used to receive user-input during completion. + +Suggested possible values are: + * `completing-read': emacs built-in completion method + * `ido-completing-read': dynamic completion within the minibuffer. + +The function is used in two contexts - +1) directly - for instance in when listing _other_ maildirs + in `mu4e-ask-maildir' +2) if `mu4e-read-option-use-builtin' is nil, it is used + as part of `mu4e-read-option' in many places. + +Set it to `completing-read' when you want to use completion +frameworks such as Helm, Ivy or Vertico. In that case, you +might want to add something like the following in your configuration. + + (setq mu4e-read-option-use-builtin nil + mu4e-completing-read-function \\='completing-read) +." + :type 'function + :options '(completing-read ido-completing-read) + :group 'mu4e) + +(defcustom mu4e-read-option-use-builtin t + "Whether to use mu4e's traditional completion for +`mu4e-read-option'. + +If nil, use the value of `mu4e-completing-read-function', integrated +into mu4e. + +Many of the third-party completion frameworks - such as Helm, Ivy +and Vertico - influence `completion-read', so to have mu4e follow +your overall settings, try the equivalent of + + (setq mu4e-read-option-use-builtin nil + mu4e-completing-read-function \\='completing-read) + +Tastes differ, but without any such frameworks, the unaugmented +Emacs `completing-read' is rather Spartan." + :type 'boolean + :group 'mu4e) + +(defcustom mu4e-use-fancy-chars nil + "When set, allow fancy (Unicode) characters for marks/threads. +You can customize the exact fancy characters used with +`mu4e-marks' and various `mu4e-headers-..-mark' and +`mu4e-headers..-prefix' variables." + :type 'boolean + :group 'mu4e) + +;; maybe move the next ones... but they're convenient +;; here because they're needed in multiple buffers. + +(defcustom mu4e-view-auto-mark-as-read t + "Automatically mark messages as read when you read them. +This is the default behavior, but can be turned off, for example +when using a read-only file-system. + +This can also be set to a function; if so, receives a message +plist which should evaluate to nil if the message should *not* be +marked as read-only, or non-nil otherwise." + :type '(choice + boolean + function) + :group 'mu4e-view) + + + +(defun mu4e-select-other-view () + "Switch between headers view and message view." + (interactive) + (let* ((other-buf + (cond + ((mu4e-current-buffer-type-p 'view) + (mu4e-get-headers-buffer)) + ((mu4e-current-buffer-type-p 'headers) + (mu4e-get-view-buffer)) + (t (mu4e-error + "This window is neither the headers nor the view window.")))) + (other-win (and other-buf (get-buffer-window other-buf)))) + (if (window-live-p other-win) + (select-window other-win) + (mu4e-message "No window to switch to")))) + + + +;;; Messages, warnings and errors +(defun mu4e-format (frm &rest args) + "Create [mu4e]-prefixed string based on format FRM and ARGS." + (concat + "[" (propertize "mu4e" 'face 'mu4e-title-face) "] " + (apply 'format frm + (mapcar (lambda (x) + (if (stringp x) + (decode-coding-string x 'utf-8) + x)) + args)))) + +(defun mu4e-message (frm &rest args) + "Display FRM with ARGS like `message' in mu4e style. +If we're waiting for user-input or if there's some message in the +echo area, don't show anything." + (unless (or (active-minibuffer-window)) + (message "%s" (apply 'mu4e-format frm args)))) + +(declare-function mu4e~loading-close "mu4e-headers") + +(defun mu4e-error (frm &rest args) + "Display an error with FRM and ARGS like `mu4e-message'. + +Create [mu4e]-prefixed error based on format FRM and ARGS. Does a +local-exit and does not return, and raises a +debuggable (backtrace) error." + (mu4e-log 'error (apply 'mu4e-format frm args)) + (error "%s" (apply 'mu4e-format frm args))) + +(defun mu4e-warn (frm &rest args) + "Create [mu4e]-prefixed warning based on format FRM and ARGS. +Does a local-exit and does not return." + (mu4e-log 'error (apply 'mu4e-format frm args)) + (user-error "%s" (apply 'mu4e-format frm args))) + +;;; Reading user input + +(defun mu4e--plist-get (lst prop) + "Get PROP from plist LST and raise an error if not present." + (or (plist-get lst prop) + (if (plist-member lst prop) + nil + (mu4e-error "Missing property %s in %s" prop lst)))) + +(defun mu4e--matching-choice (choices kar) + "Does KAR match any of the CHOICES? + +KAR is a character and CHOICES is an alist as described in +`mu4e--read-choice-builtin'. + +First try an exact match, but if there isn't, try +case-insensitive. + +Return the cdr (value) of the matching cell, if any." + (let* ((match) (match-ci)) + (catch 'found + (seq-do + (lambda (choice) + ;; first try an exact match + (let ((case-fold-search nil)) + (if (char-equal kar (caadr choice)) + (progn + (setq match choice) + (throw 'found choice)) ;; found it - quit. + ;; perhaps case-insensitive? + (let ((case-fold-search t)) + (when (and (not match-ci) (char-equal kar (caadr choice))) + (setq match-ci choice)))))) + choices)) + (if match (cdadr match) + (when match-ci (cdadr match-ci))))) + +(defun mu4e--read-choice-completing-read (prompt choices) + "Read and return one of CHOICES, prompting for PROMPT. + +PROMPT describes a multiple-choice question to the user. CHOICES +is an alist of the form + ( ( <display-string> ( <shortcut> . <value> )) + ... ) +Any input that is not one of CHOICES is ignored. This is mu4e's +version of `read-char-choice' which becomes case-insensitive +after trying an exact match. + +Return the matching choice value (cdr of the cell)." + (let* ((metadata `(metadata + (display-sort-function . ,#'identity) + (cycle-sort-function . ,#'identity))) + (quick-result) + (result + (minibuffer-with-setup-hook + (lambda () + (add-hook 'post-command-hook + (lambda () + ;; Exit directly if a quick key is pressed + (let ((prefix (minibuffer-contents-no-properties))) + (unless (string-empty-p prefix) + (setq quick-result + (mu4e--matching-choice + choices (string-to-char prefix))) + (when quick-result + (exit-minibuffer))))) + -1 'local)) + (funcall mu4e-completing-read-function + prompt + ;; Use function with metadata to disable sorting. + (lambda (input predicate action) + (if (eq action 'metadata) + metadata + (complete-with-action action choices input predicate))) + ;; Require confirmation, if the input does not match a suggestion + nil t nil nil nil)))) + (or quick-result + (cdadr (assoc result choices))))) + +(defun mu4e--read-choice-builtin (prompt choices) + "Read and return one of CHOICES, prompting for PROMPT. + +PROMPT describes a multiple-choice question to the user. CHOICES +is an alist of the fiorm + ( ( <display-string> ( <shortcut> . <value> )) + ... ) +Any input that is not one of CHOICES is ignored. This is mu4e's +version of `read-char-choice' which becomes case-insensitive +after trying an exact match. + +Return the matching choice value (cdr of the cell)." + (let ((chosen) (inhibit-quit nil) + (prompt (format "%s%s" + (mu4e-format prompt) + (mapconcat #'car choices ", ")))) + (while (not chosen) + (message nil) ;; this seems needed... + (when-let ((kar (read-char-exclusive prompt))) + (when (eq kar ?\e) (keyboard-quit)) ;; `read-char-exclusive' is a C + ;; function and doesn't check for + ;; `keyboard-quit', there we need to + ;; check if ESC is pressed + (setq chosen (mu4e--matching-choice choices kar)))) + chosen)) + +(defun mu4e-read-option (prompt options) + "Ask user for an option from a list on the input area. + +PROMPT describes a multiple-choice question to the user. OPTIONS +describe the options, and is a list of cells describing +particular options. Cells have the following structure: + + (OPTION . RESULT) + +where OPTIONS is a non-empty string describing the option. The +first character of OPTION is used as the shortcut, and obviously +all shortcuts must be different, so you can prefix the string +with an uniquifying character. + +The options are provided as a list for the user to choose from; +user can then choose by typing CHAR. Example: + (mu4e-read-option \"Choose an animal: \" + \\='((\"Monkey\" . monkey) (\"Gnu\" . gnu) (\"xMoose\" . moose))) + +User now will be presented with a list: \"Choose an animal: + [M]onkey, [G]nu, [x]Moose\". + +If optional character KEY is provied, use that instead of asking +the user. + +Function returns the value (cdr) of the matching cell." + (let* ((choices ;; ((<display> ( <key> . <value> ) ...) + (seq-map + (lambda (option) + (list + (concat ;; <display> + "[" (propertize (substring (car option) 0 1) + 'face 'mu4e-highlight-face) + "]" + (substring (car option) 1)) + (cons + (string-to-char (car option)) ;; <key> + (cdr option)))) ;; <value> + options)) + (response (funcall + (if mu4e-read-option-use-builtin + #'mu4e--read-choice-builtin + #'mu4e--read-choice-completing-read) + prompt choices))) + (or response + (mu4e-warn "invalid input")))) + +(defun mu4e-filter-single-key (lst) + "Return a list consisting of LST items with a `characterp' :key prop." + ;; This works for bookmarks and maildirs. + (seq-filter (lambda (item) + (characterp (plist-get item :key))) + lst)) + + +;;; Logging / debugging + +(defconst mu4e--log-max-size 1000000 + "Max number of characters to keep around in the log buffer.") +(defconst mu4e--log-buffer-name "*mu4e-log*" + "Name of the logging buffer.") + +(defun mu4e--get-log-buffer () + "Fetch (and maybe create) the log buffer." + (unless (get-buffer mu4e--log-buffer-name) + (with-current-buffer (get-buffer-create mu4e--log-buffer-name) + (view-mode) + (when (fboundp 'so-long-mode) + (unless (eq major-mode 'so-long-mode) + (eval '(so-long-mode)))) + (setq buffer-undo-list t))) + mu4e--log-buffer-name) + +(defun mu4e-log (type frm &rest args) + "Log a message of TYPE with format-string FRM and ARGS. +Use the mu4e log buffer for this. If the variable mu4e-debug is +non-nil. Type is a symbol, either `to-server', `from-server' or +`misc'. + +This function is meant for debugging." + (when mu4e-debug + (with-current-buffer (mu4e--get-log-buffer) + (let* ((inhibit-read-only t) + (tstamp (propertize (format-time-string "%Y-%m-%d %T.%3N" + (current-time)) + 'face 'font-lock-string-face)) + (msg-face + (pcase type + ('from-server 'font-lock-type-face) + ('to-server 'font-lock-function-name-face) + ('misc 'font-lock-variable-name-face) + ('error 'font-lock-warning-face) + (_ (mu4e-error "Unsupported log type")))) + (msg (propertize (apply 'format frm args) 'face msg-face))) + (save-excursion + (goto-char (point-max)) + (insert tstamp + (pcase type + ('from-server " <- ") + ('to-server " -> ") + ('error " !! ") + (_ " ")) + msg "\n") + ;; if `mu4e-log-max-lines is specified and exceeded, clearest the + ;; oldest lines + (when (> (buffer-size) mu4e--log-max-size) + (goto-char (- (buffer-size) mu4e--log-max-size)) + (beginning-of-line) + (delete-region (point-min) (point)))))))) + +(defun mu4e-toggle-logging () + "Toggle `mu4e-debug'. +In debug-mode, mu4e logs some of its internal workings to a +log-buffer. See `mu4e-show-log'." + (interactive) + (mu4e-log 'misc "logging disabled") + (setq mu4e-debug (not mu4e-debug)) + (mu4e-message "debug logging has been %s" + (if mu4e-debug "enabled" "disabled")) + (mu4e-log 'misc "logging enabled")) + +(defun mu4e-show-log () + "Visit the mu4e debug log." + (interactive) + (unless mu4e-debug (mu4e-toggle-logging)) + (let ((buf (get-buffer mu4e--log-buffer-name))) + (unless (buffer-live-p buf) + (mu4e-warn "No debug log available")) + (display-buffer buf))) + + + +;;; Flags +;; Converting flags->string and vice-versa + +(defun mu4e-flags-to-string (flags) + "Convert a list of Maildir[1] FLAGS into a string. + +See `mu4e-string-to-flags'. \[1\]: +http://cr.yp.to/proto/maildir.html." + (seq-sort + '< + (seq-mapcat + (lambda (flag) + (pcase flag + (`draft "D") + (`flagged "F") + (`new "N") + (`passed "P") + (`replied "R") + (`seen "S") + (`trashed "T") + (`attach "a") + (`encrypted "x") + (`signed "s") + (`unread "u") + (_ ""))) + (seq-uniq flags) 'string))) + +(defun mu4e-string-to-flags (str) + "Convert a STR with Maildir[1] flags into a list of flags. + +See `mu4e-string-to-flags'. \[1\]: +http://cr.yp.to/proto/maildir.html." + (seq-uniq + (seq-filter + 'identity + (seq-mapcat + (lambda (kar) + (list + (pcase kar + ('?D 'draft) + ('?F 'flagged) + ('?P 'passed) + ('?R 'replied) + ('?S 'seen) + ('?T 'trashed) + (_ nil)))) + str)))) + + +;;; Misc +(defun mu4e-copy-thing-at-point () + "Copy e-mail address or URL at point to the kill ring. +If there is not e-mail address at point, do nothing." + (interactive) + (let* ((thing (and (thing-at-point 'email) + (string-trim (thing-at-point 'email 'no-props) "<" ">"))) + (thing (or thing (get-text-property (point) 'shr-url))) + (thing (or thing (thing-at-point 'url 'no-props)))) + (when thing + (kill-new thing) + (mu4e-message "Copied '%s' to kill-ring" thing)))) + +(defun mu4e-display-size (size) + "Get a human-friendly string representation of SIZE (in bytes)." + (cond + ((>= size 1000000) + (format "%2.1fM" (/ size 1000000.0))) + ((and (>= size 1000) (< size 1000000)) + (format "%2.1fK" (/ size 1000.0))) + ((< size 1000) + (format "%d" size)) + (t "?"))) + +(defun mu4e-split-ranges-to-numbers (str n) + "Convert STR containing attachment numbers into a list of numbers. + +STR is a string; N is the highest possible number in the list. +This includes expanding e.g. 3-5 into 3,4,5. If the letter +\"a\" ('all')) is given, that is expanded to a list with numbers +[1..n]." + (let ((str-split (split-string str)) + beg end list) + (dolist (elem str-split list) + ;; special number "a" converts into all attachments 1-N. + (when (equal elem "a") + (setq elem (concat "1-" (int-to-string n)))) + (if (string-match "\\([0-9]+\\)-\\([0-9]+\\)" elem) + ;; we have found a range A-B, which needs converting + ;; into the numbers A, A+1, A+2, ... B. + (progn + (setq beg (string-to-number (match-string 1 elem)) + end (string-to-number (match-string 2 elem))) + (while (<= beg end) + (cl-pushnew beg list :test 'equal) + (setq beg (1+ beg)))) + ;; else just a number + (cl-pushnew (string-to-number elem) list :test 'equal))) + ;; Check that all numbers are valid. + (mapc + (lambda (x) + (cond + ((> x n) + (mu4e-warn "Attachment %d bigger than maximum (%d)" x n)) + ((< x 1) + (mu4e-warn "Attachment number must be greater than 0 (%d)" x)))) + list))) + +(defun mu4e-make-temp-file (ext) + "Create a self-destructing temporary file with extension EXT. +The file will self-destruct in a short while, enough to open it +in an external program." + (let ((tmpfile (make-temp-file "mu4e-" nil (concat "." ext)))) + (run-at-time "30 sec" nil + (lambda () (ignore-errors (delete-file tmpfile)))) + tmpfile)) + +(defun mu4e-display-manual () + "Display the mu4e manual page for the current mode. +Or go to the top level if there is none." + (interactive) + (info (pcase major-mode + ('mu4e-main-mode "(mu4e)Main view") + ('mu4e-headers-mode "(mu4e)Headers view") + ('mu4e-view-mode "(mu4e)Message view") + (_ "mu4e")))) + + +;;; bookmarks +(defun mu4e--make-bookmark-record () + "Create a bookmark for the message at point." + (let* ((msg (mu4e-message-at-point)) + (subject (or (plist-get msg :subject) "No subject")) + (date (plist-get msg :date)) + (date (if date (format-time-string "%F: " date) "")) + (title (format "%s%s" date subject)) + (msgid (or (plist-get msg :message-id) + (mu4e-error + "Cannot bookmark message without message-id")))) + `(,title + ,@(bookmark-make-record-default 'no-file 'no-context) + (message-id . ,msgid) + (handler . mu4e--jump-to-bookmark)))) + +(declare-function mu4e-view-message-with-message-id "mu4e-view") +(declare-function mu4e-message-at-point "mu4e-message") + +(defun mu4e--jump-to-bookmark (bookmark) + "View the message referred to by BOOKMARK." + (when-let ((msgid (bookmark-prop-get bookmark 'message-id))) + (mu4e-view-message-with-message-id msgid))) + + ;;; Macros + +(defmacro mu4e-setq-if-nil (var val) + "Set VAR to VAL if VAR is nil." + `(unless ,var (setq ,var ,val))) + + + +;;; Misc +(defun mu4e-join-paths (directory &rest components) + "Append COMPONENTS to DIRECTORY and return the resulting string. + +This is mu4e's version of Emacs 28's `file-name-concat' with the +difference it also handles slashes at the beginning of +COMPONENTS." + (replace-regexp-in-string + "//+" "/" + (mapconcat (lambda (part) (if (stringp part) part "")) + (cons directory components) "/"))) + +(defun mu4e-string-replace (from-string to-string in-string) + "Replace FROM-STRING with TO-STRING in IN-STRING each time it occurs. +Mu4e version of emacs 28's string-replace." + (replace-regexp-in-string (regexp-quote from-string) + to-string in-string nil 'literal)) + +(defun mu4e-plistp (object) + "Non-nil if and only if OBJECT is a valid plist. + +This is mu4e's version of Emacs 29's `plistp'." + (let ((len (proper-list-p object))) + (and len (zerop (% len 2))))) + +(defun mu4e-key-description (cmd) + "Get the textual form of current binding to interactive function CMD. +If it is unbound, return nil. If there are multiple bindings, +return the shortest. + +Roughly does what `substitute-command-keys' does, but picks +shorter keys in some cases where there are multiple bindings." + ;; not a perfect heuristic: e.g. '<up>' is longer that 'C-p' + (car-safe + (seq-sort (lambda (b1 b2) + (< (length b1) (length b2))) + (seq-map #'key-description + (where-is-internal cmd))))) + +(provide 'mu4e-helpers) +;;; mu4e-helpers.el ends here diff --git a/mu4e/mu4e-icalendar.el b/mu4e/mu4e-icalendar.el new file mode 100644 index 0000000..30feb51 --- /dev/null +++ b/mu4e/mu4e-icalendar.el @@ -0,0 +1,210 @@ +;;; mu4e-icalendar.el --- iCalendar & diary integration -*- lexical-binding: t; -*- + +;; Copyright (C) 2019-2023 Christophe Troestler + +;; Author: Christophe Troestler <Christophe.Troestler@umons.ac.be> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Keywords: email icalendar +;; Version: 0.0 + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; To install: +;; (require 'mu4e-icalendar) +;; (gnus-icalendar-setup) +;; Optional: +;; (setq mu4e-icalendar-trash-after-reply t) + +;; By default, the original message is not cited. However, if you +;; would like to reply to it, the citation is in the kill-ring (paste +;; it with `yank'). + +;; To add the event to a diary file of your choice: +;; (setq mu4e-icalendar-diary-file "/path/to/your/diary") +;; If the file specified is not your main diary file, add +;; #include "/path/to/your/diary" +;; to you main diary file to display the events. + +;; To enable optional iCalendar->Org sync functionality +;; NOTE: both the capture file and the headline(s) inside must already exist +;; (require 'org-agenda) +;; (setq gnus-icalendar-org-capture-file "~/org/notes.org") +;; (setq gnus-icalendar-org-capture-headline '("Calendar")) +;; (gnus-icalendar-org-setup) + +;;; Code: + +(require 'gnus-icalendar) +(require 'cl-lib) + +(require 'mu4e-mark) +(require 'mu4e-helpers) +(require 'mu4e-contacts) +(require 'mu4e-headers) +(require 'mu4e-obsolete) + + +;;; Configuration +;;;; Calendar + +(defgroup mu4e-icalendar nil + "Icalendar related settings." + :group 'mu4e) + +(defcustom mu4e-icalendar-trash-after-reply nil + "If non-nil, trash the icalendar invitation after replying." + :type 'boolean + :group 'mu4e-icalendar) + +(defcustom mu4e-icalendar-diary-file nil + "If non-nil, the file in which to add events upon reply." + :type '(choice (const :tag "Do not insert a diary entry" nil) + (string :tag "Insert a diary entry in this file")) + :group 'mu4e-icalendar) + + +(defun mu4e--icalendar-has-email (email list) + "Check that EMAIL is in LIST." + (let ((email (downcase email))) + (cl-find-if (lambda (c) (let ((e (mu4e-contact-email c))) + (and (stringp e) (string= email (downcase e))))) + list))) + +(declare-function mu4e--view-mode-p "mu4e-view") +(defun mu4e--icalendar-reply (orig data) + "Wrapper for using either `mu4e-icalender-reply' or the ORIG function." + (funcall (if (mu4e--view-mode-p) #'mu4e-icalendar-reply orig) data)) + +(advice-add #'gnus-icalendar-reply :around #'mu4e--icalendar-reply) +;;(advice-remove #'gnus-icalendar-reply #'mu4e--icalendar-reply) + +(defun mu4e-icalendar-reply (data) + "Reply to the text/calendar event present in DATA." + ;; Based on `gnus-icalendar-reply'. + (let* ((handle (car data)) + (status (cadr data)) + (event (caddr data)) + (gnus-icalendar-additional-identities + (mu4e-personal-addresses 'no-regexp)) + (reply (gnus-icalendar-with-decoded-handle + handle + (gnus-icalendar-event-reply-from-buffer + (current-buffer) status (gnus-icalendar-identities)))) + (msg (mu4e-message-at-point 'noerror)) + (charset (cdr (assoc 'charset (mm-handle-type handle))))) + (when reply + (cl-labels + ((fold-icalendar-buffer + () + (goto-char (point-min)) + (while (re-search-forward "^\\(.\\{72\\}\\)\\(.+\\)$" nil t) + (replace-match "\\1\n \\2") + (goto-char (line-beginning-position))))) + + (let ((ical-name gnus-icalendar-reply-bufname)) + (with-current-buffer (get-buffer-create ical-name) + (delete-region (point-min) (point-max)) + (insert reply) + (fold-icalendar-buffer) + (when (and charset (string= (downcase charset) "utf-8")) + (decode-coding-region (point-min) (point-max) 'utf-8))) + + (save-excursion ;; Compose the reply message. + (let* ((message-signature nil) + (organizer (gnus-icalendar-event:organizer event)) + (organizer (when (and organizer + (not (string-empty-p organizer))) + organizer)) + (organizer + (or organizer + (plist-get (car (plist-get msg :reply-to)) :email) + (plist-get (car (plist-get msg :from)) :email) + (mu4e-warn "Cannot find organizer"))) + (message-cite-function #'mu4e-message-cite-nothing)) + (mu4e-compose-reply-to organizer) + (message-goto-body) + (mml-insert-multipart "alternative") + (mml-insert-empty-tag 'part 'type "text/plain") + (mml-attach-buffer ical-name + "text/calendar; method=REPLY; charset=UTF-8") + (when mu4e-icalendar-trash-after-reply + ;; Override `mu4e-sent-handler' set by `mu4e-compose-mode' to + ;; also trash the message (thus must be appended to hooks). + (add-hook 'message-sent-hook + (mu4e--icalendar-trash-message-hook msg) 90 t)) + + (when gnus-icalendar-org-enabled-p + (if (gnus-icalendar-find-org-event-file event) + (gnus-icalendar--update-org-event event status) + (gnus-icalendar:org-event-save event status))) + (when mu4e-icalendar-diary-file + (mu4e--icalendar-insert-diary event status + mu4e-icalendar-diary-file))))))))) + +(declare-function mu4e-view-headers-next "mu4e-view") +(defun mu4e--icalendar-trash-message (original-msg) + "Trash the message ORIGINAL-MSG and move to the next one." + (lambda (docid path) + "See `mu4e-sent-handler' for DOCID and PATH." + (mu4e-sent-handler docid path) + (let* ((docid (mu4e-message-field original-msg :docid)) + (markdescr (assq 'trash mu4e-marks)) + (action (plist-get (cdr markdescr) :action)) + (target (mu4e-get-trash-folder original-msg))) + (with-current-buffer (mu4e-get-headers-buffer) + (run-hook-with-args 'mu4e-mark-execute-pre-hook 'trash original-msg) + (funcall action docid original-msg target)) + (when (and (mu4e~headers-view-this-message-p docid) + (buffer-live-p (mu4e-get-view-buffer))) + (mu4e-display-buffer (mu4e-get-view-buffer)) + (or (mu4e-view-headers-next) + (kill-buffer-and-window)))))) + +(defun mu4e--icalendar-trash-message-hook (original-msg) + "Trash the iCalendar message ORIGINAL-MSG." + (lambda () + (setq mu4e-sent-func + (mu4e--icalendar-trash-message original-msg)))) + +(defun mu4e--icalendar-insert-diary (event reply-status filename) + "Insert a diary entry for the EVENT in file named FILENAME. +REPLY-STATUS is the status of the reply. The possible values are +given in the doc of `gnus-icalendar-event-reply-from-buffer'." + ;; FIXME: handle recurring events + (let* ((beg (gnus-icalendar-event:start-time event)) + (beg-date (format-time-string "%d/%m/%Y" beg)) + (beg-time (format-time-string "%H:%M" beg)) + (end (gnus-icalendar-event:end-time event)) + (end-date (format-time-string "%d/%m/%Y" end)) + (end-time (format-time-string "%H:%M" end)) + (summary (gnus-icalendar-event:summary event)) + (location (gnus-icalendar-event:location event)) + (status (capitalize (symbol-name reply-status))) + (txt (if location + (format "%s (%s)\n %s " summary status location) + (format "%s (%s)" summary status)))) + (with-temp-buffer + (if (string= beg-date end-date) + (insert beg-date " " beg-time "-" end-time " " txt "\n") + (insert beg-date " " beg-time " Start of: " txt "\n") + (insert beg-date " " end-time " End of: " txt "\n")) + (write-region (point-min) (point-max) filename t)))) + +;;; +(provide 'mu4e-icalendar) +;;; mu4e-icalendar.el ends here diff --git a/mu4e/mu4e-lists.el b/mu4e/mu4e-lists.el new file mode 100644 index 0000000..f19239f --- /dev/null +++ b/mu4e/mu4e-lists.el @@ -0,0 +1,170 @@ +;;; mu4e-lists.el --- Get names for mailing lists -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; In this file, we create a table of list-id -> shortname for mailing lists. +;; The shortname (friendly) should a at most 8 characters, camel-case + +;;; Code: +(require 'mu4e-message) +(require 'mu4e-helpers) + + + ;;; Helpers +(defmacro mu4e-message-id-url(base-url) + "Construct lambda to get an archive URL for message. +This is based on some BASE-URL to which the message-id is concatenated; +e.g. public-inbox-based archives." + `(lambda (msg) (concat ,base-url "/" (plist-get msg :message-id)))) + +(defmacro mu4e-x-seq-url (base-url) + "Construct x-seq archive URL for MSG or nil if not found." + `(lambda (msg) + (when-let ((xseq (mu4e-fetch-field msg "X-Seq"))) + (concat ,base-url "/" xseq)))) + +;;; Configuration +(defvar mu4e-mailing-lists + `( (:list-id "bbdb-info.lists.sourceforge.net" :name "BBDB") + (:list-id "boost-announce.lists.boost.org" :name "Boost") + (:list-id "boost-interest.lists.boost.org" :name "Boost") + (:list-id "curl-library.cool.haxx.se" :name "Curl") + (:list-id "dbus.lists.freedesktop.org" :name "DBus") + (:list-id "desktop-devel-list.gnome.org" :name "Gnome") + (:list-id "discuss-webrtc.googlegroups.com" :name "WebRTC") + (:list-id "emacs-devel.gnu.org" :name "EmacsDev" + :archive ,(mu4e-message-id-url "https://yhetil.org/emacs-devel")) + (:list-id "emacs-orgmode.gnu.org" :name "Orgmode" + :archive ,(mu4e-message-id-url "https://list.orgmode.org")) + (:list-id "emms-help.gnu.org" :name "Emms") + (:list-id "gcc-help.gcc.gnu.org" :name "Gcc") + (:list-id "gmime-devel-list.gnome.org" :name "GMime") + (:list-id "gnome-shell-list.gnome.org" :name "Gnome") + (:list-id "gnu-emacs-sources.gnu.org" :name "Emacs") + (:list-id "gnupg-users.gnupg.org" :name "Gnupg") + (:list-id "gstreamer-devel.lists.freedesktop.org" :name "GstDev") + (:list-id "gtk-devel-list.gnome.org" :name "GtkDev") + (:list-id "guile-devel.gnu.org" :name "Guile" + :archive ,(mu4e-message-id-url "https://yhetil.org/guile-devel")) + (:list-id "guile-user.gnu.org" :name "Guile" + :archive ,(mu4e-message-id-url "https://yhetil.org/guile-user")) + (:list-id "help-gnu-emacs.gnu.org" :name "EmacsUsr" + :archive ,(mu4e-message-id-url "https://yhetil.org/emacs-user")) + (:list-id "mu-discuss.googlegroups.com" :name "Mu") + (:list-id "nautilus-list.gnome.org" :name "Nautilus") + (:list-id "notmuch.notmuchmail.org" :name "Notmuch" + :archive ,(mu4e-message-id-url "https://yhetil.org/notmuch")) + (:list-id "sqlite-announce.sqlite.org" :name "SQlite") + (:list-id "sqlite-dev.sqlite.org" :name "SQLite") + (:list-id "xapian-discuss.lists.xapian.org" :name "Xapian") + (:list-id "xdg.lists.freedesktop.org" :name "XDG") + (:list-id "wl-en.lists.airs.net" :name "WdrLust") + (:list-id "wl-en.ml.gentei.org" :name "WdrLust") + (:list-id "xapian-devel.lists.xapian.org" :name "Xapian") + (:list-id "zsh-users.zsh.org" :name "Zsh" + :archive ,(mu4e-x-seq-url "https://www.zsh.org/users"))) + "List of plists with keys: +- `:list-id' - the mailing list id +- `:name' - the display name +- `:archive' - (optional) a function taking a MSG and + returning an URL to to the online-location of + the message. +After changes, use `mu4e-mailing-list-info-refresh' to update the +corresponding data-structures.") + +(defgroup mu4e-lists nil "Configuration for mailing lists." + :group 'mu4e) + +(defcustom mu4e-user-mailing-lists nil + "A list with plists like `mu4e-mailing-lists'. +These are used in addition to the built-in list +`mu4e-mailing-lists'. + +The older format, a list of cons cells, + (LIST-ID . NAME) +is still supported for backward compatibility. + +After changing, use `mu4e-mailing-list-info-refresh' to make mu4e +use the new values." + :group 'mu4e-headers + :type '(repeat (plist))) + +(defcustom mu4e-mailing-list-patterns '("\\([^.]*\\)\\.") + "A list of regexps to capture a shortname out of a list-id. +For the first regex that matches, its first match-group will be +used as the shortname." + :group 'mu4e-headers + :type '(repeat (regexp))) + +(defvar mu4e--lists-hash nil + "Hash-table of list-id => plist. +Based on `mu4e-mailing-lists' and `mu4e-user-mailing-lists'.") + +(defun mu4e-mailing-list-info-refresh () + "Refresh the mailing list info. +Based on the current value of `mu4e-mailing-lists' and +`mu4e-user-mailing-lists'." + (interactive) + (setq mu4e--lists-hash (make-hash-table :test 'equal)) + (seq-do (lambda (item) + (if (mu4e-plistp item) + ;; the new format + (puthash (plist-get item :list-id) item mu4e--lists-hash) + ;; backward compatibility + (puthash (car item) (cdr item) mu4e--lists-hash))) + (append mu4e-mailing-lists + mu4e-user-mailing-lists)) + mu4e--lists-hash) + +(defun mu4e-mailing-list-info (list-id) + "Get mailing list info for LIST-ID. +Return nil if not found." + (unless mu4e--lists-hash (mu4e-mailing-list-info-refresh)) + (gethash list-id mu4e--lists-hash)) + + +(defun mu4e-get-mailing-list-shortname (list-id) + "Get the shortname for a mailing-list with list-id LIST-ID. +Either we know about this mailing list, or otherwise +we guess one." + (or ;; 1. perhaps we have it in one of our lists? + (plist-get (mu4e-mailing-list-info list-id) :name) + ;; 2. see if it matches some pattern + (if (seq-find (lambda (p) (string-match p list-id)) + mu4e-mailing-list-patterns) + (match-string 1 list-id) + ;; 3. otherwise, just return the whole thing + list-id))) + +(defun mu4e-mailing-list-archive-url (&optional msg) + "Get the mailing-list archive URL for MSG. +If MSG is nil, use the message at point." + (when-let* ((msg (or msg (mu4e-message-at-point))) + (list-id (plist-get msg :list)) + (list-info (and list-id (mu4e-mailing-list-info list-id))) + (func (plist-get list-info :archive))) + (when func + (funcall func msg)))) + +(provide 'mu4e-lists) +;;; mu4e-lists.el ends here diff --git a/mu4e/mu4e-main.el b/mu4e/mu4e-main.el new file mode 100644 index 0000000..ffe22a5 --- /dev/null +++ b/mu4e/mu4e-main.el @@ -0,0 +1,435 @@ +;;; mu4e-main.el --- The Main interface for mu4e -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;;; Code: + +(require 'smtpmail) +(require 'mu4e-helpers) +(require 'mu4e-context) +(require 'mu4e-compose) +(require 'mu4e-bookmarks) +(require 'mu4e-folders) +(require 'mu4e-update) +(require 'mu4e-contacts) +(require 'mu4e-search) +(require 'mu4e-vars) ;; mu-wide variables +(require 'mu4e-window) +(require 'mu4e-query-items) + +(declare-function mu4e-compose-new "mu4e-compose") +(declare-function mu4e-quit "mu4e") + +(require 'cl-lib) + + +;; Configuration + +(defcustom mu4e-main-hide-personal-addresses nil + "Whether to hide the personal address in the main view. + + This can be useful to avoid the noise when there are many, and +also hides the warning if your `user-mail-address' is not part of +the personal addresses." + :type 'boolean + :group 'mu4e-main) + +(defcustom mu4e-main-hide-fully-read nil + "Whether to hide bookmarks or maildirs without unread messages." + :type 'boolean + :group 'mu4e-main) + +(defcustom mu4e-main-rendered-hook nil + "Hook run after the main-view has been rendered." + :type 'hook + :group 'mu4e-main) + + +;;; Mode +(define-derived-mode mu4e-org-mode org-mode "mu4e:org" + "Major mode for mu4e documents.") + +(defun mu4e-info (path) + "Show a buffer with the information (an org-file) at PATH." + (unless (file-exists-p path) + (mu4e-error "Cannot find %s" path)) + (let ((curbuf (current-buffer))) + (find-file path) + (mu4e-org-mode) + (setq buffer-read-only t) + (define-key mu4e-org-mode-map (kbd "q") + `(lambda () + (interactive) + (bury-buffer) + (switch-to-buffer ,curbuf))))) + +(defun mu4e-about () + "Show the mu4e \"About\" page." + (interactive) + (mu4e-info (mu4e-join-paths mu4e-doc-dir "mu4e-about.org"))) + +(defun mu4e-news () + "Show page with news for the current version of mu4e." + (interactive) + (mu4e-info (mu4e-join-paths mu4e-doc-dir "NEWS.org"))) + +(defun mu4e-baseline-time () + "Show the baseline time." + (interactive) + (mu4e-message "Baseline time: %s" (mu4e--baseline-time-string))) + +(defun mu4e--baseline-time-string () + "Calculate the baseline time string." + (let* ((baseline-t mu4e--query-items-baseline-tstamp) + (updated-t (plist-get mu4e-index-update-status :tstamp)) + (delta-t (and baseline-t updated-t + (float-time (time-subtract updated-t baseline-t))))) + (if (and delta-t (> delta-t 0)) + (format-seconds "%Y %D %H %M %z%S since latest" delta-t) + (if baseline-t + (current-time-string baseline-t) + "Never")))) + +(defvar mu4e-main-mode-map + (let ((map (make-sparse-keymap))) + + (define-key map "q" #'mu4e-quit) + (define-key map "C" #'mu4e-compose-new) + + (define-key map "m" #'mu4e--main-toggle-mail-sending-mode) + (define-key map "f" #'smtpmail-send-queued-mail) + ;; + (define-key map (kbd "C-S-u") #'mu4e-update-mail-and-index) + ;; for terminal users + (define-key map (kbd "C-c C-u") #'mu4e-update-mail-and-index) + (define-key map "U" #'mu4e-update-mail-and-index) + (define-key map "S" #'mu4e-kill-update-mail) + (define-key map ";" #'mu4e-context-switch) + (define-key map "$" #'mu4e-show-log) + (define-key map "A" #'mu4e-about) + (define-key map "N" #'mu4e-news) + (define-key map "H" #'mu4e-display-manual) + map) + "Keymap for the *mu4e-main* buffer.") + +(easy-menu-define mu4e-main-mode-menu + mu4e-main-mode-map "Menu for mu4e's main view." + (append + '("Mu4e" ;;:visible mu4e-headers-mode + "--" + ["Update mail and index" mu4e-update-mail-and-index] + ["Flush queued mail" smtpmail-send-queued-mail] + "--" + ["Show debug log" mu4e-show-log] + ) + mu4e--compose-menu-items + mu4e--search-menu-items + '( + "--" + ["Quit" mu4e-quit :help "Quit mu4e"]))) + +(declare-function mu4e--server-bookmarks-queries "mu4e") + +(define-derived-mode mu4e-main-mode special-mode "mu4e:main" + "Major mode for the mu4e main screen. + +This mode is a bit special when it comes to keybinding, since it +shows those keybindings. + +For the rebinding the mu4e functions (such as +`mu4e-search-bookmark' and `mu4e-search-maildir') to different +keys, note that mu4e determines the bindings when drawing the +screen, which is *after* we enable the mode. Thus, the +keybindings must be known when this happens. + +Binding the existing bindings (such as \='s') to different +functions, is *not* really supported, and we still display the +default binding for the original function; which should still do +the reasonable thing in most cases. + +Still, such a rebinding *only* affects the key, and not e.g. the +mouse-bindings." + (setq truncate-lines t + overwrite-mode 'overwrite-mode-binary) + (mu4e-context-minor-mode) + (mu4e-search-minor-mode) + (mu4e-update-minor-mode) + (setq-local revert-buffer-function + (lambda (_ignore-auto _noconfirm) + ;; reset the baseline and get updated results. + (mu4e--query-items-refresh 'reset-baseline)))) + + +(defun mu4e--main-action (title cmd &optional bindstr alt) + "Produce main view action string with TITLE. + +When activated, invoke interactive function CMD. + +In the result, used the TITLE string, with the first occurrence +of [@] replaced by a textual replacement of a binding to CMD as +per `mu4e-key-description', or, if specified, BINDSTR. + +If a string ALT is specified, and BINDSTR is longer than a single +character, use ALT as a substitute. ALT should be a string of +length 1. + +If the first letter after the [@] is equal to the last letter of the +binding representation, remove that first letter." + (let* ((bindstr (or bindstr (mu4e-key-description cmd) alt + (mu4e-error "No binding for %s" cmd))) + (bindstr + (if (and alt (> (length bindstr) 1)) alt bindstr)) + (title ;; remove first letter afrer [] if it equal last of binding + (mu4e-string-replace + (concat "[@]" (substring bindstr -1)) "[@]" title)) + (title ;; insert binding in [@] + (mu4e-string-replace + "[@]" (format "[%s]" (propertize bindstr 'face 'mu4e-highlight-face)) + title)) + (map (make-sparse-keymap))) + (define-key map [mouse-2] cmd) + (define-key map (kbd "RET") cmd) + (propertize title 'keymap map))) + +(defun mu4e--main-items (item-type max-length) + "Produce the string with menu-items for ITEM-TYPE. +ITEM-TYPE is a symbol, either `bookmarks' or `maildirs'. + +MAX-LENGTH is the maximum length of the item titles; this is used +for aligning them." + (mapconcat + (lambda (item) + (cl-destructuring-bind + (&key hide name key favorite query &allow-other-keys) item + ;; hide items explicitly hidden, without key or wrong category. + (if hide + "" + (let ((item-info + ;; note, we have a function for the binding, + ;; and perhaps a different one for the lambda. + (cond + ((eq item-type 'maildirs) + (list #'mu4e-search-maildir #'mu4e-search + query)) + ((eq item-type 'bookmarks) + (list #'mu4e-search-bookmark #'mu4e-search-bookmark + (mu4e-get-bookmark-query key))) + (t + (mu4e-error "Invalid item-type %s" item-type))))) + (concat + (mu4e--main-action + ;; main title + (format "\t* [@] %s " + (propertize + name + 'face (if favorite 'mu4e-header-key-face nil) + 'help-echo query)) + ;; function to call when activated + (lambda () (interactive) + (funcall (nth 1 item-info) + (nth 2 item-info))) + ;; custom key binding string + (concat (mu4e-key-description (nth 0 item-info)) (string key))) + ;; counts + (format "%s%s\n" + (make-string (- max-length (string-width name)) ?\s) + (mu4e--query-item-display-counts item))))))) + ;; only items which have a single-character :key + (mu4e-filter-single-key (mu4e-query-items item-type)) "")) + +(defun mu4e--key-val (key val &optional unit) + "Show a KEY / VAL pair, with optional UNIT." + (concat + "\t* " + (propertize (format "%-20s" key) 'face 'mu4e-header-title-face) + ": " + (propertize val 'face 'mu4e-header-key-face) + (if unit + (propertize (concat " " unit) 'face 'mu4e-header-title-face) + "") + "\n")) + +(defun mu4e--main-baseline-time-string () + "Calculate the baseline time string for use in the main-" + (let* ((baseline-t mu4e--query-items-baseline-tstamp) + (updated-t (plist-get mu4e-index-update-status :tstamp)) + (delta-t (and baseline-t updated-t + (float-time (time-subtract updated-t baseline-t))))) + (if (and delta-t (> delta-t 0)) + (format-seconds "%Y %D %H %M %z%S ago" delta-t) + (if baseline-t + (current-time-string baseline-t) + "Never")))) + +(defun mu4e--main-redraw () + "Redraw the main buffer if there is one. +Otherwise, do nothing." + (when-let* ((buffer (get-buffer mu4e-main-buffer-name)) + (buffer (and (buffer-live-p buffer) buffer))) + (with-current-buffer buffer + (let* ((inhibit-read-only t) + (pos (point)) + (addrs (mu4e-personal-addresses)) + (max-length (seq-reduce (lambda (a b) + (max a (length (plist-get b :name)))) + (mu4e-query-items) 0))) + (mu4e-main-mode) + (erase-buffer) + (insert + "* " + (propertize "mu4e" 'face 'mu4e-header-key-face) + (propertize " - mu for emacs version " 'face 'mu4e-title-face) + (propertize mu4e-mu-version 'face 'mu4e-header-key-face) + "\n\n" + (propertize " Basics\n\n" 'face 'mu4e-title-face) + (mu4e--main-action + "\t* [@]jump to some maildir\n" #'mu4e-search-maildir nil "j") + (mu4e--main-action + "\t* enter a [@]search query\n" #'mu4e-search nil "s") + (mu4e--main-action + "\t* [@]Compose a new message\n" #'mu4e-compose-new nil "C") + "\n" + (propertize " Bookmarks\n\n" 'face 'mu4e-title-face) + (mu4e--main-items 'bookmarks max-length) + "\n" + (propertize " Maildirs\n\n" 'face 'mu4e-title-face) + (mu4e--main-items 'maildirs max-length) + "\n" + (propertize " Misc\n\n" 'face 'mu4e-title-face) + (mu4e--main-action "\t* [@]Choose query\n" + #'mu4e-search-query nil "c") + (mu4e--main-action "\t* [@]Switch context\n" + #'mu4e-context-switch nil ";") + (mu4e--main-action "\t* [@]Update email & database\n" + #'mu4e-update-mail-and-index nil "U") + ;; show the queue functions if `smtpmail-queue-dir' is defined + (if (file-directory-p smtpmail-queue-dir) + (mu4e--main-view-queue) + "") + "\n" + (mu4e--main-action "\t* [@]News\n" #'mu4e-news nil "N") + (mu4e--main-action "\t* [@]About mu4e\n" #'mu4e-about nil "A") + (mu4e--main-action "\t* [@]Help\n" #'mu4e-display-manual nil "H") + (mu4e--main-action "\t* [@]quit\n" #'mu4e-quit nil "q") + "\n" + (propertize " Info\n\n" 'face 'mu4e-title-face) + (mu4e--key-val "last updated" + (current-time-string + (plist-get mu4e-index-update-status :tstamp))) + (mu4e--key-val "database-path" (mu4e-database-path)) + (mu4e--key-val "maildir" (mu4e-root-maildir)) + (mu4e--key-val "in store" + (format "%d" (plist-get mu4e--server-props :doccount)) + "messages") + (if mu4e-main-hide-personal-addresses "" + (mu4e--key-val "personal addresses" + (if addrs (mapconcat #'identity addrs ", " ) "none")))) + + (if mu4e-main-hide-personal-addresses "" + (unless (mu4e-personal-address-p user-mail-address) + (mu4e-message (concat + "Tip: `user-mail-address' ('%s') is not part " + "of mu's addresses; add it with 'mu init + --my-address='") user-mail-address))) + (goto-char pos))))) + +(defun mu4e--main-view-queue () + "Display queue-related actions in the main view." + (concat + (mu4e--main-action "\t* toggle [@]mail sending mode " + #'mu4e--main-toggle-mail-sending-mode) + "(currently " + (propertize (if smtpmail-queue-mail "queued" "direct") + 'face 'mu4e-header-key-face) + ")\n" + (let ((queue-size (mu4e--main-queue-size))) + (if (zerop queue-size) + "" + (mu4e--main-action + (format "\t* [@]flush %s queued %s\n" + (propertize (int-to-string queue-size) + 'face 'mu4e-header-key-face) + (if (> queue-size 1) "mails" "mail")) + 'smtpmail-send-queued-mail))))) + +(defun mu4e--main-queue-size () + "Return, as an int, the number of emails in the queue." + (condition-case nil + (with-temp-buffer + (insert-file-contents (expand-file-name smtpmail-queue-index-file + smtpmail-queue-dir)) + (count-lines (point-min) (point-max))) + (error 0))) + +(declare-function mu4e--start "mu4e") + +(defun mu4e--main-view () + "(Re)create the mu4e main-view, and switch to it. + +If `mu4e-split-view' equals \\='single-window, show a mu4e menu +instead." + (if (eq mu4e-split-view 'single-window) + (mu4e--main-menu) + (let ((buf (get-buffer-create mu4e-main-buffer-name)) + (inhibit-read-only t)) + (with-current-buffer buf + (mu4e--main-redraw)) + (mu4e-display-buffer buf t) + (run-hooks 'mu4e-main-rendered-hook))) + (goto-char (point-min))) + +;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +;; Interactive functions +;; Toggle mail sending mode without switching +(defun mu4e--main-toggle-mail-sending-mode () + "Toggle sending mail mode, either queued or direct." + (interactive) + (unless (file-directory-p smtpmail-queue-dir) + (mu4e-error "`smtpmail-queue-dir' does not exist")) + (setq smtpmail-queue-mail (not smtpmail-queue-mail)) + (message (concat "Outgoing mail will now be " + (if smtpmail-queue-mail "queued" "sent directly"))) + (unless (or (eq mu4e-split-view 'single-window) + (not (buffer-live-p (get-buffer mu4e-main-buffer-name)))) + (mu4e--main-redraw))) + +(defun mu4e--main-menu () + "The mu4e main menu in the mini-buffer." + (let ((func (mu4e-read-option + "Do: " + '(("jump" . mu4e~headers-jump-to-maildir) + ("search" . mu4e-search) + ("Compose" . mu4e-compose-new) + ("bookmarks" . mu4e-search-bookmark) + (";Switch context" . mu4e-context-switch) + ("Update" . mu4e-update-mail-and-index) + ("News" . mu4e-news) + ("About" . mu4e-about) + ("Help " . mu4e-display-manual))))) + (call-interactively func) + (when (eq func 'mu4e-context-switch) + (sit-for 1) + (mu4e--main-menu)))) + +(provide 'mu4e-main) +;;; mu4e-main.el ends here diff --git a/mu4e/mu4e-mark.el b/mu4e/mu4e-mark.el new file mode 100644 index 0000000..c3947a7 --- /dev/null +++ b/mu4e/mu4e-mark.el @@ -0,0 +1,471 @@ +;;; mu4e-mark.el --- Marking messages -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2024 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; In this file are function related to marking messages; they assume we are +;; currently in the headers buffer. + +;;; Code: + +(require 'mu4e-server) +(require 'mu4e-message) +(require 'mu4e-folders) + +;; keep byte-compiler happy +(declare-function mu4e~headers-mark "mu4e-headers") +(declare-function mu4e~headers-goto-docid "mu4e-headers") +(declare-function mu4e-headers-next "mu4e-headers") + +;;; Variables & constants + +(defcustom mu4e-headers-leave-behavior 'ask + "What to do when user leaves the current headers view. + +\"Leaving\" here means quitting the headers views, refreshing it +or even quitting mu4e or Emacs. + +Value is one of the following symbols: +- `ask' ask user whether to ignore the marks +- `apply' automatically apply the marks before doing anything else +- `ignore' automatically ignore the marks without asking" + :type '(choice (const :tag "ask user whether to ignore marks" ask) + (const :tag "apply marks without asking" apply) + (const :tag "ignore marks without asking" ignore)) + :group 'mu4e-headers) + +(defcustom mu4e-mark-execute-pre-hook nil + "Hook run just *before* a mark is applied to a message. +The hook function is called with two arguments, the mark being +executed and the message itself." + :type 'hook + :group 'mu4e-headers) + +(defvar mu4e-headers-show-target t + "Whether to show targets (such as \"-> delete\", \"-> /archive\") +when marking message. Normally, this is useful information for +the user, however, when you often mark large numbers (thousands) +of message, showing the target makes this quite a bit +slower (showing the target uses Emacs overlays, which can be slow +when overused).") + +;;; Insert stuff + +(defvar mu4e--mark-map nil + "Contains a mapping of docid->markinfo. +When a message is marked, the information is added here. markinfo +is a cons cell consisting of the following: (mark . target) where +MARK is the type of mark (move, trash, delete) TARGET (optional) +is the target directory (for \"move\")") + +;; the mark-map is specific for the current header buffer +;; currently, there can't be more than one, but we never know what will +;; happen in the future + +;; the fringe is the space on the left of headers, where we put marks below some +;; handy definitions; only `mu4e-mark-fringe-len' should be change (if ever), +;; the others follow from that. +(defconst mu4e--mark-fringe-len 2 + "Width of the fringe for marks on the left.") +(defconst mu4e--mark-fringe (make-string mu4e--mark-fringe-len ?\s) + "The space on the left of message headers to put marks.") +(defconst mu4e--mark-fringe-format (format "%%-%ds" mu4e--mark-fringe-len) + "Format string to set a mark and leave remaining space.") + +(defun mu4e--mark-initialize () + "Initialize the marks-subsystem." + (set (make-local-variable 'mu4e--mark-map) (make-hash-table)) + ;; ask user when kill buffer / emacs with live marks. + ;; (subject to mu4e-headers-leave-behavior) + (add-hook 'kill-buffer-query-functions + #'mu4e-mark-handle-when-leaving nil t) + (add-hook 'kill-emacs-query-functions + #'mu4e-mark-handle-when-leaving nil t)) + +(defun mu4e--mark-clear () + "Clear the marks-subsystem." + (clrhash mu4e--mark-map)) + +(defun mu4e--mark-find-headers-buffer () + "Find the headers buffer, if any." + (seq-find (lambda (_) + (mu4e-current-buffer-type-p 'headers)) + (buffer-list))) + +(defmacro mu4e--mark-in-context (&rest body) + "Evaluate BODY in the context of the headers buffer. +The current buffer must be either a headers or view buffer." + `(cond + ((mu4e-current-buffer-type-p 'headers) ,@body) + ((mu4e-current-buffer-type-p 'view) + (when (buffer-live-p (mu4e-get-headers-buffer)) + (let* ((msg (mu4e-message-at-point)) + (docid (mu4e-message-field msg :docid))) + (with-current-buffer (mu4e-get-headers-buffer) + (when (mu4e~headers-goto-docid docid) + ,@body + ))))))) + +(defconst mu4e-marks + '((refile + :char ("r" . "▶") + :prompt "refile" + :dyn-target (lambda (target msg) (mu4e-get-refile-folder msg)) + :action (lambda (docid msg target) + (mu4e--server-move docid (mu4e--mark-check-target target) "-N"))) + (delete + :char ("D" . "x") + :prompt "Delete" + :show-target (lambda (target) "delete") + :action (lambda (docid msg target) (mu4e--server-remove docid))) + (flag + :char ("+" . "✚") + :prompt "+flag" + :show-target (lambda (target) "flag") + :action (lambda (docid msg target) + (mu4e--server-move docid nil "+F-u-N"))) + (move + :char ("m" . "▷") + :prompt "move" + :ask-target mu4e--mark-get-move-target + :action (lambda (docid msg target) + (mu4e--server-move docid (mu4e--mark-check-target target) "-N"))) + (read + :char ("!" . "◼") + :prompt "!read" + :show-target (lambda (target) "read") + :action (lambda (docid msg target) (mu4e--server-move docid nil "+S-u-N"))) + (trash + :char ("d" . "▼") + :prompt "dtrash" + :dyn-target (lambda (target msg) (mu4e-get-trash-folder msg)) + :action (lambda (docid msg target) + (mu4e--server-move docid + (mu4e--mark-check-target target) "+T-N"))) + (unflag + :char ("-" . "➖") + :prompt "-unflag" + :show-target (lambda (target) "unflag") + :action (lambda (docid msg target) (mu4e--server-move docid nil "-F-N"))) + (untrash + :char ("=" . "▲") + :prompt "=untrash" + :show-target (lambda (target) "untrash") + :action (lambda (docid msg target) (mu4e--server-move docid nil "-T"))) + (unread + :char ("?" . "◻") + :prompt "?unread" + :show-target (lambda (target) "unread") + :action (lambda (docid msg target) (mu4e--server-move docid nil "-S+u-N"))) + (unmark + :char " " + :prompt "unmark" + :action (mu4e-error "No action for unmarking")) + (action + :char ( "a" . "◯") + :prompt "action" + :ask-target (lambda () (mu4e-read-option "Action: " mu4e-headers-actions)) + :action (lambda (docid msg actionfunc) + (save-excursion + (when (mu4e~headers-goto-docid docid) + (mu4e-headers-action actionfunc))))) + (something + :char ("*" . "✱") + :prompt "*something" + :action (mu4e-error "No action for deferred mark"))) + + "The list of all the possible marks. +This is an alist mapping mark symbols to their properties. The +properties are: + :char (string) or (basic . fancy) The character to display in + the headers view. Either a single-character string, or a + dotted-pair cons cell where the second item will be used if + `mu4e-use-fancy-chars' is t, otherwise we'll use + the first one. It can also be a plain string for backwards + compatibility since we didn't always support + `mu4e-use-fancy-chars' here. + :prompt (string) The prompt to use when asking for marks (used for + example when marking a whole thread) + :ask-target (function returning a string) Get the target. This + function run once per bulk-operation, and thus is suitable + for user-interaction. If nil, the target is nil. + :dyn-target (function from (TARGET MSG) to string). Compute + the dynamic target. This is run once per message, which is + passed as MSG. The default is to just return the target. + :show-target (function from TARGET to string) How to display + the target. + :action (function taking (DOCID MSG TARGET)). The action to + apply on the message.") + +(defun mu4e-mark-at-point (mark target) + "Mark (or unmark) message at point. +MARK specifies the mark-type. For `move'-marks and `trash'-marks +the TARGET argument is non-nil and specifies to which maildir the +message is to be moved/trashed. The function works in both +headers buffers and message buffers. + +The following marks are available, and the corresponding props: + + MARK TARGET description + ---------------------------------------------------------- + `refile' y mark this message for archiving + `something' n mark this message for *something* (decided later) + `delete' n remove the message + `flag' n mark this message for flagging + `move' y move the message to some folder + `read' n mark the message as read + `trash' y trash the message to some folder + `unflag' n mark this message for unflagging + `untrash' n remove the `trashed' flag from a message + `unmark' n unmark this message + `unread' n mark the message as unread + `action' y mark the message for some action." + (interactive) + (let* ((msg (mu4e-message-at-point)) + (docid (mu4e-message-field msg :docid)) + ;; get a cell with the mark char and the "move" already has a target + ;; (the target folder) the other ones get a pseudo "target", as info + ;; for the user. + (markdesc (cdr (or (assq mark mu4e-marks) + (mu4e-error "Invalid mark %S" mark)))) + (get-markkar + (lambda (char) + (if (listp char) + (if mu4e-use-fancy-chars (cdr char) (car char)) + char))) + (markkar (funcall get-markkar (plist-get markdesc :char))) + (target (mu4e--mark-get-dyn-target mark target)) + (show-fct (plist-get markdesc :show-target)) + (shown-target (if show-fct + (funcall show-fct target) + (if target (format "%S" target))))) + (unless docid (mu4e-warn "No message on this line")) + (unless (eq major-mode 'mu4e-headers-mode) + (mu4e-error "Not in headers-mode")) + (save-excursion + (when (mu4e~headers-mark docid markkar) + ;; update the hash -- remove everything current, and if add the new + ;; stuff, unless we're unmarking + (remhash docid mu4e--mark-map) + ;; remove possible mark overlays + (remove-overlays (line-beginning-position) (line-end-position) + 'mu4e-mark t) + ;; now, let's set a mark (unless we were unmarking) + (unless (eql mark 'unmark) + (puthash docid (cons mark target) mu4e--mark-map) + ;; when we have a target (ie., when moving), show the target folder in + ;; an overlay + (when (and shown-target mu4e-headers-show-target) + (let* ((targetstr (propertize (concat "-> " shown-target " ") + 'face 'mu4e-system-face)) + ;; mu4e~headers-goto-docid docid t \will take us just after + ;; the docid cookie and then we skip the mu4e--mark-fringe + (start (+ (length mu4e--mark-fringe) + (mu4e~headers-goto-docid docid t))) + (overlay (make-overlay start (+ start (length targetstr))))) + (overlay-put overlay 'display targetstr) + (overlay-put overlay 'mu4e-mark t) + (overlay-put overlay 'evaporate t) + docid))))))) + +(defun mu4e--mark-get-move-target () + "Ask for a move target, and propose to create it if it does not exist." + (let* ((target (mu4e-ask-maildir "Move message to: ")) + (target (if (string= (substring target 0 1) "/") + target + (concat "/" target))) + (fulltarget (mu4e-join-paths (mu4e-root-maildir) target))) + (when (mu4e-create-maildir-maybe fulltarget) + target))) + +(defun mu4e--mark-ask-target (mark) + "Ask the target for MARK, if the user should be asked the target." + (let ((getter (plist-get (cdr (assq mark mu4e-marks)) :ask-target))) + (and getter (funcall getter)))) + +(defun mu4e--mark-get-dyn-target (mark target) + "Get the dynamic TARGET for MARK. +The result may depend on the message at point." + (let ((getter (plist-get (cdr (assq mark mu4e-marks)) :dyn-target))) + (if getter + (funcall getter target (mu4e-message-at-point)) + target))) + +(defun mu4e-mark-set (mark &optional target) + "Mark the header at point with MARK or all in the region. +Optionally, provide TARGET (for moves)." + (unless target + (setq target (mu4e--mark-ask-target mark))) + (if (not (use-region-p)) + ;; single message + (mu4e-mark-at-point mark target) + ;; mark all messages in the region. + (save-excursion + (let ((cant-go-further) (eor (region-end))) + (goto-char (region-beginning)) + (while (and (< (point) eor) (not cant-go-further)) + (mu4e-mark-at-point mark target) + (setq cant-go-further (not (mu4e-headers-next)))))))) + +(defun mu4e-mark-restore (docid) + "Restore the visual mark for the message with DOCID." + (let ((markcell (gethash docid mu4e--mark-map))) + (when markcell + (save-excursion + (when (mu4e~headers-goto-docid docid) + (mu4e-mark-at-point (car markcell) (cdr markcell))))))) + +(defun mu4e--mark-get-markpair (prompt &optional allow-something) + "Ask user with PROMPT for a mark and return (MARK . TARGET). +If ALLOW-SOMETHING is non-nil, allow the `something' pseudo mark +as well." + (let* ((marks (mapcar (lambda (markdescr) + (cons (plist-get (cdr markdescr) :prompt) + (car markdescr))) + mu4e-marks)) + (marks + (if allow-something + marks (seq-remove (lambda (m) (eq 'something (cdr m))) marks))) + (mark (mu4e-read-option prompt marks)) + (target (mu4e--mark-ask-target mark))) + (cons mark target))) + +(defun mu4e-mark-resolve-deferred-marks () + "Check if there are any deferred ('something') mark-instances. +If there are such marks, replace them with a _real_ mark (ask the +user which one)." + (interactive) + (mu4e--mark-in-context + (let ((markpair)) + (maphash + (lambda (docid val) + (let ((mark (car val))) + (when (eql mark 'something) + (unless markpair + (setq markpair + (mu4e--mark-get-markpair "Set deferred mark(s) to: " nil))) + (save-excursion + (when (mu4e~headers-goto-docid docid) + (mu4e-mark-set (car markpair) (cdr markpair))))))) + mu4e--mark-map)))) + +(defun mu4e--mark-check-target (target) + "Check if TARGET exists; if not, offer to create it." + (let ((fulltarget (mu4e-join-paths (mu4e-root-maildir) target))) + (if (not (mu4e-create-maildir-maybe fulltarget)) + (mu4e-error "Target dir %s does not exist " fulltarget) + target))) + +(defun mu4e-mark-execute-all (&optional no-confirmation) + "Execute the actions for all marked messages in this buffer. +After the actions have been executed successfully, the affected +messages are *hidden* from the current header list. Since the +headers are the result of a search, we cannot be certain that the +messages no longer match the current one - to get that +certainty, we need to rerun the search, but we don't want to do +that automatically, as it may be too slow and/or break the user's +flow. Therefore, we hide the message, which in practice seems to +work well. + +If NO-CONFIRMATION is non-nil, don't ask user for confirmation." + (interactive "P") + (mu4e--mark-in-context + (let* ((marknum (mu4e-mark-marks-num)) + (prompt (format "Are you sure you want to execute %d mark%s?" + marknum (if (> marknum 1) "s" "")))) + (if (zerop marknum) + (mu4e-warn "Nothing is marked") + (mu4e-mark-resolve-deferred-marks) + (when (or no-confirmation (y-or-n-p prompt)) + (maphash + (lambda (docid val) + (let* ((mark (car val)) (target (cdr val)) + (markdescr (assq mark mu4e-marks)) + (msg (save-excursion + (mu4e~headers-goto-docid docid) + (mu4e-message-at-point)))) + ;; note: whenever you do something with the message, + ;; it looses its N (new) flag + (if markdescr + (progn + (run-hook-with-args + 'mu4e-mark-execute-pre-hook mark msg) + (funcall (plist-get (cdr markdescr) :action) + docid msg target)) + (mu4e-error "Unrecognized mark %S" mark)))) + mu4e--mark-map)) + (mu4e-mark-unmark-all 'no-confirm) + (message nil))))) + +(defun mu4e-mark-unmark-all (&optional no-confirmation) + "Unmark all marked messages." + (interactive) + (mu4e--mark-in-context + (when (zerop (mu4e-mark-marks-num)) + (mu4e-warn "Nothing is marked")) + (let* ((marknum (hash-table-count mu4e--mark-map)) + (prompt (format "Are you sure you want to unmark %d message%s?" + marknum (if (> marknum 1) "s" "")))) + (when (or no-confirmation (y-or-n-p prompt)) + (maphash + (lambda (docid _val) + (save-excursion + (when (mu4e~headers-goto-docid docid) + (mu4e-mark-set 'unmark)))) + mu4e--mark-map) + ;; in any case, clear the marks map + (mu4e--mark-clear))))) + +(defun mu4e-mark-docid-marked-p (docid) + "Is the given DOCID marked?" + (when (gethash docid mu4e--mark-map) t)) + +(defun mu4e-mark-marks-num () + "Return the number of mark-instances in the current buffer." + (mu4e--mark-in-context + (if mu4e--mark-map (hash-table-count mu4e--mark-map) 0))) + +(defun mu4e-mark-handle-when-leaving () + "Handle any mark-instances in the current buffer when leaving. +This is done according to the value of +`mu4e-headers-leave-behavior'. This function is to be called +before any further action (like searching, quitting the buffer) +is taken; returning t means \"take the following action\", return +nil means \"don't do anything\"." + (mu4e--mark-in-context + (let ((marknum (mu4e-mark-marks-num)) + (what mu4e-headers-leave-behavior)) + (unless (zerop marknum) ;; nothing to do? + (when (eq what 'ask) + (setq what (mu4e-read-option + (format "There are %d existing mark(s); should we: " + marknum) + '( ("apply marks" . apply) + ("ignore marks?" . ignore))))) + ;; we determined what to do... now do it + (when (eq what 'apply) + (mu4e-mark-execute-all t))))) + t) ;; return t for compat with `kill-buffer-query-functions + +;;; _ +(provide 'mu4e-mark) +;;; mu4e-mark.el ends here diff --git a/mu4e/mu4e-message.el b/mu4e/mu4e-message.el new file mode 100644 index 0000000..95f8aff --- /dev/null +++ b/mu4e/mu4e-message.el @@ -0,0 +1,247 @@ +;;; mu4e-message.el --- Working with mu4e-message plists -*- lexical-binding: t -*- + +;; Copyright (C) 2012-2022 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Functions to get data from mu4e-message plist structure + +;;; Code: + +(require 'mu4e-vars) +(require 'mu4e-contacts) +(require 'mu4e-window) +(require 'flow-fill) +(require 'shr) +(require 'pp) + +(declare-function mu4e-error "mu4e-helpers") +(declare-function mu4e-warn "mu4e-helpers") +(declare-function mu4e-personal-address-p "mu4e-contacts") +(declare-function mu4e-make-temp-file "mu4e-helpers") + +;;; Message fields + +(defsubst mu4e-message-field-raw (msg field) + "Retrieve FIELD from message plist MSG. + +See \"mu fields\" for the full list of field, in particular the +\"sexp\" column. + +Returns nil if the field does not exist. + +A message plist looks something like: +\(:docid 32461 + :from ((:name \"Nikola Tesla\" :email \"niko@example.com\")) + :to ((:name \"Thomas Edison\" :email \"tom@example.com\")) + :cc ((:name \"Rupert The Monkey\" :email \"rupert@example.com\")) + :subject \"RE: what about the 50K?\" + :date (20369 17624 0) + :size 4337 + :message-id \"238C8233AB82D81EE81AF0114E4E74@123213.mail.example.com\" + :path \"/home/tom/Maildir/INBOX/cur/133443243973_1.10027.atlas:2,S\" + :maildir \"/INBOX\" + :priority normal + :flags (seen) +\)). +Some notes on the format: +- The address fields are lists of plist (:name NAME :email EMAIL), + where the :name part can be absent. The `mu4e-contact-name' and + `mu4e-contact-email' accessors can be useful for this. +- The date is in format emacs uses in `current-time' +- Attachments are a list of elements with fields :index (the number of + the MIME-part), :name (the file name, if any), :mime-type (the + MIME-type, if any) and :size (the size in bytes, if any). +- Messages in the Headers view come from the database and do not have + :attachments, :body-txt or :body-html fields. Message in the + Message view use the actual message file, and do include these fields." + ;; after all this documentation, the spectacular implementation + (if msg + (plist-get msg field) + (mu4e-error "Message must be non-nil"))) + +(defsubst mu4e-message-field (msg field) + "Retrieve FIELD from message plist MSG. +Like `mu4e-message-field-nil', but will sanitize nil values: +- all string field except body-txt/body-html: nil -> \"\" +- numeric fields + dates : nil -> 0 +- all others : return the value +Thus, function will return nil for empty lists, non-existing body-txt +or body-html." + (let ((val (mu4e-message-field-raw msg field))) + (cond + (val + val) ;; non-nil -> just return it + ((member field '(:subject :message-id :path :maildir :in-reply-to)) + "") ;; string fields except body-txt, body-html: nil -> "" + ((member field '(:body-html :body-txt)) + val) + ((member field '(:docid :size)) + 0) ;; numeric type: nil -> 0 + (t + val)))) ;; otherwise, just return nil + +(defsubst mu4e-message-has-field (msg field) + "If MSG has a FIELD return t, nil otherwise." + (plist-member msg field)) + +(defsubst mu4e-message-at-point (&optional noerror) + "Get the message s-expression for the message at point. +Either the headers buffer or the view buffer, or nil if there is +no such message. If optional NOERROR is non-nil, do not raise an +error when there is no message at point." + (or (cond + ((eq major-mode 'mu4e-headers-mode) (get-text-property (point) 'msg)) + ((eq major-mode 'mu4e-view-mode) mu4e--view-message)) + (unless noerror (mu4e-warn "No message at point")))) + +(defsubst mu4e-message-field-at-point (field) + "Get the field FIELD from the message at point. +This is equivalent to: + (mu4e-message-field (mu4e-message-at-point) FIELD)." + (mu4e-message-field (mu4e-message-at-point) field)) + +(defun mu4e-message-contact-field-matches (msg cfield rx) + "Does MSG's contact-field CFIELD match regexp RX? +Check if any of the of the CFIELD in MSG matches RX. I.e. +anything in field CFIELD (either :to, :from, :cc or :bcc, or a +list of those) of msg MSG matches (with their name or e-mail +address) regular expressions RX. If there is a match, return +non-nil; otherwise return nil. RX can also be a list of regular +expressions, in which case any of those are tried for a match." + (cond + ((null cfield)) + ((listp cfield) + (seq-find (lambda (cf) (mu4e-message-contact-field-matches msg cf rx)) + cfield)) + ((listp rx) + ;; if rx is a list, try each one of them for a match + (seq-find + (lambda (a-rx) (mu4e-message-contact-field-matches msg cfield a-rx)) + rx)) + (t + ;; not a list, check the rx + (seq-find + (lambda (ct) + (let ((name (mu4e-contact-name ct)) + (email (mu4e-contact-email ct)) + ;; the 'rx' may be some `/rx/` from mu4e-personal-addresses; + ;; so let's detect and extract in that case. + (rx (if (string-match-p "^\\(.*\\)/$" rx) + (substring rx 1 -1) rx))) + (or + (and name (string-match rx name)) + (and email (string-match rx email))))) + (mu4e-message-field msg cfield))))) + +(defun mu4e-message-contact-field-matches-me (msg cfield) + "Does contact-field CFIELD in MSG match me? +Checks whether any +of the of the contacts in field CFIELD (either :to, :from, :cc or +:bcc) of msg MSG matches *me*, that is, any of the addresses for +which `mu4e-personal-address-p' return t. Returns the contact +cell that matched, or nil." + (seq-find (lambda (cell) + (mu4e-personal-address-p (mu4e-contact-email cell))) + (mu4e-message-field msg cfield))) + +(defun mu4e-message-sent-by-me (msg) + "Is this MSG (to be) sent by me? +Checks if the from field matches user's personal addresses." + (mu4e-message-contact-field-matches-me msg :from)) + +(defun mu4e-message-personal-p (msg) + "Does MSG have user's personal address? +In any of the contact + fields?" + (seq-some + (lambda (field) + (mu4e-message-contact-field-matches-me msg field)) + '(:from :to :cc :bcc))) + +(defsubst mu4e-message-part-field (msgpart field) + "Get some FIELD from MSGPART. +A part would look something like: + (:index 2 :name \"photo.jpg\" :mime-type \"image/jpeg\" :size 147331)." + (plist-get msgpart field)) + +;; backward compatibility ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; +(defalias 'mu4e-msg-field 'mu4e-message-field) + +(defun mu4e-field-at-point (field) + "Get FIELD for the message at point. +Either in the headers buffer or the view buffer. Field is a +symbol, see `mu4e-header-info'." + (plist-get (mu4e-message-at-point) field)) + +(defun mu4e-message-readable-path (&optional msg) + "Get a readable path to MSG or raise an error. +If MSG is nil, use `mu4e-message-at-point'." + (let ((path (plist-get (or msg (mu4e-message-at-point)) :path))) + (unless (file-readable-p path) + (mu4e-error "No readable message at %s; database outdated?" path)) + path)) + +(defun mu4e-copy-message-path () + "Copy the message-path of message at point to the kill ring." + (interactive) + (let ((path (mu4e-message-field-at-point :path))) + (kill-new path) + (mu4e-message "Saved '%s' to kill-ring" path))) + +(defun mu4e-sexp-at-point () + "Show or hide the s-expression for the message-at-point, if any." + (interactive) + (if-let ((win (get-buffer-window mu4e--sexp-buffer-name))) + (delete-window win) + (when-let ((msg (mu4e-message-at-point 'noerror))) + (when (buffer-live-p mu4e--sexp-buffer-name) + (kill-buffer mu4e--sexp-buffer-name)) + (with-current-buffer-window + (get-buffer-create mu4e--sexp-buffer-name) nil nil + (if (fboundp 'lisp-data-mode) + (lisp-data-mode) + (lisp-mode)) + (insert (pp-to-string msg)) + (font-lock-ensure) + ;; add basic `quit-window' bindings + (view-mode 1))))) + +(declare-function mu4e--decoded-message "mu4e-compose") + +(defun mu4e-fetch-field (msg hdr &optional first) + "Find the value for an arbitrary header field HDR from MSG. + +If the header appears multiple times, the field values are +concatenated, unless FIRST is non-nil, in which case only the +first value is returned. See `message-field-value' and +`nessage-fetch-field' for details. + +Note: this loads the full message file such that any available +message header can be used. If the header is part of the MSG +plist, it is much more efficient to get the information from that +plist." + (with-temp-buffer + (insert (mu4e--decoded-message msg 'headers-only)) + (message-field-value hdr first))) +;;; +(provide 'mu4e-message) +;;; mu4e-message.el ends here diff --git a/mu4e/mu4e-mime-parts.el b/mu4e/mu4e-mime-parts.el new file mode 100644 index 0000000..73a1742 --- /dev/null +++ b/mu4e/mu4e-mime-parts.el @@ -0,0 +1,486 @@ +;;; mu4e-mime-parts.el --- Dealing with MIME-parts & URLs -*- lexical-binding: t -*- + +;; Copyright (C) 2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Implements functions and variables for dealing with MIME-parts and URLs. + + +;;; TODO: +;; [~] mime part candidate sorting -> is his even possible generally? +;; [ ] URL support + +;;; Code: + + +(require 'mu4e-vars) +(require 'mu4e-folders) +(require 'gnus-art) + + + +(defcustom mu4e-view-open-program + (pcase system-type + ('darwin "open") + ('cygwin "cygstart") + (_ "xdg-open")) + "Tool to open the correct program for a given file or MIME-type. +May also be a function of a single argument, the file to be +opened. + +In the function-valued case a likely candidate is +`mailcap-view-file' although note that there was an Emacs bug up +to Emacs 29 which prevented opening a file if `mailcap-mime-data' +specified a function as viewer." + :type '(choice string function) + :group 'mu4e-view) + + +;; remember the mime-handles, so we can clean them up when +;; we quit this buffer. +(defvar-local mu4e~gnus-article-mime-handles nil) +(put 'mu4e~gnus-article-mime-handles 'permanent-local t) + +(defun mu4e--view-kill-mime-handles () + "Kill cached MIME-handles, if any." + (when mu4e~gnus-article-mime-handles + (mm-destroy-parts mu4e~gnus-article-mime-handles) + (setq mu4e~gnus-article-mime-handles nil))) + + +;;; MIME-parts +(defvar-local mu4e--view-mime-parts nil + "Cached MIME parts for this message.") + + +(defun mu4e-view-mime-parts() + "Get the list of MIME parts for this message. +The list is a list of plists, one for each MIME-part. + +The plists have the properties: + + :part-index : Gnus index number + :mime-type : MIME-type (string) or nil + :encoding : Content encoding (string) or nil + :disposition : Content disposition (attachment\" or inline\") or nil + :filename : The file name if it has one, or an invented one + otherwise + +There are some internal fields as well, e.g. ; subject to change: + + :target-dir : Target directory for saving + :attachment-like : When it has a filename, we can save it + :handle : Gnus handle." + (or mu4e--view-mime-parts + (setq + mu4e--view-mime-parts + (let ((parts) (indices)) + (save-excursion + (goto-char (point-min)) + (while (not (eobp)) + (when-let ((part (get-text-property (point) 'gnus-data)) + (index (get-text-property (point) 'gnus-part))) + (when (and part (numberp index) (not (member index indices))) + (let* ((disp (mm-handle-disposition part)) + (fname (mm-handle-filename part)) + (mime-type (mm-handle-media-type part)) + (info + `(:part-index ,index + :mime-type ,mime-type + :encoding ,(mm-handle-encoding part) + :disposition ,(car-safe disp) + + ;; if there's no file-name, invent one + ;; XXX perhaps guess extension based on mime-type + :filename ,(or fname + (format "mime-part-%02d" index)) + + ;; below are internal + + :target-dir ,(mu4e-determine-attachment-dir + fname mime-type) + ;; 'attachment-like' just means it has its own + ;; filename an we thus we can save it through + ;; `mu4e-view-save-attachments', even if it has an + ;; 'inline' disposition. + :attachment-like ,(if fname t nil) + :handle ,part))) + (push index indices) + (push info parts)))) + (goto-char (or (next-single-property-change (point) 'gnus-part) + (point-max))))) + ;; sort by the GNU's part-index, so the order is the same as + ;; in the message on screen + (seq-sort (lambda (p1 p2) (< (plist-get p1 :part-index) + (plist-get p2 :part-index))) parts))))) + +;; https://emacs.stackexchange.com/questions/74547/completing-read-search-also-in-annotationsxc + +(defun mu4e--uniqify-file-name (fname) + "Return a non-yet-existing filename based on FNAME. +If FNAME does not yet exist, return it unchanged. +Otherwise, return a file with a unique number appended to the base-name." + (let ((num 1) (orig-name fname)) + (while (file-exists-p fname) + (setq fname (format "%s(%d)%s%s" + (file-name-sans-extension orig-name) + num + (if (file-name-extension orig-name) "." "") + (file-name-extension orig-name))) + (cl-incf num))) + fname) + +(defvar mu4e--completions-table nil) + +(defun mu4e-view-complete-all () + "Pick all current candidates." + (interactive) + (if (bound-and-true-p helm-mode) + (mu4e-warn "Not supported with helm") + (when mu4e--completions-table + (insert (string-join + (seq-map #'car mu4e--completions-table) ", "))))) + +(defvar mu4e-view-completion-minor-mode-map + (let ((map (make-sparse-keymap))) + (define-key map (kbd "C-c C-a") #'mu4e-view-complete-all) + ;; XXX perhaps a binding for clearing all? + map) + "Keybindings for mu4e-view completion.") + +(define-minor-mode mu4e-view-completion-minor-mode + "Minor-mode for completing mu4e mime parts." + :global nil + :init-value nil ;; disabled by default + :group 'mu4e + :lighter "" + :keymap mu4e-view-completion-minor-mode-map) + +(defun mu4e--part-annotation (candidate part type longest-filename) + "Calculate the annotation candidates as per +`:annotation-function' (see `completion-extra-properties') + +CANDIDATE is the value to annotate. + +PART is the matching MIME-part for the annotation, (as per +`mu4e-view-mime-part'). + +TYPE is the of what to annotate, a symbol, either ATTACHMENT or +MIME-PART. + +LONGEST-FILENAME is the length of the longest filename; this +information' is used for alignment." + (let* ((filename (propertize (or (plist-get part :filename) "") + 'face 'mu4e-header-key-face)) + (mimetype (propertize (or (plist-get part :mime-type) "") + 'face 'mu4e-header-value-face)) + (target (propertize (or (plist-get part :target-dir) "") + 'face 'mu4e-system-face))) + + ;; Sadly, we need too align by hand; this makes some assumptions + ;; such a mono-type font and enough space in the minibuffer; and + ;; mixing values and representation; ideally Emacs would allow + ;; just take some columns and align them (since it knows the display + ;; details). + + (pcase type + ('attachment + ;; in case we're annotating an attachment, the filename is + ;; the candidate (completion), so we don't need it in the + ;; the annotation. We just need to but some space at beginning + ;; for alignment + (concat + (make-string (- (+ longest-filename 2) + (length (format "%s" candidate))) ?\s) + (format "%20s" mimetype) + " " + (format "%s" (concat "-> " target)))) + ('mime-part + ;; when we're annotating a mime-part, the candidate is just a number, + ;; and the filename is part of the annotation. + (concat + " " + filename + (make-string (- (+ longest-filename 2) + (length filename)) ?\s) + (format "%20s" mimetype) + " " + (format "%s" (concat "-> " target)))) + (_ (mu4e-error "Unsupported annotation type %s" type))))) + + +(defvar helm-comp-read-use-marked) +(defun mu4e--completing-read-real (prompt candidates multi) + "Call the appropriate completion-read function. +- PROMPT is a string informing the user what to complete +- CANDIDATES is an alist of candidates of the form + (id . part) +- MULTI if t, allow for completing _multiple_ candidates." + (cond + ((bound-and-true-p helm-mode) + ;; tweaks for "helm"; it's not nice to have to special-case for + ;; completion frameworks, but this has been supported for while. + ;; basically, with helm, helm-comp-read-use-marked + completing-read + ;; is preferred over completing-read-multiple + (let ((helm-comp-read-use-marked t)) + (completing-read prompt candidates))) + (multi + (completing-read-multiple prompt candidates)) + (t + (completing-read prompt candidates)))) + +(defun mu4e--completing-read (prompt candidates type &optional multi) + "Read the part-id of some MIME-type in this message. + +Presents the user with completions for the MIME-parts in +the current message. + +- PROMPT is a string informing the user what to complete +- CANDIDATES is an alist of candidates of the form + (id . part) +- TYPE is the annotation type to uses as per `mu4e--part-annotation'. +Optionally, +- MULTI if t, allow for completing _multiple_ candidates." + (cl-assert candidates) + (let* ((longest-filename (seq-max + (seq-map (lambda (c) + (length (plist-get (cdr c) :filename))) + candidates))) + (annotation-func (lambda (candidate) + (mu4e--part-annotation candidate + (cdr-safe + (assoc candidate candidates)) + type longest-filename))) + (completion-extra-properties + `(;; :affixation-function requires emacs 28 + :annotation-function ,annotation-func + :exit-function (lambda (_a _b) (setq mu4e--completions-table nil))))) + (setq mu4e--completions-table candidates) + (minibuffer-with-setup-hook + (lambda () + (mu4e-view-completion-minor-mode)) + (mu4e--completing-read-real prompt candidates multi)))) + +(defun mu4e-view-save-attachments (&optional ask-dir) + "Save files from the current view buffer. +This applies to all MIME-parts that are \"attachment-like\" (have a filename), +regardless of their disposition. + +With ASK-DIR is non-nil, user can specify the target-directory; otherwise +one is determined using `mu4e-attachment-dir'." + (interactive "P") + (let* ((parts (mu4e-view-mime-parts)) + (candidates (seq-map + (lambda (fpart) + (cons ;; (filename . annotation) + (plist-get fpart :filename) + fpart)) + (seq-filter + (lambda (part) (plist-get part :attachment-like)) + parts))) + (candidates (or candidates + (mu4e-warn "No attachments for this message"))) + (files (mu4e--completing-read "Save file(s): " candidates + 'attachment 'multi)) + (custom-dir (when ask-dir (read-directory-name + "Save to directory: ")))) + ;; we have determined what files to save, and where. + (seq-do (lambda (fname) + (let* ((part (cdr (assoc fname candidates))) + (path (mu4e--uniqify-file-name + (mu4e-join-paths + (or custom-dir (plist-get part :target-dir)) + (plist-get part :filename))))) + (mm-save-part-to-file (plist-get part :handle) path))) + files))) + +(defvar mu4e-view-mime-part-actions + '( + ;; + ;; some basic ones + ;; + + ;; save MIME-part to a file + (:name "save" :handler gnus-article-save-part :receives index) + ;; pipe MIME-part to some arbitrary shell command + (:name "|pipe" :handler gnus-article-pipe-part :receives index) + ;; open with the default handler, if any + (:name "open" :handler mu4e--view-open-file :receives temp) + ;; open with some custom file. + (:name "wopen-with" :handler (lambda (file)(mu4e--view-open-file file t)) + :receives temp) + + ;; + ;; some more examples + ;; + + ;; import GPG key + (:name "gpg" :handler epa-import-keys :receives temp) + ;; open in this emacs instance; tries to use the attachment name, + ;; so emacs can use specific modes etc. + (:name "emacs" :handler find-file-read-only :receives temp) + ;; open in this emacs instance, "raw" + (:name "raw" :handler (lambda (str) + (let ((tmpbuf + (get-buffer-create " *mu4e-raw-mime*"))) + (with-current-buffer tmpbuf + (insert str) + (view-mode) + (goto-char (point-min))) + (display-buffer tmpbuf))) :receives pipe)) + + "Specifies actions for MIME-parts. + +Each of the actions is a plist with keys +`(:name <name> ;; name of the action; shortcut is first letter of name + + :handler ;; one of: + ;; - a function receiving the index/temp/pipe + ;; - a string, which is taken as a shell command + + :receives ;; a symbol specifying what the handler receives + ;; - index: the index number of the mime part (default) + ;; - temp: the full path to the mime part in a + ;; temporary file, which is deleted immediately + ;; after the handler returns + ;; - pipe: the attachment is piped to some shell command + ;; or as a string parameter to a function +).") + + +(defun mu4e--view-mime-part-to-temp-file (handle) + "Write MIME-part HANDLE to a temporary file and return the file name. +The filename is deduced from the MIME-part's filename, or +otherwise random; the result is placed in a temporary directory +with a unique name. Returns the full path for the file created. +The directory and file are self-destructed." + (let* ((tmpdir (make-temp-file "mu4e-temp-" t)) + (fname (mm-handle-filename handle)) + (fname (and fname + (gnus-map-function mm-file-name-rewrite-functions + (file-name-nondirectory fname)))) + (fname (if fname + (concat tmpdir "/" (replace-regexp-in-string "/" "-" fname)) + (let ((temporary-file-directory tmpdir)) + (make-temp-file "mimepart"))))) + (mm-save-part-to-file handle fname) + (run-at-time "30 sec" nil + (lambda () (ignore-errors (delete-directory tmpdir t)))) + fname)) + +(defun mu4e--view-open-file (file &optional force-ask) + "Open FILE with default handler, if any. +Otherwise, or if FORCE-ASK is set, ask user for the program to +open with." + (if (and (not force-ask) + (functionp mu4e-view-open-program)) + (funcall mu4e-view-open-program file) + (let ((opener + (or (and (not force-ask) mu4e-view-open-program + (executable-find mu4e-view-open-program)) + (read-shell-command "Open MIME-part with: ")))) + (call-process opener nil 0 nil file)))) + +(defun mu4e-view-mime-part-action (&optional n) + "Apply some action to MIME-part N in the current message. +If N is not specified, ask for it. For instance, '3 A o' opens +the third MIME-part." + ;; (interactive + ;; (list (read-number "Number of MIME-part: "))) + (interactive) + (let* ((parts (mu4e-view-mime-parts)) + (candidates (seq-map + (lambda (part) + (cons (number-to-string + (plist-get part :part-index)) part)) + parts)) + (candidates (or candidates + (mu4e-warn "No MIME-parts for this message"))) + (ids (seq-map #'string-to-number + (if n (list (number-to-string n)) + (mu4e--completing-read "MIME-part(s) to operate on: " + candidates + 'mime-part 'multi)))) + (options + (mapcar (lambda (action) `(,(plist-get action :name) . ,action)) + mu4e-view-mime-part-actions)) + (action + (or (and options (mu4e-read-option "Action: " options)) + (mu4e-error "No such action"))) + (handler + (or (plist-get action :handler) + (mu4e-error "No :handler item found for action %S" action))) + (receives + (or (plist-get action :receives) + (mu4e-error "No :receives item found for action %S" action)))) + + ;; Apply the action to all selected MIME-parts + (seq-do (lambda (id) + (cl-assert (numberp id)) + (let* ((part (or (cdr-safe (assoc (number-to-string id) candidates)) + (mu4e-error "No part found for id %s" id))) + (handle (plist-get part :handle))) + (save-excursion + (cond + ((functionp handler) + (cond + ((eq receives 'index) (funcall handler id)) + ((eq receives 'pipe) + (funcall handler (mm-with-unibyte-buffer + (mm-insert-part handle) + (buffer-string)))) + ((eq receives 'temp) + (funcall handler + (mu4e--view-mime-part-to-temp-file handle))) + (t (mu4e-error "Invalid :receive for %S" action)))) + ((stringp handler) + (cond + ((eq receives 'index) + (shell-command + (concat handler " " (shell-quote-argument id)))) + ((eq receives 'pipe) + (progn + (mm-pipe-part handle handler))) + ((eq receives 'temp) + (shell-command + (shell-command + (concat + handler " " + (shell-quote-argument + (mu4e--view-mime-part-to-temp-file handle)))))) + (t (mu4e-error "Invalid action %S" action)))))))) + ids))) + +(defun mu4e-process-file-through-pipe (path pipecmd) + "Process file at PATH through a pipe with PIPECMD." + (let ((buf (get-buffer-create "*mu4e-output"))) + (with-current-buffer buf + (let ((inhibit-read-only t)) + (erase-buffer) + (call-process-shell-command pipecmd path t t) + (view-mode))) + (display-buffer buf))) + + + +(provide 'mu4e-mime-parts) +;;; mu4e-mime-parts.el ends here diff --git a/mu4e/mu4e-modeline.el b/mu4e/mu4e-modeline.el new file mode 100644 index 0000000..f1e668d --- /dev/null +++ b/mu4e/mu4e-modeline.el @@ -0,0 +1,141 @@ +;;; mu4e-modeline.el --- Modeline for mu4e -*- lexical-binding: t -*- + +;; Copyright (C) 2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; This file contains functionality for putting mu4e-related information in the +;; Emacs modeline, both buffer-specific and globally. + +;;; Code: + +(require 'cl-lib) + + +(defcustom mu4e-modeline-max-width 42 + "Determines the maximum length of the local modeline string. +If the string exceeds this limit, it will be truncated to fit. + +Note: this only affects the local modeline items (such as the context, +the search properties and the last query), not the global items +(such as the favorite bookmark results)." + :type 'integer + :group 'mu4e-modeline) + +(defcustom mu4e-modeline-prefer-bookmark-name t + "Show bookmark name rather than query in modeline. + +If non-nil, if the current search query matches some bookmark, +display the bookmark name rather than the query." + :type 'boolean + :group 'mu4e-modeline) + +(defcustom mu4e-modeline-show-global t + "Whether to populate global modeline segments. + +If non-nil, show both buffer-specific and global modeline items, +otherwise only present buffer-specific information." + :type 'boolean + :group 'mu4e-modeline) + +(defvar-local mu4e--modeline-buffer-items nil + "List of buffer-local items for the mu4e modeline. +Each element is function that evaluates to a string.") + +(defvar mu4e--modeline-global-items nil + "List of items for the global modeline. +Each element is function that evaluates to a string.") + +(defun mu4e--modeline-register (func &optional global) + "Register FUNC for calculating some mu4e modeline part. +If GLOBAL is non-nil, add to the global-modeline; otherwise use +the buffer-local one." + (add-to-list + (if global + 'mu4e--modeline-global-items + 'mu4e--modeline-buffer-items) + func 'append)) + +(defun mu4e--modeline-quote-and-truncate (str) + "Quote STR to be used literally in the modeline. +The string is truncaed to fit if its length exceeds +`mu4e-modeline-max-width'." + (replace-regexp-in-string + "%" "%%" + (truncate-string-to-width str mu4e-modeline-max-width 0 nil t))) + +(defvar mu4e--modeline-item nil + "Mu4e item for the global-mode-line.") + +(defvar mu4e--modeline-global-string-cached nil + "Cached version of the _global_ modeline string. +Note that we don't cache the local parts, so that the modeline +gets updated when we leave the buffer from which the local parts +originate.") + +(defun mu4e--modeline-string () + "Get the current mu4e modeline string." + (let* ((collect + (lambda (lst) + (mapconcat + (lambda (func) (or (funcall func) "")) lst " "))) + (global-string ;; global string is _cached_ as it may be expensive. + (and + mu4e-modeline-show-global + (or mu4e--modeline-global-string-cached + (setq mu4e--modeline-global-string-cached + (funcall collect mu4e--modeline-global-items)))))) + (concat + ;; (local) buffer items are _not_ cached, so they'll get update + ;; automatically when leaving the buffer. + (mu4e--modeline-quote-and-truncate + (funcall collect mu4e--modeline-buffer-items)) + (and global-string " ") + global-string))) + +(define-minor-mode mu4e-modeline-mode + "Minor mode for showing mu4e information on the modeline." + ;; This is a bit special 'global' mode, since it consists of both + ;; buffer-specific parts (mu4e--modeline-buffer-items) and global items + ;; (mu4e--modeline-global-items). + :global t + :group 'mu4e + :lighter nil + (if mu4e-modeline-mode + (progn + (setq mu4e--modeline-item '(:eval (mu4e--modeline-string))) + (add-to-list 'global-mode-string mu4e--modeline-item) + (mu4e--modeline-update)) + (progn + (setq global-mode-string + (seq-remove (lambda (item) (equal item mu4e--modeline-item)) + global-mode-string))) + (force-mode-line-update))) + +(defun mu4e--modeline-update () + "Recalculate and force-update the modeline." + (when mu4e-modeline-mode + (setq mu4e--modeline-global-string-cached nil) + (force-mode-line-update))) + +(provide 'mu4e-modeline) + +;;; mu4e-modeline.el ends here diff --git a/mu4e/mu4e-notification.el b/mu4e/mu4e-notification.el new file mode 100644 index 0000000..9ad9638 --- /dev/null +++ b/mu4e/mu4e-notification.el @@ -0,0 +1,99 @@ +;;; mu4e-notification.el --- Showing mail notifications -*- lexical-binding: t-*- +;; +;; Copyright (C) 1996-2023 Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;;; Commentary: +;;; Generic support for showing new-mail notifications. + +;;; Code: + +(require 'mu4e-query-items) +(require 'mu4e-bookmarks) + +;; for emacs' built-in desktop notifications to work, we need +;; dbus +(when (featurep 'dbus) + (require 'notifications)) + +(defcustom mu4e-notification-filter #'mu4e--default-notification-filter + "Function for determining if a notification is to be emitted. + +If this is the case, the function should return non-nil. +The function must accept an optional single parameter, unused for +now." + :type 'function + :group 'mu4e-notification) + +(defcustom mu4e-notification-function + #'mu4e--default-notification-function + "Function to emit a notification. + +The function is invoked when we need to emit a new-mail +notification in some system-specific way. The function is invoked +when the query-items have been updated and +`mu4e-notification-filter' returns non-nil. + +The function must accept an optional single parameter, unused for +now." + :type 'function + :group 'mu4e-notification) + +(defvar mu4e--notification-id nil + "The last notification id, so we can replace it.") + +(defun mu4e--default-notification-filter (&optional _) + "Return t if a notification should be shown. + +This default implementation does so when the number of unread +messages changed since the last notification and it is greater +than zero." + (when-let* ((fav (mu4e-bookmark-favorite)) + (delta-unread (plist-get fav :delta-unread))) + (when (and (> delta-unread 0) + (not (= delta-unread mu4e--last-delta-unread))) + (setq mu4e--last-delta-unread delta-unread) ;; update + t ;; do show notification + ))) + +(defun mu4e--default-notification-function (&optional _) + "Default function for handling notifications. +The default implementation uses emacs' built-in dbus-notification +support." + (when-let* ((fav (mu4e-bookmark-favorite)) + (title "mu4e found new mail") + (delta-unread (or (plist-get fav :delta-unread) 0)) + (body (format "%d new message%s in %s" + delta-unread + (if (= delta-unread 1) "" "s") + (plist-get fav :name)))) + (cond + ((fboundp 'do-applescript) + (do-applescript + (format "display notification %S with title %S" body title))) + ((fboundp 'notifications-notify) + ;; notifications available + (setq mu4e--notification-id + (notifications-notify + :title title + :body body + :app-name "mu4e@emacs" + :replaces-id mu4e--notification-id + ;; a custom mu4e icon would be nice... + ;; :app-icon (ignore-errors + ;; (image-search-load-path + ;; "gnus/gnus.png")) + :actions '("Show" "Favorite bookmark" + "default" "Favorite bookmark") + :on-action (lambda (_1 _2) (mu4e-jump-to-favorite))))) + ;; ... TBI: other notifications ... + (t ;; last resort + (mu4e-message "%s: %s" title body))))) + +(defun mu4e--notification () + "Function called when the query items have been updated." + (when (and (funcall mu4e-notification-filter) + (functionp mu4e-notification-function)) + (funcall mu4e-notification-function))) + +(provide 'mu4e-notification) +;;; mu4e-notification.el ends here diff --git a/mu4e/mu4e-obsolete.el b/mu4e/mu4e-obsolete.el new file mode 100644 index 0000000..c040e1a --- /dev/null +++ b/mu4e/mu4e-obsolete.el @@ -0,0 +1,283 @@ +;;; mu4e-obsolete.el --- Obsolete things -*- lexical-binding: t -*- + +;; Copyright (C) 2022-2024 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Obsolete variable & function aliases go here, so we don't clutter up the +;; code. + +;;; Code: + + +;; mu4e-draft/compose + +(make-obsolete-variable 'mu4e-reply-to-address + 'mu4e-compose-reply-to-address + "v0.9.9") + +(make-obsolete-variable 'mu4e-auto-retrieve-keys "no longer used." "1.3.1") + +(make-obsolete-variable 'mu4e-compose-func "no longer used" "1.11.26") + +(make-obsolete-variable 'mu4e-compose-crypto-reply-encrypted-policy "The use of the + 'mu4e-compose-crypto-reply-encrypted-policy' variable is deprecated. + 'mu4e-compose-crypto-policy' should be used instead" "2020-03-06") + +(make-obsolete-variable 'mu4e-compose-crypto-reply-plain-policy "The use of the + 'mu4e-compose-crypto-reply-plain-policy' variable is deprecated. + 'mu4e-compose-crypto-policy' should be used instead" + "2020-03-06") + +(make-obsolete-variable 'mu4e-compose-crypto-reply-policy "The use of the + 'mu4e-compose-crypto-reply-policy' variable is deprecated. + 'mu4e-compose-crypto-reply-plain-policy' and + 'mu4e-compose-crypto-reply-encrypted-policy' should be used instead" + "2017-09-02") + +(make-obsolete-variable 'mu4e-compose-auto-include-date + "This is done unconditionally now" "1.3.5") + +(make-obsolete-variable 'mu4e-compose-signature-auto-include + "Usage message-signature directly" "1.11.22") + +(define-obsolete-variable-alias + 'mu4e-compose-signature 'message-signature "1.11.22") +(define-obsolete-variable-alias + 'mu4e-compose-cite-function 'message-cite-function "1.11.22") +(define-obsolete-variable-alias + 'mu4e-compose-in-new-frame 'mu4e-compose-switch "1.11.22") + +(define-obsolete-variable-alias 'mu4e-compose-hidden-headers + 'mu4e-draft-hidden-headers "1.12.5") + + +;; mu4e-message + +(make-obsolete-variable 'mu4e-html2text-command "No longer in use" "1.7.0") +(make-obsolete-variable 'mu4e-view-prefer-html "No longer in use" "1.7.0") +(make-obsolete-variable 'mu4e-view-html-plaintext-ratio-heuristic + "No longer in use" "1.7.0") +(make-obsolete-variable 'mu4e-message-body-rewrite-functions + "No longer in use" "1.7.0") +;;; Html2Text +(make-obsolete 'mu4e-shr2text "No longer in use" "1.7.0") + + + +;; old message view +(make-obsolete-variable 'mu4e-view-show-addresses + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-view-wrap-lines nil "0.9.9-dev7") +(make-obsolete-variable 'mu4e-view-hide-cited nil "0.9.9-dev7") +(make-obsolete-variable 'mu4e-view-date-format + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-view-image-max-width + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-view-image-max-height + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-save-multiple-attachments-without-asking + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-view-attachment-assoc + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-view-attachment-actions + "See mu4e-view-mime-part-actions" "1.7.0") +(make-obsolete-variable 'mu4e-view-header-field-keymap + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-view-header-field-keymap + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-view-contacts-header-keymap + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-view-attachments-header-keymap + "Unused with the new message view" "1.7.0") +(make-obsolete-variable 'mu4e-imagemagick-identify nil "1.7.0") +(make-obsolete-variable 'mu4e-view-show-images + "No longer used" "1.7.0") +(make-obsolete-variable 'mu4e-view-gnus "Old view is gone" "1.7.0") +(make-obsolete-variable 'mu4e-view-use-gnus "Gnus view is the default" "1.5.10") + +(make-obsolete-variable 'mu4e-cited-regexp "No longer used" "1.7.0") + +(define-obsolete-variable-alias 'mu4e-view-blocked-images 'gnus-blocked-images + "1.5.12") +(define-obsolete-variable-alias 'mu4e-view-inhibit-images 'gnus-inhibit-images + "1.5.12") + +(define-obsolete-variable-alias 'mu4e-after-view-message-hook + 'mu4e-view-rendered-hook "1.9.7") + + +;; mu4e-org +(define-obsolete-function-alias 'org-mu4e-open 'mu4e-org-open "1.3.6") +(define-obsolete-function-alias 'org-mu4e-store-and-capture + 'mu4e-org-store-and-capture "1.3.6") + + +;; mu4e-search +(define-obsolete-variable-alias 'mu4e-headers-results-limit + 'mu4e-search-results-limit "1.7.0") +(define-obsolete-variable-alias 'mu4e-headers-full-search + 'mu4e-search-full "1.7.0") +(define-obsolete-variable-alias 'mu4e-headers-show-threads + 'mu4e-search-threads "1.7.0") +(define-obsolete-variable-alias + 'mu4e-headers-search-bookmark-hook + 'mu4e-search-bookmark-hook "1.7.0") +(define-obsolete-variable-alias 'mu4e-headers-search-hook + 'mu4e-search-hook "1.7.0") +(define-obsolete-function-alias 'mu4e-headers-search 'mu4e-search "1.7.0") +(define-obsolete-function-alias 'mu4e-headers-search-edit + 'mu4e-search-edit "1.7.0") +(define-obsolete-function-alias 'mu4e-headers-search-bookmark + 'mu4e-search-bookmark "1.7.0") +(define-obsolete-function-alias 'mu4e-headers-search-bookmark-edit + 'mu4e-search-bookmark-edit "1.7.0") +(define-obsolete-function-alias 'mu4e-headers-search-narrow + 'mu4e-search-narrow "1.7.0") +(define-obsolete-function-alias 'mu4e-headers-rerun-search + 'mu4e-search-rerun "1.7.0") +(define-obsolete-function-alias 'mu4e-headers-query-next + 'mu4e-search-next "1.7.0") +(define-obsolete-function-alias 'mu4e-headers-query-prev + 'mu4e-search-prev "1.7.0") +(define-obsolete-function-alias 'mu4e-headers-forget-queries + 'mu4e-search-forget "1.7.0") +(define-obsolete-function-alias 'mu4e-read-query + 'mu4e-search-read-query "1.7.0") + +(make-obsolete-variable 'mu4e-display-update-status-in-modeline + "No longer used" "1.9.11") + +;; mu4e-headers +(make-obsolete-variable 'mu4e-headers-field-properties-function + "not used" "1.6.1") + +(define-obsolete-function-alias 'mu4e-headers-toggle-setting + 'mu4e-headers-toggle-property "1.9.5") +(define-obsolete-function-alias 'mu4e-headers-toggle-threading + 'mu4e-headers-toggle-property "1.9.5") +(define-obsolete-function-alias 'mu4e-headers-toggle-full-search + 'mu4e-headers-toggle-property "1.9.5") +(define-obsolete-function-alias 'mu4e-headers-toggle-include-related + 'mu4e-headers-toggle-property "1.9.5") +(define-obsolete-function-alias 'mu4e-headers-toggle-skip-duplicates + 'mu4e-headers-toggle-property "1.9.5") + +(define-obsolete-function-alias 'mu4e-headers-change-sorting + 'mu4e-search-change-sorting "1.9.11") +(define-obsolete-function-alias 'mu4e-headers-toggle-property + 'mu4e-search-toggle-property "1.9.11") + +(define-obsolete-variable-alias 'mu4e-headers-include-related + 'mu4e-search-include-related "1.9.11") +(define-obsolete-variable-alias 'mu4e-headers-skip-duplicates + 'mu4e-search-skip-duplicates "1.9.11") +(define-obsolete-variable-alias 'mu4e-headers-sort-field + 'mu4e-search-sort-field "1.9.11") +(define-obsolete-variable-alias 'mu4e-headers-sort-direction + 'mu4e-search-sort-direction "1.9.11") + +(define-obsolete-variable-alias 'mu4e-headers-hide-predicate + 'mu4e-search-hide-predicate "1.9.11") +(define-obsolete-variable-alias 'mu4e-headers-hide-enabled + 'mu4e-search-hide-enabled "1.9.11") + +(define-obsolete-variable-alias 'mu4e-headers-threaded-label + 'mu4e-search-threaded-label "1.9.12") +(define-obsolete-variable-alias 'mu4e-headers-full-label + 'mu4e-search-full-label "1.9.12") +(define-obsolete-variable-alias 'mu4e-headers-related-label + 'mu4e-search-related-label "1.9.12") +(define-obsolete-variable-alias 'mu4e-headers-skip-duplicates-label + 'mu4e-search-skip-duplicates-label "1.9.12") +(define-obsolete-variable-alias 'mu4e-headers-hide-label + 'mu4e-search-hide-label "1.9.12") +;; by exception, add alias for internal func +(define-obsolete-function-alias 'mu4e~headers-jump-to-maildir + 'mu4e-search-maildir "1.9.13") + + +;; mu4e-main +(define-obsolete-variable-alias + 'mu4e-main-buffer-hide-personal-addresses + 'mu4e-main-hide-personal-addresses "1.5.7") + + +;; mu4e-server + +(make-obsolete-variable + 'mu4e-maildir + "determined by server; see `mu4e-root-maildir'." "1.3.8") + +(make-obsolete-variable 'mu4e-header-func "mu4e-headers-append-func" "1.7.4") +(make-obsolete-variable 'mu4e-temp-func "No longer used" "1.7.0") + + +;; mu4e-update +(define-obsolete-function-alias 'mu4e-interrupt-update-mail + 'mu4e-kill-update-mail "1.0-alpha0") + +;; mu4e-helpers +(define-obsolete-function-alias 'mu4e-quote-for-modeline + 'mu4e--modeline-quote-and-truncate "1.9.16") + +;; mu4e-folder +(make-obsolete-variable 'mu4e-cache-maildir-list "No longer used" "1.11.15") + +;; mu4e-contacts + +(define-obsolete-function-alias 'mu4e-user-mail-address-p + 'mu4e-personal-address-p "1.5.5") + +;; don't use the older vars anymore +(make-obsolete-variable 'mu4e-user-mail-address-regexp + 'mu4e-user-mail-address-list "0.9.9.x") +(make-obsolete-variable 'mu4e-my-email-addresses + 'mu4e-user-mail-address-list "0.9.9.x") +(make-obsolete-variable 'mu4e-user-mail-address-list + "determined by server; see `mu4e-personal-addresses'." + "1.3.8") +(make-obsolete-variable 'mu4e-contact-rewrite-function + "mu4e-contact-process-function (see docstring)" + "1.3.2") +(make-obsolete-variable 'mu4e-compose-complete-ignore-address-regexp + "mu4e-contact-process-function (see docstring)" + "1.3.2") + +(make-obsolete-variable 'mu4e-compose-reply-recipients + "use mu4e-compose-reply / mu4e-compose-wide-reply" + "1.11.23") +(make-obsolete-variable 'mu4e-compose-reply-ignore-address + "see: message-prune-recipient-rules" "1.11.23") + +;; this is only a _rough_ +(make-obsolete-variable 'mu4e-compose-dont-reply-to-self + "message-dont-reply-to-names" + "1.11.24") +;; calendar +(define-obsolete-function-alias 'mu4e-icalendar-setup + 'gnus-icalendar-setup '"1.11.22") + +;; mu4e. +(define-obsolete-function-alias 'mu4e-clear-caches #'ignore "1.11.15") + +(provide 'mu4e-obsolete) +;;; mu4e-obsolete.el ends here diff --git a/mu4e/mu4e-org.el b/mu4e/mu4e-org.el new file mode 100644 index 0000000..18d9b66 --- /dev/null +++ b/mu4e/mu4e-org.el @@ -0,0 +1,146 @@ +;;; mu4e-org --- Org-links to mu4e messages/queries -*- lexical-binding: t -*- + +;; Copyright (C) 2012-2024 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Keywords: outlines, hypermedia, calendar, mail + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of 1the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; The expect version here is org 9.x. + +;;; Code: + +(require 'org) +(require 'mu4e-view) +(require 'mu4e-contacts) + + +(defgroup mu4e-org nil + "Settings for the Org mode related functionality in mu4e." + :group 'mu4e + :group 'org) + +(defcustom mu4e-org-link-desc-func + (lambda (msg) (or (plist-get msg :subject) "No subject")) + "Function that takes a msg and returns a description. +This can be used in org capture templates and storing links. + +Example usage: + + (defun my-link-descr (msg) + (let ((subject (or (plist-get msg :subject) + \"No subject\")) + (date (or (format-time-string mu4e-headers-date-format + (mu4e-msg-field msg :date)) + \"No date\"))) + (concat subject \" \" date))) + + (setq mu4e-org-link-desc-func \\='my-link-descr)" + :type '(function) + :group 'mu4e-org) + +(defvar mu4e-org-link-query-in-headers-mode nil + "Prefer linking to the query rather than to the message. +If non-nil, `org-store-link' in `mu4e-headers-mode' links to the +the current query; otherwise, it links to the message at point.") + +;; backward compat until org >= 9.3 is univeral. +(defalias 'mu4e--org-link-store-props + (if (fboundp 'org-link-store-props) + #'org-link-store-props + (with-no-warnings + #'org-store-link-props))) + +(defun mu4e--org-store-link-query () + "Store a link to a mu4e query." + (setq org-store-link-plist nil) ; reset + (mu4e--org-link-store-props + :type "mu4e" + :query (mu4e-last-query) + :date (format-time-string "%FT%T") ;; avoid error + :link (concat "mu4e:query:" (mu4e-last-query)) + :description (format "[%s]" (mu4e-last-query)))) + +(defun mu4e--org-store-link-message (&optional msg) + "Store a link to a mu4e message. +If MSG is non-nil, store a link to MSG, otherwise use `mu4e-message-at-point'." + (setq org-store-link-plist nil) + (let* ((msg (or msg (mu4e-message-at-point))) + (from (car-safe (plist-get msg :from))) + (to (car-safe (plist-get msg :to))) + (date (format-time-string "%FT%T" (plist-get msg :date))) + (msgid (or (plist-get msg :message-id) + (mu4e-error "Cannot link message without message-id"))) + (props `(:type "mu4e" + :date ,date + :from ,(mu4e-contact-full from) + :fromname ,(mu4e-contact-name from) + :fromnameoraddress ,(or (mu4e-contact-name from) + (mu4e-contact-email from)) ;; mu4e-specific + :maildir ,(plist-get msg :maildir) + :message-id ,msgid + :path ,(plist-get msg :path) + :subject ,(plist-get msg :subject) + :to ,(mu4e-contact-full to) + :tonameoraddress ,(or (mu4e-contact-name to) + (mu4e-contact-email to)) ;; mu4e-specific + :link ,(concat "mu4e:msgid:" msgid) + :description ,(funcall mu4e-org-link-desc-func msg)))) + (apply #'mu4e--org-link-store-props props))) + +(defun mu4e-org-store-link () + "Store a link to a mu4e message or query. +It links to the last known query when in `mu4e-headers-mode' with +`mu4e-org-link-query-in-headers-mode' set; otherwise it links to +a specific message, based on its message-id, so that links stay +valid even after moving the message around." + (cond + ((derived-mode-p 'mu4e-view-mode) (mu4e--org-store-link-message)) + ((derived-mode-p 'mu4e-headers-mode) + (if mu4e-org-link-query-in-headers-mode + (mu4e--org-store-link-query) + (mu4e--org-store-link-message))))) + +(defun mu4e-org-open (link) + "Open the org LINK. +Open the mu4e message (for links starting with \"msgid:\") or run +the query (for links starting with \"query:\")." + (require 'mu4e) + (cond + ((string-match "^msgid:\\(.+\\)" link) + (mu4e-view-message-with-message-id (match-string 1 link))) + ((string-match "^query:\\(.+\\)" link) + (mu4e-search (match-string 1 link) current-prefix-arg)) + (t (mu4e-error "Unrecognized link type '%s'" link)))) + +(defun mu4e-org-store-and-capture () + "Store a link to the current message or query. +\(depending on `mu4e-org-link-query-in-headers-mode', and capture +it with org)." + (interactive) + (call-interactively 'org-store-link) + (org-capture)) + +;; install mu4e-link support. +(org-link-set-parameters "mu4e" + :follow #'mu4e-org-open + :store #'mu4e-org-store-link) +(provide 'mu4e-org) +;;; mu4e-org.el ends here diff --git a/mu4e/mu4e-pkg.el.in b/mu4e/mu4e-pkg.el.in new file mode 100644 index 0000000..ed8e733 --- /dev/null +++ b/mu4e/mu4e-pkg.el.in @@ -0,0 +1,7 @@ +;; -*- no-byte-compile: t; -*- +(define-package "mu4e" "@VERSION@" + "part of mu4e, the mu mail user agent" + '((emacs "@EMACS_MIN_VERSION@")) + :authors '(("Dirk-Jan C. Binnema" . "djcb@djcbsoftware.nl")) + :maintainer '("Dirk-Jan C. Binnema" . "djcb@djcbsoftware.nl") + :keywords '("email")) diff --git a/mu4e/mu4e-query-items.el b/mu4e/mu4e-query-items.el new file mode 100644 index 0000000..b2523e7 --- /dev/null +++ b/mu4e/mu4e-query-items.el @@ -0,0 +1,254 @@ +;;; mu4e-query-items.el --- Manage query results -*- lexical-binding: t -*- + +;; Copyright (C) 2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: +;; +;; Managing the last query results / baseline, which we use to get the +;; unread-counts, i.e., query items. `mu4e-query-items` delivers these items, +;; aggregated from various sources. + + +;;; Code: + +;;; Last & baseline query results for bookmarks. +(require 'cl-lib) +(require 'mu4e-helpers) +(require 'mu4e-server) + +(defcustom mu4e-query-rewrite-function 'identity + "Function to rewrite a query. + +It takes a search expression string, and returns a possibly + changed search expression string. + +This function is applied on the search expression just before +searching, and allows users to modify the query. + +For instance, we could change any instance of \"workmail\" into +\"maildir:/long-path-to-work-related-emails\", by setting the function + +\\=(setq mu4e-query-rewrite-function + (lambda(expr) + (replace-regexp-in-string \"workmail\" + \"maildir:/long-path-to-work-related-emails\" expr))) + +It is good to remember that the replacement does not understand +anything about the query, it just does text replacement. + +A word of caution: the function should be deterministic and +always return the same result for a given query (at least within +some \"context\" (see `mu4e-context'). If not, you may get incorrect results +for the various unread counts." + :type 'function + :group 'mu4e-search) + +(defvar mu4e--query-items-baseline nil + "Some previous version of the query-items. +This is used as the baseline to track updates by comparing it to +the latest query-items.") +(defvar mu4e--query-items-baseline-tstamp nil + "Timestamp for when the query-items baseline was updated.") +(defvar mu4e--last-delta-unread 0 "Last notified number.") + +(defun mu4e--bookmark-query (bm) + "Get the query string for some bookmark BM." + (when bm + (let* ((query (or (plist-get bm :query) + (mu4e-warn "No query in %S" bm))) + ;; queries being functions is deprecated, but for now we + ;; still support it. + (query (if (functionp query) (funcall query) query))) + (unless (stringp query) + (mu4e-warn "Could not get query string from %s" bm)) + ;; apparently, non-UTF8 queries exist, i.e., + ;; with maildir names. + (decode-coding-string query 'utf-8 t)))) + +(defun mu4e--query-items-pick-favorite (items) + "Pick the :favorite querty item. +If ITEMS does not yet have a favorite item, pick the first." + (unless (seq-find + (lambda (item) (plist-get item :favorite)) items) + (plist-put (car items) :favorite t)) + items) + +(defvar mu4e--bookmark-items-cached nil "Cached bookmarks query items.") +(defvar mu4e--maildir-items-cached nil "Cached maildirs query items.") + +(declare-function mu4e-bookmarks "mu4e-bookmarks") +(declare-function mu4e-maildir-shortcuts "mu4e-folders") + +(defun mu4e--query-item-display-counts (item) + "Get the count display string for some query-data ITEM." + ;; purely for display, but we need it in the main menu, modeline + ;; so let's keep it consistent. + (cl-destructuring-bind (&key unread hide-unread delta-unread count + &allow-other-keys) item + (if hide-unread + "" + (concat + (propertize (number-to-string unread) + 'face 'mu4e-header-key-face + 'help-echo "Number of unread") + (if (<= delta-unread 0) "" + (propertize (format "(%+d)" delta-unread) 'face + 'mu4e-unread-face)) + "/" + (propertize (number-to-string count) + 'help-echo "Total number"))))) + +(defun mu4e--query-items-refresh (&optional reset-baseline) + "Get the latest query data from the mu4e server. +With RESET-BASELINE, reset the baseline first." + (when reset-baseline + (setq mu4e--query-items-baseline nil + mu4e--query-items-baseline-tstamp nil + mu4e--bookmark-items-cached nil + mu4e--maildir-items-cached nil + mu4e--last-delta-unread 0)) + (mu4e--server-queries + ;; note: we must apply the rewrite function here, since the query does not go + ;; through mu4e-search. + (mapcar (lambda (bm) + (funcall mu4e-query-rewrite-function + (mu4e--bookmark-query bm))) + (seq-filter (lambda (item) + (and (not (or (plist-get item :hide) + (plist-get item :hide-unread))))) + (mu4e-query-items))))) + +(defun mu4e--query-items-queries-handler (_sexp) + "Handler for queries responses from the mu4e-server. +I.e. what we get in response to mu4e--query-items-refresh." + ;; if we cleared the baseline (in mu4e--query-items-refresh) + ;; set it to the latest now. + (unless mu4e--query-items-baseline + (setq mu4e--query-items-baseline (mu4e-server-query-items) + mu4e--query-items-baseline-tstamp (current-time))) + + (setq mu4e--bookmark-items-cached nil + mu4e--maildir-items-cached nil) + (mu4e-query-items) ;; for side-effects + ;; tell the world. + (run-hooks 'mu4e-query-items-updated-hook)) + +;; this makes for O(n*m)... but with typically small(ish) n,m. Perhaps use a +;; hash for last-query-items and baseline-results? +(defun mu4e--query-find-item (query data) + "Find the item in DATA for the given QUERY." + (seq-find (lambda (item) + (equal query (mu4e--bookmark-query item))) + data)) + +(defun mu4e--make-query-items (data type) + "Map the items in DATA to plists with aggregated query information. + +DATA is either the bookmarks or maildirs (user-defined). + +LAST-RESULTS-DATA contains unread/counts we received from the +server, while BASELINE-DATA contains the same but taken at some +earier time. + +The TYPE denotes the category for the query item, a symbol +bookmark or maildir." + (seq-map + (lambda (item) + (let* ((maildir (plist-get item :maildir)) + ;; for maildirs, construct the query + (query (if (equal type 'maildirs) + (format "maildir:\"%s\"" maildir) + (plist-get item :query))) + (query (if (functionp query) (funcall query) query)) + (name (plist-get item :name)) + ;; it is possible that the user has a rewrite function + (effective-query (funcall mu4e-query-rewrite-function query)) + ;; maildir items may have an implicit name + ;; which is the maildir value. + (name (or name (and (equal type 'maildirs) maildir))) + (last-results (mu4e-server-query-items)) + (baseline mu4e--query-items-baseline) + ;; we use the _effective_ query to find the results, + ;; since that's what the server will give to us. + (baseline-item + (mu4e--query-find-item effective-query baseline)) + (last-results-item + (mu4e--query-find-item effective-query last-results)) + (count (or (plist-get last-results-item :count) 0)) + (unread (or (plist-get last-results-item :unread) 0)) + (baseline-count (or (plist-get baseline-item :count) count)) + (baseline-unread (or (plist-get baseline-item :unread) unread)) + (delta-unread (- unread baseline-unread)) + (value + (list + :name name + :query query + :key (plist-get item :key) + :count count + :unread unread + :delta-count (- count baseline-count) + :delta-unread delta-unread))) + ;; remember the *effective* query too; we don't really need it, but + ;; useful for debugging. + (unless (string= query effective-query) + (plist-put value :effective-query effective-query)) + + ;; nil props bring me discomfort + (when (plist-get item :favorite) + (plist-put value :favorite t)) + (when (plist-get item :hide) + (plist-put value :hide t)) + (when (plist-get item :hide-unread) + (plist-put value :hide-unread t)) + value)) + data)) + +(defun mu4e-query-items (&optional type) + "Grab query items of TYPE. + +TYPE is symbol; either bookmarks or maildirs, or nil for both. + +This combines: + - the latest queries data (i.e., `(mu4e-server-query-items)') + - baseline queries data (i.e. `mu4e-baseline') + with the combined queries for `(mu4e-bookmarks)' and + `(mu4e-maildir-shortcuts)' in bookmarks-compatible plists. + +This packages the aggregated information in a format that is convenient +for use in various places." + (cond + ((equal type 'bookmarks) + (or mu4e--bookmark-items-cached + (setq mu4e--bookmark-items-cached + (mu4e--query-items-pick-favorite + (mu4e--make-query-items (mu4e-bookmarks) 'bookmarks))))) + ((equal type 'maildirs) + (or mu4e--maildir-items-cached + (setq mu4e--maildir-items-cached + (mu4e--make-query-items (mu4e-maildir-shortcuts) 'maildirs)))) + ((not type) + (append (mu4e-query-items 'bookmarks) + (mu4e-query-items 'maildirs))) + (t + (mu4e-error "No such type %s" type)))) + +(provide 'mu4e-query-items) +;;; mu4e-query-items.el ends here diff --git a/mu4e/mu4e-search.el b/mu4e/mu4e-search.el new file mode 100644 index 0000000..f42a981 --- /dev/null +++ b/mu4e/mu4e-search.el @@ -0,0 +1,635 @@ +;;; mu4e-search.el --- Search-related functions -*- lexical-binding: t -*- + +;; Copyright (C) 2021,2022 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Search-related functions and a minor-mode. + +;;; Code: + +(require 'seq) +(require 'mu4e-helpers) +(require 'mu4e-message) +(require 'mu4e-bookmarks) +(require 'mu4e-contacts) +(require 'mu4e-lists) +(require 'mu4e-mark) +(require 'mu4e-query-items) + + +;;; Configuration +(defgroup mu4e-search nil + "Search-related settings." + :group 'mu4e) + +(defcustom mu4e-search-results-limit 500 + "Maximum number of results to show. +This affects performance, especially when +`mu4e-summary-include-related' is non-nil. +Set to -1 for no limits." + :type '(choice (const :tag "Unlimited" -1) + (integer :tag "Limit")) + :group 'mu4e-search) + +(defcustom mu4e-search-full nil + "Whether to search for all results. +If this is nil, search for up to `mu4e-search-results-limit')" + :type 'boolean + :group 'mu4e-search) + +(defcustom mu4e-search-threads t + "Whether to calculate threads for the search results." + :type 'boolean + :group 'mu4e-search) + +(defcustom mu4e-search-include-related t + "Whether to include \"related\" messages in queries. +With this option set to non-nil, not just return the matches for +a searches, but also messages that are related (through their +references) to these messages. This can be useful e.g. to include +sent messages into message threads." + :type 'boolean + :group 'mu4e-search) + +(defcustom mu4e-search-skip-duplicates t + "Whether to skip duplicate messages. +With this option set to non-nil, show only one of duplicate +messages. This is useful when you have multiple copies of the same +message, which is a common occurrence for example when using Gmail +and offlineimap." + :type 'boolean + :group 'mu4e-search) + +(defvar mu4e-search-hide-predicate nil + "Predicate function to hide matching headers. +Either nil or a function taking one message plist parameter and +which which return non-nil for messages that should be hidden from +the search results. Also see `mu4e-search-hide-enabled'. + +Example that hides all trashed messages: + + (setq mu4e-search-hide-predicate + (lambda (msg) + (member \\='trashed (mu4e-message-field msg :flags)))).") + +(defvar mu4e-search-hide-enabled t + "Whether `mu4e-search-hide-predicate' should be active. +This can be used to toggle use of the predicate through + `mu4e-search-toggle-property'.") + + +(defcustom mu4e-search-sort-field :date + "Field to sort the headers by. A symbol: +one of: `:date', `:subject', `:size', `:prio', `:from', `:to.', +`:list'. + +Note that when threading is enabled (through +`mu4e-search-threads'), the headers are exclusively sorted +chronologically (`:date') by the newest message in the thread." + :type '(radio (const :date) + (const :subject) + (const :size) + (const :prio) + (const :from) + (const :to) + (const :list)) + :group 'mu4e-search) + +(defcustom mu4e-search-sort-direction 'descending + "Direction to sort by; a symbol either `descending' (sorting + Z->A) or `ascending' (sorting A->Z)." + :type '(radio (const ascending) + (const descending)) + :group 'mu4e-search) + +;; mu4e-query-rewrite-function lives in mu4e-query-items.el +;; to avoid circular deps. + +(defcustom mu4e-search-bookmark-hook nil + "Hook run just after invoking a bookmarked search. + +This function receives the query as its parameter, before any +rewriting as per `mu4e-query-rewrite-function' has taken place. + +The reason to use this instead of `mu4e-search-hook' is +if you only want to execute a hook when a search is entered via a +bookmark, e.g. if you'd like to treat the bookmarks as a custom +folder and change the options for the search." + :type 'hook + :group 'mu4e-search) + +(defcustom mu4e-search-hook nil + "Hook run just before executing a new search operation. +This function receives the query as its parameter, before any +rewriting as per `mu4e-query-rewrite-function' has taken place + +This is a more general hook facility than the +`mu4e-search-bookmark-hook'. It gets called on every +executed search, not just those that are invoked via bookmarks, +but also manually invoked searches." + :type 'hook + :group 'mu4e-search) + +;; Internals + +;;; History +(defvar mu4e--search-query-past nil + "Stack of queries before the present one.") +(defvar mu4e--search-query-future nil "Stack of queries after the present one.") +(defvar mu4e--search-query-stack-size 20 + "Maximum size for the query stacks.") +(defvar mu4e--search-last-query nil + "The present (most recent) query.") + + + +;;; Interactive functions +(declare-function mu4e--search-execute "mu4e-headers") + +(defvar mu4e--search-view-target nil + "Whether to automatically view (open) the target message.") +(defvar mu4e--search-msgid-target nil + "Message-id to jump to after the search has finished.") + + +(defun mu4e-search (&optional expr prompt edit ignore-history msgid show) + "Search for query EXPR. + +Switch to the output buffer for the results. This is an +interactive function which ask user for EXPR. PROMPT, if non-nil, +is the prompt used by this function (default is \"Search for:\"). +If EDIT is non-nil, instead of executing the query for EXPR, let +the user edit the query before executing it. + +If IGNORE-HISTORY is true, do *not* update the query history +stack. If MSGID is non-nil, attempt to move point to the first +message with that message-id after searching. If SHOW is non-nil, +show the message with MSGID." + (interactive) + (let* ((prompt (mu4e-format (or prompt "Search for: "))) + (expr + (if (or (null expr) edit) + (mu4e-search-read-query prompt expr) + expr))) + (mu4e-mark-handle-when-leaving) + (mu4e--search-execute expr ignore-history) + (setq mu4e--search-msgid-target msgid + mu4e--search-view-target show) + (mu4e--modeline-update))) + +(defun mu4e-search-edit () + "Edit the last search expression." + (interactive) + (mu4e-search mu4e--search-last-query nil t)) + +(defun mu4e-search-bookmark (&optional expr edit) + "Search using some bookmarked query EXPR. +If EDIT is non-nil, let the user edit the bookmark before starting +the search." + (interactive) + (let* ((expr + (or expr + (mu4e-ask-bookmark + (if edit "Select bookmark: " "Bookmark: ")))) + (expr (if (functionp expr) (funcall expr) expr)) + (fav (mu4e--bookmark-query (mu4e-bookmark-favorite)))) + ;; reset baseline when searching for the favorite bookmark query + (when (and fav (string= fav expr)) + (mu4e--query-items-refresh 'reset-baseline)) + + (run-hook-with-args 'mu4e-search-bookmark-hook expr) + (mu4e-search expr (when edit "Edit query: ") edit))) + +(defun mu4e-search-bookmark-edit () + "Edit an existing bookmark before executing it." + (interactive) + (mu4e-search-bookmark nil t)) + + +(defun mu4e-search-maildir (maildir &optional edit) + "Search the messages in MAILDIR. +The user is prompted to ask what maildir. If prefix-argument EDIT +is given, offer to edit the search query before executing it." + (interactive + (let ((maildir (mu4e-ask-maildir "Jump to maildir: "))) + (list maildir current-prefix-arg))) + (when maildir + (let* ((query (format "maildir:\"%s\"" maildir)) + (query (if edit + (mu4e-search-read-query "Refine query: " query) query))) + (mu4e-mark-handle-when-leaving) + (mu4e-search query)))) + +(defun mu4e-search-narrow(&optional filter) + "Narrow the last search. +Do so by appending search expression FILTER to the last search +expression. Note that you can go back to the previous +query (effectively, \"widen\" it), with `mu4e-search-prev'." + (interactive + (let ((filter + (read-string (mu4e-format "Narrow down to: ") + nil 'mu4e~headers-search-hist nil t))) + (list filter))) + (unless mu4e--search-last-query + (mu4e-warn "There's nothing to filter")) + (mu4e-search (format "(%s) AND (%s)" mu4e--search-last-query filter))) + +(defun mu4e--search-push-query (query where) + "Push QUERY to one of the query stacks. +WHERE is a symbol telling us where to push; it's a symbol, either +`future' or `past'. Also removes duplicates and truncates to +limit the stack size." + (let ((stack + (pcase where + ('past mu4e--search-query-past) + ('future mu4e--search-query-future)))) + ;; only add if not the same item + (unless (and stack (string= (car stack) query)) + (push query stack) + ;; limit the stack to `mu4e--search-query-stack-size' elements + (when (> (length stack) mu4e--search-query-stack-size) + (setq stack (cl-subseq stack 0 mu4e--search-query-stack-size))) + ;; remove all duplicates of the new element + (seq-remove (lambda (elm) (string= elm (car stack))) (cdr stack)) + ;; update the stacks + (pcase where + ('past (setq mu4e--search-query-past stack)) + ('future (setq mu4e--search-query-future stack)))))) + +(defun mu4e--search-pop-query (whence) + "Pop a query from the stack. +WHENCE is a symbol telling us where to get it from, either `future' +or `past'." + (pcase whence + ('past + (unless mu4e--search-query-past + (mu4e-warn "No more previous queries")) + (pop mu4e--search-query-past)) + ('future + (unless mu4e--search-query-future + (mu4e-warn "No more next queries")) + (pop mu4e--search-query-future)))) + +(defun mu4e-search-rerun () + "Re-run the search for the last search expression." + (interactive) + ;; if possible, try to return to the same message + (let* ((msg (mu4e-message-at-point t)) + (msgid (and msg (mu4e-message-field msg :message-id)))) + (mu4e-search mu4e--search-last-query nil nil t msgid))) + +(defun mu4e--search-query-navigate (whence) + "Execute the previous query from the query stacks. +WHENCE determines where the query is taken from and is a symbol, +either `future' or `past'." + (let ((query (mu4e--search-pop-query whence)) + (where (if (eq whence 'future) 'past 'future))) + (when query + (mu4e--search-push-query mu4e--search-last-query where) + (mu4e-search query nil nil t)))) + +(defun mu4e-search-next () + "Execute the next query from the query stack." + (interactive) + (mu4e--search-query-navigate 'future)) + +(defun mu4e-search-prev () + "Execute the previous query from the query stacks." + (interactive) + (mu4e--search-query-navigate 'past)) + +;; forget the past so we don't repeat it :/ +(defun mu4e-search-forget () + "Forget the search history." + (interactive) + (setq mu4e--search-query-past nil + mu4e--search-query-future nil) + (mu4e-message "Query history cleared")) + +(defun mu4e-last-query () + "Get the most recent query or nil if there is none." + mu4e--search-last-query) + +;;; Completion for queries + +(defvar mu4e--search-hist nil "History list of searches.") +(defvar mu4e-minibuffer-search-query-map + (let ((map (copy-keymap minibuffer-local-map))) + (define-key map (kbd "TAB") #'completion-at-point) + map) + "The keymap for reading a search query.") + +(defun mu4e-search-read-query (prompt &optional initial-input) + "Read a query with completion using PROMPT and INITIAL-INPUT." + (minibuffer-with-setup-hook + (lambda () + (setq-local completion-at-point-functions + #'mu4e--search-query-completion-at-point) + (use-local-map mu4e-minibuffer-search-query-map)) + (read-string prompt initial-input 'mu4e--search-hist))) + +(defconst mu4e--search-query-keywords + '("and" "or" "not" + "from:" "to:" "cc:" "bcc:" "contact:" "recip:" "date:" "subject:" "body:" + "list:" "maildir:" "flag:" "mime:" "file:" "prio:" "tag:" "msgid:" + "size:" "embed:")) + +(defun mu4e--search-completion-contacts-action (match _status) + "Delete contact alias from contact autocompletion, leaving just email address. +Implements the `completion-extra-properties' :exit-function' which +requires a function with arguments string MATCH and completion +status, STATUS." + (let ((contact-email (replace-regexp-in-string "^.*<\\|>$" "" match))) + (delete-char (- (length match))) + (insert contact-email))) + +(defun mu4e--search-query-completion-at-point () + "Provide completion when entering search expressions." + (cond + ((not (looking-back "[:\"][^ \t]*" nil)) + (let ((bounds (bounds-of-thing-at-point 'word))) + (list (or (car bounds) (point)) + (or (cdr bounds) (point)) + mu4e--search-query-keywords))) + ((looking-back "flag:\\(\\w*\\)" nil) + (list (match-beginning 1) + (match-end 1) + '("attach" "draft" "flagged" "list" "new" "passed" "replied" + "seen" "trashed" "unread" "encrypted" "signed" "personal"))) + ((looking-back "maildir:\\([a-zA-Z0-9/.]*\\)" nil) + (list (match-beginning 1) + (match-end 1) + (mapcar (lambda (dir) + ;; Quote maildirs with whitespace in their name, e.g., + ;; maildir:"Foobar/Junk Mail". + (if (string-match-p "[[:space:]]" dir) + (concat "\"" dir "\"") + dir)) + (mu4e-get-maildirs)))) + ((looking-back "prio:\\(\\w*\\)" nil) + (list (match-beginning 1) + (match-end 1) + (list "high" "normal" "low"))) + ((looking-back "mime:\\([a-zA-Z0-9/-]*\\)" nil) + (list (match-beginning 1) + (match-end 1) + (when (fboundp 'mailcap-mime-types) (mailcap-mime-types)))) + ((looking-back "\\(from\\|to\\|cc\\|bcc\\|contact\\|recip\\):\\([a-zA-Z0-9/.@]*\\)" nil) + (list (match-beginning 2) + (match-end 2) + mu4e--contacts-set + :exit-function + #'mu4e--search-completion-contacts-action)) + ((looking-back "list:\\([a-zA-Z0-9/.@]*\\)" nil) + (list (match-beginning 1) + (match-end 1) + mu4e--lists-hash)))) + +;;; Interactive functions +(defun mu4e-search-change-sorting (&optional field dir) + "Change the sorting/threading parameters. +FIELD is the field to sort by; DIR is a symbol: either +`ascending', `descending', t (meaning: if FIELD is the same as +the current sortfield, change the sort-order) or nil (ask the +user). + +When threads are enabled (`mu4e-search-threads'), you can only sort +by the `:date' field." + (interactive) + (let* ((choices ;; with threads enabled, you can only sort by *date* + (if mu4e-search-threads + '(("date" . :date)) + '(("date" . :date) + ("from" . :from) + ("list" . :list) + ("maildir" . :maildir) + ("prio" . :prio) + ("zsize" . :size) + ("subject" . :subject) + ("to" . :to)))) + (field + (or field + (mu4e-read-option "Sortfield: " choices))) + ;; note: 'sortable' is either a boolean (meaning: if non-nil, this is + ;; sortable field), _or_ another field (meaning: sort by this other + ;; field). + (sortable (plist-get (cdr (assoc field mu4e-header-info)) :sortable)) + ;; error check + (sortable + (if sortable + sortable + (mu4e-error "Not a sortable field"))) + (sortfield (if (booleanp sortable) field sortable)) + (dir + (cl-case dir + ((ascending descending) dir) + ;; change the sort order if field = curfield + (t + (if (eq sortfield mu4e-search-sort-field) + (if (eq mu4e-search-sort-direction 'ascending) + 'descending 'ascending) + 'descending))))) + (setq + mu4e-search-sort-field sortfield + mu4e-search-sort-direction dir) + (mu4e-message "Sorting by %s (%s)" + (symbol-name sortfield) + (symbol-name mu4e-search-sort-direction)) + (mu4e-search-rerun))) + +(defun mu4e-search-toggle-property (&optional dont-refresh) + "Toggle some aspect of search. +When prefix-argument DONT-REFRESH is non-nil, do not refresh the +last search with the new setting." + (interactive "P") + (let* ((toggles '(("fFull-search" . mu4e-search-full) + ("rInclude-related" . mu4e-headers-include-related) + ("tShow threads" . mu4e-search-threads) + ("uSkip duplicates" . mu4e-search-skip-duplicates) + ("pHide-predicate" . mu4e-search-hide-enabled))) + (toggles (seq-map + (lambda (cell) + (cons + (concat (car cell) + (format" (%s)" + (if (symbol-value (cdr cell)) "on" "off"))) + (cdr cell))) + toggles)) + (choice (mu4e-read-option "Toggle property " toggles))) + (when choice + (set choice (not (symbol-value choice))) + (mu4e-message "Set `%s' to %s" (symbol-name choice) (symbol-value choice)) + (mu4e--modeline-update) + (unless dont-refresh + (mu4e-search-rerun))))) + +(defvar mu4e-search-threaded-label '("T" . "Ⓣ") + "Non-fancy and fancy labels to indicate threaded search in the mode-line.") +(defvar mu4e-search-full-label '("F" . "Ⓕ") + "Non-fancy and fancy labels to indicate full search in the mode-line.") +(defvar mu4e-search-related-label '("R" . "Ⓡ") + "Non-fancy and fancy labels to indicate related search in the mode-line.") +(defvar mu4e-search-skip-duplicates-label '("U" . "Ⓤ") ;; 'U' for 'unique' + "Non-fancy and fancy labels for include-related search in the mode-line.") +(defvar mu4e-search-hide-label '("H" . "Ⓗ") + "Non-fancy and fancy labels to indicate header-hiding is active in +the mode-line.") + +(defun mu4e--search-modeline-item () + "Get mu4e-search modeline item." + (let* ((label (lambda (label-cons) + (if mu4e-use-fancy-chars + (cdr label-cons) (car label-cons)))) + (props + `((,mu4e-search-full ,mu4e-search-full-label + "Full search") + (,mu4e-search-include-related + ,mu4e-search-related-label + "Include related messages") + (,mu4e-search-threads + ,mu4e-search-threaded-label + "Show message threads") + (,mu4e-search-skip-duplicates + ,mu4e-search-skip-duplicates-label + "Skip duplicate messages") + (,mu4e-search-hide-enabled + ,mu4e-search-hide-label + "Enable message hide predicate"))) + ;; can we fin find a bookmark corresponding + ;; with this query? + (bookmark + (and mu4e-modeline-prefer-bookmark-name + (seq-find (lambda (item) + (string= + mu4e--search-last-query + (or (plist-get item :effective-query) + (plist-get item :query)))) + (mu4e-query-items 'bookmarks))))) + (concat + (propertize + (mapconcat + (lambda (cell) + (when (nth 0 cell) (funcall label (nth 1 cell)))) + props "") + 'help-echo (concat "mu4e search properties legend\n\n" + (mapconcat + (lambda (cell) + (format "%s %s (%s)" + (funcall label (nth 1 cell)) + (nth 2 cell) + (if (nth 0 cell) "yes" : "no"))) + props "\n"))) + " [" + (propertize + (if bookmark ;; show the bookmark name instead of the query? + (plist-get bookmark :name) + mu4e--search-last-query) + 'face 'mu4e-title-face + 'help-echo (format "mu4e query:\n\t%s" mu4e--search-last-query)) + "]"))) + +(defun mu4e-search-query (&optional edit) + "Select a search query through `completing-read'. + +If prefix-argument EDIT is non-nil, allow for editing the chosen +query before submitting it." + (interactive "P") + (let* ((candidates (seq-map (lambda (item) + (cons (plist-get item :name) item)) + (mu4e-query-items))) + (longest-name + (seq-max (seq-map (lambda (c) (length (car c))) candidates))) + (longest-query + (seq-max (seq-map (lambda (c) (length (plist-get (cdr c) :query))) + candidates))) + + (annotation-func + (lambda (candidate) + (let* ((item (cdr-safe (assoc candidate candidates))) + (name (propertize (or (plist-get item :name) "") + 'face 'mu4e-header-key-face)) + (query (propertize (or (plist-get item :query) "") + 'face 'mu4e-header-value-face))) + (concat + " " + (make-string (- longest-name (length name)) ?\s) + query + (make-string (- longest-query (length query)) ?\s) + " " + (mu4e--query-item-display-counts item))))) + (completion-extra-properties + `(:annotation-function ,annotation-func)) + (chosen (completing-read "Query: " candidates)) + (query (or (plist-get (cdr-safe (assoc chosen candidates)) :query) + (mu4e-warn "No query for %s" chosen)))) + (mu4e-search-bookmark query edit))) + +(defvar mu4e-search-minor-mode-map + (let ((map (make-sparse-keymap))) + (define-key map "s" #'mu4e-search) + (define-key map "S" #'mu4e-search-edit) + (define-key map "/" #'mu4e-search-narrow) + + (define-key map (kbd "<M-left>") #'mu4e-search-prev) + (define-key map (kbd "\\") #'mu4e-search-prev) + (define-key map (kbd "<M-right>") #'mu4e-search-next) + + (define-key map "O" #'mu4e-search-change-sorting) + (define-key map "P" #'mu4e-search-toggle-property) + + (define-key map "b" #'mu4e-search-bookmark) + (define-key map "B" #'mu4e-search-bookmark-edit) + + (define-key map "c" #'mu4e-search-query) + + (define-key map "j" #'mu4e-search-maildir) + map) + "Keymap for mu4e-search-minor-mode.") + +(define-minor-mode mu4e-search-minor-mode + "Mode for searching for messages." + :global nil + :init-value nil ;; disabled by default + :group 'mu4e + :lighter "" + :keymap mu4e-search-minor-mode-map) + +(defvar mu4e--search-menu-items + '("--" + ["Search" mu4e-search + :help "Search using expression"] + ["Search bookmark" mu4e-search-bookmark + :help "Show messages matching some bookmark query"] + ["Search maildir" mu4e-search-maildir + :help "Show messages in some maildir"] + ["Choose query" mu4e-search-query + :help "Show messages for some query"] + ["Previous query" mu4e-search-prev + :help "Run previous query"] + ["Next query" mu4e-search-next + :help "Run next query"] + ["Narrow search" mu4e-search-narrow + :help "Narrow the search query"]) + "Easy menu items for search.") + +(provide 'mu4e-search) +;;; mu4e-search.el ends here diff --git a/mu4e/mu4e-server.el b/mu4e/mu4e-server.el new file mode 100644 index 0000000..a280f09 --- /dev/null +++ b/mu4e/mu4e-server.el @@ -0,0 +1,712 @@ +;;; mu4e-server.el --- Control mu server from mu4e -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;;; Code: + +(require 'mu4e-helpers) + + +;;; Configuration +(defcustom mu4e-mu-home nil + "Location of an alternate mu home dir. +If not set, use the defaults, based on the XDG Base Directory +Specification. + +Changes to this value only take effect after (re)starting the mu +session." + :group 'mu4e + :type '(choice (const :tag "Default location" nil) + (directory :tag "Specify location")) + :safe 'stringp) + +(defcustom mu4e-mu-binary (executable-find "mu") + "Path to the mu-binary to use. + +Changes to this value only take effect after (re)starting the mu +session." + :type '(file :must-match t) + :group 'mu4e + :safe 'stringp) + +(defcustom mu4e-mu-debug nil + "Whether to run the mu binary in debug-mode. +Setting this to t increases the amount of information in the log. + +Changes to this value only take effect after (re)starting the mu +session." + :type 'boolean + :group 'mu4e) + +(defcustom mu4e-change-filenames-when-moving nil + "Change message file names when moving them. + +When moving messages to different folders, normally mu/mu4e keep +the base filename the same (the flags-part of the filename may +change still). With this option set to non-nil, mu4e instead +changes the filename. + +This latter behavior works better with some +IMAP-synchronization programs such as mbsync; the default works +better with e.g. offlineimap." + :type 'boolean + :group 'mu4e + :safe 'booleanp) + +(defcustom mu4e-mu-allow-temp-file nil + "Allow using temp-files for optimizing mu <-> mu4e communication. + +Some commands - in particular \"find\" and \"contacts\" - return +big s-expressions; and it turns out that reading those is faster +by passing them through a temp file rather than through normal +stdin/stdout channel - esp. on the (common case) where the +file-system for temp-files is in-memory. + +To see if the helps, you can benchmark the rendering with + (setq mu4e-headers-report-render-time t) + +and compare the results with `mu4e-mu-allow-temp' set and unset. + +Note: for a change to this variable to take effect, you need to +stop/start mu4e." + :type 'boolean + :group 'mu4e + :safe 'booleanp) + + +;; Cached data +(defvar mu4e-maildir-list) + + +;; Handlers are not strictly internal, but are not meant +;; for overriding outside mu4e. The are mainly for breaking +;; dependency cycles. + +(defvar mu4e-error-func nil + "Function called for each error received. +The function is passed an error plist as argument. See +`mu4e--server-filter' for the format.") + +(defvar mu4e-update-func nil + "Function called for each :update sexp returned. +The function is passed a msg sexp as argument. +See `mu4e--server-filter' for the format.") + +(defvar mu4e-remove-func nil + "Function called for each :remove sexp returned. +This happens when some message has been deleted. The function is +passed the docid of the removed message.") + +(defvar mu4e-sent-func nil + "Function called for each :sent sexp received. +This happens when some message has been sent. The function is +passed the docid and the draft-path of the sent message.") + +(defvar mu4e-view-func nil + "Function called for each single-message sexp. +The function is passed a message sexp as argument. See +`mu4e--server-filter' for the format.") + +(defvar mu4e-headers-append-func nil + "Function called with a list of headers to append. +The function is passed a list of message plists as argument. See +See `mu4e--server-filter' for the details.") + +(defvar mu4e-found-func nil + "Function called for when we received a :found sexp. +This happens after the headers have been returned, to report on +the number of matches. See `mu4e--server-filter' for the format.") + +(defvar mu4e-erase-func nil + "Function called we receive an :erase sexp. +This before new headers are displayed, to clear the current +headers buffer. See `mu4e--server-filter' for the format.") + +(defvar mu4e-info-func nil + "Function called for each (:info type ....) sexp received. +from the server process.") + +(defvar mu4e-pong-func nil + "Function called for each (:pong type ....) sexp received.") + +(defvar mu4e-queries-func nil + "Function called for each (:queries type ....) sexp received.") + +(defvar mu4e-contacts-func nil + "A function called for each (:contacts (<list-of-contacts>)) +sexp received from the server process.") + + +;;; Dealing with Server properties +(defvar mu4e--server-props nil + "Metadata we receive from the mu4e server.") + +(defun mu4e-server-properties () + "Get the server metadata plist." + mu4e--server-props) + +(defun mu4e-root-maildir() + "Get the root maildir." + (or (and mu4e--server-props + (plist-get mu4e--server-props :root-maildir)) + (mu4e-error "Root maildir unknown; did you start mu4e?"))) + +(defun mu4e-database-path() + "Get the root maildir." + (or (and mu4e--server-props + (plist-get mu4e--server-props :database-path)) + (mu4e-error "Root maildir unknown; did you start mu4e?"))) + +(defun mu4e-server-version() + "Get the root maildir." + (or (and mu4e--server-props + (plist-get mu4e--server-props :version)) + (mu4e-error "Version unknown; did you start mu4e?"))) + +;;; remember queries result. +(defvar mu4e--server-query-items nil + "Query items results we receive from the mu4e server. +Those are the results from the counting-queries +for bookmarks and maildirs.") + +(defun mu4e-server-query-items () + "Get the latest server query items." + mu4e--server-query-items) + + +;;; Handling raw server data + +(defvar mu4e--server-buf nil + "Buffer (string) for data received from the backend.") +(defconst mu4e--server-name " *mu4e-server*" + "Name of the server process, buffer.") +(defvar mu4e--server-process nil + "The mu-server process.") + +;; dealing with the length cookie that precedes expressions +(defconst mu4e--server-cookie-pre "\376" + "Each expression starts with a length cookie: +<`mu4e--server-cookie-pre'><length-in-hex><`mu4e--server-cookie-post'>.") +(defconst mu4e--server-cookie-post "\377" + "Each expression starts with a length cookie: +<`mu4e--server-cookie-pre'><length-in-hex><`mu4e--server-cookie-post'>.") +(defconst mu4e--server-cookie-matcher-rx + (concat mu4e--server-cookie-pre "\\([[:xdigit:]]+\\)" + mu4e--server-cookie-post) + "Regular expression matching the length cookie. +Match 1 will be the length (in hex).") + +(defun mu4e-running-p () + "Whether mu4e is running. +Checks whether the server process is live." + (and mu4e--server-process + (memq (process-status mu4e--server-process) + '(run open listen connect stop)) t)) + +(defsubst mu4e--server-eat-sexp-from-buf () + "Eat the next s-expression from `mu4e--server-buf'. +Note: this is a string, not an emacs-buffer. `mu4e--server-buf gets +its contents from the mu-servers in the following form: + <`mu4e--server-cookie-pre'><length-in-hex><`mu4e--server-cookie-post'> +Function returns this sexp, or nil if there was none. +`mu4e--server-buf' is updated as well, with all processed sexp data +removed." + (ignore-errors ;; the server may die in the middle... + (let ((b (string-match mu4e--server-cookie-matcher-rx mu4e--server-buf)) + (sexp-len) (objcons)) + (when b + (setq sexp-len (string-to-number (match-string 1 mu4e--server-buf) 16)) + ;; does mu4e--server-buf contain the full sexp? + (when (>= (length mu4e--server-buf) (+ sexp-len (match-end 0))) + ;; clear-up start + (setq mu4e--server-buf (substring mu4e--server-buf (match-end 0))) + ;; note: we read the input in binary mode -- here, we take the part + ;; that is the sexp, and convert that to utf-8, before we interpret + ;; it. + (setq objcons (read-from-string + (decode-coding-string + (substring mu4e--server-buf 0 sexp-len) + 'utf-8 t))) + (when objcons + (setq mu4e--server-buf (substring mu4e--server-buf sexp-len)) + (car objcons))))))) + +(defun mu4e--server-plist-get (plist key) + "Like `plist-get' but load data from file if it is a string. + +I.e. (mu4e--server-plist-get (:foo bar) :foo) + => bar +but + (mu4e--server-plist-get (:foo \"/tmp/data.eld\") :foo) + => evaluates the contents of /tmp/data.eld + (and deletes the file afterward). + +This for the few sexps we get from the mu server that support this +(headers, contacts, maildirs)." + ;; XXX: perhaps re-use the same buffer? + (let ((val (plist-get plist key))) + (if (stringp val) + (with-temp-buffer + (insert-file-contents val) + (goto-char (point-min)) + (delete-file val) + (read (current-buffer))) + val))) + + +(defun mu4e--server-filter (_proc str) + "Filter string STR from PROC. +This processes the \"mu server\" output. It accumulates the +strings into valid sexpsv and evaluating those. + +The server output is as follows: + + 1. an error + (:error 2 :message \"unknown command\") + ;; eox + => passed to `mu4e-error-func'. + + 2a. a header exp looks something like: + (:headers + ( ;; message 1 + :docid 1585 + :from ((\"Donald Duck\" . \"donald@example.com\")) + :to ((\"Mickey Mouse\" . \"mickey@example.com\")) + :subject \"Wicked stuff\" + :date (20023 26572 0) + :size 15165 + :references (\"200208121222.g7CCMdb80690@msg.id\") + :in-reply-to \"200208121222.g7CCMdb80690@msg.id\" + :message-id \"foobar32423847ef23@pluto.net\" + :maildir: \"/archive\" + :path \"/home/mickey/Maildir/inbox/cur/1312_3.32282.pluto,4cd5bd4e9:2,\" + :priority high + :flags (new unread) + :meta <meta-data> + ) + ( .... more messages ) +) +;; eox + => this will be passed to `mu4e-headers-append-func'. + + 2b. After the list of headers has been returned (see 2a.), + we'll receive a sexp that looks like + (:found <n>) with n the number of messages found. The <n> will be + passed to `mu4e-found-func'. + + 3. a view looks like: + (:view <msg-sexp>) + => the <msg-sexp> (see 2.) will be passed to `mu4e-view-func'. + the <msg-sexp> also contains :body-txt and/or :body-html + + 4. a database update looks like: + (:update <msg-sexp> :move <nil-or-t>) + like :header + + => the <msg-sexp> (see 2.) will be passed to + `mu4e-update-func', :move tells us whether this is a move to + another maildir, or merely a flag change. + + 5. a remove looks like: + (:remove <docid>) + => the docid will be passed to `mu4e-remove-func' + + 6. a compose looks like: + (:compose <reply|forward|edit|new> [:original<msg-sexp>] [:include <attach>]) + `mu4e-compose-func'. :original looks like :view." + (mu4e-log 'misc "* Received %d byte(s)" (length str)) + (setq mu4e--server-buf (concat mu4e--server-buf str)) ;; update our buffer + (let ((sexp (mu4e--server-eat-sexp-from-buf))) + (with-local-quit + (while sexp + (mu4e-log 'from-server "%s" sexp) + (cond + ;; a list of messages (after a find command) + ((plist-get sexp :headers) + (funcall mu4e-headers-append-func + (mu4e--server-plist-get sexp :headers))) + + ;; the found sexp, we receive after getting all the headers + ((plist-get sexp :found) + (funcall mu4e-found-func (plist-get sexp :found))) + + ;; viewing a specific message + ((plist-get sexp :view) + (funcall mu4e-view-func (plist-get sexp :view))) + + ;; receive an erase message + ((plist-get sexp :erase) + (funcall mu4e-erase-func)) + + ;; receive a :sent message + ((plist-get sexp :sent) + (funcall mu4e-sent-func + (plist-get sexp :docid) + (plist-get sexp :path))) + + ;; received a pong message + ((plist-get sexp :pong) + (setq mu4e--server-props (plist-get sexp :props)) + (funcall mu4e-pong-func sexp)) + + ;; receive queries info + ((plist-get sexp :queries) + (setq mu4e--server-query-items (plist-get sexp :queries)) + (funcall mu4e-queries-func sexp)) + + ;; received a contacts message + ;; note: we use 'member', to match (:contacts nil) + ((plist-member sexp :contacts) + (funcall mu4e-contacts-func + (mu4e--server-plist-get sexp :contacts) + (plist-get sexp :tstamp))) + + ;; something got moved/flags changed + ((plist-get sexp :update) + (funcall mu4e-update-func + (plist-get sexp :update) + (plist-get sexp :move) + (plist-get sexp :maybe-view))) + + ;; a message got removed + ((plist-get sexp :remove) + (funcall mu4e-remove-func (plist-get sexp :remove))) + + ;; get some info + ((plist-get sexp :info) + (funcall mu4e-info-func sexp)) + + ;; get some data + ((plist-get sexp :maildirs) + (setq mu4e-maildir-list (mu4e--server-plist-get sexp :maildirs))) + + ;; receive an error + ((plist-get sexp :error) + (funcall mu4e-error-func + (plist-get sexp :error) + (plist-get sexp :message))) + + (t (mu4e-message "Unexpected data from server [%S]" sexp))) + + (setq sexp (mu4e--server-eat-sexp-from-buf)))))) + +(defun mu4e--kill-stale () + "Kill stale mu4e server process. +As per issue #2198." + (seq-each + (lambda(proc) + (when (and (process-live-p proc) + (string-prefix-p mu4e--server-name (process-name proc))) + (mu4e-message "killing stale mu4e server") + (ignore-errors + (signal-process proc 'SIGINT) ;; nicely + (sit-for 1.0) + (signal-process proc 'SIGKILL)))) ;; forcefully + (process-list))) + +(defun mu4e--server-args() + "Return the command line args for the command to start the mu4e-server." + ;; [--debug] server [--muhome=..] + (seq-filter #'identity ;; filter out nil + `(,(when mu4e-mu-debug "--debug") + "server" + ,(when mu4e-mu-allow-temp-file "--allow-temp-file") + ,(when mu4e-mu-home (format "--muhome=%s" mu4e-mu-home))))) + +(defun mu4e--version-check () + ;; sanity-check 1 + (let ((default-directory temporary-file-directory)) ;;ensure it's local. + (unless (and mu4e-mu-binary (file-executable-p mu4e-mu-binary)) + (mu4e-error + "Cannot find mu, please set `mu4e-mu-binary' to the mu executable path")) + ;; sanity-check 2 + (let ((version (let ((s (shell-command-to-string + (concat mu4e-mu-binary " --version")))) + (and (string-match "version \\([.0-9]+\\)" s) + (match-string 1 s))))) + (if (not (string= version mu4e-mu-version)) + (mu4e-error + (concat + "Found mu version %s, but mu4e needs version %s" + "; please set `mu4e-mu-binary' " + "accordingly") + version mu4e-mu-version) + (mu4e-message "Found mu version %s" version))))) + +(defun mu4e-server-repl () + "Start a mu4e-server repl. + +This is meant for debugging/testing - the repl is designed for +machines, not for humans. + +You cannot run the repl when mu4e is running (or vice-versa)." + (interactive) + (if (mu4e-running-p) + (mu4e-error "Cannot run repl when mu4e is running") + (progn + (mu4e--version-check) + (let ((cmd (string-join (cons mu4e-mu-binary (mu4e--server-args)) " "))) + (term cmd) + (rename-buffer "*mu4e-repl*" 'unique) + (message "invoked: '%s'" cmd))))) + +(defun mu4e--server-start () + "Start the mu server process." + (mu4e--version-check) + ;; kill old/stale servers, if any. + (mu4e--kill-stale) + (let* ((process-connection-type nil) ;; use a pipe + (args (mu4e--server-args))) + (setq mu4e--server-buf "") + (mu4e-log 'misc "* invoking '%s' with parameters %s" mu4e-mu-binary + (mapconcat (lambda (arg) (format "'%s'" arg)) args " ")) + (setq mu4e--server-process (apply 'start-process + mu4e--server-name mu4e--server-name + mu4e-mu-binary args)) + ;; register a function for (:info ...) sexps + (unless mu4e--server-process + (mu4e-error "Failed to start the mu4e backend")) + (set-process-query-on-exit-flag mu4e--server-process nil) + (set-process-coding-system mu4e--server-process 'binary 'utf-8-unix) + (set-process-filter mu4e--server-process 'mu4e--server-filter) + (set-process-sentinel mu4e--server-process 'mu4e--server-sentinel))) + +(defun mu4e--server-kill () + "Kill the mu server process." + (let* ((buf (get-buffer mu4e--server-name)) + (proc (and (buffer-live-p buf) (get-buffer-process buf)))) + (when proc + (mu4e-message "shutting down") + (set-process-filter mu4e--server-process nil) + (set-process-sentinel mu4e--server-process nil) + (let ((delete-exited-processes t)) + (mu4e--server-call-mu '(quit))) + ;; try sending SIGINT (C-c) to process, so it can exit gracefully + (ignore-errors + (signal-process proc 'SIGINT)))) + (setq + mu4e--server-process nil + mu4e--server-buf nil)) + +;; error codes are defined in src/mu-util +;;(defconst mu4e-xapian-empty 19 "Error code: xapian is empty/non-existent") + +(defun mu4e--server-sentinel (proc _msg) + "Function called when the server process PROC terminates with MSG." + (let ((status (process-status proc)) (code (process-exit-status proc))) + (mu4e-log 'misc "* famous last words from server: '%s'" mu4e--server-buf) + (setq mu4e--server-process nil) + (setq mu4e--server-buf "") ;; clear any half-received sexps + (cond + ((eq status 'signal) + (cond + ((or(eq code 9) (eq code 2)) (message nil)) + ;;(message "the mu server process has been stopped")) + (t (mu4e-error (format "server process received signal %d" code))))) + ((eq status 'exit) + (cond + ((eq code 0) + (message nil)) ;; don't do anything + ((eq code 11) + (error "schema mismatch; please re-init mu from command-line")) + ((eq code 19) + (error "mu database is locked by another process")) + (t (error "mu server process ended with exit code %d" code)))) + (t + (error "something bad happened to the mu server process"))))) + +(defun mu4e--server-call-mu (form) + "Call the mu server with some command FORM." + (unless (mu4e-running-p) (mu4e--server-start)) + (let* ((print-length nil) (print-level nil) + (cmd (format "%S" form))) + (mu4e-log 'to-server "%s" cmd) + (process-send-string mu4e--server-process (concat cmd "\n")))) + +(defun mu4e--server-add (path) + "Add the message at PATH to the database. +On success, we receive `'(:info add :path <path> :docid <docid>)' +as well as `'(:update <msg-sexp>)`'; otherwise, we receive an error." + (mu4e--server-call-mu `(add :path ,path))) + +(defun mu4e--server-contacts (personal after maxnum tstamp) + "Ask for contacts with PERSONAL AFTER MAXNUM TSTAMP. + +S-expression (:contacts (<list>) :tstamp \"<tstamp>\") +is expected in response. + +If PERSONAL is non-nil, only get personal contacts, if AFTER is +non-nil, get only contacts seen AFTER (the time_t value). If MAX is non-nil, +get at most MAX contacts." + (mu4e--server-call-mu + `(contacts + :personal ,(and personal t) + :after ,(or after nil) + :tstamp ,(or tstamp nil) + :maxnum ,(or maxnum nil)))) + +(defun mu4e--server-data (kind) + "Request data of some KIND. +KIND is a symbol. Currently supported kinds: maildirs." + (mu4e--server-call-mu + `(data :kind ,kind))) + +(defun mu4e--server-find (query threads sortfield sortdir maxnum skip-dups + include-related) + "Run QUERY with THREADS SORTFIELD SORTDIR MAXNUM SKIP-DUPS INCLUDE-RELATED. + +If THREADS is non-nil, show results in threaded fashion, +SORTFIELD is a symbol describing the field to sort by (or nil); +see `mu4e~headers-sortfield-choices'. If SORT is `descending', +sort Z->A, if it's `ascending', sort A->Z. MAXNUM determines the +maximum number of results to return, or nil for unlimited. If +SKIP-DUPS is non-nil, show only one of duplicate messages (see +`mu4e-headers-skip-duplicates'). If INCLUDE-RELATED is non-nil, +include messages related to the messages matching the search +query (see `mu4e-headers-include-related'). + +For each result found, a function is called, depending on the +kind of result. The variables `mu4e-error-func' contain the +function that to be be called for, resp., a message (header) +or an error." + (mu4e--server-call-mu + `(find + :query ,query + :threads ,(and threads t) + :sortfield ,sortfield + :descending ,(if (eq sortdir 'descending) t nil) + :maxnum ,maxnum + :skip-dups ,(and skip-dups t) + :include-related ,(and include-related t)))) + +(defun mu4e--server-index (&optional cleanup lazy-check) + "Index messages. +If CLEANUP is non-nil, remove messages which are in the database +but no longer in the filesystem. If LAZY-CHECK is non-nil, only +consider messages for which the time stamp (ctime) of the +directory they reside in has not changed since the previous +indexing run. This is much faster than the non-lazy check, but +won't update messages that have change (rather than having been +added or removed), since merely editing a message does not update +the directory time stamp." + (mu4e--server-call-mu + `(index :cleanup ,(and cleanup t) + :lazy-check ,(and lazy-check t)))) + +(defun mu4e--server-mkdir (path &optional update) + "Create a new maildir-directory at filesystem PATH. +When UPDATE is non-nil, send a update when completed. +PATH must be below the root-maildir." + ;; handle maildir cache + (if (not (string-prefix-p (mu4e-root-maildir) path)) + (mu4e-error "Cannot create maildir outside root-maildir") + (add-to-list 'mu4e-maildir-list ;; update cache + (substring path (length (mu4e-root-maildir))))) + (mu4e--server-call-mu `(mkdir + :path ,path + :update ,(or update nil)))) + +(defun mu4e--server-move (docid-or-msgid &optional maildir flags no-view) + "Move message identified by DOCID-OR-MSGID. +Optionally to MAILDIR and optionally setting FLAGS. If MAILDIR is +nil, message will be moved within the same maildir. + +At least one of MAILDIR and FLAGS must be specified. Note that +even when MAILDIR is nil, this is still a filesystem move, since +a change in flags implies a change in message filename. + +MAILDIR must be a maildir, that is, the part _without_ cur/ or new/ +or the root-maildir-prefix. E.g. \"/archive\". This directory must +already exist. + +The FLAGS parameter can have the following forms: + 1. a list of flags such as `(passed replied seen)' + 2. a string containing the one-char versions of the flags, e.g. \"PRS\" + 3. a delta-string specifying the changes with +/- and the one-char flags, + e.g. \"+S-N\" to set Seen and remove New. + +The flags are any of `deleted', `flagged', `new', `passed', `replied' `seen' or +`trashed', or the corresponding \"DFNPRST\" as defined in [1]. See +`mu4e-string-to-flags' and `mu4e-flags-to-string'. +The server reports the results for the operation through +`mu4e-update-func'. + +If the variable `mu4e-change-filenames-when-moving' is +non-nil, moving to a different maildir generates new names forq +the target files; this helps certain tools (such as mbsync). + +If NO-VIEW is non-nil, do not update the view. + +Returns either (:update ... ) or (:error ) sexp, which are handled my +`mu4e-update-func' and `mu4e-error-func', respectively." + (unless (or maildir flags) + (mu4e-error "At least one of maildir and flags must be specified")) + (unless (or (not maildir) + (file-exists-p + (mu4e-join-paths (mu4e-root-maildir) maildir))) + (mu4e-error "Target directory does not exist")) + (mu4e--server-call-mu + `(move + :docid ,(if (stringp docid-or-msgid) nil docid-or-msgid) + :msgid ,(if (stringp docid-or-msgid) docid-or-msgid nil) + :flags ,(or flags nil) + :maildir ,(or maildir nil) + :rename ,(and maildir mu4e-change-filenames-when-moving t) + :no-view ,(and no-view t)))) + +(defun mu4e--server-ping () + "Sends a ping to the mu server, expecting a (:pong ...) in response." + (mu4e--server-call-mu `(ping))) + +(defun mu4e--server-queries (queries) + "Sends queries to the mu server, expecting a (:queries ...) sexp in response. +QUERIES is a list of queries for the number of results with +read/unread status are returned in the pong-response." + (mu4e--server-call-mu `(queries :queries ,queries))) + +(defun mu4e--server-remove (docid-or-path) + "Remove message with either DOCID or PATH. +The results are reported through either (:update ... ) +or (:error) sexps." + (if (stringp docid-or-path) + (mu4e--server-call-mu `(remove :path ,docid-or-path)) + (mu4e--server-call-mu `(remove :docid ,docid-or-path)))) + +(defun mu4e--server-view (docid-or-msgid &optional mark-as-read) + "View a message referred to by DOCID-OR-MSGID. +Optionally, if MARK-AS-READ is non-nil, the backend marks the +message as \"read\" before returning, if not already. The result +will be delivered to the function registered as `mu4e-view-func'." + (mu4e--server-call-mu + `(view + :docid ,(if (stringp docid-or-msgid) nil docid-or-msgid) + :msgid ,(if (stringp docid-or-msgid) docid-or-msgid nil) + :mark-as-read ,(and mark-as-read t) + ;; when moving (due to mark-as-read), change filenames + ;; if so configured. Note: currently this *ignored* + ;; because mbsync seems to get confused. + :rename ,(and mu4e-change-filenames-when-moving t)))) + + +(provide 'mu4e-server) +;;; mu4e-server.el ends here diff --git a/mu4e/mu4e-speedbar.el b/mu4e/mu4e-speedbar.el new file mode 100644 index 0000000..c2c414e --- /dev/null +++ b/mu4e/mu4e-speedbar.el @@ -0,0 +1,133 @@ +;;; mu4e-speedbar --- Speedbar support for mu4e -*- lexical-binding: t -*- + +;; Copyright (C) 2012-2021 Antono Vasiljev, Dirk-Jan C. Binnema + +;; Author: Antono Vasiljev <self@antono.info> +;; Version: 0.1 +;; Keywords: file, tags, tools + +;; This file is not part of GNU Emacs. + +;; This program is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation; either version 3, or (at your option) +;; any later version. + +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: +;; +;; Speedbar provides a frame in which files, and locations in files +;; are displayed. These functions provide mu4e specific support, +;; showing maildir list in the side-bar. +;; +;; This file requires speedbar. + +;;; Code: + +(require 'speedbar) +(require 'mu4e-vars) +(require 'mu4e-headers) +(require 'mu4e-context) +(require 'mu4e-bookmarks) + +(defvar mu4e-main-speedbar-key-map nil + "Keymap used when in mu4e display mode.") +(defvar mu4e-headers-speedbar-key-map nil + "Keymap used when in mu4e display mode.") +(defvar mu4e-view-speedbar-key-map nil + "Keymap used when in mu4e display mode.") + +(defvar mu4e-main-speedbar-menu-items nil + "Additional menu-items to add to speedbar frame.") +(defvar mu4e-headers-speedbar-menu-items nil + "Additional menu-items to add to speedbar frame.") +(defvar mu4e-view-speedbar-menu-items nil + "Additional menu-items to add to speedbar frame.") + + +(defun mu4e-speedbar-install-variables () + "Install those variables used by speedbar to enhance mu4e." + (add-hook 'mu4e-context-changed-hook + #'mu4e~speedbar-context-changed-hook-fn) + (dolist (keymap + '( mu4e-main-speedbar-key-map + mu4e-headers-speedbar-key-map + mu4e-view-speedbar-key-map)) + (unless keymap + (setq keymap (speedbar-make-specialized-keymap)) + (define-key keymap "RET" 'speedbar-edit-line) + (define-key keymap "e" 'speedbar-edit-line)))) + +(defun mu4e~speedbar-context-changed-hook-fn () + (when (buffer-live-p speedbar-buffer) + (with-current-buffer speedbar-buffer + (let ((inhibit-read-only t)) + (mu4e-speedbar-buttons))))) + +(with-eval-after-load 'speedbar + (mu4e-speedbar-install-variables)) + +(defun mu4e~speedbar-render-maildir-list () + "Insert the list of maildirs in the speedbar." + (interactive) + (when (buffer-live-p speedbar-buffer) + (with-current-buffer speedbar-buffer + (mapcar (lambda (maildir-name) + (speedbar-insert-button + (concat " " maildir-name) + 'mu4e-highlight-face + 'highlight + 'mu4e~speedbar-maildir + maildir-name)) + (mu4e-get-maildirs))))) + +(defun mu4e~speedbar-maildir (&optional _text token _ident) + "Jump to maildir TOKEN. TEXT and INDENT are not used." + (dframe-with-attached-buffer + (mu4e-search (concat "\"maildir:" token "\"") current-prefix-arg))) + +(defun mu4e~speedbar-render-bookmark-list () + "Insert the list of bookmarks in the speedbar" + (interactive) + (mapcar (lambda (bookmark) + (unless (plist-get bookmark :hide) + (speedbar-insert-button + (concat " " (plist-get bookmark :name)) + 'mu4e-highlight-face + 'highlight + 'mu4e~speedbar-bookmark + (plist-get bookmark :query)))) + (mu4e-bookmarks))) + +(defun mu4e~speedbar-bookmark (&optional _text token _ident) + "Run bookmarked query TOKEN. TEXT and INDENT are not used." + (dframe-with-attached-buffer + (mu4e-search token current-prefix-arg))) + +;;;###autoload +(defun mu4e-speedbar-buttons (&optional _buffer) + "Create buttons for any mu4e BUFFER." + (interactive) + (erase-buffer) + (insert (propertize "* mu4e\n\n" 'face 'mu4e-title-face)) + + (insert (propertize " Bookmarks\n" 'face 'mu4e-title-face)) + (mu4e~speedbar-render-bookmark-list) + (insert "\n") + (insert (propertize " Maildirs\n" 'face 'mu4e-title-face)) + (mu4e~speedbar-render-maildir-list)) + +(defun mu4e-main-speedbar-buttons (buffer) (mu4e-speedbar-buttons buffer)) +(defun mu4e-headers-speedbar-buttons (buffer) (mu4e-speedbar-buttons buffer)) +(defun mu4e-view-speedbar-buttons (buffer) (mu4e-speedbar-buttons buffer)) + +;;; _ +(provide 'mu4e-speedbar) +;;; mu4e-speedbar.el ends here diff --git a/mu4e/mu4e-thread.el b/mu4e/mu4e-thread.el new file mode 100644 index 0000000..c973745 --- /dev/null +++ b/mu4e/mu4e-thread.el @@ -0,0 +1,295 @@ +;;; mu4e-thread.el --- Thread folding support -*- lexical-binding: t -*- + +;; Copyright (C) 2023 Nicolas P. Rougier + +;; Author: Nicolas P. Rougier <Nicolas.Rougier@inria.fr> +;; Keywords: mail + +;; This program is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; mu4e-thread.el is a library that allows to fold and unfold threads in mu4e +;; headers mode. Folding works by creating an overlay over thread children that +;; display a summary (number of hidden messages and possibly number of unread +;; messages). + +;; Folding is performed just-in-time such that it is quite fast to +;; fold/unfold threads. When a thread has unread messages, the folding stops at +;; the first unread message unless `mu4e-thread-fold-unread` has been set to t. + +;; Similarly, when a thread has marked messages, the folding stops at the first +;; marked message. + +;; Note, you can only use these functions when threads are available, roughly +;; when `mu4e-search-threads' in non-nil. + +;;; Usage example: +;; +;; After a search, mu4e-thread-mode will be enable when threads +;; are available; so, to automatically sort them: +;; (add-hook 'mu4e-thread-mode-hook #'mu4e-thread-fold-apply-all) + +;;; Code: + +(require 'mu4e-vars) +(require 'mu4e-message) +(require 'mu4e-mark) + +(defcustom mu4e-thread-fold-unread nil + "Whether to fold unread messages in a thread." + :type 'boolean + :group 'mu4e-headers) + +(defcustom mu4e-thread-fold-single-children nil + "When non-nil, fold a thread even if there is only a single child. +Otherwise, do not not fold single children since would simply +hide the single child." + :type 'boolean + :group 'mu4e-headers) + +(defface mu4e-thread-fold-face + `((t :inherit mu4e-highlight-face)) + "Face for the information line of a folded thread." + :group 'mu4e-faces) + +(defvar-local mu4e-thread--fold-status nil + "Global folding status.") + +(defvar-local mu4e-thread--docids nil + "Thread list whose folding has been set individually.") + +(defvar mu4e-headers-fields) ;; defined in mu4e-headers.el +(defun mu4e-thread-fold-info (count unread) + "Text to be displayed for a folded thread. +There are COUNT hidden and UNREAD messages overall." + (let ((size (+ 2 (apply #'+ (mapcar (lambda (item) (or (cdr item) 0)) + mu4e-headers-fields)))) + (msg (concat (format"[%d hidden messages%s]\n" count + (if (> unread 0) + (format ", %d unread" unread) + ""))))) + (propertize (concat " " (make-string size ?•) " " msg)))) + +(defun mu4e-thread-message-folded-p () + "Is point in a folded area?" + (when-let* ((overlay (mu4e-thread-is-folded)) + (beg (overlay-start overlay)) + (end (overlay-end overlay))) + (and (>= (point) beg) (< (point) end)))) + +(declare-function 'mu4e~headers-thread-root-p "mu4e-headers") +(defalias 'mu4e-thread-is-root 'mu4e~headers-thread-root-p) + +(defun mu4e-thread-goto-root () + "Go to the root of the current thread." + (interactive) + (goto-char (mu4e-thread-root)) + (beginning-of-line)) + +(defun mu4e-thread-root () + "Get the root of the current thread." + (interactive) + (let ((point)) + (save-excursion + (while (and (not (bobp)) + (not (mu4e-thread-is-root))) + (forward-line -1)) + (setq point (point))) + point)) + +(declare-function 'mu4e-headers-prev-thread "mu4e-headers") +(declare-function 'mu4e-headers-next-thread "mu4e-headers") + +(defalias 'mu4e-thread-goto-prev 'mu4e-headers-prev-thread) +(defalias 'mu4e-thread-goto-next 'mu4e-headers-next-thread) + +(defun mu4e-thread-prev () + "Get the root of the previous thread (if any)." + (save-excursion + (when (mu4e-thread-goto-prev) + (mu4e-thread-root)))) + +(defun mu4e-thread-next() + "Get the root of the next thread (if any)." + (save-excursion + (when (mu4e-thread-goto-next) + (mu4e-thread-root)))) + +(defun mu4e-thread-is-folded () + "Test if thread at point is folded." + (interactive) + (let* ((thread-beg (mu4e-thread-root)) + (thread-end (or (mu4e-thread-next) (point-max))) + (overlays (overlays-in thread-beg thread-end))) + (catch 'folded + (dolist (overlay overlays) + (when (overlay-get overlay 'mu4e-thread-folded) + (throw 'folded overlay)))))) + +(defun mu4e-thread-fold-toggle-all () + "Toggle all threads folding unconditionally. +Reset individual folding states." + (interactive) + (setq mu4e-thread--docids nil) + (if mu4e-thread--fold-status + (mu4e-thread-unfold-all) + (mu4e-thread-fold-all))) + +(defun mu4e-thread-fold-apply-all () + "Apply global folding status to all threads not set individually." + (interactive) + ;; Global fold status + (if mu4e-thread--fold-status + (mu4e-thread-fold-all) + (mu4e-thread-unfold-all)) + ;; Individual fold status + (save-excursion + (goto-char (point-min)) + (catch 'end-search + (while (not (eobp)) + (when-let* ((msg (get-text-property (point) 'msg)) + (docid (mu4e-message-field msg :docid)) + (state (cdr (assoc docid mu4e-thread--docids)))) + (if (eq state 'folded) + (mu4e-thread-fold) + (mu4e-thread-unfold))) + (unless (mu4e-thread-next) + (throw 'end-search t)) + (mu4e-thread-goto-next))))) + +(defun mu4e-thread-fold-all () + "Fold all threads unconditionally." + (interactive) + (setq mu4e-thread--fold-status t) + + (save-excursion + (goto-char (point-min)) + (catch 'done + (while (not (eobp)) + (mu4e-thread-fold t) + (unless (mu4e-thread-goto-next) + (throw 'done t)))))) + +(defun mu4e-thread-unfold-all () + "Unfold all threads unconditionally." + (interactive) + (setq mu4e-thread--fold-status nil) + (remove-overlays (point-min) (point-max) 'mu4e-thread-folded t)) + +(defun mu4e-thread-fold-toggle () + "Toggle folding for thread at point." + (interactive) + (if (mu4e-thread-is-folded) + (mu4e-thread-unfold) + (mu4e-thread-fold))) + +(defun mu4e-thread-fold-toggle-goto-next () + "Toggle folding for thread at point and go to next thread." + (interactive) + (if (mu4e-thread-is-folded) + (mu4e-thread-unfold-goto-next) + (mu4e-thread-fold-goto-next))) + +(defun mu4e-thread-unfold (&optional no-save) + "Unfold thread at point and store state unless NO-SAVE is t." + (interactive) + (unless (eq (line-end-position) (point-max)) + (when-let ((overlay (mu4e-thread-is-folded))) + (unless no-save + (mu4e-thread--save-state 'unfolded)) + (delete-overlay overlay)))) + +(defun mu4e-thread--save-state (state) + "Save the folding STATE of thread at point." + (save-excursion + (mu4e-thread-goto-root) + (when-let* ((msg (get-text-property (point) 'msg)) + (docid (mu4e-message-field msg :docid))) + (setf (alist-get docid mu4e-thread--docids) state)))) + +(defun mu4e-thread-fold (&optional no-save) + "Fold thread at point and store state unless NO-SAVE is t." + (interactive) + (unless (eq (line-end-position) (point-max)) + (let* ((thread-beg (mu4e-thread-root)) + (thread-end (mu4e-thread-next)) + (thread-end (if thread-end (1- thread-end) (point-max))) + (unread-count 0) + (fold-beg (save-excursion + (goto-char thread-beg) + (forward-line) + (point))) + (fold-end (save-excursion + (goto-char thread-beg) + (forward-line) + (catch 'fold-end + (while (and (not (eobp)) + (get-text-property (point) 'msg) + (and thread-end (< (point) thread-end))) + (let* ((msg (get-text-property (point) 'msg)) + (docid (mu4e-message-field msg :docid)) + (flags (mu4e-message-field msg :flags)) + (unread (memq 'unread flags))) + (when (mu4e-mark-docid-marked-p docid) + (throw 'fold-end (point))) + (when unread + (unless mu4e-thread-fold-unread + (throw 'fold-end (point))) + (setq unread-count (+ 1 unread-count)))) + (forward-line))) + (point)))) + (unless no-save + (mu4e-thread--save-state 'folded)) + (let ((child-count (count-lines fold-beg fold-end)) + (unread-count (if mu4e-thread-fold-unread unread-count 0))) + (when (> child-count (if mu4e-thread-fold-single-children 0 1)) + (let ((inhibit-read-only t) + (overlay (make-overlay fold-beg fold-end)) + (info (mu4e-thread-fold-info child-count unread-count))) + (add-text-properties fold-beg (+ fold-beg 1) + '(face mu4e-thread-fold-face)) + (overlay-put overlay 'mu4e-thread-folded t) + (overlay-put overlay 'display info))))))) + +(defun mu4e-thread-fold-goto-next () + "Fold the thread at point and go to next thread." + (interactive) + (unless (eq (line-end-position) (point-max)) + (mu4e-thread-fold) + (mu4e-thread-goto-next))) + +(defun mu4e-thread-unfold-goto-next () + "Unfold the thread at point and go to next thread." + (interactive) + (unless (eq (line-end-position) (point-max)) + (mu4e-thread-unfold) + (mu4e-thread-goto-next))) + +(define-minor-mode mu4e-thread-mode + "Mode for thread-support." + :global nil + :init-value nil ;; disabled by default + :group 'mu4e + :lighter "" + :keymap + (let ((map (make-sparse-keymap))) + (define-key map (kbd "<S-left>") #'mu4e-thread-goto-root) + (define-key map (kbd "<tab>") #'mu4e-thread-fold-toggle-goto-next) + (define-key map (kbd "<C-tab>") #'mu4e-thread-fold-toggle-goto-next) + (define-key map (kbd "<backtab>") #'mu4e-thread-fold-toggle-all) + map)) + +(provide 'mu4e-thread) +;;; mu4e-thread.el ends here diff --git a/mu4e/mu4e-update.el b/mu4e/mu4e-update.el new file mode 100644 index 0000000..5bb9e1d --- /dev/null +++ b/mu4e/mu4e-update.el @@ -0,0 +1,335 @@ +;;; mu4e-update.el --- Update the mu4e message store -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; Updating the mu4e message store: calling a mail retrieval program and +;; re-running the index. + +;;; Code: + +(require 'mu4e-helpers) +(require 'mu4e-server) + +;;; Customization + +(defcustom mu4e-get-mail-command "true" + "Shell command for retrieving new mail. +Common values are \"offlineimap\", \"fetchmail\" or \"mbsync\", but +arbitrary shell-commands can be used. + +When set to the literal string \"true\" (the default), the +command simply finishes successfully (running the \"true\" +command) without retrieving any mail. This can be useful when +mail is already retrieved in another way, such as a local MDA." + :type 'string + :group 'mu4e + :safe 'stringp) + +(defcustom mu4e-index-update-error-warning t + "Whether to display warnings during the retrieval process. +This depends on the `mu4e-get-mail-command' exit code." + :type 'boolean + :group 'mu4e + :safe 'booleanp) + +(defcustom mu4e-index-update-error-continue t + "Whether to continue with indexing after an error during retrieval." + :type 'boolean + :group 'mu4e + :safe 'booleanp) + +(defcustom mu4e-index-update-in-background t + "Whether to retrieve mail in the background." + :type 'boolean + :group 'mu4e + :safe 'booleanp) + +(defcustom mu4e-index-cleanup t + "Whether to run a cleanup phase after indexing. + +That is, validate that each message in the message store has a +corresponding message file in the filesystem. + +Having this option as t ensures that no non-existing messages are +shown but can slow with large message stores on slow file-systems." + :type 'boolean + :group 'mu4e + :safe 'booleanp) + +(defcustom mu4e-index-lazy-check nil + "Whether to only use a \"lazy\" check during reindexing. +This influences how we decide whether a message +needs (re)indexing or not. + +When this is set to non-nil, mu only uses the directory +timestamps to decide whether it needs to check the messages +beneath it. This makes indexing much faster, but might miss some +changes. For this, you might want to occasionally call +`mu4e-update-index-nonlazy'; `mu4e-update-pre-hook' can be used +to automate this." + :type 'boolean + :group 'mu4e + :safe 'booleanp) + +(defcustom mu4e-update-interval nil + "Number of seconds between mail retrieval/indexing. +If nil, don't update automatically. Note, changes in +`mu4e-update-interval' only take effect after restarting mu4e. + +Important, the automatic update *only* works when `mu4e' is +running." + :type '(choice (const :tag "No automatic update" nil) + (integer :tag "Seconds")) + :group 'mu4e + :safe 'integerp) + +(defvar mu4e-update-pre-hook nil + "Hook run just *before* the mail-retrieval / database updating process starts. +You can use this hook for example to `mu4e-get-mail-command' with +some specific setting.") + +(defcustom mu4e-hide-index-messages nil + "Whether to hide the \"Indexing...\" and contacts messages." + :type 'boolean + :group 'mu4e) + +(defvar mu4e-index-updated-hook nil + "Hook run when the indexing process has completed. +The variable `mu4e-index-update-status' can be used to get +information about what changed.") + +(defvar mu4e-message-changed-hook nil + "Hook run when there is a message changed in the data store. +For new messages, it depends on `mu4e-index-updated-hook'. This +can be used as a simple way to invoke some action when a message +changed") + +(defvar mu4e-index-update-status nil + "Last-seen completed update status, based on server status messages. + +If non-nil, this is a plist of the form: +\( +:checked <number of messages processed> (checked whether up-to-date) +:updated <number of messages updated/added +:cleaned-up <number of stale messages removed from store +:stamp <emacs (current-time) timestamp for the status)") + +(defconst mu4e-last-update-buffer "*mu4e-last-update*" + "Name of buffer with cloned from the last update buffer. +Useful for diagnosing update problems.") + + +;;; Internal variables / const +(defconst mu4e--update-name " *mu4e-update*" + "Name of the process and buffer to update mail.") +(defvar mu4e--progress-reporter nil + "Internal, the progress reporter object.") +(defvar mu4e--update-timer nil + "The mu4e update timer.") +(defconst mu4e--update-buffer-height 8 + "Height of the mu4e message retrieval/update buffer.") +(defvar mu4e--get-mail-ask-password "mu4e get-mail: Enter password: " + "Query string for `mu4e-get-mail-command' password.") +(defvar mu4e--get-mail-password-regexp "^Remote: Enter password: $" + "Regexp for a `mu4e-get-mail-command' password query.") + + +(defun mu4e--get-mail-process-filter (proc msg) + "Filter the MSG output of the `mu4e-get-mail-command' PROC. + +Currently the filter only checks if the command asks for a +password by matching the output against +`mu4e~get-mail-password-regexp'. The messages are inserted into +the process buffer. + +Also scrolls to the final line, and update the progress +throbber." + (when mu4e--progress-reporter + (progress-reporter-update mu4e--progress-reporter)) + + (when (string-match mu4e--get-mail-password-regexp msg) + (if (process-get proc 'x-interactive) + (process-send-string proc + (concat (read-passwd mu4e--get-mail-ask-password) + "\n")) + ;; TODO kill process? + (mu4e-error "Unrecognized password request"))) + (when (process-buffer proc) + (let ((inhibit-read-only t) + (procwin (get-buffer-window (process-buffer proc)))) + ;; Insert at end of buffer. Leave point alone. + (with-current-buffer (process-buffer proc) + (goto-char (point-max)) + (if (string-match ".*\r\\(.*\\)" msg) + (progn + ;; kill even with \r + (end-of-line) + (let ((end (point))) + (beginning-of-line) + (delete-region (point) end)) + (insert (match-string 1 msg))) + (insert msg))) + ;; Auto-scroll unless user is interacting with the window. + (when (and (window-live-p procwin) + (not (eq (selected-window) procwin))) + (with-selected-window procwin + (goto-char (point-max))))))) + +(defun mu4e-index-message (frm &rest args) + "Display FRM with ARGS like `mu4e-message' for index messages. +However, if `mu4e-hide-index-messages' is non-nil, do not display anything." + (unless mu4e-hide-index-messages + (apply 'mu4e-message frm args))) + +(defun mu4e-update-index () + "Update the mu4e index." + (interactive) + (mu4e--server-index mu4e-index-cleanup mu4e-index-lazy-check)) + +(defun mu4e-update-index-nonlazy () + "Update the mu4e index non-lazily. +This is just a convenience wrapper for indexing the non-lazy way +if you otherwise want to use `mu4e-index-lazy-check'." + (interactive) + (let ((mu4e-index-cleanup t) (mu4e-index-lazy-check nil)) + (mu4e-update-index))) + +(defvar mu4e--update-buffer nil + "The buffer of the update process when updating.") + +(define-derived-mode mu4e--update-mail-mode special-mode "mu4e:update" + "Major mode used for retrieving new e-mail messages in `mu4e'.") + +(define-key mu4e--update-mail-mode-map (kbd "q") 'mu4e-kill-update-mail) + +(defun mu4e--temp-window (buf height) + "Create a temporary window with HEIGHT at the bottom BUF. + +This function uses `display-buffer' with a default preset. + +To override this behavior, customize `display-buffer-alist'." + (display-buffer buf `(display-buffer-at-bottom + (preserve-size . (nil . t)) + (height . ,height) + (inhibit-same-window . t) + (window-height . fit-window-to-buffer))) + (set-window-buffer (get-buffer-window buf) buf)) + +(defun mu4e--update-sentinel-func (proc _msg) + "Sentinel function for the update process PROC." + (when mu4e--progress-reporter + (progress-reporter-done mu4e--progress-reporter) + (setq mu4e--progress-reporter nil)) + (unless mu4e-hide-index-messages + (message nil)) + (if (or (not (eq (process-status proc) 'exit)) + (/= (process-exit-status proc) 0)) + (progn + (when mu4e-index-update-error-warning + (mu4e-message "Update process returned with non-zero exit code") + (sit-for 5)) + (when mu4e-index-update-error-continue + (mu4e-update-index))) + (mu4e-update-index)) + (when (buffer-live-p mu4e--update-buffer) + (delete-windows-on mu4e--update-buffer) + ;; clone the update buffer for diagnosis + (when (get-buffer mu4e-last-update-buffer) + (kill-buffer mu4e-last-update-buffer)) + (with-current-buffer mu4e--update-buffer + (special-mode) + (clone-buffer mu4e-last-update-buffer)) + ;; and kill the buffer itself; the cloning is needed + ;; so the temp window handling works as expected. + (kill-buffer mu4e--update-buffer))) + +;; complicated function, as it: +;; - needs to check for errors +;; - (optionally) pop-up a window +;; - (optionally) check password requests +(defun mu4e--update-mail-and-index-real (run-in-background) + "Get a new mail by running `mu4e-get-mail-command'. +If +RUN-IN-BACKGROUND is non-nil (or called with prefix-argument), +run in the background; otherwise, pop up a window." + (let* ((process-connection-type t) + (proc (start-process-shell-command + mu4e--update-name mu4e--update-name + mu4e-get-mail-command)) + (buf (process-buffer proc)) + (win (or run-in-background + (mu4e--temp-window buf mu4e--update-buffer-height)))) + (set-process-query-on-exit-flag proc nil) + (setq mu4e--update-buffer buf) + (when (window-live-p win) + (with-selected-window win + (erase-buffer) + (insert "\n") ;; FIXME -- needed so output starts + (mu4e--update-mail-mode))) + (setq mu4e--progress-reporter + (unless mu4e-hide-index-messages + (make-progress-reporter + (mu4e-format "Retrieving mail...")))) + (set-process-sentinel proc 'mu4e--update-sentinel-func) + ;; if we're running in the foreground, handle password requests + (unless run-in-background + (process-put proc 'x-interactive (not run-in-background)) + (set-process-filter proc 'mu4e--get-mail-process-filter)))) + +(defun mu4e-update-mail-and-index (run-in-background) + "Retrieve new mail by running `mu4e-get-mail-command'. +If RUN-IN-BACKGROUND is non-nil (or called with prefix-argument), +run in the background; otherwise, pop up a window." + (interactive "P") + (unless mu4e-get-mail-command + (mu4e-error "`mu4e-get-mail-command' is not defined")) + (if (and (buffer-live-p mu4e--update-buffer) + (process-live-p (get-buffer-process mu4e--update-buffer))) + (mu4e-message "Update process is already running") + (progn + (run-hooks 'mu4e-update-pre-hook) + (mu4e--update-mail-and-index-real run-in-background)))) + +(defun mu4e-kill-update-mail () + "Stop the update process by killing it." + (interactive) + (let* ((proc (and (buffer-live-p mu4e--update-buffer) + (get-buffer-process mu4e--update-buffer)))) + (when (process-live-p proc) + (kill-process proc t)))) + +(define-minor-mode mu4e-update-minor-mode + "Mode for triggering mu4e updates." + :global nil + :init-value nil ;; disabled by default + :group 'mu4e + :lighter "" + :keymap + (let ((map (make-sparse-keymap))) + (define-key map (kbd "C-S-u") #'mu4e-update-mail-and-index) + ;; for terminal users + (define-key map (kbd "C-c C-u") #'mu4e-update-mail-and-index) + map)) + +(provide 'mu4e-update) +;;; mu4e-update.el ends here diff --git a/mu4e/mu4e-vars.el b/mu4e/mu4e-vars.el new file mode 100644 index 0000000..6a95c32 --- /dev/null +++ b/mu4e/mu4e-vars.el @@ -0,0 +1,392 @@ +;;; mu4e-vars.el --- Variables and faces for mu4e -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;;; Code: + +(require 'message) +(require 'mu4e-helpers) + +;;; Configuration +(defgroup mu4e nil + "Mu4e - an email-client for Emacs." + :group 'mail) + +(defcustom mu4e-confirm-quit t + "Whether to confirm to quit mu4e." + :type 'boolean + :group 'mu4e) + +(defcustom mu4e-modeline-support t + "Support for showing information in the modeline." + :type 'boolean + :group 'mu4e) + +(defcustom mu4e-notification-support nil + "Support for new-message notifications." + :type 'boolean + :group 'mu4e) + +(defcustom mu4e-org-support t + "Support Org-mode links." + :type 'boolean + :group 'mu4e) + +(defcustom mu4e-speedbar-support nil + "Support having a speedbar to navigate folders/bookmarks." + :type 'boolean + :group 'mu4e) + +(defcustom mu4e-eldoc-support nil + "Support eldoc help in the headers-view." + :type 'boolean + :group 'mu4e) + +(defcustom mu4e-date-format-long "%c" + "Date format to use in the message view. +Follows the format of `format-time-string'." + :type 'string + :group 'mu4e) + +(defcustom mu4e-dim-when-loading t + "Dim buffer text when loading new data. +If non-nil, dim some buffers during data retrieval and rendering, +and show some \"Loading\" banner." + :type 'boolean + :group 'mu4e) + + +;;; Faces + +(defgroup mu4e-faces nil + "Type faces (fonts) used in mu4e." + :group 'mu4e + :group 'faces) + +(defface mu4e-unread-face + '((t :inherit font-lock-keyword-face :weight bold)) + "Face for an unread message header." + :group 'mu4e-faces) + +(defface mu4e-trashed-face + '((t :inherit font-lock-comment-face :strike-through t)) + "Face for an message header in the trash folder." + :group 'mu4e-faces) + +(defface mu4e-draft-face + '((t :inherit font-lock-string-face)) + "Face for a draft message header. +I.e. a message with the draft flag set." + :group 'mu4e-faces) + +(defface mu4e-flagged-face + '((t :inherit font-lock-constant-face :weight bold)) + "Face for a flagged message header." + :group 'mu4e-faces) + +(defface mu4e-replied-face + '((t :inherit font-lock-builtin-face :weight normal :slant normal)) + "Face for a replied message header." + :group 'mu4e-faces) + +(defface mu4e-forwarded-face + '((t :inherit font-lock-builtin-face :weight normal :slant normal)) + "Face for a passed (forwarded) message header." + :group 'mu4e-faces) + +(defface mu4e-header-face + '((t :inherit default)) + "Face for a header without any special flags." + :group 'mu4e-faces) + +(defface mu4e-related-face + '((t :inherit default :slant italic)) + "Face for a \='related' header." :group 'mu4e-faces) + +(defface mu4e-header-title-face + '((t :inherit font-lock-type-face)) + "Face for a header title in the headers view." + :group 'mu4e-faces) + +(defface mu4e-header-highlight-face + `((t :inherit hl-line :weight bold :underline t + ,@(and (>= emacs-major-version 27) '(:extend t)))) + "Face for the header at point." + :group 'mu4e-faces) + +(defface mu4e-header-marks-face + '((t :inherit font-lock-preprocessor-face)) + "Face for the mark in the headers list." + :group 'mu4e-faces) + +(defface mu4e-header-key-face + '((t :inherit message-header-name :weight bold)) + "Face used to highlight items in various places." + :group 'mu4e-faces) + +(defface mu4e-header-field-face + '((t :weight bold)) + "Face for a header field name (such as \"Subject:\" in \"Subject:\ +Foo\")." + :group 'mu4e-faces) + +(defface mu4e-header-value-face + '((t :inherit font-lock-type-face)) + "Face for a header value (such as \"Re: Hello!\")." + :group 'mu4e-faces) + +(defface mu4e-special-header-value-face + '((t :inherit font-lock-builtin-face)) + "Face for special header values." + :group 'mu4e-faces) + +(defface mu4e-link-face + '((t :inherit link)) + "Face for showing URLs and attachments in the message view." + :group 'mu4e-faces) + +(defface mu4e-contact-face + '((t :inherit font-lock-variable-name-face)) + "Face for showing URLs and attachments in the message view." + :group 'mu4e-faces) + +(defface mu4e-highlight-face + '((t :inherit highlight)) + "Face for highlighting things." + :group 'mu4e-faces) + +(defface mu4e-title-face + '((t :inherit font-lock-type-face :weight bold)) + "Face for a header title in the headers view." + :group 'mu4e-faces) + +(defface mu4e-modeline-face + '((t :inherit font-lock-string-face :weight bold)) + "Face for the query in the mode-line." + :group 'mu4e-faces) + +(defface mu4e-footer-face + '((t :inherit font-lock-comment-face)) + "Face for message footers (signatures)." + :group 'mu4e-faces) + +(defface mu4e-url-number-face + '((t :inherit font-lock-constant-face :weight bold)) + "Face for the number tags for URLs." + :group 'mu4e-faces) + +(defface mu4e-system-face + '((t :inherit font-lock-comment-face :slant italic)) + "Face for system message (such as the footers for message headers)." + :group 'mu4e-faces) + +(defface mu4e-ok-face + '((t :inherit font-lock-comment-face :weight bold :slant normal)) + "Face for things that are okay." + :group 'mu4e-faces) + +(defface mu4e-warning-face + '((t :inherit font-lock-warning-face :weight bold :slant normal)) + "Face for warnings / error." + :group 'mu4e-faces) + +(defface mu4e-compose-separator-face + '((t :inherit message-separator :slant italic)) + "Face for the headers/message separator in mu4e-compose-mode." + :group 'mu4e-faces) + +(defface mu4e-region-code + '((t (:background "DarkSlateGray"))) + "Face for highlighting marked region in mu4e-view buffer." + :group 'mu4e-faces) + +;;; Header information + +(defconst mu4e-header-info + '((:bcc + . (:name "Bcc" + :shortname "Bcc" + :help "Blind Carbon-Copy recipients for the message" + :sortable t)) + (:cc + . (:name "Cc" + :shortname "Cc" + :help "Carbon-Copy recipients for the message" + :sortable t)) + (:changed + . (:name "Changed" + :shortname "Chg" + :help "Date/time when the message was changed most recently" + :sortable t)) + (:date + . (:name "Date" + :shortname "Date" + :help "Date/time when the message was sent" + :sortable t)) + (:human-date + . (:name "Date" + :shortname "Date" + :help "Date/time when the message was sent" + :sortable :date)) + (:flags + . (:name "Flags" + :shortname "Flgs" + :help "Flags for the message" + :sortable nil)) + (:from + . (:name "From" + :shortname "From" + :help "The sender of the message" + :sortable t)) + (:from-or-to + . (:name "From/To" + :shortname "From/To" + :help "Sender of the message if it's not me; otherwise the recipient" + :sortable nil)) + (:maildir + . (:name "Maildir" + :shortname "Maildir" + :help "Maildir for this message" + :sortable t)) + (:list + . (:name "List-Id" + :shortname "List" + :help "Mailing list id for this message" + :sortable t)) + (:mailing-list + . (:name "List" + :shortname "List" + :help "Mailing list friendly name for this message" + :sortable :list)) + (:message-id + . (:name "Message-Id" + :shortname "MsgID" + :help "Message-Id for this message" + :sortable nil)) + (:path + . (:name "Path" + :shortname "Path" + :help "Full filesystem path to the message" + :sortable t)) + (:size + . (:name "Size" + :shortname "Size" + :help "Size of the message" + :sortable t)) + (:subject + . (:name "Subject" + :shortname "Subject" + :help "Subject of the message" + :sortable t)) + (:tags + . (:name "Tags" + :shortname "Tags" + :help "Tags for the message" + ;; sort by _first_ tag. + :sortable t)) + (:thread-subject + . (:name "Subject" + :shortname "Subject" + :help "Subject of the thread" + :sortable :subject)) + (:to + . (:name "To" + :shortname "To" + :help "Recipient of the message" + :sortable t))) + + "An alist of all possible header fields and information about them. + +This is used in the user-interface (the column headers in the +header list, and the fields the message view). + +Most fields should be self-explanatory. A special one is +`:from-or-to', which is equal to `:from' unless `:from' matches +one of the addresses in `(mu4e-personal-addresses)', in which +case it will be equal to `:to'. + +Furthermore, the property `:sortable' determines whether we can +sort by this field. This can be either a boolean (nil or t), or a +symbol for /another/ field. For example, the `:human-date' field +uses `:date' for that. + +Note, `:sortable' is not supported for custom header fields.") + +(defvar mu4e-header-info-custom + '( + ;; some examples & debug helpers. + + (:thread-path + . ;; Shows the internal thread-path + ( :name "Thread-path" + :shortname "Thp" + :help "The thread-path" + :function (lambda (msg) + (let ((thread (mu4e-message-field msg :thread))) + (or (and thread (plist-get thread :path)) ""))))) + + (:thread-date + . ;; Shows the internal thread-date + ( :name "Thread-date" + :shortname "Thd" + :help "The thread-date" + :function (lambda (msg) + (let* ((thread (mu4e-message-field msg :thread)) + (tdate (and thread (plist-get thread :date-tstamp)))) + (format-time-string "%F %T " (or tdate 0)))))) + (:recipnum + . + ( :name "Number of recipients" + :shortname "Recip#" + :help "Number of recipients for this message" + :function + (lambda (msg) + (format "%d" + (+ (length (mu4e-message-field msg :to)) + (length (mu4e-message-field msg :cc)))))))) + + "An alist of custom (user-defined) headers. +The format is similar to `mu4e-header-info', but adds a :function +property, which should point to a function that takes a message +plist as argument, and returns a string. See the default value of +`mu4e-header-info-custom for an example. + +Note that when using the gnus-based view, you only have access to +a limited set of message fields: only the ones used in the +header-view, not including, for instance, the message body.") + +;;; Internals + +(defvar-local mu4e~headers-view-win nil + "The view window connected to this headers view.") + +;; It's useful to have the current view message available to +;; `mu4e-view-mode-hooks' functions, and we set up this variable +;; before calling `mu4e-view-mode'. However, changing the major mode +;; clobbers any local variables. Work around that by declaring the +;; variable permanent-local. +(defvar-local mu4e--view-message nil "The message being viewed in view mode.") +(put 'mu4e--view-message 'permanent-local t) +;;; _ +(provide 'mu4e-vars) +;;; mu4e-vars.el ends here diff --git a/mu4e/mu4e-view.el b/mu4e/mu4e-view.el new file mode 100644 index 0000000..033540a --- /dev/null +++ b/mu4e/mu4e-view.el @@ -0,0 +1,1167 @@ +;;; mu4e-view.el --- Mode for viewing e-mail messages -*- lexical-binding: t -*- + +;; Copyright (C) 2021-2024 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;; In this file we define mu4e-view-mode (+ helper functions), which is used for +;; viewing e-mail messages + +;;; Code: + +(require 'cl-lib) +(require 'calendar) +(require 'gnus-art) +(require 'comint) +(require 'browse-url) +(require 'button) +(require 'epa) +(require 'epg) +(require 'thingatpt) + +(require 'mu4e-actions) +(require 'mu4e-compose) +(require 'mu4e-context) +(require 'mu4e-headers) +(require 'mu4e-mark) +(require 'mu4e-message) +(require 'mu4e-server) +(require 'mu4e-search) +(require 'mu4e-mime-parts) + +;; utility functions +(require 'mu4e-contacts) +(require 'mu4e-vars) + +;;; Options + +(defcustom mu4e-view-scroll-to-next t + "Move to the next message with `mu4e-view-scroll-up-or-next'. +When at the end of a message, move to the next one, if any. +Otherwise, don't move to the next message." + :type 'boolean + :group 'mu4e-view) + +(defcustom mu4e-view-fields + '(:from :to :cc :subject :flags :date :maildir :mailing-list :tags) + "Header fields to display in the message view buffer. + +For the complete list of available headers, see +`mu4e-header-info'. + +Note, you can use this to add fields that are not otherwise +shown; you can further tweak the other fields using e.g., +`gnus-visible-headers' and `gnus-ignored-headers' - see the gnus +documentation for details." + :type '(repeat symbol) + :group 'mu4e-view) + +(defcustom mu4e-view-actions + (delq nil `(("capture message" . mu4e-action-capture-message) + ("view in browser" . mu4e-action-view-in-browser) + ("browse online archive" . mu4e-action-browse-list-archive) + ,(when (fboundp 'xwidget-webkit-browse-url) + '("xview in xwidget" . mu4e-action-view-in-xwidget)) + ("show this thread" . mu4e-action-show-thread))) + "List of actions to perform on messages in view mode. +The actions are cons-cells of the form: + (NAME . FUNC) +where: +* NAME is the name of the action (e.g. \"Count lines\") +* FUNC is a function which receives a message plist as an argument. + +The first letter of NAME is used as a shortcut character." + :group 'mu4e-view + :type '(alist :key-type string :value-type function)) + +(defcustom mu4e-view-max-specpdl-size 4096 + "The value of `max-specpdl-size' for displaying messages with Gnus." + :type 'integer + :group 'mu4e-view) + + + +(defconst mu4e--view-raw-buffer-name " *mu4e-raw-view*" + "Name for the raw message view buffer.") + +(defun mu4e-view-raw-message () + "Display the raw contents of message at point in a new buffer." + (interactive) + (let ((path (mu4e-message-readable-path)) + (buf (get-buffer-create mu4e--view-raw-buffer-name))) + (with-current-buffer buf + (let ((inhibit-read-only t)) + (erase-buffer) + (mu4e-raw-view-mode) + (insert-file-contents path) + (goto-char (point-min)))) + (mu4e-display-buffer buf t))) + +(defun mu4e-view-pipe (cmd) + "Pipe the message at point through shell command CMD. +Then, display the results." + (interactive "sShell command: ") + (let ((path (mu4e-message-readable-path))) + (mu4e-process-file-through-pipe path cmd))) + +(defmacro mu4e--view-in-headers-context (&rest body) + "Evaluate BODY in the context of the headers buffer." + `(progn + (let* ((msg (mu4e-message-at-point)) + (buffer (cond + ;; are we already inside a headers buffer? + ((mu4e-current-buffer-type-p 'headers) (current-buffer)) + ;; if not, are we inside a view buffer, and does + ;; it have linked headers buffer? + ((mu4e-current-buffer-type-p 'view) + (when (mu4e--view-detached-p (current-buffer)) + (mu4e-error + "Cannot navigate in a detached view buffer.")) + (mu4e-get-headers-buffer)) + ;; fallback; but what would trigger this? + (t (mu4e-get-headers-buffer)))) + (docid (mu4e-message-field msg :docid))) + (unless docid + (mu4e-error "Message without docid: action is not possible")) + + ;; make sure to select the window if possible, or jumping won't be + ;; reflected. + (with-selected-window (or (get-buffer-window buffer) + (get-buffer-window)) + (with-current-buffer buffer + (mu4e-thread-unfold-all) + (if (or (mu4e~headers-goto-docid docid) + ;; TODO: Is this the best way to find another + ;; relevant docid for a view buffer? + ;; + ;; If you attach a view buffer to another headers + ;; buffer that does not contain the current docid + ;; then `mu4e~headers-goto-docid' returns nil and we + ;; get an error. This "hack" instead gets its + ;; now-changed headers buffer's current message as a + ;; docid + (mu4e~headers-goto-docid + (with-current-buffer buffer + (mu4e-message-field (mu4e-message-at-point) :docid)))) + ,@body + (mu4e-error "Cannot find message in headers buffer"))))))) + +(defun mu4e-view-headers-next (&optional n) + "Move point to the next message header. +If this succeeds, return the new docid. Otherwise, return nil. +Optionally, takes an integer N (prefix argument), to the Nth next +header." + (interactive "P") + (mu4e--view-in-headers-context + (mu4e~headers-move (or n 1)))) + +(defun mu4e-view-headers-prev (&optional n) + "Move point to the previous message header. +If this succeeds, return the new docid. Otherwise, return nil. +Optionally, takes an integer N (prefix argument), to the Nth +previous header." + (interactive "P") + (mu4e--view-in-headers-context + (mu4e~headers-move (- (or n 1))))) + +(defun mu4e--view-prev-or-next (func backwards) + "Move point to the next or previous message. +Go to the previous message if BACKWARDS is non-nil. +unread message header in the headers buffer connected with this +message view. If this succeeds, return the new docid. Otherwise, +return nil." + (mu4e--view-in-headers-context (funcall func backwards)) + (mu4e-select-other-view) + (mu4e-headers-view-message)) + +(defun mu4e-view-headers-prev-unread () + "Move point to the previous unread message header. +If this succeeds, return the new docid. Otherwise, return nil." + (interactive) + (mu4e--view-prev-or-next #'mu4e~headers-prev-or-next-unread t)) + +(defun mu4e-view-headers-next-unread () + "Move point to the next unread message header. +If this succeeds, return the new docid. Otherwise, return nil." + (interactive) + (mu4e--view-prev-or-next #'mu4e~headers-prev-or-next-unread nil)) + +(defun mu4e-view-headers-prev-thread() + "Move point to the previous thread. +If this succeeds, return the new docid. Otherwise, return nil." + (interactive) + (mu4e--view-prev-or-next #'mu4e~headers-prev-or-next-thread t)) + +(defun mu4e-view-headers-next-thread() + "Move point to the previous thread. +If this succeeds, return the new docid. Otherwise, return nil." + (interactive) + (mu4e--view-prev-or-next #'mu4e~headers-prev-or-next-thread nil)) + +(defun mu4e-view-thread-goto-root () + "Move to thread root." + (interactive) + (mu4e--view-in-headers-context (mu4e-thread-goto-root))) + +(defun mu4e-view-thread-fold-toggle-goto-next () + "Toggle threading or go to next." + (interactive) + (mu4e--view-in-headers-context (mu4e-thread-fold-toggle-goto-next))) + +(defun mu4e-view-thread-fold-toggle-all () + "Toggle all threads." + (interactive) + (mu4e--view-in-headers-context (mu4e-thread-fold-toggle-all))) + + +;;; Interactive functions +(defun mu4e-view-action (&optional msg) + "Ask user for some action to apply on MSG, then do it. +If MSG is nil apply action to message returned +bymessage-at-point. The actions are specified in +`mu4e-view-actions'." + (interactive) + (let* ((msg (or msg (mu4e-message-at-point))) + (actionfunc (mu4e-read-option "Action: " mu4e-view-actions))) + (funcall actionfunc msg))) + +(defun mu4e-view-mark-pattern () + "Mark messages that match a certain pattern. +Ask user for a kind of mark, (move, delete etc.), a field to +match and a regular expression to match with. Then, mark all +matching messages with that mark." + (interactive) + (mu4e--view-in-headers-context (mu4e-headers-mark-pattern))) + +(defun mu4e-view-mark-thread (&optional markpair) + "Mark whole thread with a certain mark. +Ask user for a kind of mark (move, delete etc.), and apply it +to all messages in the thread at point in the headers view. The +optional MARKPAIR can also be used to provide the mark +selection." + (interactive) + (mu4e--view-in-headers-context + (if markpair (mu4e-headers-mark-thread nil markpair) + (call-interactively 'mu4e-headers-mark-thread)))) + +(defun mu4e-view-mark-subthread (&optional markpair) + "Mark subthread with a certain mark. +Ask user for a kind of mark (move, delete etc.), and apply it +to all messages in the subthread at point in the headers view. +The optional MARKPAIR can also be used to provide the mark +selection." + (interactive) + (mu4e--view-in-headers-context + (if markpair (mu4e-headers-mark-subthread markpair) + (mu4e-headers-mark-subthread)))) + +(defun mu4e-view-search-narrow () + "Run `mu4e-headers-search-narrow' in the headers buffer." + (interactive) + (mu4e--view-in-headers-context (mu4e-search-narrow))) + +(defun mu4e-view-search-edit () + "Run `mu4e-search-edit' in the headers buffer." + (interactive) + (mu4e--view-in-headers-context (mu4e-search-edit))) + +(defun mu4e-mark-region-code () + "Highlight region marked with `message-mark-inserted-region'. +Add this function to `mu4e-view-mode-hook' to enable this feature." + (require 'message) + (let (beg end ov-beg ov-end ov-inv) + (save-excursion + (goto-char (point-min)) + (while (re-search-forward + (concat "^" message-mark-insert-begin) nil t) + (setq ov-beg (match-beginning 0) + ov-end (match-end 0) + ov-inv (make-overlay ov-beg ov-end) + beg ov-end) + (overlay-put ov-inv 'invisible t) + (overlay-put ov-inv 'mu4e-overlay t) + (when (re-search-forward + (concat "^" message-mark-insert-end) nil t) + (setq ov-beg (match-beginning 0) + ov-end (match-end 0) + ov-inv (make-overlay ov-beg ov-end) + end ov-beg) + (overlay-put ov-inv 'invisible t)) + (when (and beg end) + (let ((ov (make-overlay beg end))) + (overlay-put ov 'mu4e-overlay t) + (overlay-put ov 'face 'mu4e-region-code)) + (setq beg nil end nil)))))) + +;;; View Utilities + +(defun mu4e-view-mark-custom () + "Run some custom mark function." + (mu4e--view-in-headers-context + (mu4e-headers-mark-custom))) + +(defun mu4e--view-split-view-p () + "Return t if we're in split-view, nil otherwise." + (member mu4e-split-view '(horizontal vertical))) + + +(defun mu4e-view-detach () + "Detach the view buffer from its headers buffer." + (interactive) + (unless mu4e-linked-headers-buffer + (mu4e-error "This view buffer is already detached.")) + (mu4e-message "Detached view buffer from %s" + (progn mu4e-linked-headers-buffer + (with-current-buffer mu4e-linked-headers-buffer + (when (eq (selected-window) mu4e~headers-view-win) + (setq mu4e~headers-view-win nil))) + (setq mu4e-linked-headers-buffer nil) + ;; automatically rename mu4e-view-article buffer when + ;; detaching; will get renamed back when reattaching + (rename-buffer (make-temp-name (buffer-name)) t)))) + +(defun mu4e-view-attach (headers-buffer) + "Attaches a view buffer to a headers buffer." + (interactive + (list (get-buffer (read-buffer + "Select a headers buffer to attach to: " nil t + (lambda (buf) (with-current-buffer (car buf) + (mu4e-current-buffer-type-p 'headers))))))) + (mu4e-message "Attached view buffer to %s" headers-buffer) + (setq mu4e-linked-headers-buffer headers-buffer) + (with-current-buffer headers-buffer + (setq mu4e~headers-view-win (selected-window)))) + +;;; Scroll commands + +(defun mu4e-view-scroll-up-or-next () + "Scroll-up the current message. +If `mu4e-view-scroll-to-next' is non-nil, and we cannot scroll up +any further, go the next message." + (interactive) + (condition-case nil + (scroll-up) + (error + (when mu4e-view-scroll-to-next + (mu4e-view-headers-next))))) + +(defun mu4e-scroll-up () + "Scroll text of selected window up one line." + (interactive) + (scroll-up 1)) + +(defun mu4e-scroll-down () + "Scroll text of selected window down one line." + (interactive) + (scroll-down 1)) + +;;; Mark commands + +(defun mu4e-view-unmark-all () + "If we're in split-view, unmark all messages. +Otherwise, warn user that unmarking only works in the header +list." + (interactive) + (if (mu4e--view-split-view-p) + (mu4e--view-in-headers-context (mu4e-mark-unmark-all)) + (mu4e-message "Unmarking needs to be done in the header list view"))) + +(defun mu4e-view-unmark () + "If we're in split-view, unmark message at point. +Otherwise, warn user that unmarking only works in the header +list." + (interactive) + (if (mu4e--view-split-view-p) + (mu4e-view-mark-for-unmark) + (mu4e-message "Unmarking needs to be done in the header list view"))) + +(defmacro mu4e--view-defun-mark-for (mark) + "Define a function mu4e-view-mark-for- MARK." + (let ((funcname (intern (format "mu4e-view-mark-for-%s" mark))) + (docstring (format "Mark the current message for %s." mark))) + `(progn + (defun ,funcname () ,docstring + (interactive) + (mu4e--view-in-headers-context + (mu4e-headers-mark-and-next ',mark))) + (put ',funcname 'definition-name ',mark)))) + +(mu4e--view-defun-mark-for move) +(mu4e--view-defun-mark-for refile) +(mu4e--view-defun-mark-for delete) +(mu4e--view-defun-mark-for flag) +(mu4e--view-defun-mark-for unflag) +(mu4e--view-defun-mark-for unmark) +(mu4e--view-defun-mark-for something) +(mu4e--view-defun-mark-for read) +(mu4e--view-defun-mark-for unread) +(mu4e--view-defun-mark-for trash) +(mu4e--view-defun-mark-for untrash) + +(defun mu4e-view-marked-execute () + "Execute the marked actions." + (interactive) + (mu4e--view-in-headers-context + (mu4e-mark-execute-all))) + + +;;; URL handling + +(defvar mu4e--view-link-map nil + "A map of some number->url so we can jump to url by number.") +(put 'mu4e--view-link-map 'permanent-local t) + +(defvar mu4e-view-active-urls-keymap + (let ((map (make-sparse-keymap))) + (define-key map (kbd "<mouse-2>") #'mu4e--view-browse-url-from-binding) + (define-key map (kbd "M-<return>") #'mu4e--view-browse-url-from-binding) + map) + "Keymap used for the URLs inside the body.") + +(defvar mu4e--view-beginning-of-url-regexp + "https?\\://\\|mailto:" + "Regexp that matches the beginning of certain URLs. +Match-string 1 will contain the matched URL, if any.") + + +(defun mu4e--view-browse-url-from-binding (&optional url) + "View in browser the url at point, or click location. +If the optional argument URL is provided, browse that instead. +If the url is mailto link, start writing an email to that address." + (interactive) + (let* (( url (or url (mu4e--view-get-property-from-event 'mu4e-url)))) + (when url + (if (string-match-p "^mailto:" url) + (browse-url-mail url) + (browse-url url))))) + +(defun mu4e--view-get-property-from-event (prop) + "Get the property PROP at point, or the location of the mouse. +The action is chosen based on the `last-command-event'. +Meant to be evoked from interactive commands." + (if (and (eventp last-command-event) + (mouse-event-p last-command-event)) + (let ((posn (event-end last-command-event))) + (when (numberp (posn-point posn)) + (get-text-property + (posn-point posn) + prop + (window-buffer (posn-window posn))))) + (get-text-property (point) prop))) + +;; this is fairly simplistic... +(defun mu4e--view-activate-urls () + "Turn things that look like URLs into clickable things. +Also number them so they can be opened using `mu4e-view-go-to-url'." + (let ((num 0)) + (save-excursion + (setq mu4e--view-link-map ;; buffer local + (make-hash-table :size 32 :weakness nil)) + (goto-char (point-min)) + (while (re-search-forward mu4e--view-beginning-of-url-regexp nil t) + (let ((bounds (thing-at-point-bounds-of-url-at-point))) + (when bounds + (let* ((url (thing-at-point-url-at-point)) + (ov (make-overlay (car bounds) (cdr bounds)))) + (puthash (cl-incf num) url mu4e--view-link-map) + (add-text-properties + (car bounds) + (cdr bounds) + `(face mu4e-link-face + mouse-face highlight + mu4e-url ,url + keymap ,mu4e-view-active-urls-keymap + help-echo + "[mouse-1] or [M-RET] to open the link")) + (overlay-put ov 'mu4e-overlay t) + (overlay-put ov 'after-string + (propertize (format "\u200B[%d]" num) + 'face 'mu4e-url-number-face))))))))) + + +(defun mu4e--view-get-urls-num (prompt &optional multi) + "Ask the user with PROMPT for an URL number for MSG. +The number is [1..n] for URLs \[0..(n-1)] in the message. If +MULTI is nil, return the number for the URL; otherwise (MULTI is +non-nil), accept ranges of URL numbers, as per +`mu4e-split-ranges-to-numbers', and return the corresponding +string." + (let* ((count (hash-table-count mu4e--view-link-map)) (def)) + (when (zerop count) (mu4e-error "No links for this message")) + (if (not multi) + (if (= count 1) + (read-number (mu4e-format "%s: " prompt) 1) + (read-number (mu4e-format "%s (1-%d): " prompt count))) + (progn + (setq def (if (= count 1) "1" (format "1-%d" count))) + (read-string (mu4e-format "%s (default %s): " prompt def) + nil nil def))))) + +(defun mu4e-view-go-to-url (&optional multi) + "Offer to go visit one or more URLs. +If MULTI (prefix-argument) is non-nil, offer to go to a range of URLs." + (interactive "P") + (mu4e--view-handle-urls "URL to visit" + multi + (lambda (url) (mu4e--view-browse-url-from-binding url)))) + +(defun mu4e-view-save-url (&optional multi) + "Offer to save URLs to the kill ring. +If MULTI (prefix-argument) is nil, save a single one, otherwise, offer +to save a range of URLs." + (interactive "P") + (mu4e--view-handle-urls "URL to save" multi + (lambda (url) + (kill-new url) + (mu4e-message "Saved %s to the kill-ring" url)))) + +(defun mu4e-view-fetch-url (&optional multi) + "Offer to fetch (download) URLs. +If MULTI (prefix-argument) is nil, +download a single one, otherwise, offer to fetch a range of +URLs. The urls are fetched to `mu4e-attachment-dir'." + (interactive "P") + (mu4e--view-handle-urls + "URL to fetch" multi + (lambda (url) + (let ((target (concat (mu4e-determine-attachment-dir url) "/" + (file-name-nondirectory url)))) + (url-copy-file url target) + (mu4e-message "Fetched %s -> %s" url target))))) + +(defun mu4e--view-handle-urls (prompt multi urlfunc) + "Handle URLs. +If MULTI is nil, apply URLFUNC to a single uri, otherwise, apply +it to a range of uris. PROMPT is the query to present to the user." + (if multi + (mu4e--view-handle-multi-urls prompt urlfunc) + (mu4e--view-handle-single-url prompt urlfunc))) + +(defun mu4e--view-handle-single-url (prompt urlfunc &optional num) + "Apply URLFUNC to some URL with NUM in the current message. +Prompting the user with PROMPT for the number." + (let* ((num (or num (mu4e--view-get-urls-num prompt))) + (url (gethash num mu4e--view-link-map))) + (unless url (mu4e-warn "Invalid number for URL")) + (funcall urlfunc url))) + +(defun mu4e--view-handle-multi-urls (prompt urlfunc) + "Apply URLFUNC to a a range of URLs in the current message. + +Prompting the user with PROMPT for the numbers. + +Default is to apply it to all URLs, [1..n], where n is the number +of urls. You can type multiple values separated by space, e.g. 1 +3-6 8 will visit urls 1,3,4,5,6 and 8. + +Furthermore, there is a shortcut \"a\" which means all urls, but as +this is the default, you may not need it." + (let* ((linkstr (mu4e--view-get-urls-num + "URL number range (or 'a' for 'all')" t)) + (count (hash-table-count mu4e--view-link-map)) + (linknums (mu4e-split-ranges-to-numbers linkstr count))) + (dolist (num linknums) + (mu4e--view-handle-single-url prompt urlfunc num)))) + +(defun mu4e-view-for-each-uri (func) + "Evaluate FUNC(uri) for each uri in the current message." + (maphash (lambda (_num uri) (funcall func uri)) mu4e--view-link-map)) + +(defun mu4e-view-message-with-message-id (msgid) + "View message with message-id MSGID. +This (re)creates a +headers-buffer with a search for MSGID, then open a view for that +message." + (mu4e-search (concat "msgid:" msgid) nil nil t msgid t)) + +;;; Variables + +(defvar gnus-icalendar-additional-identities) +(defvar-local mu4e--view-rendering nil) + +(defun mu4e-view (msg) + "Display the message MSG in a new buffer, and keep in sync with HDRSBUF. +\"In sync\" here means that moving to the next/previous message +in the the message view affects HDRSBUF, as does marking etc. + +As a side-effect, a message that is being viewed loses its +`unread' marking if it still had that." + ;; update headers, if necessary. + (mu4e~headers-update-handler msg nil nil) + ;; Create a new view buffer (if needed) as it is not + ;; feasible to recycle an existing buffer due to buffer-specific + ;; state (buttons, etc.) that can interfere with message rendering + ;; in gnus. + ;; + ;; Unfortunately that does create its own issues: namely ensuring + ;; buffer-local state that *must* survive is correctly copied + ;; across. + (let ((linked-headers-buffer)) + (when-let ((existing-buffer (mu4e-get-view-buffer nil nil))) + ;; required; this state must carry over from the killed buffer + ;; to the new one. + (setq linked-headers-buffer mu4e-linked-headers-buffer) + (if (memq mu4e-split-view '(horizontal vertical)) + (delete-windows-on existing-buffer t)) + (kill-buffer existing-buffer)) + (setq gnus-article-buffer (mu4e-get-view-buffer nil t)) + (with-current-buffer gnus-article-buffer + (when linked-headers-buffer + (setq mu4e-linked-headers-buffer linked-headers-buffer)) + (let ((inhibit-read-only t) + (gnus-unbuttonized-mime-types '(".*/.*")) + (gnus-buttonized-mime-types + (append (list "multipart/signed" "multipart/encrypted") + gnus-buttonized-mime-types)) + (gnus-inhibit-mime-unbuttonizing t)) + (remove-overlays (point-min)(point-max) 'mu4e-overlay t) + (erase-buffer) + (insert-file-contents-literally + (mu4e-message-readable-path msg) nil nil nil t) + ;; some messages have ^M which causes various rendering + ;; problems later (#2260, #2508), so let's remove those + (article-remove-cr) + (setq-local mu4e--view-message msg) + (mu4e--view-render-buffer msg)) + (mu4e-loading-mode 0))) + (unless (mu4e--view-detached-p gnus-article-buffer) + (with-current-buffer mu4e-linked-headers-buffer + ;; We need this here as we want to avoid displaying the buffer until + ;; the last possible moment --- after the message is rendered in the + ;; view buffer. + ;; + ;; Otherwise, `mu4e-display-buffer' may adjust the view buffer's + ;; window height based on a buffer that has no text in it yet! + (setq-local mu4e~headers-view-win + (mu4e-display-buffer gnus-article-buffer nil)) + (unless (window-live-p mu4e~headers-view-win) + (mu4e-error "Cannot get a message view")) + (select-window mu4e~headers-view-win))) + (with-current-buffer gnus-article-buffer + (let ((inhibit-read-only t)) + (run-hooks 'mu4e-view-rendered-hook)) + ;; only needed on some setups; #2683 + (goto-char (point-min)))) + +(defun mu4e-view-message-text (msg) + "Return the rendered MSG as a string." + (with-temp-buffer + (insert-file-contents-literally + (mu4e-message-readable-path msg) nil nil nil t) + (let ((gnus-inhibit-mime-unbuttonizing nil) + (gnus-unbuttonized-mime-types '(".*/.*")) + (mu4e-view-fields '(:from :to :cc :subject :date))) + (mu4e--view-render-buffer msg) + (buffer-substring-no-properties (point-min) (point-max))))) + +(defun mu4e-action-view-in-browser (msg &optional skip-headers) + "Show current MSG in browser if it includes an HTML-part. +If SKIP-HEADERS is set, do not show include message headers. +The variables `browse-url-browser-function', +`browse-url-handlers', and `browse-url-default-handlers' +determine which browser function to use." + (with-temp-buffer + (insert-file-contents-literally + (mu4e-message-readable-path msg) nil nil nil t) + ;; just continue if some of the decoding fails. + (ignore-errors (run-hooks 'gnus-article-decode-hook)) + (let ((header (unless skip-headers + (cl-loop for field in '("from" "to" "cc" "date" "subject") + when (message-field-value field) + concat (format "%s: %s\n" (capitalize field) it)))) + (parts (mm-dissect-buffer t t))) + ;; If singlepart, enforce a list. + (when (and (bufferp (car parts)) + (stringp (car (mm-handle-type parts)))) + (setq parts (list parts))) + ;; Process the list + (unless (gnus-article-browse-html-parts parts header) + (mu4e-warn "Message does not contain a \"text/html\" part")) + (mm-destroy-parts parts)))) + +(defun mu4e-action-view-in-xwidget (msg) + "Show current MSG in an embedded xwidget, if available." + (unless (fboundp 'xwidget-webkit-browse-url) + (mu4e-error "No xwidget support available")) + (let ((browse-url-handlers nil) + (browse-url-browser-function + (lambda (url &optional _rest) + (xwidget-webkit-browse-url url)))) + (mu4e-action-view-in-browser msg))) + +(defun mu4e--view-render-buffer (msg) + "Render current buffer with MSG using Gnus' article mode." + (setq gnus-summary-buffer (get-buffer-create " *appease-gnus*")) + (let* ((inhibit-read-only t) + (max-specpdl-size mu4e-view-max-specpdl-size) + (mm-decrypt-option 'known) + (ct (mail-fetch-field "Content-Type")) + (ct (and ct (mail-header-parse-content-type ct))) + (charset (mail-content-type-get ct 'charset)) + (charset (and charset (intern charset))) + (mu4e--view-rendering t); Needed if e.g. an ics file is buttonized + (gnus-article-emulate-mime nil) ;; avoid perf problems + (gnus-newsgroup-charset + (if (and charset (coding-system-p charset)) charset + (detect-coding-region (point-min) (point-max) t))) + ;; Possibly add headers (before "Attachments") + (gnus-display-mime-function (mu4e--view-gnus-display-mime msg))) + (condition-case err + (progn + (mm-enable-multibyte) + ;; just continue if some of the decoding fails. + (ignore-errors (run-hooks 'gnus-article-decode-hook)) + (gnus-article-prepare-display) + (mu4e--view-activate-urls) + ;; `gnus-summary-bookmark-make-record' does not work properly when "appeased." + (kill-local-variable 'bookmark-make-record-function) + (setq mu4e~gnus-article-mime-handles gnus-article-mime-handles + gnus-article-decoded-p gnus-article-decode-hook) + (set-buffer-modified-p nil) + (add-hook 'kill-buffer-hook #'mu4e--view-kill-mime-handles)) + (epg-error + (mu4e-warn "EPG error: %s; fall back to raw view" + (error-message-string err)))))) + +(defun mu4e-view-refresh () + "Refresh the message view." + ;;; XXX: sometimes, side-effect: increase the header-buffers size + (interactive) + (when-let ((msg (and (derived-mode-p 'mu4e-view-mode) + mu4e--view-message))) + (mu4e-view-quit) + (mu4e-view msg))) + +(defun mu4e-view-toggle-show-mime-parts() + "Toggle whether to show all MIME-parts." + (interactive) + (setq gnus-inhibit-mime-unbuttonizing + (not gnus-inhibit-mime-unbuttonizing)) + (mu4e-view-refresh)) + +(defun mu4e-view-toggle-fill-flowed() + "Toggle flowed-message text filling." + (interactive) + (setq mm-fill-flowed (not mm-fill-flowed)) + (mu4e-view-refresh)) + +(defun mu4e-view-toggle-emulate-mime() + "Toggle GNUs MIME-emulation. +Note that for some messages, this can trigger high CPU load." + (interactive) + (setq gnus-article-emulate-mime (not gnus-article-emulate-mime)) + (mu4e-view-refresh)) + +(defun mu4e--view-gnus-display-mime (msg) + "Like `gnus-display-mime', but include mu4e headers to MSG." + (lambda (&optional ihandles) + (gnus-display-mime ihandles) + (unless ihandles + (save-restriction + (article-goto-body) + (forward-line -1) + (narrow-to-region (point) (point)) + (dolist (field mu4e-view-fields) + (let ((fieldval (mu4e-message-field msg field))) + (pcase field + ((or ':path ':maildir ':list) + (mu4e--view-gnus-insert-header field fieldval)) + (':message-id + (when-let ((msgid (plist-get msg :message-id))) + (mu4e--view-gnus-insert-header field (format "<%s>" msgid)))) + (':mailing-list + (let ((list (plist-get msg :list))) + (if list (mu4e-get-mailing-list-shortname list) ""))) + ((or ':flags ':tags) + (let ((flags (mapconcat (lambda (flag) + (if (symbolp flag) + (symbol-name flag) + flag)) fieldval ", "))) + (mu4e--view-gnus-insert-header field flags))) + (':size (mu4e--view-gnus-insert-header + field (mu4e-display-size fieldval))) + ((or ':subject ':to ':from ':cc ':bcc ':from-or-to + ':user-agent ':date ':attachments + ':signature ':decryption)) ;; handled by Gnus + (_ + (mu4e--view-gnus-insert-header-custom msg field))))) + (let ((gnus-treatment-function-alist + '((gnus-treat-highlight-headers + gnus-article-highlight-headers)))) + (gnus-treat-article 'head)))))) + +(defun mu4e--view-gnus-insert-header (field val) + "Insert a header FIELD with value VAL." + (let* ((info (cdr (assoc field mu4e-header-info))) + (key (plist-get info :name)) + (help (plist-get info :help))) + (if (and val (> (length val) 0)) + (insert (propertize (concat key ":") 'help-echo help) + " " val "\n")))) + +(defun mu4e--view-gnus-insert-header-custom (msg field) + "Insert MSG's custom FIELD." + (let* ((info (cdr-safe (or (assoc field mu4e-header-info-custom) + (mu4e-error "Custom field %S not found" field)))) + (key (plist-get info :name)) + (func (or (plist-get info :function) + (mu4e-error "No :function defined for custom field %S %S" + field info))) + (val (funcall func msg)) + (help (plist-get info :help))) + (when (and val (> (length val) 0)) + (insert (propertize (concat key ":") 'help-echo help) " " val "\n")))) + +(define-advice gnus-icalendar-event-from-handle + (:filter-args (handle-attendee) mu4e--view-fix-missing-charset) + "Avoid error when displaying an ical attachment without a charset." + (if (and (boundp 'mu4e--view-rendering) mu4e--view-rendering) + (let* ((handle (car handle-attendee)) + (attendee (cadr handle-attendee)) + (buf (mm-handle-buffer handle)) + (ty (mm-handle-type handle)) + (rest (cddr handle))) + ;; Put the fallback at the end: + (setq ty (append ty '((charset . "utf-8")))) + (setq handle (cons buf (cons ty rest))) + (list handle attendee)) + handle-attendee)) + +(defun mu4e--view-mode-p () + "Is the buffer in mu4e-view-mode or one of its descendants?" + (or (eq major-mode 'mu4e-view-mode) + (derived-mode-p '(mu4e-view-mode)))) + +(defun mu4e--view-nop (func &rest args) + "Do not invoke FUNC with ARGS when in mu4e-view-mode. +This is useful for advising some Gnus-functionality that does not work in mu4e." + (unless (mu4e--view-mode-p) + (apply func args))) + +(defun mu4e--view-button-reply (func &rest args) + "Advise FUNC with ARGS to make `gnus-button-reply' links work in mu4e." + (if (mu4e--view-mode-p) + (mu4e-compose-reply) + (apply func args))) + +(defun mu4e--view-button-message-id (func &rest args) + "Advise FUNC with ARGS to make `gnus-button-message-id' links work in mu4e." + (if (and (mu4e--view-mode-p) (stringp (car-safe args))) + (mu4e-view-message-with-message-id (car args)) + (apply func args))) + +(defun mu4e--view-msg-mail (func &rest args) + "Advise FUNC with ARGS to make `gnus-msg-mail' links compose with mu4e." + (if (mu4e--view-mode-p) + (apply 'mu4e-compose-mail args) + (apply func args))) + +(defun mu4e-view-quit () + "Quit the mu4e-view buffer." + (interactive) + (if (memq mu4e-split-view '(horizontal vertical)) + (ignore-errors ;; try, don't error out. + (kill-buffer-and-window)) + ;; single-window case + (let ((docid (mu4e-field-at-point :docid))) + (when mu4e-linked-headers-buffer ;; re-use mu4e-view-detach? + (with-current-buffer mu4e-linked-headers-buffer + (when (eq (selected-window) mu4e~headers-view-win) + (setq mu4e~headers-view-win nil))) + (setq mu4e-linked-headers-buffer nil) + (kill-buffer) + ;; attempt to move point to just-viewed message. + (when docid + (ignore-errors + (mu4e~headers-goto-docid docid))))))) + +(defvar mu4e-view-mode-map + (let ((map (make-keymap))) + (define-key map (kbd "C-S-u") #'mu4e-update-mail-and-index) + (define-key map (kbd "C-c C-u") #'mu4e-update-mail-and-index) + + (define-key map "q" #'mu4e-view-quit) + + (define-key map "z" #'mu4e-view-detach) + (define-key map "Z" #'mu4e-view-attach) + + (define-key map "%" #'mu4e-view-mark-pattern) + (define-key map "t" #'mu4e-view-mark-subthread) + (define-key map "T" #'mu4e-view-mark-thread) + + (define-key map "g" #'mu4e-view-go-to-url) + (define-key map "k" #'mu4e-view-save-url) + (define-key map "f" #'mu4e-view-fetch-url) + + (define-key map "." #'mu4e-view-raw-message) + (define-key map "," #'mu4e-sexp-at-point) + (define-key map "|" #'mu4e-view-pipe) + (define-key map "a" #'mu4e-view-action) + (define-key map "A" #'mu4e-view-mime-part-action) + (define-key map "e" #'mu4e-view-save-attachments) + + ;; change the number of headers + (define-key map (kbd "C-+") #'mu4e-headers-split-view-grow) + (define-key map (kbd "C--") #'mu4e-headers-split-view-shrink) + (define-key map (kbd "<C-kp-add>") #'mu4e-headers-split-view-grow) + (define-key map (kbd "<C-kp-subtract>") #'mu4e-headers-split-view-shrink) + + ;; intra-message navigation + (define-key map (kbd "S-SPC") #'scroll-down) + (define-key map (kbd "SPC") #'mu4e-view-scroll-up-or-next) + (define-key map (kbd "RET") #'mu4e-scroll-up) + (define-key map (kbd "<backspace>") #'mu4e-scroll-down) + + ;; navigation between messages + (define-key map "p" #'mu4e-view-headers-prev) + (define-key map "n" #'mu4e-view-headers-next) + ;; the same + (define-key map (kbd "<M-down>") #'mu4e-view-headers-next) + (define-key map (kbd "<M-up>") #'mu4e-view-headers-prev) + + (define-key map (kbd "[") #'mu4e-view-headers-prev-unread) + (define-key map (kbd "]") #'mu4e-view-headers-next-unread) + (define-key map (kbd "{") #'mu4e-view-headers-prev-thread) + (define-key map (kbd "}") #'mu4e-view-headers-next-thread) + + ;; ;; threads + ;; TODO: find some binding that don't conflict + ;; (define-key map (kbd "<S-left>") #'mu4e-view-thread-goto-root) + ;; ;; <tab> is taken already + ;; (define-key map (kbd "<C-S-tab>") #'mu4e-view-thread-fold-toggle-goto-next) + ;; (define-key map (kbd "<backtab>") #'mu4e-view-thread-fold-toggle-all) + + + ;; switching from view <-> headers (when visible) + (define-key map "y" #'mu4e-select-other-view) + + ;; marking/unmarking + (define-key map "d" #'mu4e-view-mark-for-trash) + (define-key map (kbd "<delete>") #'mu4e-view-mark-for-delete) + (define-key map (kbd "<deletechar>") #'mu4e-view-mark-for-delete) + (define-key map (kbd "D") #'mu4e-view-mark-for-delete) + (define-key map (kbd "m") #'mu4e-view-mark-for-move) + (define-key map (kbd "r") #'mu4e-view-mark-for-refile) + + (define-key map (kbd "?") #'mu4e-view-mark-for-unread) + (define-key map (kbd "!") #'mu4e-view-mark-for-read) + + (define-key map (kbd "+") #'mu4e-view-mark-for-flag) + (define-key map (kbd "-") #'mu4e-view-mark-for-unflag) + (define-key map (kbd "=") #'mu4e-view-mark-for-untrash) + (define-key map (kbd "&") #'mu4e-view-mark-custom) + + (define-key map (kbd "*") #'mu4e-view-mark-for-something) + (define-key map (kbd "<kp-multiply>") #'mu4e-view-mark-for-something) + (define-key map (kbd "<insert>") #'mu4e-view-mark-for-something) + (define-key map (kbd "<insertchar>") #'mu4e-view-mark-for-something) + + (define-key map ";" #'mu4e-context-switch) + + (define-key map (kbd "#") #'mu4e-mark-resolve-deferred-marks) + ;; misc + (define-key map "M" #'mu4e-view-massage) + + (define-key map "w" #'visual-line-mode) + (define-key map "h" #'mu4e-view-toggle-html) + (define-key map (kbd "M-q") #'article-fill-long-lines) + + (define-key map "c" #'mu4e-copy-thing-at-point) + + ;; next 3 only warn user when attempt in the message view + (define-key map "u" #'mu4e-view-unmark) + (define-key map "U" #'mu4e-view-unmark-all) + (define-key map "x" #'mu4e-view-marked-execute) + + (define-key map "$" #'mu4e-show-log) + (define-key map "H" #'mu4e-display-manual) + + ;; Make 0..9 shortcuts for digit-argument. Actually, none of the bound + ;; functions seem to use a prefix arg but those bindings existed because we + ;; used to use `suppress-keymap'. And possibly users added their own + ;; prefix arg consuming commands. + (dotimes (i 10) + (define-key map (kbd (format "%d" i)) #'digit-argument)) + + (set-keymap-parent map special-mode-map) + (set-keymap-parent map button-buffer-map) + map) + "Keymap for mu4e-view mode.") + +(easy-menu-define mu4e-view-mode-menu + mu4e-view-mode-map "Menu for mu4e's view-mode." + (append + '("View" + "--" + ["Toggle wrap lines" visual-line-mode] + ["View raw" mu4e-view-raw-message] + ["Pipe through shell" mu4e-view-pipe] + "--" + ["Mark for deletion" mu4e-view-mark-for-delete] + ["Mark for untrash" mu4e-view-mark-for-untrash] + ["Mark for trash" mu4e-view-mark-for-trash] + ["Mark for move" mu4e-view-mark-for-move] + ) + mu4e--compose-menu-items + mu4e--search-menu-items + mu4e--context-menu-items + '( + "--" + ["Quit" mu4e-view-quit + :help "Quit the view"] + ))) + +(defcustom mu4e-raw-view-mode-hook nil + "Hook run when entering \\[mu4e-raw-view] mode." + :options '() + :type 'hook + :group 'mu4e-view) + +(defcustom mu4e-view-mode-hook nil + "Hook run when entering \\[mu4e-view] mode." + :options '(turn-on-visual-line-mode) + :type 'hook + :group 'mu4e-view) + +(defcustom mu4e-view-rendered-hook '(mu4e-resize-linked-headers-window) + "Hook run by `mu4e-view' after a message is rendered." + :type 'hook + :group 'mu4e-view) + +(define-derived-mode mu4e-raw-view-mode fundamental-mode "mu4e:raw-view" + (view-mode)) + +;; "Define the major-mode for the mu4e-view." +(define-derived-mode mu4e-view-mode gnus-article-mode "mu4e:view" + "Major mode for viewing an e-mail message in mu4e. +Based on Gnus' article-mode." + ;; some external tools (bbdb) depend on this + (setq gnus-article-buffer (current-buffer)) + + ;; ;; turn off gnus modeline changes and menu items + (advice-add 'gnus-set-mode-line :around #'mu4e--view-nop) + (advice-add 'gnus-button-reply :around #'mu4e--view-button-reply) + (advice-add 'gnus-button-message-id :around #'mu4e--view-button-message-id) + (advice-add 'gnus-msg-mail :around #'mu4e--view-msg-mail) + + ;; advice gnus-block-private-groups to always return "." + ;; so that by default we block images. + (advice-add 'gnus-block-private-groups :around + (lambda(func &rest args) + (if (mu4e--view-mode-p) + "." (apply func args)))) + (use-local-map mu4e-view-mode-map) + (mu4e-context-minor-mode) + (mu4e-search-minor-mode) + (mu4e-compose-minor-mode) + (setq buffer-undo-list t) ;; don't record undo info + + ;; support bookmarks. + (set (make-local-variable 'bookmark-make-record-function) + 'mu4e--make-bookmark-record) + + ;; autopair mode gives error when pressing RET + ;; turn it off + (when (boundp 'autopair-dont-activate) + (setq autopair-dont-activate t))) + +;;; Massaging the message view + +(defcustom mu4e-view-massage-options + '( ("ctoggle citations" . gnus-article-hide-citation) + ("htoggle headers" . gnus-article-hide-headers) + ("ytoggle crypto" . gnus-article-hide-pem) + ("ftoggle fill-flowed" . mu4e-view-toggle-fill-flowed) + ("mtoggle show all MIME parts" . mu4e-view-toggle-show-mime-parts) + ("Mtoggle show emulate MIME" . mu4e-view-toggle-emulate-mime)) +"Various options for \"massaging\" the message view. See `(gnus) +Article Treatment' for more options." + :group 'mu4e-view + :type '(alist :key-type string :value-type function)) + +(defun mu4e-view-massage() + "Massage current message view as per `mu4e-view-massage-options'." + (interactive) + (funcall (mu4e-read-option "Massage: " mu4e-view-massage-options))) + + +(defun mu4e-view-toggle-html () + "Toggle html-display of the first html-part found." + (interactive) + ;; This function assumes `gnus-article-mime-handle-alist' is sorted by + ;; pertinence, i.e. the first HTML part found in it is the most important one. + (save-excursion + (if-let ((html-part + (seq-find (lambda (handle) + (equal (mm-handle-media-type (cdr handle)) + "text/html")) + gnus-article-mime-handle-alist)) + (text-part + (seq-find (lambda (handle) + (equal (mm-handle-media-type (cdr handle)) + "text/plain")) + gnus-article-mime-handle-alist))) + (gnus-article-inline-part (car html-part)) + (mu4e-warn "Cannot switch; no html and/or text part in this message")))) + +;;; Bug Reference mode support + +;; Due to mu4e's view buffer handling (mu4e-view-mode is called long before the +;; actual mail text is inserted into the buffer), one should activate +;; bug-reference-mode in mu4e-after-view-message-hook, not mu4e-view-mode-hook. + +;; This is Emacs 28 stuff but there is no need to guard it with some (f)boundp +;; checks (which would return nil if bug-reference.el is not loaded before +;; mu4e) since the function definition doesn't hurt and `add-hook' works fine +;; for not yet defined variables (by creating them). +(declare-function bug-reference-maybe-setup-from-mail "ext:bug-reference") + +(defvar mu4e--view-bug-reference-checked-headers + '("list" "list-id" "to" "from" "cc" "subject" "reply-to") + "List of mail headers whose values are passed to bug-reference's auto-setup.") + +(defun mu4e--view-try-setup-bug-reference-mode () + "Try to guess bug-reference setup from the current mu4e mail. +Looks at the maildir and the mail headers in +`mu4e--view-bug-reference-checked-headers' and tries to guess suitable +values for `bug-reference-bug-regexp' and +`bug-reference-url-format' by matching the maildir name against +GROUP-REGEXP and each header value against HEADER-REGEXP in +`bug-reference-setup-from-mail-alist'." + (when (derived-mode-p 'mu4e-view-mode) + (let (header-values) + (save-excursion + (goto-char (point-min)) + (dolist (field mu4e--view-bug-reference-checked-headers) + (let ((val (mail-fetch-field field))) + (when val + (push val header-values))))) + (bug-reference-maybe-setup-from-mail + (mail-fetch-field "maildir") + header-values)))) + +(with-eval-after-load 'bug-reference + (add-hook 'bug-reference-auto-setup-functions + #'mu4e--view-try-setup-bug-reference-mode)) + + +(provide 'mu4e-view) +;;; mu4e-view.el ends here diff --git a/mu4e/mu4e-window.el b/mu4e/mu4e-window.el new file mode 100644 index 0000000..af2e933 --- /dev/null +++ b/mu4e/mu4e-window.el @@ -0,0 +1,383 @@ +;;; mu4e-window.el --- Window management -*- lexical-binding: t; -*- + +;; Copyright (C) 2022 Mickey Petersen +;; Copyright (C) 2023-2024 Dirk-Jan C. Binnema + +;; Author: Mickey Petersen <mickey@masteringemacs.org> +;; Keywords: mail + +;; This program is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with this program. If not, see <https://www.gnu.org/licenses/>. + +;;; Commentary: + +;;; Code: + +;;; Buffer names for internal use + +(defconst mu4e--sexp-buffer-name "*mu4e-sexp-at-point*" + "Buffer name for sexp buffers.") + +(defvar mu4e-main-buffer-name "*mu4e-main*" + "Name of the mu4e main buffer.") + +(defvar mu4e-embedded-buffer-name " *mu4e-embedded*" + "Name for the embedded message view buffer.") + +;; Buffer names for public use + +(defvar mu4e-headers-buffer-name "*mu4e-headers*" + "Name of the buffer for message headers.") + +(defvar mu4e-view-buffer-name "*mu4e-article*" + "Name of the view buffer.") + +(defvar mu4e-headers-buffer-name-func nil + "Function used to name the headers buffers.") + +(defvar mu4e-view-buffer-name-func nil + "Function used to name the view buffers. + +The function is given one argument, the headers buffer it is +linked to.") + +(defvar-local mu4e-linked-headers-buffer nil + "Holds the headers buffer object that ties it to a view.") + +(defcustom mu4e-split-view 'horizontal + "How to show messages / headers. +A symbol which is either: + * `horizontal': split horizontally (headers on top) + * `vertical': split vertically (headers on the left). + * `single-window': view and headers in one window (mu4e will try not to + touch your window layout), main view in minibuffer + * anything else: don't split (show either headers or messages, + not both). + +Also see `mu4e-headers-visible-lines' and +`mu4e-headers-visible-columns'. + +Note that in older mu4e version, the value could also be +function; this is no longer supported; instead you can use +`display-buffer-alist'." + :type '(choice (const :tag "Split horizontally" horizontal) + (const :tag "Split vertically" vertical) + (const :tag "Single window" single-window) + (const :tag "Don't split" nil)) + :group 'mu4e-headers) + +(defcustom mu4e-headers-visible-lines 10 + "Number of lines to display in the header view when using the +horizontal split-view. This includes the header-line at the top, +and the mode-line." + :type 'integer + :group 'mu4e-headers) + +(defcustom mu4e-headers-visible-columns 30 + "Number of columns to display for the header view when using the +vertical split-view." + :type 'integer + :group 'mu4e-headers) + +(defcustom mu4e-compose-switch nil + "Where to display the new message? +A symbol: +- nil : default (new buffer) +- window : compose in new window +- frame or t : compose in new frame +- display-buffer: use `display-buffer' / `display-buffer-alist' + (for fine-tuning). + +For backward compatibility with `mu4e-compose-in-new-frame', t is +treated as =\\'frame." + :type 'symbol + :group 'mu4e-compose) + +(declare-function mu4e-view-mode "mu4e-view") +(declare-function mu4e-error "mu4e-helpers") +(declare-function mu4e-warn "mu4e-helpers") +(declare-function mu4e-message "mu4e-helpers") + +(defun mu4e-get-headers-buffer (&optional buffer-name create) + "Return a related headers buffer optionally named BUFFER-NAME. + +If CREATE is non-nil, the headers buffer is created if the +generated name does not already exist." + (let* ((buffer-name + (or + ;; buffer name generator func. If a user wants + ;; to supply its own naming scheme, we use that + ;; in lieu of our own heuristic. + (and mu4e-headers-buffer-name-func + (funcall mu4e-headers-buffer-name-func)) + ;; if we're supplied a buffer name for a + ;; headers buffer then try to use that one. + buffer-name + ;; if we're asking for a headers buffer from a + ;; view, then we get our linked buffer. If + ;; there is no such linked buffer -- it is + ;; detached -- raise an error. + (and (mu4e-current-buffer-type-p 'view) + mu4e-linked-headers-buffer) + ;; if we're already in a headers buffer then + ;; that is the one we use. + (and (mu4e-current-buffer-type-p 'headers) + (current-buffer)) + ;; default name to use if all other checks fail. + mu4e-headers-buffer-name)) + (buffer (get-buffer buffer-name))) + (when (and (not (buffer-live-p buffer)) create) + (setq buffer (get-buffer-create buffer-name))) + ;; This may conceivably return a non-existent buffer if `create' + ;; and `buffer-live-p' are nil. + ;; + ;; This is seemingly "OK" as various parts of the code check for + ;; buffer liveness themselves. + buffer)) + +(defun mu4e-get-view-buffers (pred) + "Filter all known view buffers and keep those where PRED return non-nil. + +The PRED function is called from inside the buffer that is being +tested." + (seq-filter + (lambda (buf) + (with-current-buffer buf + (and (mu4e-current-buffer-type-p 'view) + (and pred (funcall pred buf))))) + (buffer-list))) + +(defun mu4e--view-detached-p (buffer) + "Return non-nil if BUFFER is a detached view buffer." + (with-current-buffer buffer + (unless (mu4e-current-buffer-type-p 'view) + (mu4e-error "Buffer `%s' is not a valid mu4e view buffer" buffer)) + (null mu4e-linked-headers-buffer))) + +(defun mu4e--get-current-buffer-type () + "Return an internal symbol that corresponds to each mu4e major mode." + (cond ((or (derived-mode-p 'mu4e-view-mode) + (derived-mode-p 'mu4e-raw-view-mode)) 'view) + ((derived-mode-p 'mu4e-headers-mode) 'headers) + ((derived-mode-p 'mu4e-compose-mode) 'compose) + ((derived-mode-p 'mu4e-main-mode) 'main) + (t 'unknown))) + +(defun mu4e-current-buffer-type-p (type) + "Return non-nil if the current buffer is a mu4e buffer of TYPE. + +Where TYPE is `view', `headers', `compose', `main' or `unknown'. + +Checks are performed using `derived-mode-p' and the current +buffer's major mode." + (eq (mu4e--get-current-buffer-type) type)) + + +;; backward-compat; buffer-local-boundp was introduced in emacs 28. +(defun mu4e--buffer-local-boundp (symbol buffer) + "Return non-nil if SYMBOL is bound in BUFFER. +Also see `local-variable-p'." + (condition-case nil + (buffer-local-value symbol buffer) + (:success t) + (void-variable nil))) + + +(defun mu4e-get-view-buffer (&optional headers-buffer create) + "Return a view buffer belonging optionally to HEADERS-BUFFER. + +If HEADERS-BUFFER is nil, the most likely (and available) headers +buffer is used. + +Detached view buffers are ignored; that may result in a new view buffer +being created if CREATE is non-nil." + ;; If `headers-buffer' is nil, then the caller does not have a + ;; headers buffer preference. + ;; + ;; In that case, we request the most plausible headers buffer from + ;; `mu4e-get-headers-buffer'. + (when (setq headers-buffer (or headers-buffer (mu4e-get-headers-buffer))) + (let ((buffer) + ;; If `mu4e-view-buffer-name-func' is non-nil, then use that + ;; to source the name of the view buffer to create or re-use. + (buffer-name + (or (and mu4e-view-buffer-name-func + (funcall mu4e-view-buffer-name-func headers-buffer)) + ;; If the variable is nil, use the default + ;; name + mu4e-view-buffer-name)) + ;; Search all view buffers and return those that are linked to + ;; `headers-buffer'. + (linked-buffer + (mu4e-get-view-buffers + (lambda (buf) + (and (mu4e--buffer-local-boundp 'mu4e-linked-headers-buffer buf) + (eq mu4e-linked-headers-buffer headers-buffer)))))) + ;; If such a linked buffer exists and its buffer is live, we use that + ;; buffer. + (if (and linked-buffer (buffer-live-p (car linked-buffer))) + ;; NOTE: It's possible for there to be more than one linked view + ;; buffer. + ;; + ;; What, if anything, should the heuristic be to pick the + ;; one to use? Presently `car' is used, but there are better + ;; ways, no doubt. Perhaps preferring those with live windows? + (setq buffer (car linked-buffer)) + (setq buffer (get-buffer buffer-name)) + ;; check if `buffer' is already live *and* detached. If it is, + ;; we'll generate a new, unique name. + (when (and (buffer-live-p buffer) (mu4e--view-detached-p buffer)) + (setq buffer (generate-new-buffer-name buffer-name))) + (when (and (not (buffer-live-p buffer)) create) + (setq buffer (get-buffer-create (or buffer buffer-name))) + (with-current-buffer buffer + (mu4e-view-mode)))) + (when (and buffer (buffer-live-p buffer)) + ;; Required. Callers expect the view buffer to be set. + (set-buffer buffer) + ;; Required. The call chain of `mu4e-view-mode' ends up + ;; calling `kill-all-local-variables', which destroys the + ;; local binding. + (set (make-local-variable 'mu4e-linked-headers-buffer) headers-buffer)) + buffer))) + +;; backward compat: `display-buffer-full-frame' only appears in emacs 29. +(unless (fboundp 'display-buffer-full-frame) + (defun display-buffer-full-frame (buffer alist) + "Display BUFFER in the current frame, taking the entire frame. +ALIST is an association list of action symbols and values. See +Info node `(elisp) Buffer Display Action Alists' for details of +such alists. + +This is an action function for buffer display, see Info +node `(elisp) Buffer Display Action Functions'. It should be +called only by `display-buffer' or a function directly or +indirectly called by the latter." + (when-let ((window (or (display-buffer-reuse-window buffer alist) + (display-buffer-same-window buffer alist) + (display-buffer-pop-up-window buffer alist) + (display-buffer-use-some-window buffer alist)))) + (delete-other-windows window) + window))) + + +(defun mu4e-display-buffer (buffer-or-name &optional select) + "Display BUFFER-OR-NAME as per `mu4e-split-view'. + +If SELECT is non-nil, the final window (and thus BUFFER-OR-NAME) +is selected. + +This function internally uses `display-buffer' (or +`pop-to-buffer' if SELECT is non-nil). + +It is therefore possible to change the display behavior by +modifying `display-buffer-alist'. + +If `mu4e-split-view' is a function, then it must return a live window +for BUFFER-OR-NAME to be displayed in." + ;; For now, using a function for mu4e-split-view is not behaving well + ;; Turn off. + (when (functionp mu4e-split-view) + (mu4e-message "Function for `mu4e-split-view' not supported; fallback") + (setq mu4e-split-view 'horizontal)) + + (let* ((buffer-name (or (get-buffer buffer-or-name) + (mu4e-error "Buffer `%s' does not exist" + buffer-or-name))) + (buffer-type + (with-current-buffer buffer-name (mu4e--get-current-buffer-type))) + (direction (cons 'direction + (pcase (cons buffer-type mu4e-split-view) + ;; views or headers can display + ;; horz/vert depending on the value of + ;; `mu4e-split-view' + (`(,(or 'view 'headers) . horizontal) 'below) + (`(,(or 'view 'headers) . vertical) 'right) + (`(,_ . t) nil)))) + (window-size + (pcase (cons buffer-type mu4e-split-view) + ;; views or headers can display + ;; horz/vert depending on the value of + ;; `mu4e-split-view' + ('(view . horizontal) + '((window-height . shrink-window-if-larger-than-buffer))) + ('(view . vertical) + '((window-min-width . fit-window-to-buffer))) + (`(,_ . t) nil))) + (window-action (cond + ;; main-buffer + ((eq buffer-type 'main) + '(display-buffer-reuse-window + display-buffer-reuse-mode-window + display-buffer-full-frame)) + ;; compose-buffer + ((eq buffer-type 'compose) + (pcase mu4e-compose-switch + ('window #'display-buffer-pop-up-window) + ((or 'frame 't) #'display-buffer-pop-up-frame) + (_ '(display-buffer-reuse-window + display-buffer-reuse-mode-window + display-buffer-same-window)))) + ;; headers buffer + ((memq buffer-type '(headers)) + '(display-buffer-reuse-window + display-buffer-reuse-mode-window + display-buffer-same-window)) + + ((memq mu4e-split-view '(horizontal vertical)) + '(display-buffer-in-direction)) + + ((memq mu4e-split-view '(single-window)) + '(display-buffer-reuse-window + display-buffer-reuse-mode-window + display-buffer-same-window)) + ;; I cannot discern a difference between + ;; `single-window' and "anything else" in + ;; `mu4e-split-view'. + (t '(display-buffer-reuse-window + display-buffer-reuse-mode-window + display-buffer-same-window)))) + (arg `((,@window-action) + ,@window-size + ,direction))) + (funcall (if select #'pop-to-buffer #'display-buffer) + buffer-name + arg))) + +(defun mu4e-resize-linked-headers-window () + "Resizes the linked headers window belonging to a view. + +Resizes the current headers view according to `mu4e-split-view' +and `mu4e-headers-visible-lines' or +`mu4e-headers-visible-columns'. + +This function is best called from the hook +`mu4e-view-rendered-hook'." + (unless (mu4e-current-buffer-type-p 'view) + (mu4e-error "Cannot resize as this is not a valid view buffer.")) + (when-let (win (and mu4e-linked-headers-buffer + (get-buffer-window mu4e-linked-headers-buffer))) + ;; This can fail for any number of reasons. If it does, we do + ;; nothing. If the user has customized the window display we may + ;; find it impossible to resize the window, and that should not be + ;; cause for error. + (ignore-errors + (cond ((eq mu4e-split-view 'vertical) + (window-resize win (- mu4e-headers-visible-columns + (window-width win nil)) + t t nil)) + ((eq mu4e-split-view 'horizontal) + (set-window-text-height win mu4e-headers-visible-lines)))))) + +(provide 'mu4e-window) +;;; mu4e-window.el ends here diff --git a/mu4e/mu4e.el b/mu4e/mu4e.el new file mode 100644 index 0000000..d202a3c --- /dev/null +++ b/mu4e/mu4e.el @@ -0,0 +1,266 @@ +;;; mu4e.el --- Mu4e, the mu mail user agent -*- lexical-binding: t -*- + +;; Copyright (C) 2011-2023 Dirk-Jan C. Binnema + +;; Author: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Maintainer: Dirk-Jan C. Binnema <djcb@djcbsoftware.nl> +;; Keywords: email + +;; This file is not part of GNU Emacs. + +;; mu4e is free software: you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation, either version 3 of the License, or +;; (at your option) any later version. + +;; mu4e is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. + +;; You should have received a copy of the GNU General Public License +;; along with mu4e. If not, see <http://www.gnu.org/licenses/>. + +;;; Commentary: + +;;; Code: +(require 'mu4e-obsolete) + +(require 'mu4e-vars) +(require 'mu4e-window) +(require 'mu4e-helpers) +(require 'mu4e-folders) +(require 'mu4e-context) +(require 'mu4e-contacts) +(require 'mu4e-headers) +(require 'mu4e-search) +(require 'mu4e-view) +(require 'mu4e-compose) +(require 'mu4e-bookmarks) +(require 'mu4e-update) +(require 'mu4e-main) +(require 'mu4e-notification) +(require 'mu4e-server) ;; communication with backend + + + +(when mu4e-speedbar-support + (require 'mu4e-speedbar)) ;; support for speedbar +(when mu4e-org-support + (require 'mu4e-org)) ;; support for org-mode links + +;; We can't properly use compose buffers that are revived using +;; desktop-save-mode; so let's turn that off. +(with-eval-after-load 'desktop + (eval '(add-to-list 'desktop-modes-not-to-save 'mu4e-compose-mode))) + + +;;;###autoload +(defun mu4e (&optional background) + "If mu4e is not running yet, start it. +Then, show the main window, unless BACKGROUND (prefix-argument) +is non-nil." + (interactive "P") + (if (not (mu4e-running-p)) + (progn + (mu4e--init-handlers) + (mu4e--start (unless background #'mu4e--main-view))) + ;; mu4e already running; show unless BACKGROUND + (unless background + (if (buffer-live-p (get-buffer mu4e-main-buffer-name)) + (switch-to-buffer mu4e-main-buffer-name) + (mu4e--main-view))))) + +(defun mu4e-quit(&optional bury) + "Quit the mu4e session or bury the buffer. + +If prefix-argument BURY is non-nil, merely bury the buffer. +Otherwise, completely quit mu4e, including automatic updating." + (interactive "P") + (if bury + (bury-buffer) + (if mu4e-confirm-quit + (when (y-or-n-p (mu4e-format "Are you sure you want to quit?")) + (mu4e--stop)) + (mu4e--stop)))) + +;;; Internals + +(defun mu4e--check-requirements () + "Check for the settings required for running mu4e." + (unless (>= emacs-major-version 25) + (mu4e-error "Emacs >= 25.x is required for mu4e")) + (when (mu4e-server-properties) + (unless (string= (mu4e-server-version) mu4e-mu-version) + (mu4e-error "The mu server has version %s, but we need %s" + (mu4e-server-version) mu4e-mu-version))) + (unless (and mu4e-mu-binary (file-executable-p mu4e-mu-binary)) + (mu4e-error "Please set `mu4e-mu-binary' to the full path to the mu + binary")) + (dolist (var '(mu4e-sent-folder mu4e-drafts-folder + mu4e-trash-folder)) + (unless (and (boundp var) (symbol-value var)) + (mu4e-error "Please set %S" var)) + (unless (functionp (symbol-value var)) ;; functions are okay, too + (let* ((dir (symbol-value var)) + (path (mu4e-join-paths (mu4e-root-maildir) dir))) + (unless (string= (substring dir 0 1) "/") + (mu4e-error "%S must start with a '/'" dir)) + (unless (mu4e-create-maildir-maybe path) + (mu4e-error "%s (%S) does not exist" path var)))))) + +;;; Starting / getting mail / updating the index + +(defun mu4e--pong-handler (_data func) + "Handle \"pong\" responses from the mu server. +Invoke FUNC if non-nil." + (let ((doccount (plist-get (mu4e-server-properties) :doccount))) + (mu4e--check-requirements) + (when func (funcall func)) + (when (zerop doccount) + (mu4e-message "Store is empty; try indexing (M-x mu4e-update-index).")) + (when (and mu4e-update-interval (null mu4e--update-timer)) + (setq mu4e--update-timer + (run-at-time 0 mu4e-update-interval + (lambda () (mu4e-update-mail-and-index + mu4e-index-update-in-background))))))) + +(defun mu4e--start (&optional func) + "Start mu4e. +If `mu4e-contexts' have been defined, but we don't have a context +yet, switch to the matching one, or none matches, the first. If +mu4e is already running, invoke FUNC (if non-nil). + +Otherwise, check requirements, then start mu4e. When successful, invoke + FUNC (if non-nil) afterwards." + (unless (mu4e-context-current) + (mu4e--context-autoswitch nil mu4e-context-policy)) + (setq mu4e-pong-func + (lambda (info) (mu4e--pong-handler info func))) + ;; show some notification? + (when mu4e-notification-support + (add-hook 'mu4e-query-items-updated-hook #'mu4e--notification)) + ;; modeline support + (when mu4e-modeline-support + (mu4e--modeline-register #'mu4e--bookmarks-modeline-item 'global) + (mu4e-modeline-mode) + (add-hook 'mu4e-query-items-updated-hook #'mu4e--modeline-update)) + (mu4e-modeline-mode (if mu4e-modeline-support 1 -1)) + ;; redraw main buffer if there is one. + (add-hook 'mu4e-query-items-updated-hook #'mu4e--main-redraw) + (mu4e--query-items-refresh 'reset-baseline) + (mu4e--server-ping) + ;; ask for the maildir-list + (mu4e--server-data 'maildirs) + ;; maybe request the list of contacts, automatically refreshed after + ;; re-indexing + (unless mu4e--contacts-set + (mu4e--request-contacts-maybe))) + +(defun mu4e--stop () + "Stop mu4e." + (when mu4e--update-timer + (cancel-timer mu4e--update-timer) + (setq mu4e--update-timer nil)) + + (setq ;; clear some caches + mu4e-maildir-list nil + mu4e--contacts-set nil + mu4e--contacts-tstamp "0") + + (remove-hook 'mu4e-query-items-updated-hook #'mu4e--main-redraw) + (remove-hook 'mu4e-query-items-updated-hook #'mu4e--modeline-update) + (remove-hook 'mu4e-query-items-updated-hook #'mu4e--notification) + (mu4e-kill-update-mail) + (mu4e-modeline-mode -1) + (mu4e--server-kill) + ;; kill all mu4e buffers + (mapc + (lambda (buf) + ;; the view buffer has the kill-buffer-hook function + ;; mu4e--view-kill-mime-handles which kills the mm-* buffers created by + ;; Gnus' article mode. Those have been returned by `buffer-list' but might + ;; already be deleted in case the view buffer has been killed first. So we + ;; need a `buffer-live-p' check here. + (when (buffer-live-p buf) + (with-current-buffer buf + (when (member major-mode + '(mu4e-headers-mode mu4e-view-mode mu4e-main-mode)) + (kill-buffer))))) + (buffer-list))) + +;;; Handlers +(defun mu4e--default-handler (&rest args) + "Dummy handler function with arbitrary ARGS." + (mu4e-error "Not handled: %s" args)) + +(defun mu4e--error-handler (errcode errmsg) + "Handler function for showing an error with ERRCODE and ERRMSG." + ;; don't use mu4e-error here; it's running in the process filter context + (pcase errcode + ('4 (mu4e-warn "No matches for this search query.")) + ('110 (display-warning 'mu4e errmsg :error)) ;; schema version. + (_ (mu4e-error "Error %d: %s" errcode errmsg)))) + +(defun mu4e--update-status (info) + "Update the status message with INFO." + (setq mu4e-index-update-status + `(:tstamp ,(current-time) + :checked ,(plist-get info :checked) + :updated ,(plist-get info :updated) + :cleaned-up ,(plist-get info :cleaned-up)))) + +(defun mu4e--info-handler (info) + "Handler function for (:INFO ...) sexps received from server." + (let* ((type (plist-get info :info)) + (checked (plist-get info :checked)) + (updated (plist-get info :updated)) + (cleaned-up (plist-get info :cleaned-up))) + (cond + ((eq type 'add) t) ;; do nothing + ((eq type 'index) + (if (eq (plist-get info :status) 'running) + (mu4e-index-message + "Indexing... checked %d, updated %d" checked updated) + (progn ;; i.e. 'complete + (mu4e--update-status info) + (mu4e-index-message + "%s completed; checked %d, updated %d, cleaned-up %d" + (if mu4e-index-lazy-check "Lazy indexing" "Indexing") + checked updated cleaned-up) + ;; index done; grab updated queries + (mu4e--query-items-refresh) + (run-hooks 'mu4e-index-updated-hook) + ;; backward compatibility... + (unless (zerop (+ updated cleaned-up)) + mu4e-message-changed-hook) + (unless (and (not (string= mu4e--contacts-tstamp "0")) + (zerop (plist-get info :updated))) + (mu4e--request-contacts-maybe) + (mu4e--server-data 'maildirs)) ;; update maildir list + (mu4e--main-redraw)))) + ((plist-get info :message) + (mu4e-index-message "%s" (plist-get info :message)))))) + +(defun mu4e--init-handlers() + "Initialize the server message handlers. +Only set set them if they were nil before, so overriding has a +chance." + (mu4e-setq-if-nil mu4e-error-func #'mu4e--error-handler) + (mu4e-setq-if-nil mu4e-update-func #'mu4e~headers-update-handler) + (mu4e-setq-if-nil mu4e-remove-func #'mu4e~headers-remove-handler) + (mu4e-setq-if-nil mu4e-view-func #'mu4e~headers-view-handler) + (mu4e-setq-if-nil mu4e-headers-append-func #'mu4e~headers-append-handler) + (mu4e-setq-if-nil mu4e-found-func #'mu4e~headers-found-handler) + (mu4e-setq-if-nil mu4e-erase-func #'mu4e~headers-clear) + + (mu4e-setq-if-nil mu4e-sent-func #'mu4e--default-handler) + (mu4e-setq-if-nil mu4e-contacts-func #'mu4e--update-contacts) + (mu4e-setq-if-nil mu4e-info-func #'mu4e--info-handler) + (mu4e-setq-if-nil mu4e-pong-func #'mu4e--default-handler) + + (mu4e-setq-if-nil mu4e-queries-func #'mu4e--query-items-queries-handler)) + +;;; +(provide 'mu4e) +;;; mu4e.el ends here diff --git a/mu4e/mu4e.texi b/mu4e/mu4e.texi new file mode 100644 index 0000000..e3e226c --- /dev/null +++ b/mu4e/mu4e.texi @@ -0,0 +1,4948 @@ +\input texinfo.tex @c -*-texinfo-*- +@documentencoding UTF-8 +@include version.texi +@c %**start of header +@setfilename mu4e.info +@settitle Mu4e @value{VERSION} user manual + +@c Use proper quote and backtick for code sections in PDF output +@c Cf. Texinfo manual 14.2 +@set txicodequoteundirected +@set txicodequotebacktick +@c %**end of header + +@copying +Copyright @copyright{} 2012-@value{UPDATED-YEAR} Dirk-Jan C. Binnema + +@quotation +Permission is granted to copy, distribute and/or modify this document +under the terms of the GNU Free Documentation License, Version 1.3 or +any later version published by the Free Software Foundation; with no +Invariant Sections, no Front-Cover Texts, and no Back-Cover Texts. A +copy of the license is included in the section entitled ``GNU Free +Documentation License.'' +@end quotation +@end copying + +@titlepage +@title @t{Mu4e} --- an e-mail client for GNU Emacs +@subtitle version @value{VERSION}, @value{UPDATED} +@author Dirk-Jan C. Binnema + +@c The following two commands start the copyright page. +@page +@vskip 0pt plus 1filll +@insertcopying +@end titlepage + +@dircategory Emacs +@direntry +* Mu4e: (Mu4e). An email client for GNU Emacs. +@end direntry + +@contents + +@ifnottex +@node Top +@top mu4e manual for version @value{VERSION} +@end ifnottex + +@iftex +@node Welcome to mu4e +@unnumbered Welcome to mu4e +@end iftex + +Welcome to @t{mu4e}! + +@t{mu4e} (@t{mu}-for-emacs) is an e-mail client for GNU Emacs version 26.3 or +newer, built on top of the @uref{https://www.djcbsoftware.nl/code/mu,mu} e-mail +search engine. @t{mu4e} is optimized for quickly processing large amounts of +e-mail. + +Some of its highlights: +@itemize +@item Fully search-based: there are no folders@footnote{that is, instead of +folders, you use queries that match messages in a particular folder}, +only queries. +@item Fully documented, with example configurations +@item User-interface optimized for speed, with quick key strokes for common actions +@item Support for non-English languages (so ``angstrom'' matches ``Ångström'') +@item Asynchronous: heavy actions don't block @t{emacs}@footnote{currently, +the only exception to this is @emph{sending mail}; there are solutions +for that though --- see the @ref{FAQ}} +@item Support for cryptography --- signing, encrypting and decrypting +@item Address auto-completion based on the contacts in your messages +@item Extendable with your own snippets of elisp +@end itemize + +In this manual, we go through the installation of @t{mu4e}, do some +basic configuration and explain its daily use. We also show you how you +can customize @t{mu4e} for your special needs. + +At the end of the manual, there are some example configurations, to get +you up to speed quickly: @ref{Example configurations}. There's also a section +with answers to frequently asked questions, @ref{FAQ}. + +@menu +* Introduction:: Where to begin +* Getting started:: Setting things up +* Main view:: The @t{mu4e} overview +* Headers view:: Lists of message headers +* Message view:: Viewing specific messages +* Composer:: Creating and editing messages +* Searching:: Some more background on searching/queries` +* Marking:: Marking messages and performing actions +* Contexts:: Defining contexts and switching between them +* Dynamic folders:: Folders that change based on circumstances +* Actions:: Defining and using custom actions +* Extending mu4e:: Writing code for @t{mu4e} +* Integration:: Integrating @t{mu4e} with Emacs facilities + +Appendices +* Other tools:: mu4e and the rest of the world +* Example configurations:: Some examples to set you up quickly +* FAQ:: Common questions and answers +* Tips and Tricks:: Useful tips +* How it works:: Some notes about the implementation of @t{mu4e} +* Debugging:: How to debug problems in @t{mu4e} +* GNU Free Documentation License:: The license of this manual + +Indices +@c * Command Index:: An item for each standard command name. +@c * Variable Index:: An item for each variable documented in this manual. +* Concept Index:: Index of @t{mu4e} concepts and other general subjects. + +@end menu + +@node Introduction +@chapter Introduction + +Let's get started +@menu +* Why another e-mail client::Aren't there enough already +* Other mail clients::Where @t{mu4e} takes its inspiration from +* What mu4e does not do::Focus on the core-business, delegate the rest +* Becoming a mu4e user::Joining the club +@end menu + +@node Why another e-mail client +@section Why another e-mail client? + +I (@t{mu4e}'s author) spend a @emph{lot} of time dealing with e-mail, +both professionally and privately. Having an efficient e-mail client +is essential. Since none of the existing ones worked the way I wanted, +I thought about creating my own. + +Emacs is an integral part of my workflow, so it made a lot of +sense to use it for e-mail as well. And as I had already written an +e-mail search engine (@t{mu}), it seemed only logical to use that as a +basis. + +@node Other mail clients +@section Other mail clients + +Under the hood, @t{mu4e} is fully search-based, similar to programs like +@uref{https://notmuchmail.org/,notmuch} and +@uref{https://sup-heliotrope.github.io/,sup}. + +However, @t{mu4e}'s user-interface is quite different. @t{mu4e}'s mail handling +(deleting, moving, etc.)@: is inspired by +@uref{http://www.gohome.org/wl/,Wanderlust} (another Emacs-based e-mail client), +@uref{http://www.mutt.org/,mutt} and the @t{dired} file-manager for emacs. + +@t{mu4e} keeps all the `state' in your maildirs, so you can easily +switch between clients, synchronize over @abbr{IMAP}, backup with +@t{rsync} and so on. The Xapian-database that @t{mu} maintains is +merely a @emph{cache}; if you delete it, you won't lose any +information. + +@node What mu4e does not do +@section What @t{mu4e} does not do + +There are a number of things that @t{mu4e} does @b{not} do, by design: +@itemize +@item @t{mu}/@t{mu4e} do @emph{not} get your e-mail messages from +a mail server. Nor does it sync-back any changes. Those tasks are delegated to +other tools, such as @uref{https://www.offlineimap.org/,offlineimap}, +@uref{http://isync.sourceforge.net/,mbsync} or +@uref{http://www.fetchmail.info/,fetchmail}; As long as the messages end up in a +maildir, @t{mu4e} and @t{mu} are happy to deal with them. +@item @t{mu4e} also does @emph{not} implement sending of messages; instead, it depends on +@ref{(smtpmail) Top}, which is part of Emacs. In addition, @t{mu4e} piggybacks on +Gnus' message editor. +@end itemize + +Thus, many of the things an e-mail client traditionally needs to do, are +delegated to other tools. This leaves @t{mu4e} to concentrate on what it does +best: quickly finding the mails you are looking for, and handle them as +efficiently as possible. + +@node Becoming a mu4e user +@section Becoming a @t{mu4e} user + +If @t{mu4e} sounds like something for you, give it a shot! We're trying +hard to make it as easy as possible to set up and use; and while you can +use elisp in various places to augment @t{mu4e}, a lot of knowledge +about programming or elisp shouldn't be required. The idea is to provide +sensible defaults, and allow for customization. + +When you take @t{mu4e} into use, it's a good idea to subscribe to the +@uref{https://groups.google.com/group/mu-discuss,mu/mu4e mailing list}. + +Sometimes, you might encounter some unexpected behavior while using @t{mu4e}, or +have some idea on how it could work better. To report this, you can use the +@uref{https://github.com/djcb/mu/issues,bug-tracker}. Please always include the +following information: + +@itemize +@item what did you expect or wish to happen? what actually happened? +@item can you provide some exact steps to reproduce? +@item what version of @t{mu4e} and @t{emacs} were you using? What operating system? +@item can you reproduce it with @command{emacs -q} and only loading @t{mu4e}? +@item if the problem is related to some specific message, please include the raw message file (appropriately anonymized, of course) +@end itemize + +@node Getting started +@chapter Getting started + +In this chapter, we go through the installation of @t{mu4e} and its basic setup. +After we have succeeded in @ref{Getting mail}, and @pxref{Indexing your +messages}, we discuss the @ref{Basic configuration}. + +After these steps, @t{mu4e} should be ready to go! + +@menu +* Requirements:: What is needed +* Versions:: Available stable and development versions +* Installation:: How to install @t{mu} and @t{mu4e} +* Getting mail:: Getting mail from a server +* Initializing the message store:: Settings things up +* Indexing your messages:: Creating and maintaining the index +* Basic configuration:: Settings for @t{mu4e} +* Folders:: Setting up standard folders +* Retrieval and indexing:: Doing it from @t{mu4e} +* Sending mail:: How to send mail +* Running mu4e:: Overview of the @t{mu4e} views + +@end menu + +@node Requirements +@section Requirements + +@t{mu}/@t{mu4e} are known to work on a wide variety of Unix- and Unix-like +systems, including many Linux distributions, OS X and FreeBSD. Emacs 26.3 or +higher is required, as well as @uref{https://xapian.org/,Xapian} and +@uref{http://spruce.sourceforge.net/gmime/,GMime}. + +@t{mu} has optional support for the Guile (Scheme) programming language (version +3.0 or higher). There are also some GUI-toys, which require GTK+ 3.x and Webkit. + +If you intend to compile @t{mu} yourself, you need to have the typical +development tools, such as C and C++17 compilers (both @command{gcc} and +@command{clang} work), @command{meson} and @command{make}, and the development +packages for GMime 3.x, GLib and Xapian. Optionally, you also need the +development packages for GTK+, Webkit and Guile. + +@node Versions +@section Versions + +The stable (release) versions have even minor version numbers, while the +development versions have odd ones. So, for example, 1.10.5 is a stable version, +while the 1.11.9 is the development version. + +The stable versions only receive bug fixes after being released, while the +development versions get new features, fixes, and, perhaps, bugs, and are meant +for people with a tolerance for that. + +There is support for one release branch; so, when the 1.10 release is available +(and a new 1.11 development series start), no more changes are expected for the +1.8 releases. + +@node Installation +@section Installation + +@t{mu4e} is part of @t{mu} --- by installing the latter, the former is +installed as well. Some Linux distributions provide packaged versions of +@t{mu}/@t{mu4e}; if you can use those, there is no need to compile +anything yourself. However, if there are no packages for your +distribution, if they are outdated, or if you want to use the latest +development versions, you can follow the steps below. + +@subsection Dependencies + +The first step is to get some build dependencies. The details depend a +bit on your system's setup / distribution. +@itemize +@item On Debian/Ubuntu and derivatives: +@example +$ sudo apt-get install git meson libgmime-3.0-dev libxapian-dev emacs +@end example +@item On Fedora and related: +@example +$ sudo dnf install git meson gmime30-devel xapian-core-devel emacs +@end example +@item Otherwise, install the equivalent of the above on your system +@end itemize + + +@subsection Getting mu + +The next step is to get the @t{mu} sources. There are two alternatives: +@itemize +@item @emph{Use a stable release} -- download a release from +@url{https://github.com/djcb/mu/releases} +@item @emph{Use an experimental development version} -- get it from the repository, +and @t{git clone https://github.com/djcb/mu.git} +@end itemize + +@subsection Building mu + +What all that in place, let's build and install @t{mu} and @t{mu4e}. +Enter the directory where you unpacked or cloned @t{mu}. Then: + +@example +$ ./configure && make +$ sudo make install +@end example + +Note: if you are familiar with @t{meson}, you can of course use its +commands directly; the @t{make} commands are just a thin wrapper around +that. + +@subsection Installation + +After this, @t{mu} and @t{mu4e} should be installed @footnote{there's a +hard dependency between versions of @t{mu4e} and @t{mu} --- you cannot +combine different versions} on your system, and be available from the +command line and in Emacs. + +You may need to restart Emacs, so it can find @t{mu4e} in its +@code{load-path}. If, even after restarting, Emacs cannot find @t{mu4e}, +you may need to add it to your @code{load-path} explicitly; check where +@t{mu4e} is installed, and add something like the following to your +configuration before trying again: +@lisp +;; the exact path may differ --- check it +(add-to-list 'load-path "/usr/local/share/emacs/site-lisp/mu4e") +@end lisp + +@subsection mu4e and emacs customization + +There is some support for using the Emacs customization system in +@t{mu4e}, but for now, we recommend setting the values manually. Please +refer to @ref{Example configurations} for a couple of examples of this; here we +go through things step-by-step. + +@node Getting mail +@section Getting mail + +In order for @t{mu} (and, by extension, @t{mu4e}) to work, you need to have your +e-mail messages stored in a +@uref{https://en.wikipedia.org/wiki/Maildir, Maildir}; in this manual we use the +term `maildir' for both the standard and the hierarchy of maildirs that store +your messages --- a specific directory structure with one-file-per-message. + +If you are already using a maildir, you are lucky. If not, some setup +is required: +@itemize +@item @emph{Using an external IMAP or POP server} --- if you are using an +@abbr{IMAP} or @abbr{POP} server, you can use tools like @t{getmail}, +@t{fetchmail}, @t{offlineimap} or @t{isync} to download your messages +into a maildir (@file{~/Maildir}, often). Because it is such a common +case, there is a full example of setting @t{mu4e} up with +@t{offlineimap} and Gmail; @pxref{Gmail configuration}. +@item @emph{Using a local mail server} --- if you are using a local mail- server +(such as @t{postfix} or @t{qmail}), you can teach them to deliver into +a maildir as well, maybe in combination with @t{procmail}. A bit of +googling should be able to provide you with the details. +@end itemize + +While a @t{mu} only supports a single Maildir, it can be spread across +different file-systems; and symbolic links are supported. + +@node Initializing the message store +@section Initializing the message store + +The first time you run @t{mu}, you need to initialize its store +(database). The default location for that is @t{~/.cache/mu/xapian}, but +you can change this using the @t{--muhome} option, and remember to pass +that to the other commands as well. Alternatively, you can use an +environment variable @t{MUHOME}. + +Assuming that your maildir is at @file{~/Maildir}, we issue the +following command: +@example + $ mu init --maildir=~/Maildir +@end example + +You can add some e-mail addresses, so @t{mu} recognizes them as yours: + +@example + $ mu init --maildir=~/Maildir --my-address=jim@@example.com \ + --my-address=bob@@example.com +@end example + +@t{mu} remembers the maildir and your addresses and uses them when +indexing messages. If you want to change them, you need to @t{init} +once again. + +The addresses may also be basic PCRE regular expressions, wrapped in +slashes, for example: + +@example + $ mu init --maildir=~/Maildir '--my-address=/foo-.*@@example\.com/' +@end example + +If you want to see the values for your message-store, you can use +@command{mu info}. + +@node Indexing your messages +@section Indexing your messages + +After you have succeeded in @ref{Getting mail} and initialized the +message database, we need to @emph{index} the messages. That is --- we +need to scan the messages in the maildir and store the information +about them in a special database. + +We can do that from @t{mu4e} --- @ref{Main view}, but the first time, +it is a good idea to run it from the command line, which makes it +easier to verify that everything works correctly. + +Assuming that your maildir is at @file{~/Maildir}, we issue the +following command: +@example + $ mu index +@end example + +This should scan your messages and fill the database, and give +progress information while doing so. + +The indexing process may take a few minutes the first time you do it +(for thousands of e-mails); afterwards it is much faster, since @t{mu} +only scans messages that are new or have changed. Indexing is discussed +in full detail in the @t{mu-index} man-page. + +After the indexing process has finished, you can quickly test if +everything worked, by trying some command-line searches, for example +@example + $ mu find hello +@end example + +which lists all messages that match @t{hello}. For more examples of +searches, see @ref{Queries}, or check the @t{mu-find} and @t{mu-easy} +man pages. If all of this worked well, we are well on our way setting +things up; the next step is to do some basic configuration for @t{mu4e}. + +@node Basic configuration +@section Basic configuration + +Before we can start using @t{mu4e}, we need to tell Emacs to load +it. So, add to your @file{~/.emacs} (or its moral equivalent, such as +@file{~/.emacs.d/init.el}) something like: + +@lisp +(require 'mu4e) +@end lisp + +If Emacs complains that it cannot find @t{mu4e}, check your +@code{load-path} and make sure that @t{mu4e}'s installation directory is +part of it. If not, you can add it: + +@lisp +(add-to-list 'load-path MU4E-PATH) +@end lisp + +with @t{MU4E-PATH} replaced with the actual path. + +@node Folders +@section Folders + +The next step is to tell @t{mu4e} where it can find your Maildir, and +some special folders. + +So, for example@footnote{Note that the folders (@t{mu4e-sent-folder}, +@t{mu4e-drafts-folder}, @t{mu4e-trash-folder} and +@t{mu4e-refile-folder}) can also be @emph{functions} that are evaluated +at runtime. This allows for dynamically changing them depending on the +situation. See @ref{Dynamic folders} for details.}: +@lisp +;; these are actually the defaults +(setq + mu4e-sent-folder "/sent" ;; folder for sent messages + mu4e-drafts-folder "/drafts" ;; unfinished messages + mu4e-trash-folder "/trash" ;; trashed messages + mu4e-refile-folder "/archive") ;; saved messages +@end lisp + +The folder (maildir) names are all relative to the root-maildir (see the +output of @command{mu info}). If you use @t{mu4e-context}, see +@ref{Contexts and special folders} for what that means for these special +folders. + +@node Retrieval and indexing +@section Retrieval and indexing with mu4e +@cindex mail retrieval +@cindex indexing +As we have seen, we can do all of the mail retrieval @emph{outside} of +Emacs/@t{mu4e}. However, you can also do it from within +@t{mu4e}. + +@subsection Basics + +To set up mail-retrieval from within @t{mu4e}, set the variable +@code{mu4e-get-mail-command} to the program or shell command you want to +use for retrieving mail. You can then get your e-mail using @kbd{M-x +mu4e-update-mail-and-index}, or @kbd{C-S-u} in all @t{mu4e}-views; +alternatively, you can use @kbd{C-c C-u}, which may be more convenient +if you use emacs in a terminal. + +You can kill the (foreground) update process with @kbd{q}. + +It is possible to update your mail and index periodically in the +background or foreground, by setting the variable +@code{mu4e-update-interval} to the number of seconds between these +updates. If set to @code{nil}, it won't update at all. After you make +changes to @code{mu4e-update-interval}, @t{mu4e} must be restarted +before the changes take effect. By default, this will run in +background and to change it to run in foreground, set +@code{mu4e-index-update-in-background} to @code{nil}. + +After updating has completed, @t{mu4e} keeps the output in a buffer +@t{*mu4e-last-update*}, which you can use for diagnosis if needed. + +@subsection Handling errors during mail retrieval + +If the mail-retrieval process returns with a non-zero exit code, +@t{mu4e} shows a warning (unless @code{mu4e-index-update-error-warning} +is set to @code{nil}), but then try to index your maildirs anyway +(unless @code{mu4e-index-update-error-continue} is set to @code{nil}). + +Reason for these defaults is that some of the mail-retrieval programs +may return non-zero, even when the updating process succeeded; however, +it is hard to tell such pseudo-errors from real ones like `login +failed'. + +If you need more refinement, it may be useful to wrap the mail-retrieval +program in a shell-script, for example @t{fetchmail} returns 1 to +indicate `no mail'; we can handle that with: +@lisp +(setq mu4e-get-mail-command "fetchmail -v || [ $? -eq 1 ]") +@end lisp +A similar approach can be used with other mail retrieval programs, +although not all of them have their exit codes documented. + +@subsection Implicit mail retrieval + +If you don't have a specific command for getting mail, for example +because you are running your own mail-server, you can leave +@code{mu4e-get-mail-command} at @t{"true"} (the default), in which case +@t{mu4e} won't try to get new mail, but still re-index your messages. + +@subsection Speeding up indexing + +If you have a large number of e-mail messages in your store, +(re)indexing might take a while. The defaults for indexing are to +ensure that we always have correct, up-to-date information about your +messages, even if other programs have modified the Maildir. + +The downside of this thoroughness (which is the default) is that it is +relatively slow, something that can be noticeable with large e-mail +corpora on slow file-systems. For a faster approach, you can use the +following: + +@lisp +(setq + mu4e-index-cleanup nil ;; don't do a full cleanup check + mu4e-index-lazy-check t) ;; don't consider up-to-date dirs +@end lisp + +In many cases, the mentioned thoroughness might not be needed, and +these settings give a very significant speed-up. If it does not work +for you (e.g., @t{mu4e} fails to find some new messages), simply leave +at the default. + +Note that you can occasionally run a thorough indexing round using +@code{mu4e-update-index-nonlazy}. + +For further details, please refer to the @t{mu-index} manpage; in +particular, see @t{.noindex} and @t{.noupdate} which can help reducing +the indexing time. + +@subsection Example setup + +A simple setup could look something like: + +@lisp +(setq + mu4e-get-mail-command "offlineimap" ;; or fetchmail, or ... + mu4e-update-interval 300) ;; update every 5 minutes +@end lisp + +A hook @code{mu4e-update-pre-hook} is available which is run right +before starting the process. That can be useful, for example, to +influence, @code{mu4e-get-mail-command} based on the the current +situation (location, time of day, ...). + +It is possible to get notifications when the indexing process does any +updates --- for example when receiving new mail. See +@code{mu4e-index-updated-hook} and some tips on its usage in the +@ref{FAQ}. + +@node Sending mail +@section Sending mail + +@t{mu4e} uses Emacs's @ref{(message) Top,,message-mode} for writing mail. + +For sending mail using @abbr{SMTP}, @t{mu4e} uses @ref{(smtpmail) +Top,,smtpmail}. This package supports many different ways to send mail; please +refer to its documentation for the details. + +Here, we only provide some simple examples --- for more, see @ref{Example +configurations}. + +A very minimal setup: + +@lisp +;; tell message-mode how to send mail +(setq message-send-mail-function 'smtpmail-send-it) +;; if our mail server lives at smtp.example.org; if you have a local +;; mail-server, simply use 'localhost' here. +(setq smtpmail-smtp-server "smtp.example.org") +@end lisp + +Since @t{mu4e} (re)uses the same @t{message mode} and @t{smtpmail} that +Gnus uses, many settings for those also apply to @t{mu4e}. + +@subsection Dealing with sent messages + +By default, @t{mu4e} puts a copy of messages you sent in the folder +determined by @code{mu4e-sent-folder}. In some cases, this may not be +what you want - for example, when using Gmail-over-@abbr{IMAP}, this +interferes with Gmail's handling of the sent messages folder, and you +may end up with duplicate messages. + +You can use the variable @code{mu4e-sent-messages-behavior} to customize +what happens with sent messages. The default is the symbol @code{sent} +which, as mentioned, causes the message to be copied to your +sent-messages folder. Other possible values are the symbols @code{trash} +(the sent message is moved to the trash-folder +(@code{mu4e-trash-folder}), and @code{delete} to simply discard the sent +message altogether (so Gmail can deal with it). + +For Gmail-over-@abbr{IMAP}, you could add the following to your +settings: +@verbatim +;; don't save messages to Sent Messages, Gmail/IMAP takes care of this +(setq mu4e-sent-messages-behavior 'delete) +@end verbatim +And that's it! We should now be ready to go. + +For more complex needs, @code{mu4e-sent-messages-behavior} can also be +a parameter-less function that returns one of the mentioned symbols; +see the built-in documentation for the variable. + +@node Running mu4e +@section Running mu4e + +After following the steps in this chapter, we now (hopefully!) have a +working @t{mu4e} setup. Great! In the next chapters, we walk you +through the various views in @t{mu4e}. + +For your orientation, the diagram below shows how the views relate to each +other, and the default key-bindings to navigate between them. + +@cartouche +@verbatim + + [C] +--------+ [RFCE] + --------> | editor | <-------- + / +--------+ \ + / [RFCE]^ \ +/ | \ ++-------+ [sjbB]+---------+ [RET] +---------+ +| main | <---> | headers | <----> | message | ++-------+ [q] +---------+ [qbBjs] +---------+ + [sjbB] ^ +[.] | [q] + V + +-----+ + | raw | + +-----+ + +Default bindings +---------------- +R: Reply s: search .: raw view (toggle) +F: Forward j: jump-to-maildir q: quit +C: Compose b: bookmark-search +E: Edit B: edit bookmark-search + +@end verbatim +@end cartouche + +@node Main view +@chapter The main view + +After you have installed @t{mu4e} (@pxref{Getting started}), you can start it +with @kbd{M-x mu4e}. @t{mu4e} does some checks to ensure everything is set up +correctly, and then shows you the @t{mu4e} main view. Its major mode is +@code{mu4e-main-mode}. + +@menu +* Overview: MV Overview. What is the main view +* Basic actions::What can we do +* Bookmarks and Maildirs: Bookmarks and Maildirs. Jumping to other places +* Miscellaneous::Notes +@end menu + +@node MV Overview +@section Overview + +The main view looks something like the following: + +@cartouche +@verbatim +* mu4e - mu for emacs version x.y.z + + Basics + + * [j]ump to some maildir + * enter a [s]earch query + * [C]ompose a new message + + Bookmarks + + * [bu] Unread messages 13085(+3)/13085 + * [bt] Today's messages + * [bw] Last 7 days 53(+3)/128 + * [bp] Messages with images 75/2441 + + Maildirs + + * [ja] /archive 2101/18837 + * [ji] /inbox 8(+2)/10 + * [jb] /bulk 33/35 + * [jB] /bulkarchive 179/2090 + * [jm] /mu 694(+1)/17687 + * [jn] /sauron + * [js] /sent + + Misc + + * [;]Switch context + * [U]pdate email & database + * toggle [m]ail sending mode (currently direct) + * [f]lush 1 queued mail + + * [N]ews + * [A]bout mu4e + * [H]elp + * [q]uit + + Info + + * last-updated : Sat Dec 31 16:43:56 2022 + * database-path : /home/pam/.cache/mu/xapian + * maildir : /home/pam/Maildir + * in store : 86179 messages + * personal addresses : /.*example.com/, pam@@example.com +@end verbatim +@end cartouche + +Let's walk through the menu. + +@node Basic actions +@section Basic actions + +First, the @emph{Basics}: +@itemize +@item @t{[j]ump to some maildir}: after pressing @key{j} (``jump''), +@t{mu4e} asks you for a maildir to visit. These are the maildirs you +set in @ref{Basic configuration} and any of your own. If you choose +@key{o} (``other'') or @key{/}, you can choose from all maildirs under +the root-maildir. After choosing a maildir, the messages in that +maildir are listed, in the @ref{Headers view}. +@item @t{enter a [s]earch query}: after pressing @key{s}, @t{mu4e} asks +you for a search query, and after entering one, shows the results in the +@ref{Headers view}. +@item @t{[C]ompose a new message}: after pressing @key{C}, you are dropped in +the @ref{Composer} to write a new message. +@end itemize + +@node Bookmarks and Maildirs +@section Bookmarks and Maildirs + +The next two items in the Main view are @emph{Bookmarks} and @emph{Maildirs}. + +Bookmarks are predefined queries with a descriptive name and a shortcut. In the +example above, we see the default bookmarks. You can pick a bookmark by pressing +@key{b} followed by the specific bookmark's shortcut. If you want to edit the +bookmarked query before invoking it, use @key{B}. + +@cindex baseline +Next to each bookmark are some numbers that indicate the unread(delta)/all +matching messages for the given query, with the delta being the difference in +unread count since some ``baseline'', and only shown when this delta > 0. + +Note that the ``delta'' has its limitations: if you, for instance, deleted 5 +messages and received 5 new one, the ``delta'' would be 0, although there were +changes indeed. So it is mostly useful for tracking changes while you are +@emph{not} using @t{mu4e}. For this reason, you can reset the baseline manually, +e.g. by visiting the main view . + +By comparing current results with the baseline, you can quickly what new +messages have arrived since the last time you looked. + +The baseline@footnote{For debugging, it can be useful to see the time for the +baseline - for that, there is the @code{mu4e-baseline-time} command} . is reset +automatically when switching to the main view, or invoking @code{buffer-revert} +(@kbd{g}) while in the main-view. Visiting the ``favorite'' bookmark does the +same(explained below). + +Bookmarks are stored in the variable @code{mu4e-bookmarks}; you can add +your own and/or replace the default ones; @xref{Bookmarks}. For +instance: +@lisp +(add-to-list 'mu4e-bookmarks + ;; add bookmark for recent messages on the Mu mailing list. + '( :name "Mu7Days" + :key ?m + :query "list:mu-discuss.googlegroups.com AND date:7d..now")) +@end lisp + +There are optional keys @t{:hide} to hide the bookmark from the main menu, but +still have it available (using @key{b})) and @t{:hide-unread} to avoid +generating the unread-number; that can be useful if you have bookmarks for slow +queries. Note that @t{:hide-unread} is implied when the query is not a string; +this for the common case where the query function involves some user input, +which would be disruptive in this case. + +There is also the optional @code{:favorite} property, which at most one bookmark +should have; this bookmark is highlighted in the main view, and its +unread-status is shown in the modeline; @xref{Modeline}, and you can enable +desktop notifications; @xref{Desktop notifications}. We'd recommend creating +such a ``favorite'', which should match message that require your quick +attention: + +@lisp +(add-to-list 'mu4e-bookmarks + ;; bookmark for message that require quick attention + '( :name "Urgent" + :key ?u + :query "maildir:/inbox AND from:boss@@exmaple.com")) +@end lisp + +Note that @t{mu4e} resets the baseline when you are interacting with it (for +instance, when you visit the urgent bookmark, or when you go to the main view); +in such cases, there won't be any further notifications. + +The @emph{Maildirs} item is very similar to Bookmarks -- consider maildirs here +as being a special kind of bookmark query that matches a Maildir. You can +configure this using the variable @code{mu4e-maildir-shortcuts}; see its +docstring and @ref{Maildir searches} for more details. + +@node Miscellaneous +@section Miscellaneous + +Finally, there are some @emph{Misc} (miscellaneous) actions: +@itemize +@item @t{[U]pdate email & database} executes the shell-command in the variable +@code{mu4e-get-mail-command}, and afterwards updates the @t{mu} +database; see @ref{Indexing your messages} and @ref{Getting mail} for +details. +@item @t{[R]eset query-results baseline} this reset the current 'baseline' +for query and updates the screen; see @ref{Bookmarks and Maildirs}. +@item @t{toggle [m]ail sending mode (direct)} toggles between sending +mail directly, and queuing it first (for example, when you are offline), +and @t{[f]lush queued mail} flushes any queued mail. This item is +visible only if you have actually set up mail-queuing. @ref{Queuing +mail} +@item @t{[A]bout mu4e} provides general information about the program +@item @t{[H]elp} shows help information for this view +@item Finally, @t{[q]uit mu4e} quits your @t{mu4e}-session@footnote{@t{mu4e-quit}; or with a @t{C-u} +prefix argument, it merely buries the buffer} +@end itemize + +@node Headers view +@chapter The headers view + +The headers view shows the results of a query. The header-line shows the names +of the fields. Below that, there is a line with those fields, for each matching +message, followed by a footer line. The major-mode for the headers view is +@code{mu4e-headers-mode}. + +@menu +* Overview: HV Overview. What is the Header View +* Keybindings::Do things with your keyboard +* Marking: HV Marking. Selecting messages for doing things +* Sorting and threading::Influencing how headers are shown +* Folding threads:: Showing and hiding thread contents +* Custom headers: HV Custom headers. Adding your own headers +* Actions: HV Actions. Defining and using actions +* Buffer display:: How and where the buffers are displayed +@end menu + +@node HV Overview +@section Overview + +An example headers view: +@cartouche +@verbatim +Date V Flgs From/To List Subject +06:32 Nu To Edmund Dantès GstDev Gstreamer-V4L2SINK ... +15:08 Nu Abbé Busoni GstDev ├> ... +18:20 Nu Pierre Morrel GstDev │└> ... +07:48 Nu To Edmund Dantès GstDev └> ... +2013-03-18 S Jacopo EmacsUsr emacs server on win... +2013-03-18 S Mercédès EmacsUsr └> ... +2013-03-18 S Beachamp EmacsUsr Re: Copying a whole... +22:07 Nu Albert de Moncerf EmacsUsr └> ... +2013-03-18 S Gaspard Caderousse GstDev Issue with GESSimpl... +2013-03-18 Ss Baron Danglars GuileUsr Guile-SDL 0.4.2 ava... +End of search results +@end verbatim +@end cartouche + +Some notes to explain what you see in the example: + +@itemize +@item The fields shown in the headers view can be influenced by customizing +the variable @code{mu4e-headers-fields}; see @code{mu4e-header-info} for +the list of built-in fields. Apart from the built-in fields, you can +also create custom fields using @code{mu4e-header-info-custom}; see +@ref{HV Custom headers} for details. +@item By default, the date is shown with the @t{:human-date} field, which +shows the @emph{time} for today's messages, and the @emph{date} for +older messages. If you do not want to distinguish between `today' and +`older', you can use the @t{:date} field instead. +@item You can customize the date and time formats with the variable +@code{mu4e-headers-date-format} and @code{mu4e-headers-time-format}, +respectively. In the example, we use @code{:human-date}, which shows the +time when the message was sent today, and the date otherwise. +@item By default, the subject is shown using the @t{:subject} field; +however, it is also possible to use @t{:thread-subject}, which shows +the subject of a thread only once, similar to the display of the +@t{mutt} e-mail client. +@item The header field used for sorting is indicated by ``@t{V}'' or +``@t{^}''@footnote{or you can use little graphical triangles; see +variable @code{mu4e-use-fancy-chars}}, corresponding to the sort order +(descending or ascending, respectively). You can influence this by a +mouse click, or @key{O}. Not all fields allow sorting. +@item Instead of showing the @t{From:} and @t{To:} fields separately, you +can use From/To (@t{:from-or-to} in @code{mu4e-headers-fields} as a more +compact way to convey the most important information: it shows @t{From:} +@emph{except} when the e-mail was sent by the user (i.e., you) --- in +that case it shows @t{To:} (prefixed by @t{To}@footnote{You can +customize this by changing the variable +@code{mu4e-headers-from-or-to-prefix} (a cons cell)}, as in the example +above). +@item The `List' field shows the mailing-list a message is sent to; +@code{mu4e} tries to create a convenient shortcut for the mailing-list +name; the variable @code{mu4e-user-mailing-lists} can be used to add +your own shortcuts. You can use @code{mu4e-mailing-list-patterns} to +specify generic shortcuts. For instance, to shorten list names to the +part before @t{-list}, you could use: +@lisp +(setq mu4e-mailing-list-patterns '("\\`\\([-_a-z0-9.]+\\)-list")) +@end lisp +@item The letters in the `Flags' field correspond to the following: D=@emph{draft}, +F=@emph{flagged} (i.e., `starred'), N=@emph{new}, P=@emph{passed} (i.e., +forwarded), R=@emph{replied}, S=@emph{seen}, T=@emph{trashed}, +a=@emph{has-attachment}, x=@emph{encrypted}, s=@emph{signed}, +u=@emph{unread}. The tooltip for this field also contains this information. +@item The subject field also indicates the discussion threads, following +@uref{https://www.jwz.org/doc/threading.html,Jamie Zawinski's mail threading +algorithm}. +@item The headers view is @emph{automatically updated} if any changes are +found during the indexing process, and if there is no current +user-interaction. If you do not want such automatic updates, set +@code{mu4e-headers-auto-update} to @code{nil}. +@item Just before executing a search, a hook-function +@code{mu4e-search-hook} is invoked, which receives the search +expression as its parameter. +@item Also, there is a hook-function @code{mu4e-headers-found-hook} available which +is invoked just after @t{mu4e} has completed showing the messages in the +headers-view. +@end itemize + +@node Keybindings +@section Keybindings + +Using the below key bindings, you can do various things with these +messages; these actions are also listed in the @t{Headers} menu in the +Emacs menu bar. + +@verbatim +key description +=========================================================== +n,p view the next, previous message +],[ move to the next, previous unread message +},{ move to the next, previous thread +y select the message view (if visible) +RET open the message at point in the message view + +searching +--------- +s search +S edit last query +/ narrow the search +b search bookmark +B edit bookmark before search +c search query with completion +j jump to maildir +M-left,\ previous query +M-right next query + +O change sort order +P toggle search property + +marking +------- +d mark for moving to the trash folder += mark for removing trash flag ('untrash') +DEL,D mark for complete deletion +m mark for moving to another maildir folder +r mark for refiling ++,- mark for flagging/unflagging +?,! mark message as unread, read + +u unmark message at point +U unmark *all* messages + +% mark based on a regular expression +T,t mark whole thread, subthread + +<insert>,* mark for 'something' (decide later) +# resolve deferred 'something' marks + +x execute actions for the marked messages + +threads +------- +S-left goto root +TAB toggle threading at current level +S-TAB toggle all threading + +composition +----------- +R,W,F,C reply/reply-to-all/forward/compose +E edit (only allowed for draft messages) + +misc +---- +a execute some custom action on a header +| pipe message through shell command +C-+,C-- increase / decrease the number of headers shown +H get help +C-S-u update mail & reindex +C-c C-u update mail & reindex +q leave the headers buffer +@end verbatim + +Some keybindings are available through minor modes: +@itemize +@item Context; see @pxref{Contexts}. +@item Composition; see @pxref{Composer} and @t{mu4e-compose-minor-mode} +@end itemize + +@node HV Marking +@section Marking + +You can @emph{mark} messages for a certain action, such as deletion or +move. After one or more messages are marked, you can then execute +(@code{mu4e-mark-execute-all}, @key{x}) these actions. This two-step +mark-execute sequence is similar to what e.g. @t{dired} does. It is how +@t{mu4e} tries to be as quick as possible, while avoiding accidents. + +The mark/unmark commands support the @emph{region} (i.e., ``selection'') +--- so, for example, if you select some messages and press @key{DEL}, +all messages in the region are marked for deletion. + +You can mark all messages that match a certain pattern with @key{%}. In +addition, you can mark all messages in the current thread (@key{T}) or +sub-thread (@key{t}). + +When you do a new search or refresh the headers buffer while you still +have marked messages, you are asked what to do with those marks --- +whether to @emph{apply} them before leaving, or @emph{ignore} them. This +behavior can be influenced with the variable +@code{mu4e-headers-leave-behavior}. + +For more information about marking, see @ref{Marking}. + +@node Sorting and threading +@section Sorting and threading + +By default, @t{mu4e} sorts messages by date, in descending order: the +most recent messages are shown at the top. In addition, be default +@t{mu4e} shows the message @emph{threads}, i.e., the tree structure +representing a discussion thread; this also affects the sort order: +the top-level messages are sorted by the date of the @emph{newest} +message in the thread. + +The header field used for sorting is indicated by ``@t{V}'' or +``@t{^}''@footnote{or you can use little graphical triangles; see +variable @code{mu4e-use-fancy-chars}}, indicating the sort order +(descending or ascending, respectively). + +You can change the sort order by clicking the corresponding column with the +mouse, or with @kbd{M-x mu4e-headers-change-sorting} (@key{O}); note that not +all fields can be used for sorting. You can toggle threading on/off through +@kbd{M-x mu4e-headers-toggle-property} or @key{Pt}. For both of these functions, +unless you provide a prefix argument (@key{C-u}), the current search is updated +immediately using the new parameters. You can toggle full-search +(@ref{Searching}) through @kbd{M-x mu4e-headers-toggle-property} as well; or +@key{Pf}. + +Note that with threading enabled, the sorting is exclusively by date, +regardless of the column clicked. + +If you want to change the defaults for these settings, you can use the variables +@code{mu4e-search-sort-field} and @code{mu4e-search-show-threads}, as well as +@code{mu4e-search-change-sorting} to change the sorting of the current search +results. + +@node Folding threads +@section Folding threads + +It is possible to fold threads - that is, visually collapse threads into a +single line (and the reverse), by default using the @key{TAB} and @key{S-TAB} +bindings. Note that the collapsing is always for threads as a whole, not for +sub-threads. + +Folding stops at the @emph{first unread message}, unless you set +@code{mu4e-thread-fold-unread}. Similarly, when a thread has marked messages, +the folding stops at the first marked message. Marking folded messages is not +allowed as it is too error-prone. + +Thread-mode functionality is only available with @code{mu4e-search-threads} +enabled; this triggers a minor mode @code{mu4e-thread-mode} in the headers-view. +For now, this functionality is not available in the message view, due to the +conflicting key bindings. + +If you want to automatically fold all threads after a query, you can use a hook: +@lisp + (add-hook 'mu4e-thread-mode-hook #'mu4e-thread-fold-apply-all) +@end lisp + +By default, single-child threads are @emph{not} collapsed, since it would result +in replacing a single line with the collapsed one. However, if, for consistency, +you also want to fold those, you can use @t{mu4e-thread-fold-single-children}. + +@node HV Custom headers +@section Custom headers + +Sometimes the normal headers that @t{mu4e} offers (Date, From, To, +Subject, etc.)@: may not be enough. For these cases, @t{mu4e} offers +@emph{custom headers} in both the headers-view and the message-view. + +You can do so by adding a description of your custom header to +@code{mu4e-header-info-custom}, which is a list of custom headers. + +Let's look at an example --- suppose we want to add a custom header that +shows the number of recipients for a message, i.e., the sum of the +number of recipients in the @t{To:} and @t{Cc:} fields. Let's further +suppose that our function takes a message-plist as its argument +(@ref{Message functions}). + +@lisp +(add-to-list 'mu4e-header-info-custom + '(:recipnum . + ( :name "Number of recipients" ;; long name, as seen in the message-view + :shortname "Recip#" ;; short name, as seen in the headers view + :help "Number of recipients for this message" ;; tooltip + :function (lambda (msg) + (format "%d" + (+ (length (mu4e-message-field msg :to)) + (length (mu4e-message-field msg :cc)))))))) +@end lisp + +Or, let's get the contents of the Jabber-ID header. + +@lisp +(add-to-list 'mu4e-header-info-custom + '(:jabber-id . + ( :name "Jabber-ID" ;; long name, as seen in the message-view + :shortname "JID" ;; short name, as seen in the headers view + :help "The Jabber ID" ;; tooltip + ;; uses mu4e-fetch-field which is rel. slow, so only appropriate + ;; for mu4e-view-fields, and _not_ mu4e-headers-fields + :function (lambda (msg) + (or (mu4e-fetch-field msg "Jabber-ID") ""))))) +@end lisp + +You can then add the custom header to your @code{mu4e-headers-fields} or +@code{mu4e-view-fields}, just like the built-in headers. However, there is an +important caveat: when your custom header in @code{mu4e-headers-fields}, the +function is invoked for each of your message headers in search results, and if +it is slow, would dramatically slow down @t{mu4e}. + +@node HV Actions +@section Actions + +@code{mu4e-headers-action} (@key{a}) lets you pick custom actions to perform +on the message at point. You can specify these actions using the variable +@code{mu4e-headers-actions}. See @ref{Actions} for the details. + +@t{mu4e} defines some default actions. One of those is for @emph{capturing} a +message: @key{a c} `captures' the current message. Next, when you're editing +some message, you can include the previously captured message as an +attachment, using @code{mu4e-compose-attach-captured-message}. See +@file{mu4e-actions.el} in the @t{mu4e} source distribution for more example +actions. + +@node Buffer display +@section Buffer display + +By default, @t{mu4e} will attempt to manage the display of its own buffers. For +headers and message views, the variable @code{mu4e-split-view} is @t{mu4e's} +built-in way to decide how and where they are shown. + +@subsection Split view +You can control how @t{mu4e} displays its buffers, including the @ref{Headers +view} and the @ref{Message view}, by customizing @code{mu4e-split-view}. There +are several options available: + +@itemize +@item @t{horizontal} (this is the default): display the message view below the +header view. Use @code{mu4e-headers-visible-lines} the set the number of +lines shown (default: 8). +@item @t{vertical}: display the message view on the +right side of the header view. Use @code{mu4e-headers-visible-columns} to set +the number of visible columns (default: 30). +@item @t{single-window}: single window mode. Single-window mode tries to +minimize mu4e window operations (opening, killing, resizing, etc) and buffer +changes, while still retaining the view and headers buffers. In addition, it +replaces @t{mu4e}'s main view with a minibuffer-prompt containing the same +information. +@item anything else: prefer reusing the same window, where possible. +@end itemize + +Note that using a window-returning @emph{function} for @code{mu4e-split-view} is +no longer supported, instead you can use @code{display-buffer-alist}, see +the section on further display customization. + +@noindent +Some useful key bindings in the split view: +@itemize +@item @key{C-+} and @key{C--}: interactively change the number of columns or +headers shown +@item You can change the selected window from the +headers-view to the message-view and vice-versa with +@code{mu4e-select-other-view}, bound to @key{y} +@end itemize + +@subsection Further customization + +However, @t{mu4e}'s display rules are provisional; you can override them +easily by customizing @code{display-buffer-alist}, which governs how Emacs -- +and thus @t{mu4e} -- must display your buffers. + +Let's look at some examples. + +@subsection Fine-tuning the main buffer display + +By default @t{mu4e}'s main buffer occupies the complete frame, but this can be +changed to use the current window: + +@lisp +(add-to-list 'display-buffer-alist + `(,(regexp-quote mu4e-main-buffer-name) + display-buffer-same-window)) +@end lisp + +@subsection Fine-tuning headers buffer display + +You do not need to configure @code{mu4e-split-view} for this to work. In the +absence of explicit rules to the contrary, @t{mu4e} will fall back on the value +you have set in @code{mu4e-split-view}. + +Here is an example that displays the headers buffer in a side window to the +right. It occupies half of the width of the frame. + +@lisp +(add-to-list 'display-buffer-alist + `(,(regexp-quote mu4e-headers-buffer-name) + display-buffer-in-side-window + (side . right) + (window-width . 0.5))) +@end lisp + +You can type @key{C-x w s} to toggle the side windows to hide or show them at +will. + +Note that you may need to customize @code{mu4e-view-rendered-hook} as well; by +default it contains @code{mu4e-resize-linked-headers-window} but you can set it +to @code{nil} if you want to handle manually (through +@code{display-buffer-alist}. + +@node Message view +@chapter The message view + +This chapter discusses the message view, the view for reading e-mail messages. + +After selecting a message in the @ref{Headers view}, it appears in a +message view window, which shows the message headers, followed by the +message body. Its major mode is @code{mu4e-view-mode}, which derives +from @t{gnus-article-mode}. + +@menu +* Overview: MSGV Overview. What is the Message View +* Keybindings: MSGV Keybindings. Do things with your keyboard +* Rich-text and images: MSGV Rich-text and images. Reading rich-text messages +* Attachments and MIME-parts: MSGV Attachments and MIME-parts. Working with attachments and other MIME parts +* Custom headers: MSGV Custom headers. Your very own headers +* Actions: MSGV Actions. Defining and using actions +* Detaching & reattaching: MSGV Detaching and reattaching. Multiple message views. +@end menu + +@node MSGV Overview +@section Overview + +An example message view: + +@cartouche +@verbatim + From: randy@epiphyte.com + To: julia@eruditorum.org + Subject: Re: some pics + Flags: seen, attach + Date: Thu, 11 Feb 2021 12:59:30 +0200 (4 weeks, 3 days, 21 hours ago) + Maildir: /inbox + Attachments: [2. image/jpeg; DSCN4961.JPG]... [3. image/jpeg; DSCN4962.JPG]... + + Hi Julia, + + Some pics from our trip to Cerin Amroth. Enjoy! + + All the best, + Randy. + + On Sun 21 Dec 2003 09:06:34 PM EET, Julia wrote: + + [....] +@end verbatim +@end cartouche + +Some notes: +@itemize +@item The variable @code{mu4e-view-fields} determines the header fields to be +shown; see @code{mu4e-header-info} for a list of built-in fields. Apart +from the built-in fields, you can also create custom fields using +@code{mu4e-header-info-custom}; see @ref{MSGV Custom headers}. +@item For search-related operations, see @ref{Searching}. +@item You can scroll down the message using @key{SPC}; if you do this at the +end of a message,it automatically takes you to the next one. If you want +to prevent this behavior, set @code{mu4e-view-scroll-to-next} to +@code{nil}. +@end itemize + +@node MSGV Keybindings +@section Keybindings + +You can find most things you can do with this message in the @emph{View} menu, +or by using the keyboard; the default bindings are: + +@verbatim +key description +============================================================== +n,p view the next, previous message +],[ move to the next, previous unread message +},{ move to the next, previous thread +y select the headers view (if visible) + +RET scroll down +M-RET open URL at point / attachment at point + +SPC scroll down, if at end, move to next message +S-SPC scroll up + +searching +--------- +s search +S edit last query +/ narrow the search +b search bookmark +B edit bookmark before search +c search query with completion +j jump to maildir + +O change sort order +P toggle search property + +M-left previous query +M-right next query + +marking messages +---------------- +d mark for moving to the trash folder += mark for removing trash flag ('untrash') +DEL,D mark for complete deletion +m mark for moving to another maildir folder +r mark for refiling ++,- mark for flagging/unflagging + +u unmark message at point +U unmark *all* messages + +% mark based on a regular expression +T,t mark whole thread, subthread + +<insert>,* mark for 'something' (decide later) +# resolve deferred 'something' marks + +x execute actions for the marked messages + +composition +----------- +R,W,F,C reply/reply-to-all/forward/compose +E edit (only allowed for draft messages) + +actions +------- +g go to (visit) numbered URL (using `browse-url') +(or: <mouse-2> or M-RET with point on URL) +C-u g visits multiple URLs +f fetch (download )the numbered URL. +C-u f fetches multiple URLs +k save the numbered URL in the kill-ring. +C-u k saves multiple URLs + +e extract (save) one or more attachments (asks for numbers) +(or: <mouse-2> or S-RET with point on attachment) +a execute some custom action on the message +A execute some custom action on the message's MIME-parts + +misc +---- +z, Z detach (or reattach) a message view to a headers buffer +. show the raw message view. 'q' takes you back. +C-+,C-- increase / decrease the number of headers shown +H get help +C-S-u update mail & reindex +q leave the message view +@end verbatim + +Some keybindings are available through minor modes: +@itemize +@item Context; see @pxref{Contexts} +@item Composition; see @pxref{Composer} and @t{mu4e-compose-minor-mode} +@end itemize + +For the marking commands, please refer to @ref{Marking messages}. + +@node MSGV Rich-text and images +@section Reading rich-text messages +@cindex rich-text + +These days, many e-mail messages contain rich-text (typically, HTML); +either as an alternative to a text-only version, or even as the only +option. + +By default, mu4e tries to display the 'richest' option, which is the +last MIME-part of the alternatives. You can customize this to prefer +the text version, if available, with something like the following in +your configuration (and see the docstring for +@t{mm-discouraged-alternatives} for details): + +@lisp +(with-eval-after-load "mm-decode" + (add-to-list 'mm-discouraged-alternatives "text/html") + (add-to-list 'mm-discouraged-alternatives "text/richtext")) +@end lisp + +When displaying rich-text messages inline, @t{mu4e} (through @t{gnus}) +uses the @t{shr} built-in HTML-renderer. If you're using a dark color +theme, and the messages are hard to read, it can help to change the +luminosity, e.g.: +@lisp +(setq shr-color-visible-luminance-min 80) +@end lisp + +Note that you can switch between the HTML and text versions by +clicking on the relevant part in the messages headers; you can make it +even clearer by indicating them in the message itself, using: + +@lisp +(setq gnus-unbuttonized-mime-types nil) +@end lisp + +@subsection Inline images +When you run Emacs in graphical mode, by default images attached to +messages are shown inline in the message view buffer. + +To disable this, set @code{gnus-inhibit-images} to @t{t}. By default, +external images in HTML are not retrieved from external URLs because +they can be used to track you. + +Apart from that, you can also control whether to load remote images; +since loading remote images is often used for privacy violations, by +default this is not allowed. + +You can specify what URLs to block by setting +@code{gnus-blocked-images} to a regular expression or to a function +that will receive a single parameter which is not meaningful for +@t{mu4e}. + +For example, to enable images in Github notifications, you could use +the following: + +@lisp +(setq gnus-blocked-images + (lambda(&optional _ignore) + (if (mu4e-message-contact-field-matches + (mu4e-message-at-point) :from "notifications@@github.com") + nil "."))) +@end lisp + +@code{mu4e} inherits the default @t{gnus-blocked-images} from Gnus and +ensures that it works with @t{mu4e} too. However, mu4e is not Gnus, so +if you have Gnus-specific settings for @t{gnus-blocked-images}, you +should verify that they have the desired effect in @code{mu4e} as +well. + +@node MSGV Attachments and MIME-parts +@section Attachments and MIME-parts +@cindex attachments +@cindex mime-parts + +E-mail messages can be though as a series of ``MIME-parts'', which are sections +of the message. The most prominent is the 'body', that is the main message your +are reading. Many e-mail messages also contains @emph{attachments}, which +MIME-parts that contain files@footnote{Attachments come in two flavors: +@c{inline} and @c{attachment}. @t{mu4e} does not distinguish between them when +operating on them; everything that specifies a filename is considered an +attachment}. + +To save such attachments as files on your file systems, the @t{mu4e} +message-view offers the command @code{mu4e-view-save-attachments}; default +keybinding is @key{e} (think @emph{extract}). After invoking the command, you +can enter the file names to save, comma-separated, and using the completion +support. Press @key{RET} to save the chosen files to your file-system. + +With a prefix argument, you get to choose the target-directory, otherwise, +@t{mu4e} determines it following the variable @t{mu4e-attachment-dir} (which can +be file-system path or a function; see its docstring for details. + +While completing, @code{mu4e-view-completion-minor-mode} is active, which offers +@code{mu4e-view-complete-all} (bound to @key{C-c C-a} to complete @emph{all} +files@footnote{Except when using 'Helm'; in that case, use the Helm-mechanism +for selecting multiple}. + +@subsection MIME-parts + +Not all MIME-parts are message bodies or attachments, and it can be useful to +operate on those other parts as well. For that, there is the function +@code{mu4e-view-mime-part-action} (default key-binding @key{A}). You can pass +the number of the MIME-pars (as seen in the message view) as a prefix argument, +otherwise you get to get to choose from a completion menu. + +After choosing one or more MIME-parts, you are asked for an action to apply to +them; see the variable @code{mu4e-view-mime-part-actions} for the possibilities; +and you can add your own actions as well, see @ref{MIME-part actions} for some +example. + +@node MSGV Custom headers +@section Custom headers +@cindex custom headers + +Sometimes the normal headers (Date, From, To, Subject, etc.)@: may not be +enough. For these cases, @t{mu4e} offers @emph{custom headers} in both the +headers-view and the message-view. + +See @ref{HV Custom headers} for an example of this; the difference for +the message-view is that you should add your custom header to +@code{mu4e-view-fields} rather than @code{mu4e-headers-fields}. + +@node MSGV Actions +@section Actions + +You can perform custom functions (``actions'') on messages and their +attachments. For a general discussion on how to define your own, see +@ref{Actions}. + +@subsection Message actions +@code{mu4e-view-action} (@key{a}) lets you pick some custom action to perform +on the current message. You can specify these actions using the variable +@code{mu4e-view-actions}; @t{mu4e} defines a number of example actions. + +@subsection MIME-part actions +MIME-part actions allow you to act upon MIME-parts in a message - such +as attachments. For now, these actions are defined and documented in +@code{mu4e-view-mime-part-actions} and see + +@node MSGV Detaching and reattaching +@section Detaching and reattaching messages + +You can have multiple message views, but you must rename the view +buffer and detach it to stop @t{mu4e} from reusing it when you +navigate up or down in the headers buffer. If you have several view +buffers attached to a headers view, then @t{mu4e} may pick one at +random when it has to choose which one to display a message in. + +To detach the message view from its linked headers buffer, type +@key{z}. A message will appear saying it is detached (or warn you if +it is already detached.) + +Detached buffers are static; they cannot change the displayed message, +and no headers buffer will use a detached buffer to display its +messages. You can reattach a buffer to an live headers buffer by +typing @key{Z}. + +You can freely rename a message view buffer -- such as with @key{C-x x +r} -- if you want a custom, non-randomized name. + +Detached messages are often useful for workflows involving lots of +simultaneous messages. + +You can @emph{tear off} the window a message is in and place it in a +new frame by typing @key{C-x w ^ f}. You can also detach a window and +put it in its own tab with @key{C-x w ^ t}. + +@node Composer +@chapter Composer + +Writing e-mail messages takes place in the Composer. @t{mu4e}'s re-uses much of +Gnus' @t{message-mode}. + +Much of the @t{message-mode} functionality is available, as well some +@t{mu4e}-specifics. See @ref{(message) Top} for details; not every setting is +necessarily also supported in @t{mu4e}. + +The major mode for the composer is @code{mu4e-compose-mode}. + +@menu +* Composer overview: Composer overview. What is the composer good for +* Entering the composer:: How to start writing messages +* Keybindings: Composer Keybindings. Doing things with your keyboard +* Address autocompletion:: Quickly entering known addresses +* Compose hooks::Calling functions when composing +* Signing and encrypting:: Support for cryptography +* Queuing mail:: Sending mail when the time is ripe +* Message signatures:: Adding your personal footer to messages +* Other settings::Miscellaneous +@end menu + +@node Composer overview +@section Overview + +@cartouche +@verbatim + From: Rupert the Monkey <rupert@example.com> + To: Wally the Walrus <wally@example.com> + Subject: Re: Eau-qui d'eau qui? + --text follows this line-- + + On Mon 16 Jan 2012 10:18:47 AM EET, Wally the Walrus wrote: + + > Hi Rupert, + > + > Dude - how are things? + > + > Later -- Wally. +@end verbatim +@end cartouche + +@node Entering the composer +@section Entering the composer + +There are a view different ways to @emph{enter} the composer; i.e., from other +@t{mu4e} views or even completely outside. + +If you want the composer to start in a new frame or window, you can configure +the variable @t{mu4e-compose-switch}; see its docstring for details. + +@subsection New message + +You can start composing a completely new message with @t{mu4e-compose-new} (with +@kbd{N} from within @t{mu4e}. + +@subsection Reply + +You can compose a reply to an existing message with @t{mu4e-compose-reply} (with +@kbd{R} from within the headers view or when looking at some specific message. + +When you want to reply to @emph{all} recipients of a message, you can use +@t{mu4e-compose-wide-reply}, bound to @kbd{W}. This is often called +``reply-to-all'', while Gnus uses the term ``wide reply''. + +By default, the reply will cite the message being replied to. If you do not want +that, you can set (or @t{let}-bind) @t{message-cite-function} to +@t{mu4e-message-cite-nothing}. + +See @ref{(message) Reply} and @ref{(message) Wide Reply} for further +information. + +Note: in older versions, @t{mu4e-compose-reply} would @emph{ask} whether you +want to reply-to-all or not; if you are nostalgic for that old behavior, you +could add something like the following to your configuration: +@lisp +(defun compose-reply-wide-or-not-please-ask () + "Ask whether to reply-to-all or not." + (interactive) + (mu4e-compose-reply (yes-or-no-p "Reply to all?"))) + +(define-key mu4e-compose-minor-mode-map (kbd "R") + #'compose-reply-wide-or-not-please-ask) +@end lisp + +@subsection Forward + +You can forward some existing message with @t{mu4e-compose-forward} (with +@kbd{F} from within the headers view or when looking at some specific message. + +For more information, see @ref{(message) Forwarding}. + +To influence the way a message is forwarded, you can use the variables +@code{message-forward-as-mime} and @code{message-forward-show-mml}. + +@subsection Supersede + +Occasionally, it can be useful to ``supersede'' a message you sent; this drops +you into a new message that is just like the old message (and a @t{Supersedes:} +message header). You can then edit this message and send it. + +This is only possible for messages @emph{you} sent, as determined by +@code{mu4e-personal-or-alternative-address-p}. + +This wraps @code{message-supersede}. + +@subsection Resend + +You can re-send some existing message with @t{mu4e-compose-resend} from within +the headers view or when looking at some specific message. + +This re-sends the message without letting you edit it, as per @ref{(message) +Resending}. + + +@node Composer Keybindings +@section Keybindings + +@t{mu4e}'s composer derives from Gnus' message editor and shares most of +its keybindings. Here are some of the more useful ones (you can use the menu +to find more): + +@verbatim +key description +--- ----------- +C-c C-c send message +C-c C-d save to drafts and leave +C-c C-k kill the message buffer (the message remains in the draft folder) +C-c C-a attach a file (pro-tip: drag & drop works as well in graphical context) +C-c C-; switch the context + +(mu4e-specific) +C-S-u update mail & re-index +@end verbatim + +@node Address autocompletion +@section Address autocompletion + +@t{mu4e} supports autocompleting addresses when composing e-mail messages. +@t{mu4e} uses the e-mail addresses from the messages you sent or received as the +source for this. Address auto-completion is enabled by default; if you want to +disable it for some reason, set @t{mu4e-compose-complete-addresses} to @t{nil}. + +This uses the Emacs machinery for showing and cycling through the candidate +addresses; it is active when looking at one of the contact fields in the message +header area. + +It is also possible to use @t{mu4e}'s completion elsewhere in @t{emacs}. To +enable that, a function @t{mu4e-complete-contact} exists, which you can add to +@t{completion-at-point-functions}, see @ref{(elisp) Completion in Buffers}. +@t{mu4e} must be running for any completions to be available. + +@subsection Limiting the number of addresses + +If you have a lot of mail, especially from mailing lists and the like, there +can be a @emph{lot} of e-mail addresses, many of which may not be very useful +when auto-completing. For this reason, @t{mu4e} attempts to limit the number +of e-mail addresses in the completion pool by filtering out the ones that are +not likely to be relevant. The following variables are available for tuning +this: + +@itemize +@item @code{mu4e-compose-complete-only-personal} --- when set to @t{t}, +only consider addresses that were seen in @emph{personal} messages --- +that is, messages in which one of my e-mail addresses was seen in one +of the address fields. This is to exclude mailing list posts. You can +define what is considered `my e-mail address' using the +@t{--my-address} parameter to @t{mu init}. + +@item @code{mu4e-compose-complete-only-after} --- only consider e-mail +addresses last seen after some date. Parameter is a string, parseable by +@code{org-parse-time-string}. This excludes old e-mail addresses. The +default is @t{"2010-01-01"}, i.e., only consider e-mail addresses seen +since the start of 2010. +@item @code{mu4e-compose-complete-max} -- the maximum number of contacts to use. +This adds a hard limit to the 2000 (default) contacts; those are sorted by +recency / frequency etc. so should include the ones you most likely need. +@item @code{mu4e-contact-process-function} --- a function to rewrite or +exclude certain addresses. +@end itemize + +@node Compose hooks +@section Compose hooks + +If you want to change some setting, or execute some custom action before +message composition starts, you can define a @emph{hook function}. @t{mu4e} +offers two hooks: +@itemize +@item @code{mu4e-compose-pre-hook}: this hook is run @emph{before} composition +starts; if you are composing a @emph{reply}, @emph{forward} a message, or +@emph{edit} an existing message, the variable +@code{mu4e-compose-parent-message} points to the message being replied to, +forwarded or edited, and you can use @code{mu4e-message-field} to get the +value of various properties (and see @ref{Message functions}). +@item @code{mu4e-compose-mode-hook}: this hook is run just before composition +starts, when the whole buffer has already been set up. This is a good place +for editing-related settings. @code{mu4e-compose-parent-message} (see above) +is also at your disposal. +@item @code{mu4e-compose-post-hook}: this hook is run when we're done with +message compositions. See the docstring for details. +@end itemize + +@noindent +As mentioned, @code{mu4e-compose-mode-hook} is especially useful for +editing-related settings: + +Let's look at an example: +@lisp +(add-hook 'mu4e-compose-mode-hook + (defun my-do-compose-stuff () + "My settings for message composition." + (set-fill-column 72) + (flyspell-mode))) +@end lisp + +The hook is also useful for adding headers or changing headers, since the +message is fully formed when this hook runs. For example, to add a +@t{Bcc:}-header, you could add something like the following, using +@code{message-add-header} from @code{message-mode}. + +@lisp +(add-hook 'mu4e-compose-mode-hook + (defun my-add-bcc () + "Add a Bcc: header." + (save-excursion (message-add-header "Bcc: me@@example.com\n")))) +@end lisp + +Or to something context-specific: + +@lisp +(add-hook 'mu4e-compose-mode-hook + (lambda() + (let* ((ctx (mu4e-context-current)) + (name (if ctx (mu4e-context-name ctx)))) + (when name + (cond + ((string= name "account1") + (save-excursion (message-add-header "Bcc: account1@@example.com\n"))) + ((string= name "account2") + (save-excursion (message-add-header "Bcc: account2@@example.com\n")))))))) +@end lisp + +@noindent +For a more general discussion about extending @t{mu4e}, see @ref{Extending +mu4e}. + +@node Signing and encrypting +@section Signing and encrypting + +Signing and encrypting of messages is possible using @ref{(emacs-mime) Top, +emacs-mime}, most easily accessed through the @t{Attachments}-menu while +composing a message, or with @kbd{M-x mml-secure-message-encrypt-pgp}, @kbd{M-x +mml-secure-message-sign-pgp}. + +Important note: the messages are encrypted when they are @emph{sent}: this means +that draft messages are @emph{not} encrypted. So if you are using e.g. +@t{offlineimap} or @t{mbsync} to synchronize with some remote IMAP-service, make +sure the drafts folder is @emph{not} in the set of synchronized folders, for +obvious reasons. + +@node Queuing mail +@section Queuing mail + +If you cannot send mail right now, for example because you are +currently offline, you can @emph{queue} the mail, and send it when you +have restored your internet connection. You can control this from the +@ref{Main view}. + +To allow for queuing, you need to tell @t{smtpmail} where you want to store +the queued messages. For example: + +@lisp +(setq smtpmail-queue-mail t ;; start in queuing mode + smtpmail-queue-dir "~/Maildir/queue/cur") +@end lisp + +For convenience, we put the queue directory somewhere in our normal +maildir. If you want to use queued mail, you should create this directory +before starting @t{mu4e}. The @command{mu mkdir} command may be useful here, +so for example: + +@verbatim + $ mu mkdir ~/Maildir/queue + $ touch ~/Maildir/queue/.noindex +@end verbatim + +The file created by the @command{touch} command tells @t{mu} to ignore this +directory for indexing, which makes sense since it contains @t{smtpmail} +meta-data rather than normal messages; see the @t{mu-mkdir} and @t{mu-index} +man-pages for details. + +@emph{Warning}: when you switch on queued-mode, your messages @emph{won't} +reach their destination until you switch it off again; so, be careful not to +do this accidentally! + +@node Message signatures +@section Message signatures + +Message signatures are the standard footer blobs in e-mail messages where you +can put in information you want to include in every message. The text to include +is set with @code{message-signature} (older @t{mu4e} used +@code{mu4e-compose-signature}, but that has been obsoleted). + +@node Other settings +@section Other settings + +@itemize +@item If you want use @t{mu4e} as Emacs' default program for sending mail, +see @ref{Default email client}. +@item Normally, @t{mu4e} @emph{buries} the message buffer after sending; if you want +to kill the buffer instead, add something like the following to your +configuration: +@lisp +(setq message-kill-buffer-on-exit t) +@end lisp +@item If you want to exclude your own e-mail addresses when ``replying to +all'', set @code{message-dont-reply-to-names} to +@code{mu4e-personal-or-alternative-address-p}. In order for this to work +properly you need to pass your address to @command{mu init --my-address=} at +database initialization time, and/or use @t{message-alternative-emails}. +@end itemize + +@node Searching +@chapter Searching + +@t{mu4e} is fully search-based: even if you `jump to a folder', you are +executing a query for messages that happen to have the property of being in a +certain folder (maildir). + +Normally, queries return up to @code{mu4e-headers-results-limit} (default: 500) +results. That is usually more than enough, and makes things significantly +faster. Sometimes, however, you may want to show @emph{all} results; you can +enable this with @kbd{M-x mu4e-headers-toggle-property}, or by customizing the +variable @code{mu4e-headers-full-search}. This applies to all search commands. + +You can also influence the sort order and whether threads are shown or not; +see @ref{Sorting and threading}. + +@menu +* Queries:: Searching for messages. +* Bookmarks:: Remembering queries. +* Maildir searches:: Queries for maildirs. +* Other search functionality:: Some more tricks. +@end menu + +@node Queries +@section Queries + +@t{mu4e} queries are the same as the ones that @t{mu find} +understands@footnote{with the caveat that command-line queries are +subject to the shell's interpretation before @t{mu} sees them}. You can +consult the @code{mu-query} man page for the details. + +Additionally, @t{mu4e} supports @kbd{TAB}-completion for queries. There +there is completion for all search keywords such as @code{and}, +@code{from:}, or @code{date:} and also for certain values, i.e., the +possible values for @code{flag:}, @code{prio:}, @code{mime:}, and +@code{maildir:}. + +Let's look at some examples here. + +@itemize + +@item Get all messages regarding @emph{bananas}: +@verbatim +bananas +@end verbatim + +@item Get all messages regarding @emph{bananas} from @emph{John} with an attachment: +@verbatim +from:john and flag:attach and bananas +@end verbatim + +@item Get all messages with subject @emph{wombat} in June 2017 +@verbatim +subject:wombat and date:20170601..20170630 +@end verbatim + +@item Get all messages with PDF attachments in the @t{/projects} folder +@verbatim +maildir:/projects and mime:application/pdf +@end verbatim + +@item Get all messages about @emph{Rupert} in the @t{/Sent Items} folder. Note that +maildirs with spaces must be quoted. +@verbatim +"maildir:/Sent Items" and rupert +@end verbatim + +@item Get all important messages which are signed: +@verbatim +flag:signed and prio:high +@end verbatim + +@item Get all messages from @emph{Jim} without an attachment: +@verbatim +from:jim and not flag:attach +@end verbatim + +@item Get all messages with Alice in one of the contacts-fields (@t{to}, @t{from}, +@t{cc}, @t{bcc}): +@verbatim +contact:alice +@end verbatim + +@item Get all unread messages where the subject mentions Ångström: (search is +case-insensitive and accent-insensitive, so this matches Ångström, angstrom, +aNGstrøM, ...) +@verbatim +subject:Ångström and flag:unread +@end verbatim + +@item Get all unread messages between Mar-2012 and Aug-2013 about some bird: +@verbatim +date:20120301..20130831 and nightingale and flag:unread +@end verbatim + +@item Get today's messages: +@verbatim +date:today..now +@end verbatim + +@item Get all messages we got in the last two weeks regarding @emph{emacs}: +@verbatim +date:2w.. and emacs +@end verbatim + +@item Get messages from the @emph{Mu} mailing list: +@verbatim +list:mu-discuss.googlegroups.com +@end verbatim + +Note --- in the @ref{Headers view} you may see the `friendly name' for a +list; however, when searching you need the real name. You can see the +real name for a mailing list from the friendly name's tool-tip. + +@item Get messages with a subject soccer, Socrates, society, ...; note that +the `*'-wildcard can only appear as a term's rightmost character: +@verbatim +subject:soc* +@end verbatim + +@item Get all messages @emph{not} sent to a mailing-list: +@verbatim +NOT flag:list +@end verbatim + +@item Get all mails with attachments with filenames starting with @emph{pic}; note +that the `*' wildcard can only appear as the term's rightmost character: +@verbatim +file:pic* +@end verbatim + +@item Get all messages with PDF-attachments: +@verbatim +mime:application/pdf +@end verbatim + +Get all messages with image attachments, and note that the `*' wildcard can +only appear as the term's rightmost character: +@verbatim +mime:image/* +@end verbatim + +Get all messages with files that end in @t{.ppt}; this uses the +regular-expression support, which is powerful but relatively slow: +@verbatim +file:/\.ppt$/ +@end verbatim + +@end itemize + +@node Bookmarks +@section Bookmarks + +If you have queries that you use often, you may want to store them as +@emph{bookmarks}. Bookmark searches are available in the main view +(@pxref{Main view}), header view (@pxref{Headers view}), and message +view (@pxref{Message view}), using (by default) the key @key{b} +(@kbd{M-x mu4e-search-bookmark}), or @key{B} (@kbd{M-x +mu4e-search-bookmark-edit}) which lets you edit the bookmark first. + +@subsection Setting up bookmarks + +@t{mu4e} provides a number of default bookmarks. Their definition may +be instructive: + +@lisp +(defcustom mu4e-bookmarks + '(( :name "Unread messages" + :query "flag:unread AND NOT flag:trashed" + :key ?u) + ( :name "Today's messages" + :query "date:today..now" + :key ?t) + ( :name "Last 7 days" + :query "date:7d..now" + :hide-unread t + :key ?w) + ( :name "Messages with images" + :query "mime:image/*" + :key ?p)) + "List of pre-defined queries that are shown on the main screen. + +Each of the list elements is a plist with at least: +:name - the name of the query +:query - the query expression +:key - the shortcut key. + +Optionally, you add the following: +:hide - if t, bookmark is hidden from the main-view and speedbar. +:hide-unread - do not show the counts of unread/total number + of matches for the query. This can be useful if a bookmark uses + a very slow query. :hide-unread is implied from :hide. +" + :type '(repeat (plist)) + :group 'mu4e) +@end lisp + +You can replace these or add your own items, by putting in your +configuration (@file{~/.emacs}) something like: +@lisp +(add-to-list 'mu4e-bookmarks + '( :name "Big messages" + :query "size:5M..500M" + :key ?b)) + @end lisp + +This prepends your bookmark to the list, and assigns the key @key{b} to it. If +you want to @emph{append} your bookmark, you can use @code{t} as the third +argument to @code{add-to-list}. + +In the various @t{mu4e} views, pressing @key{b} lists all the bookmarks +defined in the echo area, with the shortcut key highlighted. So, to invoke the +bookmark we just defined (to get the list of "Big Messages"), all you need to +type is @kbd{bb}. + +@subsection Lisp expressions or functions as bookmarks + +Instead of using strings, it is also possible to use Lisp expressions as +bookmarks. Either the expression evaluates to a query string or the expression +is a function taking no argument that returns a query string. + +For example, to get all the messages that are at most a week old in your +inbox: + +@lisp +(add-to-list 'mu4e-bookmarks + '( :name "Inbox messages in the last 7 days" + :query (lambda () (concat "maildir:/inbox AND date:" + (format-time-string "%Y%m%d" + (subtract-time (current-time) (days-to-time 7))))) + :key ?w) t) +@end lisp + +Another example where the user is prompted how many days old messages should be +shown: + +@lisp +(defun my/mu4e-bookmark-num-days-old-query (days-old) + (interactive (list (read-number "Show days old messages: " 7))) + (let ((start-date (subtract-time (current-time) (days-to-time days-old)))) + (concat "maildir:/inbox AND date:" + (format-time-string "%Y%m%d" start-date)))) + +(add-to-list 'mu4e-bookmarks + `(:name "Inbox messages in the last 7 days" + :query ,(lambda () (call-interactively 'my/mu4e-bookmark-num-days-old-query)) + :key ?o) t) +@end lisp + +It is defining a function to make the code more readable. + +@subsection Editing bookmarks before searching + +There is also @kbd{M-x mu4e-search-bookmark-edit} (key @key{B}), which +lets you edit the bookmarked query before invoking it. This can be useful if +you have many similar queries, but need to change some parameter. For example, +you could have a bookmark @samp{"date:today..now AND "}@footnote{Not a valid +search query by itself}, which limits any result to today's messages. + +@node Maildir searches +@section Maildir searches + +Maildir searches are quite similar to bookmark searches (see @ref{Bookmarks}), +with the difference being that the target is always a maildir --- maildir +queries provide a `traditional' folder-like interface to a search-based e-mail +client. By default, maildir searches are available in the @ref{Main view}, +@ref{Headers view}, and @ref{Message view}, with the key @key{j} +(@code{mu4e-jump-to-maildir}). If a prefix argument is given, the maildir +query can be refined before execution. + +@subsection Setting up maildir shortcuts + +You can search for maildirs like any other message property +(e.g. with a query like @t{maildir:/myfolder}), but since it is so common, +@t{mu4e} offers a shortcut for this. + +For this to work, you need to set the variable +@code{mu4e-maildir-shortcuts} to the list of maildirs you want to have +quick access to, for example: + +@lisp +(setq mu4e-maildir-shortcuts + '( (:maildir "/inbox" :key ?i) + (:maildir "/archive" :key ?a) + (:maildir "/lists" :key ?l) + (:maildir "/work" :key ?w) + (:maildir "/sent" :key ?s) + (:maildir "/lists/project/project_X" :key ?x :name "Project X"))) +@end lisp + +This sets @key{i} as a shortcut for the @t{/inbox} folder --- effectively a +query @t{maildir:/inbox}. There is a special shortcut @key{o} or @key{/} for +@emph{other} (so don't use those for your own shortcuts!), which allows you to +choose from @emph{all} maildirs that you have. There is support for +autocompletion; note that the list of maildirs is determined when @t{mu4e} +starts; if there are changes in the maildirs while @t{mu4e} is running, you +need to restart @t{mu4e}. Optionally, you can specify a name to be displayed +in the main view. + +Each of the folder names is relative to your top-level maildir directory; so +if you keep your mail in @file{~/Maildir}, @file{/inbox} would refer to +@file{~/Maildir/inbox}. With these shortcuts, you can jump around your +maildirs (folders) very quickly --- for example, getting to the @t{/lists} +folder only requires you to type @kbd{jl}, then change to @t{/work} with +@kbd{jw}. + +While in queries you need to quote folder names (maildirs) with spaces in +them, you should @emph{not} quote them when used in +@code{mu4e-maildir-shortcuts}, since @t{mu4e} does that automatically for you. + +The very same shortcuts are used by @kbd{M-x mu4e-mark-for-move} (default +shortcut @key{m}); so, for example, if you want to move a message to the +@t{/archive} folder, you can do so by typing @kbd{ma}. + +@node Other search functionality +@section Other search functionality + +@subsection Navigating through search queries +You can navigate through previous/next queries using +@code{mu4e-headers-query-prev} and @code{mu4e-headers-query-next}, which are +bound to @key{M-left} and @key{M-right}, similar to what some web browsers do. + +@t{mu4e} tries to be smart and not record duplicate queries. Also, the number +of queries remembered has a fixed limit, so @t{mu4e} won't use too much +memory, even if used for a long time. However, if you want to forget +previous/next queries, you can use @kbd{M-x mu4e-headers-forget-queries}. + +@subsection Narrowing search results + +It can be useful to narrow existing search results, that is, to add some +clauses to the current query to match fewer messages. + +For example, suppose you're looking at some mailing list, perhaps by +jumping to a maildir (@kbd{M-x mu4e-headers-jump-to-maildir}, @key{j}) or +because you followed some bookmark (@kbd{M-x mu4e-search-bookmark}, +@key{b}). Now, you want to narrow things down to only those messages that have +attachments. + +This is when @kbd{M-x mu4e-search-narrow} (@key{/}) comes in handy. It +asks for an additional search pattern, which is appended to the current search +query, in effect getting you the subset of the currently shown headers that +also match this extra search pattern. @key{\} takes you back to the previous +query, so, effectively `widens' the search. Technically, narrowing the results +of query @t{x} with expression @t{y} implies doing a search @t{(x) AND (y)}. + +Note that messages that were not in your original search results because of +@code{mu4e-search-results-limit} may show up in the narrowed query. + +@subsection Including related messages +@anchor{Including related messages} + +It can be useful to not only show the messages that directly match a certain +query, but also include messages that are related to these messages. That is, +messages that belong to the same discussion threads are included in the results, +just like e.g. Gmail does it. You can enable this behavior by setting +@code{mu4e-search-include-related} to @code{t}, and you can toggle between +including/not-including using @key{P} (@code{mu4e-search-toggle-property}). + +Be careful though when e.g. deleting ranges of messages from a certain +folder --- the list may now also include messages from @emph{other} +folders. + +@subsection Skipping duplicates +@anchor{Skipping duplicates} + +Another useful feature is skipping of @emph{duplicate messages}. When you have +copies of messages, there's usually little value in including more than one in +search results. A common reason for having multiple copies of messages is the +combination of Gmail and @t{offlineimap}, since that is the way the labels / +virtual folders in Gmail are represented. You can enable skipping duplicates by +setting @code{mu4e-search-skip-duplicates} to @code{t}, and you can toggle +the value using @key{P} (@code{mu4e-search-toggle-property}). + +Note, messages are considered duplicates when they have the same +@t{Message-Id}. + +@node Marking +@chapter Marking + +In @t{mu4e}, the common way to do things with messages is a two-step process - +first you @emph{mark} them for a certain action, then you @emph{execute} +(@key{x}) those marks. This is similar to the way @t{dired} operates. Marking +can happen in both the @ref{Headers view} and the @ref{Message view}. + +@menu +* Marking messages::Selecting message do something with them +* What to mark for::What can we do with them +* Executing the marks::Do it +* Trashing messages::Exceptions for mailboxes like Gmail +* Leaving the headers buffer::Handling marks automatically when leaving +* Built-in marking functions::Helper functions for dealing with them +* Custom mark functions::Define your own mark function +* Adding a new kind of mark::Adding your own marks +@end menu + +@node Marking messages +@section Marking messages + +There are multiple ways to mark messages: +@itemize +@item @emph{message at point}: you can put a mark on the message-at-point in +either the @ref{Headers view} or @ref{Message view} +@item @emph{region}: you can put a mark on all messages in the current region +(selection) in the @ref{Headers view} +@item @emph{pattern}: you can put a mark on all messages in the @ref{Headers +view} matching a certain pattern with @kbd{M-x mu4e-headers-mark-pattern} +(@key{%}) +@item @emph{thread/subthread}: You can put a mark on all the messages in the +thread/subthread at point with @kbd{M-x mu4e-headers-mark-thread} and @kbd{M-x +mu4e-headers-mark-subthread}, respectively +@end itemize + +@node What to mark for +@section What to mark for + +@t{mu4e} supports a number of marks: + +@cartouche +@verbatim +mark for/as | keybinding | description +-------------+-------------+------------------------------ +'something' | *, <insert> | mark now, decide later +delete | D, <delete> | delete +flag | + | mark as 'flagged' ('starred') +move | m | move to some maildir +read | ! | mark as read +refile | r | mark for refiling +trash | d | move to the trash folder +untrash | = | remove 'trash' flag +unflag | - | remove 'flagged' mark +unmark | u | remove mark at point +unmark all | U | remove all marks +unread | ? | marks as unread +action | a | apply some action +@end verbatim +@end cartouche + +After marking a message, the left-most columns in the headers view indicate +the kind of mark. This is informative, but if you mark many (say, thousands) +messages, this slows things down significantly@footnote{this uses an +Emacs feature called @emph{overlays}, which are slow when used a lot +in a buffer}. For this reason, you can disable this by setting +@code{mu4e-headers-show-target} to @code{nil}. + +@t{something} is a special kind of mark; you can use it to mark messages +for `something', and then decide later what the `something' should +be@footnote{This kind of `deferred marking' is similar to the facility +in @t{dired}, @t{midnight commander} +(@url{https://www.midnight-commander.org/}) and the like, and uses the +same key binding (@key{insert}).} Later, you can set the actual mark +using @kbd{M-x mu4e-mark-resolve-deferred-marks} +(@key{#}). Alternatively, @t{mu4e} will ask you when you try to execute +the marks (@key{x}). + +@node Executing the marks +@section Executing the marks + +After you have marked some messages, you can execute them with @key{x} +(@kbd{M-x mu4e-mark-execute-all}). + +A hook, @code{mu4e-mark-execute-pre-hook}, is available which is run +right before execution of each mark. The hook is called with two +arguments, the mark and the message itself. + +@node Trashing messages +@section Trashing messages + +For regular mailboxes, trashing works like other marks: when executed, +the message is flagged as trashed. Depending on your mailbox provider, +the trash flag is used to automatically move the message to the trash +folder (@code{mu4e-trash-folder}) for instance. + +Some mailboxes behave differently however and they don't interpret the +trash flag. In cases like Gmail, the message must be @emph{moved} to +the trash folder and the trash flag must not be used. + +@node Leaving the headers buffer +@section Leaving the headers buffer + +When you quit or update a headers buffer that has marked messages (for +example, by doing a new search), @t{mu4e} asks you what to do with them, +depending on the value of the variable @code{mu4e-headers-leave-behavior} --- +see its documentation. + +@node Built-in marking functions +@section Built-in marking functions + +Some examples of @t{mu4e}'s built-in marking functions. + +@itemize +@item @emph{Mark the message at point for trashing}: press @key{d} +@item @emph{Mark all messages in the buffer as unread}: press @kbd{C-x h o} +@item @emph{Delete the messages in the current thread}: press @kbd{T D} +@item @emph{Mark messages with a subject matching ``hello'' for flagging}: +press @kbd{% s hello RET}. +@end itemize + +@node Custom mark functions +@section Custom mark functions + +Sometimes, the built-in functions to mark messages may not be sufficient for +your needs. For this, @t{mu4e} offers an easy way to define your own custom +mark functions. You can choose one of the custom marker functions by pressing +@key{&} in the @ref{Headers view} and @ref{Message view}. + +Custom mark functions are to be appended to the list +@code{mu4e-headers-custom-markers}. Each of the elements of this list +('markers') is a list with two or three elements: +@enumerate +@item The name of the marker --- a short string describing this marker. The +first character of this string determines its shortcut, so these should be +unique. If necessary, simply prefix the name with a unique character. +@item a predicate function, taking two arguments @code{msg} and @code{param}. +@code{msg} is the message plist (see @ref{Message functions}) and @code{param} is +a parameter provided by the third of the marker elements (see the next +item). The predicate function should return non-@t{nil} if the message +matches. +@item (optionally) a function that is evaluated once, and the result is passed as a +parameter to the predicate function. This is useful when user-input is needed. +@end enumerate + +Let's look at an example: suppose we want to match all messages that have more +than @emph{n} recipients --- we could do this with the following recipe: + +@lisp +(add-to-list 'mu4e-headers-custom-markers + '("More than n recipients" + (lambda (msg n) + (> (+ (length (mu4e-message-field msg :to)) + (length (mu4e-message-field msg :cc))) n)) + (lambda () + (read-number "Match messages with more recipients than: "))) t) +@end lisp + +After evaluating this expression, you can use it by pressing @key{&} in +the headers buffer to select a custom marker function, and then @key{M} +to choose this particular one (@t{M} because it is the first character +of the description). + +As you can see, it's not very hard to define simple functions to match +messages. There are more examples in the defaults for +@code{mu4e-headers-custom-markers}; see @file{mu4e-headers.el} and see +@ref{Extending mu4e} for general information about writing your own functions. + + +@node Adding a new kind of mark +@section Adding a new kind of mark + +It is possible to configure new marks, by adding elements to the list +@code{mu4e-marks}. Such an element must have the following form: + +@lisp +(SYMBOL + :char STRING + :prompt STRING + :ask-target (lambda () TARGET) + :dyn-target (lambda (TARGET MSG) DYN-TARGET) + :show-target (lambda (DYN-TARGET) STRING) + :action (lambda (DOCID MSG DYN-TARGET) nil)) +@end lisp + +The symbol can be any symbol, except for the symbols @code{unmark} and +@code{something}, which are reserved. The rest is a plist with the following +elements: + +@itemize +@item @code{:char} --- the character to display in the headers view. +@item @code{:prompt} --- the prompt to use when asking for marks +(used for example when marking a whole thread). +@item @code{:ask-target} --- a function run once per bulk-operation, and thus suitable for +querying the user about a target for move-like marks. If @t{nil}, the +@t{TARGET} passed to @code{:dyn-target} is @t{nil}. +@item @code{:dyn-target} --- a function run once per message +(The message is passed as @t{MSG} to the function). This function allows +to compute a per-message target, for refile-like marks. If @t{nil}, the +@t{DYN-TARGET} passed to the @code{:action} is the @t{TARGET} obtained as above. +@item @code{:show-target} --- how to display the target in the headers view. +If @code{:show-target} is @t{nil} the @t{DYN-TARGET} is shown (and +@t{DYN-TARGET} must be a string). +@item @code{:action} --- the action to apply on the message when the mark is executed. +@end itemize + +As an example, suppose we would like to add a mark for tagging messages +(GMail-style). We can use the following code (after loading @t{mu4e}): + +@lisp +(add-to-list 'mu4e-marks + '(tag + :char "g" + :prompt "gtag" + :ask-target (lambda () (read-string "What tag do you want to add? ")) + :action (lambda (docid msg target) + (mu4e-action-retag-message msg (concat "+" target))))) +@end lisp + +Adding elements to @code{mu4e-marks} (as in the example) allows you to use the +mark in bulk operations (for example when tagging a whole thread); if you also +want to add a key-binding for the headers view, you can use something like: + +@lisp +(defun my-mu4e-mark-add-tag() + "Add a tag to the message at point." + (interactive) + (mu4e-headers-mark-and-next 'tag)) + +(define-key mu4e-headers-mode-map (kbd "g") #'my-mu4e-mark-add-tag) +@end lisp + +@node Contexts +@chapter Contexts + +@menu +* What are contexts::Defining the concept +* Context policies::How to determine the current context +* Contexts and special folders::Using context variables to determine them +* Contexts example::How to define contexts +@end menu + +It can be useful to switch between different sets of settings in +@t{mu4e}; a typical example is the case where you have different e-mail +accounts for private and work email, each with their own values for +folders, e-mail addresses, mailservers and so on. + +The @code{mu4e-context} system is a @t{mu4e}-specific mechanism to allow +for that; users can define different @i{contexts} corresponding with +groups of setting and either manually switch between them, or let +@t{mu4e} determine the right context based on some user-provided +function. + +Note that there are a number of existing ways to switch accounts in +@t{mu4e}, for example using the method described in the @ref{Tips and +Tricks} section of this manual. Those still work --- but the new mechanism +has the benefit of being a core part of @code{mu4e}, thus allowing for +deeper integration. + +@node What are contexts +@section What are contexts + +Let's see what's contained in a context. Most of it is optional. + +A @code{mu4e-context} is Lisp object with the following members: +@itemize +@item @t{name}: the name of the context, e.g. @t{work} or @t{private} +@item @t{vars}: +an association-list (alist) of variable settings for this account. +@item @t{enter-func}: +an (optional) function that takes no parameter and is invoked when entering +the context. You can use this for extra setup etc. +@item @t{leave-func}: +an (optional) function that takes no parameter and is invoked when leaving +the context. You can use this for clearing things up. +@item @t{match-func}: +an (optional) function that takes an @t{MSG} message plist as argument, +and returns non-@t{nil} if this context matches the situation. @t{mu4e} +uses the first context that matches, in a couple of situations: +@itemize +@item when starting @t{mu4e} to determine the +starting context; in this case, @t{MSG} is nil. You can use e.g. the +host you're running or the time of day to determine which context +matches. +@item before replying to or forwarding a +message with the given message plist as parameter, or @t{nil} when +composing a brand new message. The function should return @t{t} when +this context is the right one for this message, or @t{nil} otherwise. +@item when determining the target folders for deleting, refiling etc; +see @ref{Contexts and special folders}. +@end itemize +@end itemize + +@t{mu4e} uses a variable @code{mu4e-contexts}, which is a list of such +objects. + +@node Context policies +@section Context policies + +When you have defined contexts and you start @t{mu4e} it decides which +context to use based on the variable @code{mu4e-context-policy}; +similarly, when you compose a new message, the context is determined +using @code{mu4e-compose-context-policy}. + +For both of these, you can choose one of the following policies: +@itemize +@item a symbol @code{always-ask}: unconditionally ask the user what context to pick. +@end itemize + +The other choices @b{only apply if none of the contexts match} (i.e., +none of the contexts' match-functions returns @code{t}). We have the +following options: + +@itemize +@item a symbol @code{ask}: ask the user if @t{mu4e} can't figure +things out the context by itself (through the match-function). This is a +good policy if there are no match functions, or if the match functions +don't cover all cases. +@item a symbol @code{ask-if-none}: if there's already a context, don't change it; +otherwise, ask the user. +@item a symbol @code{pick-first}: pick the first (default) context. This is a +good choice if +you want to specify context for special case, and fall back to the first +one if none match. +@item @code{nil}: don't change the context; this is useful if you don't change +contexts very often, and e.g. manually changes contexts with @kbd{M-x +mu4e-context-switch}. +@end itemize + +You can easily switch contexts manually using the @kbd{;} key from +the main screen. + +@node Contexts and special folders +@section Contexts and special folders + +As we discussed in @ref{Folders} and @ref{Dynamic folders}, @t{mu4e} +recognizes a number of special folders: @code{mu4e-sent-folder}, +@code{mu4e-drafts-folder}, @code{mu4e-trash-folder} and +@code{mu4e-refile-folder}. + +When you have a headers-buffer with messages that belong to different +contexts (say, a few different accounts), it is desirable for each of +them to use the specific folders for their own context --- so, for +instance, if you trash a message, it needs to go to the trash-folder for +the account it belongs to, which is not necessarily the current context. + +To make this easy to do, whenever @t{mu4e} needs to know the value for +such a special folder for a given message, it tries to determine the +appropriate context using @code{mu4e-context-determine} (and policy +@t{nil}; see @ref{Context policies}). If it finds a matching context, it +let-binds the @code{vars} for that account, and then determines the +value for the folder. It does not, however, call the @code{enter-func} +or @code{leave-func}, since we are not really switching contexts. + +In practice, this means that as long as each of the accounts has a good +@t{match-func}, all message operations automatically find the +appropriate folders. + +@node Contexts example +@section Example + +Let's explain how contexts work by looking at an example. We define two +contexts, `Private' and `Work' for a fictional user @emph{Alice +Derleth}. + +Note that in this case, we automatically switch to the first context +when starting; see the discussion in the previous section. + +@lisp + + (setq mu4e-contexts + `( ,(make-mu4e-context + :name "Private" + :enter-func (lambda () (mu4e-message "Entering Private context")) + :leave-func (lambda () (mu4e-message "Leaving Private context")) + ;; we match based on the contact-fields of the message + :match-func (lambda (msg) + (when msg + (mu4e-message-contact-field-matches msg + :to "aliced@@home.example.com"))) + :vars '( ( user-mail-address . "aliced@@home.example.com" ) + ( user-full-name . "Alice Derleth" ) + ( message-user-organization . "Homebase" ) + ( message-signature . + (concat + "Alice Derleth\n" + "Lauttasaari, Finland\n")))) + ,(make-mu4e-context + :name "Work" + :enter-func (lambda () (mu4e-message "Switch to the Work context")) + ;; no leave-func + ;; we match based on the maildir of the message + ;; this matches maildir /Arkham and its sub-directories + :match-func (lambda (msg) + (when msg + (string-match-p "^/Arkham" (mu4e-message-field msg :maildir)))) + :vars '( ( user-mail-address . "aderleth@@miskatonic.example.com" ) + ( user-full-name . "Alice Derleth" ) + ( message-user-organization . "Miskatonic University" ) + ( message-signature . + (concat + "Prof. Alice Derleth\n" + "Miskatonic University, Dept. of Occult Sciences\n")))) + + ,(make-mu4e-context + :name "Cycling" + :enter-func (lambda () (mu4e-message "Switch to the Cycling context")) + ;; no leave-func + ;; we match based on the maildir of the message; assume all + ;; cycling-related messages go into the /cycling maildir + :match-func (lambda (msg) + (when msg + (string= (mu4e-message-field msg :maildir) "/cycling"))) + :vars '( ( user-mail-address . "aderleth@@example.com" ) + ( user-full-name . "AliceD" ) + ( message-signature . nil))))) + + ;; set `mu4e-context-policy` and `mu4e-compose-policy` to tweak when mu4e should + ;; guess or ask the correct context, e.g. + + ;; start with the first (default) context; + ;; default is to ask-if-none (ask when there's no context yet, and none match) + ;; (setq mu4e-context-policy 'pick-first) + + ;; compose with the current context is no context matches; + ;; default is to ask + ;; (setq mu4e-compose-context-policy nil) +@end lisp + +A couple of notes about this example: +@itemize +@item You can manually switch the context use @code{M-x mu4e-context-switch}, +by default bound to @kbd{;} in headers, view and main mode. The current context +appears in the modeline by default; see @ref{Modeline} for details. +@item Normally, @code{M-x mu4e-context-switch} does not call the enter or +leave functions if the 'new' context is the same as the old one. +However, with a prefix-argument (@kbd{C-u}), you can force @t{mu4e} to +invoke those function even in that case. +@item The function @code{mu4e-context-current} returns the current-context; +the current context is also visible in the mode-line when in +headers, view or main mode. +@item You can set any kind of variable; including settings for mail servers etc. +However, settings such as @code{mu4e-mu-home} are not changeable after +they have been set without quitting @t{mu4e} first. +@item @code{leave-func} (if defined) for the context we are leaving, is invoked +before the @code{enter-func} (if defined) of the +context we are entering. +@item @code{enter-func} (if defined) is invoked before setting the variables. +@item @code{match-func} (if defined) is invoked just before @code{mu4e-compose-pre-hook}. +@item See the variables @code{mu4e-context-policy} and +@code{mu4e-compose-context-policy} to tweak what @t{mu4e} should do when +no context matches (or if you always want to be asked). +@item Finally, be careful to get the quotations right --- backticks, single quotes +and commas and note the '.' between variable name and its value. +@end itemize + +@node Dynamic folders +@chapter Dynamic folders + +In @ref{Folders}, we explained how you can set up @t{mu4e}'s special +folders: +@lisp +(setq + mu4e-sent-folder "/sent" ;; sent messages + mu4e-drafts-folder "/drafts" ;; unfinished messages + mu4e-trash-folder "/trash" ;; trashed messages + mu4e-refile-folder "/archive") ;; saved messages +@end lisp + +In some cases, having such static folders may not suffice --- perhaps you want +to change the folders depending on the context. For example, the folder for +refiling could vary, based on the sender of the message. + +To make this possible, instead of setting the standard folders to a string, +you can set them to be a @emph{function} that takes a message as its +parameter, and returns the desired folder name. This chapter shows you how to +do that. For a more general discussion of how to extend @t{mu4e} and writing +your own functions, see @ref{Extending mu4e}. + +If you use @t{mu4e-context}, see @ref{Contexts and special folders} for +what that means for these special folders. + +@menu +* Smart refiling:: Automatically choose the target folder +* Other dynamic folders:: Flexible folders for sent, trash, drafts +@end menu + + +@node Smart refiling +@section Smart refiling + +When refiling messages, perhaps to archive them, it can be useful to have +different target folders for different messages, based on some property of +those message --- smart refiling. + +To accomplish this, we can set the refiling folder (@code{mu4e-refile-folder}) +to a function that returns the actual refiling folder for the particular +message. An example should clarify this: + +@lisp +(setq mu4e-refile-folder + (lambda (msg) + (cond + ;; messages to the mu mailing list go to the /mu folder + ((mu4e-message-contact-field-matches msg :to + "mu-discuss@@googlegroups.com") + "/mu") + ;; messages sent directly to some specific address me go to /private + ((mu4e-message-contact-field-matches msg :to "me@@example.com") + "/private") + ;; messages with football or soccer in the subject go to /football + ((string-match "football\\|soccer" + (mu4e-message-field msg :subject)) + "/football") + ;; messages sent by me go to the sent folder + ((mu4e-message-sent-by-me msg + (mu4e-personal-addresses)) + mu4e-sent-folder) + ;; everything else goes to /archive + ;; important to have a catch-all at the end! + (t "/archive")))) +@end lisp + +@noindent +This can be very powerful; you can select some messages in the headers view, +then press @key{r}, and have them all marked for refiling to their particular +folders. + +Some notes: +@itemize +@item We set @code{mu4e-refile-folder} to an anonymous (@t{lambda}) function. This +function takes one argument, a message plist@footnote{a property list +describing a message}. The plist corresponds to the message at point. See +@ref{Message functions} for a discussion on how to deal with them. +@item In our function, we use a @t{cond} control structure; the function +returns the first of the clauses that matches. It's important to make the last +clause a catch-all, so we always return @emph{some} folder. +@item We use +the convenience function @code{mu4e-message-contact-field-matches}, +which evaluates to @code{t} if any of the names or e-mail addresses in a +contact field (in this case, the @t{To:}-field) matches the regular +expression. With @t{mu4e} version 0.9.16 or newer, the contact field can +in fact be a list instead of a single value, such as @code{'(:to :cc)'}. +@end itemize + +@node Other dynamic folders +@section Other dynamic folders + +Using the same mechanism, you can create dynamic sent-, trash-, and +drafts-folders. The message-parameter you receive for the sent and drafts +folder is the @emph{original} message, that is, the message you reply to, or +forward, or edit. If there is no such message (for example when composing a +brand new message) the message parameter is @t{nil}. + +Let's look at an example. Suppose you want a different trash folder for +work-email. You can achieve this with something like: + +@lisp +(setq mu4e-trash-folder +(lambda (msg) +;; the 'and msg' is to handle the case where msg is nil +(if (and msg +(mu4e-message-contact-field-matches msg :to "me@@work.example.com")) +"/trash-work" +"/trash"))) +@end lisp + +@noindent +Good to remember: +@itemize +@item The @code{msg} parameter you receive in the function refers to the +@emph{original message}, that is, the message being replied to or forwarded. +When re-editing a message, it refers to the message being edited. When you +compose a totally new message, the @code{msg} parameter is @code{nil}. +@item When re-editing messages, the value of @code{mu4e-drafts-folder} is ignored. +@end itemize + + +@node Actions +@chapter Actions + +@t{mu4e} lets you define custom actions for messages in @ref{Headers view} +and for both messages and attachments in @ref{Message view}. Custom +actions allow you to easily extend @t{mu4e} for specific needs --- for example, +marking messages as spam in a spam filter or applying an attachment with a +source code patch. + +You can invoke the actions with key @key{a} for actions on messages, and key +@key{A} for actions on attachments. + +For general information extending @t{mu4e} and writing your own functions, see +@ref{Extending mu4e}. + +@menu +* Defining actions::How to create an action +* Headers view actions::Doing things with message headers +* Message view actions::Doing things with messages +* MIME-part actions::Doing things with MIME-parts such as attachments +* Example actions::Some more examples +@end menu + +@node Defining actions +@section Defining actions + +Defining a new custom action comes down to writing an elisp-function to do the +work. Functions that operate on messages receive a @var{msg} parameter, which +corresponds to the message at point. Something like: +@lisp +(defun my-action-func (msg) + "Describe my message function." + ;; do stuff + ) +@end lisp + +@noindent +Functions that operate on attachments receive a @var{msg} parameter, which +corresponds to the message at point, and an @var{attachment-num}, which is the +number of the attachment as seen in the message view. An attachment function +looks like: +@lisp +(defun my-attachment-action-func (msg attachment-num) + "Describe my attachment function." + ;; do stuff + ) +@end lisp + +@noindent +After you have defined your function, you can add it to the list of +actions@footnote{Instead of defining the functions separately, you can +obviously also add a @code{lambda}-function directly to the list; however, +separate functions are easier to change}, either @code{mu4e-headers-actions}, +@code{mu4e-view-actions} or @code{mu4e-view-mime-part-actions}. The +format@footnote{Note, the format of the actions has changed since version +0.9.8.4, and you must change your configuration to use the new format; +@t{mu4e} warns you when you are using the old format.} of each action is a +cons-cell, @code{(DESCRIPTION . VALUE)}; see below for some examples. If your +shortcut is not also the first character of the description, simply prefix the +description with that character. + +Let's look at some examples. + +@node Headers view actions +@section Headers view actions + +Suppose we want to inspect the number of recipients for a message in the +@ref{Headers view}. We add the following to our configuration: + +@lisp +(defun show-number-of-recipients (msg) + "Display the number of recipients for the message at point." + (message "Number of recipients: %d" + (+ (length (mu4e-message-field msg :to)) + (length (mu4e-message-field msg :cc))))) + +;; define 'N' (the first letter of the description) as the shortcut +;; the 't' argument to add-to-list puts it at the end of the list +(add-to-list 'mu4e-headers-actions + '("Number of recipients" . show-number-of-recipients) t) +@end lisp + +After evaluating this, @kbd{a N} in the headers view shows the number of +recipients for the message at point. + +@node Message view actions +@section Message view actions + +As another example, suppose we would like to search for messages by the +sender of the message at point: + +@lisp +(defun search-for-sender (msg) + "Search for messages sent by the sender of the message at point." + (mu4e-search + (concat "from:" + (mu4e-contact-email (car (mu4e-message-field msg :from)))))) + +;; define 'x' as the shortcut +(add-to-list 'mu4e-view-actions + '("xsearch for sender" . search-for-sender) t) +@end lisp + +@indent +If you wonder why we use @code{car}, remember that the @t{From:}-field +is a list of @code{(:name NAME :email EMAIL)} plists; so this code gets +us the e-mail address of the first in the list. @t{From:}-fields rarely +have more that one address. + +@node MIME-part actions +@section MIME-part actions + +Finally, let's define a MIME-part action. + +The following example action counts the number of lines in an attachment, and +defines @key{n} as its shortcut key (the @key{n} is prefixed to the +description). See the the @code{mu4e-view-mime-part-actions} for the details of +the format. + +@lisp +(add-to-list 'mu4e-view-mime-part-actions + ;; count the number of lines in a MIME-part + '(:name "line-count" :handler "wc -l" :receives pipe)) +@end lisp + +Or another one, to import a calendar invitation into the venerable emacs diary: +@lisp +(add-to-list 'mu4e-view-mime-part-actions + ;; import into calendar; + '(:name "dimport-in-diary" :handler (lambda(file) (icalendar-import-file file diary-file)) + :receives temp)) +@end lisp + +@node Example actions +@section Example actions + +@t{mu4e} includes a number of example actions in the file +@file{mu4e-actions.el} in the source distribution (see @kbd{C-h f +mu4e-action-TAB}). For example, for viewing messages in an external web +browser. + +@node Extending mu4e +@chapter Extending mu4e + +@t{mu4e} is designed to be easily extensible --- that is, write your own +emacs-lisp to make @t{mu4e} behave exactly as you want. Here, we provide some +guidelines for doing so. + +@menu +* Extension points::Where to hook into @t{mu4e} +* Available functions::General helper functions +* Message functions::Working with messages +* Contact functions::Working with contacts +* Utility functions::Miscellaneous helpers +@end menu + +@node Extension points +@section Extension points + +There are a number of places where @t{mu4e} lets you plug in your own +functions: +@itemize +@item Custom functions for message header --- see @ref{HV Custom headers} +@item Using message-specific folders for drafts, trash, sent messages and +refiling, based on a function --- see @ref{Dynamic folders} +@item Using an attachment-specific download-directory --- see the +variable @code{mu4e-attachment-dir}. +@item Apply a function to a message in the headers view - +see @ref{Headers view actions} +@item Apply a function to a message in the message view --- +see @ref{Message view actions} +@item Add a new kind of mark for use in the headers view +- see @ref{Adding a new kind of mark} +@item Apply a function to a MIME-part --- see @ref{MIME-part actions} +@item Custom function to mark certain messages --- +see @ref{Custom mark functions} +@item Using various @emph{mode}-hooks, @code{mu4e-compose-pre-hook} (see +@ref{Compose hooks}), @code{mu4e-index-updated-hook} (see @ref{FAQ}) +@end itemize + +@noindent +You can also write your own functions without using the above. If you +want to do so, key useful functions are @code{mu4e-message-at-point} +(see below), @code{mu4e-headers-for-each} (to iterate over all +headers, see its docstring) and @code{mu4e-view-for-each-part} (to +iterate over all parts/attachments, see its docstring). There is also +@code{mu4e-view-for-each-uri} to iterate of all the URIs in the +current message. + +Another useful function is +@code{mu4e-headers-find-if} which searches for a message matching a +certain pattern; again, see its docstring. + +@node Available functions +@section Available functions + +The whole of @t{mu4e} consists of hundreds of elisp functions. However, +the majority of those are for @emph{internal} use only; you can +recognize them easily, because they all start with @code{mu4e~} or +@code{mu4e--}. These functions make all kinds of assumptions, and they +are subject to change, and should therefore @emph{not} be used. The same +is true for @emph{variables} with the same prefix; don't touch them. Let +me repeat that: +@verbatim +Do not use mu4e~... or mu4e-- functions or variables! +@end verbatim + +@noindent +In addition, you should use functions in the right context; functions +that start with @t{mu4e-view-} are only applicable to the message view, +while functions starting with @t{mu4e-headers-} are only applicable to +the headers view. Functions without such prefixes are applicable +everywhere. + +@node Message functions +@section Message functions + +Many functions in @t{mu4e} deal with message plists (property +lists). They contain information about messages, such as sender and +recipient, subject, date and so on. To deal with these plists, there are +a number of @code{mu4e-message-} functions (in @file{mu4e-message.el}), +such as @code{mu4e-message-field} and @code{mu4e-message-at-point}, and +a shortcut to combine the two, @code{mu4e-message-field-at-point}. + +For example, to get the subject of the message at point, in either the headers +view or the message view, you could write: +@lisp +(mu4e-message-field (mu4e-message-at-point) :subject) +@end lisp +@noindent +Note that: +@itemize +@item The contact fields (To, From, Cc, Bcc) are lists of cons-pairs +@code{(name . email)}; @code{name} may be @code{nil}. So, for example: +@lisp + (mu4e-message-field some-msg :to) + ;; => (("Jack" . "jack@@example.com") (nil . "foo@@example.com")) +@end lisp + +If you are only looking for a match in this list (e.g., ``Is Jack one of the +recipients of the message?''), there is a convenience function +@code{mu4e-message-contact-field-matches} to make this easy. +@item The message body is only available in the message view, not in the +headers view. +@end itemize + +Note that in headers-mode, you only have access to a reduced message +plist, without the information about the message-body or mime-parts; +@t{mu4e} does this for performance reasons. And even in view-mode, you +do not have access to arbitrary message-headers. + +However, it is possible to get the information indirectly, using the +raw-message and some third-party tool like @t{procmail}'s @t{formail}: + +@lisp +(defun my-mu4e-any-message-field-at-point (hdr) + "Quick & dirty way to get an arbitrary header HDR at +point. Requires the 'formail' tool from procmail." + (replace-regexp-in-string "\n$" "" + (shell-command-to-string + (concat "formail -x " hdr " -c < " + (shell-quote-argument (mu4e-message-field-at-point :path)))))) +@end lisp + +@node Contact functions +@section Contact functions + +It can sometimes be useful to discard or rewrite the contact information +that @t{mu4e} provides, for example to fix spelling errors, or omit +unwanted contacts. + +To handle this, @t{mu4e} provides @code{mu4e-contact-process-function}, +which, if defined, is applied to each contact. If the result is @t{nil}, +the contact is discarded, otherwise the (modified or not) contact +information is used. + +Each contact is a full e-mail address as you would see in a +contact-field of an e-mail message, e.g., +@verbatim +"Foo Bar" <foo.bar@example.com> +@end verbatim +or +@verbatim +cuux@example.com +@end verbatim + +An example @code{mu4e-contact-process-function} might look like: + +@lisp +(defun my-contact-processor (contact) + (cond + ;; remove unwanted + ((string-match-p "evilspammer@@example.com" contact) nil) + ((string-match-p "noreply" contact) nil) + ;; + ;; jonh smiht --> John Smith + ((string-match "jonh smiht" contact) + (replace-regexp-in-string "jonh smiht" "John Smith" contact)) + (t contact))) + +(setq mu4e-contact-process-function 'my-contact-processor) +@end lisp + + +@node Utility functions +@section Utility functions + +@file{mu4e-utils} contains a number of utility functions; we list a few +here. See their docstrings for details: +@itemize +@item @code{mu4e-read-option}: read one option from a list. For example: +@lisp +(mu4e-read-option "Choose an animal: " +'(("Monkey" . monkey) ("Gnu" . gnu) ("xMoose" . moose))) +@end lisp +The user is presented with: +@example +Choose an animal: [M]onkey, [G]nu, [x]Moose +@end example +@item @code{mu4e-ask-maildir}: ask for a maildir; try one of the +shortcuts (@code{mu4e-maildir-shortcuts}), or the full set of available +maildirs. +@item @code{mu4e-running-p}: return @code{t} if the @t{mu4e} process is +running, @code{nil} otherwise. +@item @code{(mu4e-user-mail-address-p addr)}: return @code{t} if @var{addr} is +one of the user's e-mail addresses (as per @code{(mu4e-personal-addresses)}). +@item @code{mu4e-log} logs to the @t{mu4e} debugging log if it is enabled; +see @code{mu4e-toggle-logging}. +@item @code{mu4e-message}, @code{mu4e-warning}, @code{mu4e-error} are the +@t{mu4e} equivalents of the normal elisp @code{message}, +@code{user-error} and @code{error} functions. +@end itemize + +@node Integration +@chapter Integrating @t{mu4e} with Emacs facilities + +In this chapter, we discuss how you can integrate @t{mu4e} with Emacs in various +ways. Here we focus on Emacs built-ins; for dealing with external tools, +@xref{Other tools}. + +@menu +* Default email client::Making mu4e the default emacs e-mail program +* Modeline::Showing mu4e's status in the modeline +* Desktop notifications::Get desktop notifications for new mail +* Emacs bookmarks::Using Emacs' bookmark system +* Eldoc::Information about the current header in the echo area +* Org-mode links::Adding mu4e to your organized life +* iCalendar::Enabling iCalendar invite processing +* Speedbar::A special frame with your folders +* Dired:: Attaching files using @t{dired} +@end menu + + +@node Default email client +@section Default email client + +Emacs allows you to select an e-mail program as the default program it uses when +you press @key{C-x m} (@code{compose-mail}), call @code{report-emacs-bug} and so +on; see @ref{(emacs) Mail Methods}. + +If you want to use @t{mu4e} for this, you can do so by adding the following +to your configuration: + +@lisp +(setq mail-user-agent 'mu4e-user-agent) +@end lisp + +Similarly, to specify @t{mu4e} as your preferred method for reading +mail, customize the variable @code{read-mail-command}. + +@lisp +(set-variable 'read-mail-command 'mu4e) +@end lisp + +@node Modeline +@section Modeline +@cindex modeline + +One of the most visible ways in which @t{mu4e} integrates with Emacs is through +the @emph{modeline} @xref{Mode Line,,,emacs}. The @t{mu4e} support for that is +handled through a minor-mode @code{mu4e-modeline-mode}, which is enabled by +default when @t{mu4e} is running. + +To completely turn off the modeline support, set @code{mu4e-modeline-support} to +@t{nil} before starting @t{mu4e}. + +@t{mu4e} shares information on the modeline in two ways: +@itemize +@item buffer-specific +@itemize +@item current context (as per @ref{Contexts}) +@item current query parameters (headers-mode only) +@end itemize +@item global: information about the results for the ``favorite query'' +@end itemize + +The global indicators can be disabled by setting @code{mu4e-modeline-show-global} +to @t{nil}. + +All of the bookmark items provide more details in their @code{help-echo}, +i.e., their tooltip. + +@subsection Query parameters bookmark item +The query parameters in the modeline start with the various query flags (such as +some representation of @code{mu4e-search-threads}, @code{mu4e-search-full}; the +@t{help-echo} (tool-tip) has the details. + +The query parameters are followed by the query-string use for the headers-view. +By default, if the query string matches some bookmark, the name of that bookmark +is shown instead of the query it specifies. This can be changed by setting +@code{mu4e-modeline-prefer-bookmark-name} to @t{nil}. + +@cindex favorite bookmark +@subsection Favorite bookmark modeline item +The global modeline contains the results of some specific ``favorite'' bookmark +query from @code{mu4e-bookmarks}. By default, the @emph{first} one in chosen, +but you may want to change that by using the @code{:favorite} property for a +particular query, e.g., as part of your @var{mu4e-bookmarks}: +@example + ;; Monitor the inbox folder in the modeline + (:query "maildir:/inbox" :name "Inbox" :key ?i :favorite t) +@end example + +The results of this query (the last time it was updated) is shown as some +character or emoji (depending on @var{mu4e-use-fancy-chars}) and 2 or 3 numbers, +just like what we saw in @xref{Bookmarks and Maildirs}, e.g., +@example + N:10(+5)/15 +@end example + +@cindex baseline query results +this means there are @emph{10 unread messages}, with @emph{5 new messages since +the baseline}, and @emph{15 messages in total} matching the query. + +You can customize the icon; see @var{mu4e-modeline-all-clear}, +@var{mu4e-modeline-all-read}, @var{mu4e-modeline-unread-items} and +@var{mu4e-modeline-new-items}. + +Due to the way queries work, the modeline is @emph{not} immediately updated when +you read messages; but going back to the main view (with @kbd{M-x mu4e} resets +the counts to latest known ones. When in the main-view, you can use +@code{revert-buffer} (@kbd{g}) to reset the counters explicitly. + +@node Desktop notifications +@section Desktop notifications +@cindex desktop notifications + +Depending on your desktop environment, it is possible to get notification when +there is new mail. + +The default implementation (which you can override) depends on the same system +used for the @xref{Bookmarks and Maildirs}, in the main view and the +@xref{Modeline}, and thus gives updates when there new messages compared to some +``baseline'', as discussed earlier. + +For now, notifications are implemented for desktop environments that support +DBus-based notifications, as per Emacs' notification sub-system @xref{(elisp) +Desktop Notifications}. + +You can enable mu4e's desktop notifications (provided that you are on a +supported system) by setting @code{mu4e-notification-support} to @t{t}. If you +want tweak the details, have a look at @code{mu4e-notification-filter} and +@code{mu4e-notification-function}. + +@node Emacs bookmarks +@section Emacs bookmarks +@cindex Emacs bookmarks + +Note, Emacs bookmarks are not to be confused with mu4e's bookmarks; the former +are a generic linking system across Emacs, while the latter are stored queries +within @t{mu4e}. + +@t{mu4e} supports linking to the message-at-point through the normal Emacs +built-in bookmark system. The links are based on the message's message-id, and +thus the bookmarks stay valid even if you move the message around. + +@node Eldoc +@section Eldoc +@cindex eldoc + +It is possible to get information about the current header in the echo-area. +You can enable this by setting @t{mu4e-eldoc-support} to non-@t{nil}. + +@node Org-mode links +@section Org-mode links + +It can be useful to include links to e-mail messages or search queries +in your org-mode files. @t{mu4e} supports this by default, unless you +set @t{mu4e-support-org} to @code{nil}. + +You can use the normal @t{org-mode} mechanisms to store links: +@kbd{M-x org-store-link} stores a link to a particular message when +you are in @ref{Message view}. When you are in @ref{Headers view}, +@kbd{M-x org-store-link} links to the @emph{query} if +@code{mu4e-org-link-query-in-headers-mode} is non-@code{nil}, and to +the particular message otherwise (which is the default). You can +customize the link description using @code{mu4e-org-link-desc-func}. + +You can insert this link later with @kbd{M-x org-insert-link}. From +@t{org-mode}, you can go to the query or message the link points to +with either @kbd{M-x org-agenda-open-link} in agenda buffers, or +@kbd{M-x org-open-at-point} elsewhere --- both typically bound to +@kbd{C-c C-o}. + +You can also directly @emph{capture} such links --- for example, to +add e-mail messages to your todo-list. For that, @t{mu4e-org} has a +function @code{mu4e-org-store-and-capture}. This captures the +message-at-point (or header --- see the discussion on +@code{mu4e-org-link-query-in-headers-mode} above), then calls +@t{org-mode}'s capture functionality. + +You can add some specific capture-template for this. In your capture +templates, the following mu4e-specific values are available: + +@cartouche +@verbatim +item | description +-----------------------------------------------------+------------------------ +%:date, %:date-timestamp, %:date-timestamp-inactive | date, org timestamps +%:from, %:fromname, %:fromaddress | sender, name/address +%:to, %:toname, %:toaddress | recipient, name/address +%:maildir | maildir for the message +%:message-id | message-id +%:path | file system path +%:subject | message subject +@end verbatim +@end cartouche + +For example, to add a message to your todo-list, and set a deadline +for processing it within two days, you could add this to +@code{org-capture-templates}: + +@lisp + ("P" "process-soon" entry (file+headline "todo.org" "Todo") + "* TODO %:fromname: %a %?\nDEADLINE: %(org-insert-time-stamp (org-read-date nil t \"+2d\"))") +@end lisp + +If you use the functionality a lot, you may want to define +key-bindings for that in headers and view mode: + +@lisp + (define-key mu4e-headers-mode-map (kbd "C-c c") 'mu4e-org-store-and-capture) + (define-key mu4e-view-mode-map (kbd "C-c c") 'mu4e-org-store-and-capture) +@end lisp + +@node iCalendar +@section iCalendar + +When Gnus' article-mode is chosen (@ref{Message view}), it is possible +to view and reply to iCalendar events. To enable this feature, add + +@lisp +(require 'mu4e-icalendar) +(mu4e-icalendar-setup) +@end lisp + +to your configuration. If you want that the original invitation message +be automatically trashed after sending the message created by clicking +on the buttons “Accept”, “Tentative”, or “Decline”, also add: + +@lisp +(setq mu4e-icalendar-trash-after-reply t) +@end lisp + +When you reply to an iCal event, a line may be automatically added to +the diary file of your choice. You can specify that file with + +@lisp +(setq mu4e-icalendar-diary-file "/path/to/your/diary") +@end lisp + +Note that, if the specified file is not your main diary file, add +@t{#include "/path/to/your/diary"} to you main diary file to display +the events. + +To enable optional iCalendar→Org sync functionality, add the following: + +@lisp +(setq gnus-icalendar-org-capture-file "~/org/notes.org") +(setq gnus-icalendar-org-capture-headline '("Calendar")) +(gnus-icalendar-org-setup) +@end lisp + +Both the capture file and the headline(s) inside it must already exist. + +By default, @code{gnus-icalendar-org-setup} adds a temporary capture +template to the variable @code{org-capture-templates}, with the +description ``used by gnus-icalendar-org'', and the shortcut key ``#''. +If you want to use your own template, create it using the same key and +description. This will prevent the temporary one from being installed +next time you @code{gnus-icalendar-org-setup} is called. + +The full default capture template is: + +@lisp +("#" "used by gnus-icalendar-org" entry + (file+olp ,gnus-icalendar-org-capture-file + ,gnus-icalendar-org-capture-headline) + "%i" :immediate-finish t) +@end lisp + +where the values of the variables @code{gnus-icalendar-org-capture-file} +and @code{gnus-icalendar-org-capture-headline} are inserted via macro +expansion. + +If, for example, you wanted to store ical events in a date tree, +prompting for the date, you could use the following: + +@lisp +("#" "used by gnus-icalendar-org" entry + (file+olp+datetree path-to-capture-file) + "%i" :immediate-finish t :time-prompt t) +@end lisp + +Note that the default behaviour for @code{datetree} targets in this +situation is to store the event at the date that you capture it, not at +the date that it is scheduled. That's why I've suggested using the +@code{:timeprompt t} argument. This gives you an opportunity to set the +time to the correct value yourself. + +You can extract the event time directly, and have the @code{org-capture} +functions use that to set the @code{datetree} location: + +@lisp +(defun my-catch-event-time (orig-fun &rest args) + "Set org-overriding-default-time to the start time of the capture event" + (let ((org-overriding-default-time (date-to-time + (gnus-icalendar-event:start (car args))))) + (apply orig-fun args))) + +(advice-add 'gnus-icalendar:org-event-save :around #'my-catch-event-time) +@end lisp + +If you do this, you'll want to omit the @code{:timeprompt t} setting +from your capture template. + +@node Speedbar +@section Speedbar +@cindex speedbar + +@code{speedbar} is an Emacs-extension that shows navigational +information for an Emacs buffer in a separate frame. Using +@code{mu4e-speedbar}, @t{mu4e} lists your bookmarks and maildir +folders and allows for one-click access to them. + +To enable this, add @t{(require 'mu4e-speedbar)} to your configuration; +then, all you need to do to activate it is @kbd{M-x speedbar}. Then, +when then switching to the @ref{Main view}, the speedbar-frame is +updated with your bookmarks and maildirs. + +For speed reasons, the list of maildirs is determined when @t{mu4e} +starts; if the list of maildirs changes while @t{mu4e} is running, you +need to restart @t{mu4e} to have those changes reflected in the speedbar +and in other places that use this list, such as auto-completion when +jumping to a maildir. + +@node Dired +@section Dired +@cindex dired + +It is possible to attach files to @t{mu4e} messages using @t{dired} +(@ref{Dired,,emacs}), using the following steps (based on a post on +the @t{mu-discuss} mailing list by @emph{Stephen Eglen}). + +@lisp +(add-hook 'dired-mode-hook 'turn-on-gnus-dired-mode) +@end lisp + +Then, mark the file(s) in @t{dired} you would like to attach and press +@t{C-c RET C-a}, and you'll be asked whether to attach them to an +existing message, or create a new one. + +@node Other tools +@appendix Other tools + +In this chapter, we discuss some ways in which @t{mu4e} can cooperate +with other tools. + +@menu +* Org-contacts::Hooking up with org-contacts +* BBDB::Hooking up with the Insidious Big Brother Database +* Sauron::Getting new mail notifications with Sauron +* Hydra:: Custom shortcut menus +@end menu + +@node Org-contacts +@section Org-contacts + +Note, @t{mu4e} supports built-in address autocompletion; @ref{Address +autocompletion}, and that is the recommended way to do this. However, it is also +possible to manage your addresses with @t{org-mode}, using +@uref{https://julien.danjou.info/projects/emacs-packages#org-contacts,org-contacts}. + +@t{mu4e-actions} defines a useful action (@ref{Actions}) for adding a +contact based on the @t{From:}-address in the message at point. To +enable this, add to your configuration something like: + +@lisp + (setq mu4e-org-contacts-file <full-path-to-your-org-contacts-file>) + (add-to-list 'mu4e-headers-actions + '("org-contact-add" . mu4e-action-add-org-contact) t) + (add-to-list 'mu4e-view-actions + '("org-contact-add" . mu4e-action-add-org-contact) t) +@end lisp + +@noindent +After this, you should be able to add contacts using @key{a o} in the +headers view and the message view, using the @t{org-capture} mechanism. +Note, the shortcut character @key{o} is due to the first character of +@t{org-contact-add}. + +@node BBDB +@section BBDB + +Note, @t{mu4e} supports built-in address autocompletion; @ref{Address +autocompletion}, and that is the recommended way to do this. However, it is also +possible to manage your addresses with +@uref{https://savannah.nongnu.org/projects/bbdb/,BBDB}. + +To enable BBDB, add to your @file{~/.emacs} (or its moral equivalent, +such as @file{~/.emacs.d/init.el}) the following @emph{after} the +@code{(require 'mu4e)} line: + +@lisp + ;; Load BBDB (Method 1) + (require 'bbdb-loaddefs) + ;; OR (Method 2) + ;; (require 'bbdb-loaddefs "/path/to/bbdb/lisp/bbdb-loaddefs.el") + ;; OR (Method 3) + ;; (autoload 'bbdb-insinuate-mu4e "bbdb-mu4e") + ;; (bbdb-initialize 'message 'mu4e) + + (setq bbdb-mail-user-agent 'mu4e-user-agent) + (setq mu4e-view-rendered-hook 'bbdb-mua-auto-update) + (setq mu4e-compose-complete-addresses nil) + (setq bbdb-mua-pop-up t) + (setq bbdb-mua-pop-up-window-size 5) + (setq mu4e-view-show-addresses t) +@end lisp + +For recent emacs (29 and later), address-completion may need some extra setup: +@lisp +(add-hook 'message-mode-hook + (lambda () + (add-to-list 'completion-at-point-functions + #'eudc-capf-complete))) +@end lisp +or, if that does not work: +@lisp +(add-hook 'message-mode-hook + (lambda () + (add-to-list 'completion-at-point-functions + #'message-expand-name))) +@end lisp + +@noindent +After this, you should be able to: +@itemize +@item In mu4e-view mode, add the sender of the email to BBDB with @key{C-u :} +@item Tab-complete addresses from BBDB when composing emails +@item View the BBDB contact while viewing a message +@end itemize + + + +@node Sauron +@section Sauron + +The Emacs package @uref{https://github.com/djcb/sauron,sauron} (by the same +author) can be used to get notifications about new mails. If you run something +like the below script from your @t{crontab} (or have some other way of having it +execute every @emph{n} minutes), you receive notifications in the +@t{sauron}-buffer when new messages arrive. + +@verbatim +#!/bin/sh + +# the mu binary +MU=mu + +# put the path to your Inbox folder here +CHECKDIR="/home/$LOGNAME/Maildir/Inbox" + +sauron_msg () { +DBUS_COOKIE="/home/$LOGNAME/.sauron-dbus" +if test "x$DBUS_SESSION_BUS_ADDRESS" = "x"; then + if test -e $DBUS_COOKIE; then + export DBUS_SESSION_BUS_ADDRESS="`cat $DBUS_COOKIE`" + fi +fi +if test -n "x$DBUS_SESSION_BUS_ADDRESS"; then + dbus-send --session \ + --dest="org.gnu.Emacs" \ + --type=method_call \ + "/org/gnu/Emacs/Sauron" \ + "org.gnu.Emacs.Sauron.AddMsgEvent" \ + string:shell uint32:3 string:"$1" +fi +} + +# +# -mmin -5: consider only messages that were created / changed in the +# the last 5 minutes +# +for f in `find $CHECKDIR -mmin -5 -a -type f -not -iname '.uidvalidity'`; do + subject=`$MU view $f | grep '^Subject:' | sed 's/^Subject://'` + sauron_msg "mail: $subject" +done +@end verbatim + +@noindent +You might want to put: +@lisp +(setq sauron-dbus-cookie t) +@end lisp +@noindent +in your setup, to allow the script to find the D-Bus session bus, even when +running outside its session. + + +@node Hydra +@section Hydra + +People sometimes ask about having multi-character shortcuts for bookmarks; an +easy way to achieve this, is by using an emacs package +@uref{https://github.com/abo-abo/hydra,Hydra}. + +With Hydra installed, we can add multi-character shortcuts, for instance: +@lisp +(defhydra my-mu4e-bookmarks-work (:color blue) + "work bookmarks" + ("b" (mu4e-search "banana AND maildir:/work") "banana") + ("u" (mu4e-search "flag:unread AND maildir:/work") "unread")) + +(defhydra my-mu4e-bookmarks-personal (:color blue) + "personal bookmarks" + ("c" (mu4e-search "capybara AND maildir:/personal") "capybara") + ("u" (mu4e-search "flag:unread AND maildir:/personal") "unread")) + +(defhydra my-mu4e-bookmarks (:color blue) + "mu4e bookmarks" + ("p" (my-mu4e-bookmarks-personal/body) "Personal") + ("w" (my-mu4e-bookmarks-work/body) "Work")) + +Now, you can bind a convenient key to my-mu4e-bookmarks/body. +@end lisp + +@node Example configurations +@appendix Example configurations + +In this chapter, we show some example configurations. While it is very useful +to see some working settings, we'd like to warn against blindly copying such +things. + +@menu +* Minimal configuration::Simplest configuration to get you going +* Longer configuration::A more extensive setup +* Gmail configuration::GMail-specific setup +* Other settings:CONF Other settings. Some other useful configuration + +@end menu + +@node Minimal configuration +@section Minimal configuration + +An (almost) minimal configuration for @t{mu4e} might look like this --- as you +see, most of it is commented-out. + +@lisp +;; example configuration for mu4e + +;; make sure mu4e is in your load-path +(require 'mu4e) + +;; use mu4e for e-mail in emacs +(setq mail-user-agent 'mu4e-user-agent) + +;; these must start with a "/", and must exist +;; (i.e.. /home/user/Maildir/sent must exist) +;; you use e.g. 'mu mkdir' to make the Maildirs if they don't +;; already exist + +;; below are the defaults; if they do not exist yet, mu4e offers to +;; create them. they can also functions; see their docstrings. +;; (setq mu4e-sent-folder "/sent") +;; (setq mu4e-drafts-folder "/drafts") +;; (setq mu4e-trash-folder "/trash") + +;; smtp mail setting; these are the same that `gnus' uses. +(setq + message-send-mail-function 'smtpmail-send-it + smtpmail-default-smtp-server "smtp.example.com" + smtpmail-smtp-server "smtp.example.com" + smtpmail-local-domain "example.com") +@end lisp + + +@node Longer configuration +@section Longer configuration + +A somewhat longer configuration, showing some more things that you can +customize. + +@lisp +;; example configuration for mu4e +(require 'mu4e) + +;; use mu4e for e-mail in emacs +(setq mail-user-agent 'mu4e-user-agent) + +;; the next are relative to the root maildir +;; (see `mu info`). +;; instead of strings, they can be functions too, see +;; their docstring or the chapter 'Dynamic folders' +(setq mu4e-sent-folder "/sent" + mu4e-drafts-folder "/drafts" + mu4e-trash-folder "/trash") + +;; the maildirs you use frequently; access them with 'j' ('jump') +(setq mu4e-maildir-shortcuts + '((:maildir "/archive" :key ?a) + (:maildir "/inbox" :key ?i) + (:maildir "/work" :key ?w) + (:maildir "/sent" :key ?s))) + +;; the headers to show in the headers list -- a pair of a field +;; and its width, with `nil' meaning 'unlimited' +;; (better only use that for the last field. +;; These are the defaults: +(setq mu4e-headers-fields + '( (:date . 25) ;; alternatively, use :human-date + (:flags . 6) + (:from . 22) + (:subject . nil))) ;; alternatively, use :thread-subject + +(add-to-list 'mu4e-bookmarks + ;; ':favorite t' i.e, use this one for the modeline + '(:query "maildir:/inbox" :name "Inbox" :key ?i :favorite t)) + +;; program to get mail; alternatives are 'fetchmail', 'getmail' +;; isync or your own shellscript. called when 'U' is pressed in +;; main view. + +;; If you get your mail without an explicit command, +;; use "true" for the command (this is the default) +(setq mu4e-get-mail-command "offlineimap") + +;; general emacs mail settings; used when composing e-mail +;; the non-mu4e-* stuff is inherited from emacs/message-mode +(setq mu4e-compose-reply-to-address "foo@@bar.example.com" + user-mail-address "foo@@bar.example.com" + user-full-name "Foo X. Bar") +(setq message-signature "Foo X. Bar\nhttp://www.example.com\n") + +;; smtp mail setting +(setq + message-send-mail-function 'smtpmail-send-it + smtpmail-default-smtp-server "smtp.example.com" + smtpmail-smtp-server "smtp.example.com" + smtpmail-local-domain "example.com" + + ;; if you need offline mode, set these -- and create the queue dir + ;; with 'mu mkdir', i.e.. mu mkdir /home/user/Maildir/queue + smtpmail-queue-mail nil + smtpmail-queue-dir "/home/user/Maildir/queue/cur") + +;; don't keep message buffers around +(setq message-kill-buffer-on-exit t) +@end lisp + + +@node Gmail configuration +@section Gmail configuration + +@emph{Gmail} is a popular e-mail provider; let's see how we can make it +work with @t{mu4e}. Since we are using @abbr{IMAP}, you must enable that +in the Gmail web interface (in the settings, under the ``Forwarding and +POP/IMAP''-tab). + +Gmail users may also be interested in @ref{Including related messages}, +and in @ref{Skipping duplicates}. + +@subsection Setting up offlineimap + +First of all, we need a program to get the e-mail from Gmail to our +local machine; for this we use @t{offlineimap}; on Debian (and +derivatives like Ubuntu), this is as easy as: + +@verbatim +$ sudo apt-get install offlineimap +@end verbatim + +while on Fedora (and similar) you need: +@verbatim +$ sudo yum install offlineimap +@end verbatim + +Then, we can configure @t{offlineimap} by editing @file{~/.offlineimaprc}: + +@verbatim +[general] +accounts = Gmail +maxsyncaccounts = 3 + +[Account Gmail] +localrepository = Local +remoterepository = Remote + +[Repository Local] +type = Maildir +localfolders = ~/Maildir + +[Repository Remote] +type = IMAP +remotehost = imap.gmail.com +remoteuser = USERNAME@gmail.com +remotepass = PASSWORD +ssl = yes +maxconnections = 1 +@end verbatim + +Obviously, you need to replace @t{USERNAME} and @t{PASSWORD} with your actual +Gmail username and password. After this, you should be able to download your +mail: + +@verbatim +$ offlineimap + OfflineIMAP 6.3.4 +Copyright 2002-2011 John Goerzen & contributors. +Licensed under the GNU GPL v2+ (v2 or any later version). + +Account sync Gmail: + ***** Processing account Gmail + Copying folder structure from IMAP to Maildir + Establishing connection to imap.gmail.com:993. +Folder sync [Gmail]: + Syncing INBOX: IMAP -> Maildir + Syncing [Gmail]/All Mail: IMAP -> Maildir + Syncing [Gmail]/Drafts: IMAP -> Maildir + Syncing [Gmail]/Sent Mail: IMAP -> Maildir + Syncing [Gmail]/Spam: IMAP -> Maildir + Syncing [Gmail]/Starred: IMAP -> Maildir + Syncing [Gmail]/Trash: IMAP -> Maildir +Account sync Gmail: + ***** Finished processing account Gmail +@end verbatim + +We can now run @command{mu} to make sure things work: + +@verbatim +$ mu index +mu: indexing messages under /home/foo/Maildir [/home/foo/.cache/mu/xapian] +| processing mail; checked: 520; updated/new: 520, cleaned-up: 0 +mu: elapsed: 3 second(s), ~ 173 msg/s +mu: cleaning up messages [/home/foo/.cache/mu/xapian] +/ processing mail; checked: 520; updated/new: 0, cleaned-up: 0 +mu: elapsed: 0 second(s) +@end verbatim + +We can run both the @t{offlineimap} and the @t{mu index} from within +@t{mu4e}, but running it from the command line makes it a bit easier to +troubleshoot as we are setting things up. + +Note: when using encryption, you probably do @emph{not} want to +synchronize your Drafts-folder, since it contains the unencrypted +messages. You can use OfflineIMAP's @t{folderfilter} for that. + +@subsection Settings + +Next step: let's make a @t{mu4e} configuration for this: + +@lisp +(require 'mu4e) + +;; use mu4e for e-mail in emacs +(setq mail-user-agent 'mu4e-user-agent) + +(setq mu4e-drafts-folder "/[Gmail].Drafts") +(setq mu4e-sent-folder "/[Gmail].Sent Mail") +(setq mu4e-trash-folder "/[Gmail].Trash") + +;; don't save message to Sent Messages, Gmail/IMAP takes care of this +(setq mu4e-sent-messages-behavior 'delete) + +;; (See the documentation for `mu4e-sent-messages-behavior' if you have +;; additional non-Gmail addresses and want assign them different +;; behavior.) + +;; setup some handy shortcuts +;; you can quickly switch to your Inbox -- press ``ji'' +;; then, when you want archive some messages, move them to +;; the 'All Mail' folder by pressing ``ma''. + +(setq mu4e-maildir-shortcuts + '( (:maildir "/INBOX" :key ?i) + (:maildir "/[Gmail].Sent Mail" :key ?s) + (:maildir "/[Gmail].Trash" :key ?t) + (:maildir "/[Gmail].All Mail" :key ?a))) + +(add-to-list 'mu4e-bookmarks + ;; ':favorite t' i.e, use this one for the modeline + '(:query "maildir:/inbox" :name "Inbox" :key ?i :favorite t)) + +;; allow for updating mail using 'U' in the main view: +(setq mu4e-get-mail-command "offlineimap") + +;; something about ourselves +(setq + user-mail-address "USERNAME@@gmail.com" + user-full-name "Foo X. Bar" + message-signature + (concat + "Foo X. Bar\n" + "http://www.example.com\n")) + +;; sending mail -- replace USERNAME with your gmail username +;; also, make sure the gnutls command line utils are installed +;; package 'gnutls-bin' in Debian/Ubuntu + +(require 'smtpmail) +(setq message-send-mail-function 'smtpmail-send-it + starttls-use-gnutls t + smtpmail-starttls-credentials '(("smtp.gmail.com" 587 nil nil)) + smtpmail-auth-credentials + '(("smtp.gmail.com" 587 "USERNAME@@gmail.com" nil)) + smtpmail-default-smtp-server "smtp.gmail.com" + smtpmail-smtp-server "smtp.gmail.com" + smtpmail-smtp-service 587) + +;; alternatively, for emacs-24 you can use: +;;(setq message-send-mail-function 'smtpmail-send-it +;; smtpmail-stream-type 'starttls +;; smtpmail-default-smtp-server "smtp.gmail.com" +;; smtpmail-smtp-server "smtp.gmail.com" +;; smtpmail-smtp-service 587) + +;; don't keep message buffers around +(setq message-kill-buffer-on-exit t) +@end lisp + +And that's it --- put the above in your emacs initialization file, change +@t{USERNAME} etc. to your own, restart Emacs, and run @kbd{M-x mu4e}. + +@node CONF Other settings +@section Other settings + +Finally, here are some more settings that are useful, but not enabled by +default for various reasons. + +@lisp +;; use 'fancy' non-ascii characters in various places in mu4e +(setq mu4e-use-fancy-chars t) + +;; save attachment to my desktop (this can also be a function) +(setq mu4e-attachment-dir "~/Desktop") + +;; attempt to show images when viewing messages +(setq mu4e-view-show-images t) +@end lisp + +@node FAQ +@appendix FAQ --- Frequently Asked Questions + +In this chapter we list a number of actual and anticipated questions and their +answers. + +@menu +* General::General questions and answers about @t{mu4e} +* Retrieving mail::Getting mail and indexing +* Reading messages::Dealing with incoming messages +* Writing messages::Dealing with outgoing messages +* Known issues::Limitations we know about +@end menu + +@node General +@section General + +@subsection Results from @t{mu} and @t{mu4e} differ - why? +@anchor{mu-mu4e-differ} In general, the same queries for @command{mu} +and @t{mu4e} should yield the same results. If they differ, this is +usually because one of the following reasons: +@itemize +@item different options: +@t{mu4e} defaults to having @t{mu4e-headers-include-related}, and +@t{mu4e-headers-results-limit} set to 500. However, the command-line +@command{mu find}'s corresponding @t{--include-related} is false, and +there's no limit (@t{--maxnum}). +@item reverse sorting: +The results may be different when @t{mu4e} and @command{mu find} do +not both sort their results in the same direction. +@item shell quoting issues: +Depending on the shell, various shell metacharacters in search query +(such as @t{*}) may be expanded by the shell before @command{mu} ever +sees them, and the query may not be what you think it is. Quoting is +necessary. +@end itemize + +@subsection The unread/all counts in the main-screen differ from the 'real' numbers - what's going on? +For speed reasons, the counts do not exclude messages that no longer exist in +the file-system, nor does it exclude duplicate messages; @xref{mu-mu4e-differ}. + +@subsection How can I quickly delete/move/trash a lot of messages? +You can select ('mark' in Emacs-speak) messages, just like you would select text +in a buffer; the actions you then take (e.g., @key{DEL} for delete, @key{m} for +move and @key{t} for trash) apply to all selected messages. You can also use +functions like @code{mu4e-headers-mark-thread} (@key{T}), +@code{mu4e-headers-mark-subthread} (@key{t}) to mark whole threads at the same +time, and @code{mu4e-headers-mark-pattern} (@key{%}) to mark all messages +matching a certain regular expression. + +@subsection Can I automatically apply the marks on messages when leaving the headers buffer? +Yes you can --- see the documentation for the variable +@t{mu4e-headers-leave-behavior}. + +@subsection How can I set @t{mu4e} as the default e-mail client in Emacs? +See @ref{Default email client}. + +@subsection Can @t{mu4e} use some fancy Unicode instead of these boring plain-ASCII ones? +Glad you asked! Yes, if you set @code{mu4e-use-fancy-chars} to @t{t}, @t{mu4e} +uses such fancy characters in a number of places. Since not all fonts include +all characters, you may want to install the @t{unifont} and/or @t{symbola} fonts +on your system. + +@subsection Can I start @t{mu4e} in the background? +Yes --- if you provide a prefix-argument (@key{C-u}), @t{mu4e} starts, but does +not show the main-window. + +@subsection Does @t{mu4e} support searching for CJK (Chinese-Japanese-Korean) characters? +Only partially. If you have @t{Xapian} 1.2.8 or newer, and set the environment +variable @t{XAPIAN_CJK_NGRAM} to non-empty before indexing, both when using +@t{mu} from the command-line and from @t{mu4e}. + +@subsection How can I customize the function to select a folder? +The @t{mu4e-completing-read-function} variable can be customized to select a +folder in any way. The variable can be set to a function that receives five +arguments, following @t{completing-read}. The default value is +@code{ido-completing-read}; to use emacs's default behavior, set the variable to +@code{completing-read}. Helm users can use the same value, and by enabling +@code{helm-mode} use helm-style completion. + +@subsection With a lot of Maildir folders, jumping to them can get slow. What can I do? +Set @code{mu4e-cache-maildir-list} to @code{t} (make sure to read its +docstring). + +@subsection How can I hide certain messages from the search results? +See the variables @code{mu4e-headers-hide-predicate} and +@code{mu4e-headers-hide-enabled}. The latter can be toggled through +@code{mu4e-headers-toggle-property}. + +For example, to filter out GMail's spam folder, set it to: +@lisp +(setq mu4e-headers-hide-predicate + (lambda (msg) + (string-suffix-p "Spam" (mu4e-message-field msg :maildir)))) +@end lisp + +@subsection I'm getting an error 'Variable binding depth exceeds max-specpdl-size' when using mu4e -- what can I do about it? +The error occurs because @t{mu4e} is binding more variables than +@t{emacs} allows for, by default. You can avoid this by setting a +higher value, e.g. by adding the following to your configuration: +@lisp +(setq max-specpdl-size 5000) +@end lisp +Note that Emacs 29 obsoletes this variable. + +@node Retrieving mail +@section Retrieving mail + +@subsection How can I get notifications when receiving mail? +There is @code{mu4e-index-updated-hook}, which gets triggered when the +indexing process triggered sees an update (not just new mail though). To +use this hook, put something like the following in your setup (assuming +you have @t{aplay} and some soundfile, change as needed): +@lisp +(add-hook 'mu4e-index-updated-hook + (defun new-mail-sound () + (shell-command "aplay ~/Sounds/boing.wav&"))) +@end lisp + +@subsection I'm getting mail through a local mailserver. What should I use for @code{mu4e-get-mail-command}? +Use the literal string @t{"true"} (or don't do anything, it's the +default) which then uses @t{/bin/true} (a command that does nothing and +always succeeds). This makes getting mail a no-op, but the messages are +still re-indexed. + +@subsection How can I re-index my messages without getting new mail? +Use @kbd{M-x mu4e-update-index} + +@subsection When I try to run @t{mu index} while @t{mu4e} is running I get errors +For instance: +@verbatim +mu: mu_store_new_writable: xapian error + 'Unable to get write lock on ~/.cache/mu/xapian: already locked +@end verbatim +What to do about this? You get this error because the underlying Xapian +database is locked by some other process; it can be opened only once in +read-write mode. There is not much @t{mu4e} can do about this, but if is +another @command{mu} instance that is holding the lock, you can ask it +to (gracefully) terminate: +@verbatim + pkill -2 -u $UID mu # send SIGINT + sleep 1 + mu index +@end verbatim +@t{mu4e} automatically restarts @t{mu} when it needs it. In practice, this +seems to work quite well. + +@subsection How can I disable the @t{Indexing...} messages? +Set the variable @code{mu4e-hide-index-messages} to non-@t{nil}. + +@subsection IMAP-synchronization and file-name changes +Some IMAP-synchronization programs such as @t{mbsync} (but not +@t{offlineimap}) don't like it when message files do not change their +names when they are moved to different folders. @t{mu4e} can attempt to +help with this - you can set the variable +@code{mu4e-change-filenames-when-moving} to non-@t{nil}. + +@subsection @command{offlineimap} and UTF-7 +@command{offlineimap} uses IMAP's UTF-7 for encoding non-ascii folder +names, while @command{mu} expects UTF-8 (so, e.g. @t{/まりもえ +お}@footnote{some Japanese characters} becomes @t{/&MH4wijCCMEgwSg-}). + +This is best solved by telling @command{offlineimap} to use UTF-8 instead --- +see @uref{https://github.com/djcb/mu/issues/68#issuecomment-8598652,this +ticket}. + +@subsection @command{mbsync} or @command{offlineimap} do not sync properly +Unfortunately, @command{mbsync} and/or @command{offlineimap} do not +always agree with @t{mu} about the meaning of various Maildir-flags. If +you encounter unexpected behavior, it is recommended you check before +and after a sync-operation. If the problem only shows up @emph{after} +sync'ing, the problem is with the sync-program, and it's most productive +to complain there. + +Also, you may want to ensure that @t{mu4e-index-lazy-check} is kept at +its default (@t{nil}) value, since it seems @command{mbsync} can make +changes that escape a 'lazy' check. + +Furthermore, there have been quite a few related queries on the +mailing-list; worthwhile to check out. + +@node Reading messages +@section Reading messages + +@subsection Opening messages is slower than expected - why? +@t{mu4e} is designed to be very fast, even with large amounts of mail. +However, if you experience slowdowns, here are some things to consider: +@itemize +@item opening messages while indexing: +@t{mu4e} communicates with the @t{mu} server mostly synchronously; this means +that you can do only one thing at a time. The one operation that potentially +does take a bit of time is indexing of mail. Indexing does happen +asynchronously, but still can slow down @t{mu} enough that users may notice. + +For some strategies to reduce that time, see the next question. +@item getting contact information can take some time: +especially when opening @t{mu4e} the first time and you have a +@emph{lot} of contacts, it can take a few seconds to process those. Note +that @t{mu4e} 1.3 and higher only get @emph{changed} contacts in +subsequent updates (after and indexing operation), so this should be +less of a concern. And you can tweak what contacts you get using +@var{mu4e-compose-complete-only-personal}, +@var{mu4e-compose-complete-only-after} and +@var{mu4e-compose-complete-max}. +@item decryption / sign verification: +encrypted / signed messages sometimes require network access, and this +may take a while; certainly if the needed servers cannot be found. +Part of this may be that influential environment variables are not set +in the emacs environment. +@end itemize + +If you still experience unexpected slowness, you can of course file a +ticket, but please be sure to mention the following: + +@itemize +@item are all messages slow or only some messages? +@item if it's only some messages, is there something specific about them? +@item in addition, please a (sufficiently censored version of) a message that is slow +@item is opening @emph{always} slow or only sometimes? When? +@end itemize + +@subsection How can I word-wrap long lines in when viewing a message? +You can toggle between wrapped and non-wrapped states using @key{w}. If you want +to do this automatically, invoke @code{visual-line-mode} in your +@code{mu4e-view-rendered-hook} (@code{mu4e-view-mode-hook} fires too early). +@subsection How can I perform custom actions on messages and attachments? +See @ref{Actions}. +@subsection How can I prevent @t{mu4e} from automatically marking messages as `read' when I read them? +Set @code{mu4e-view-auto-mark-as-read} to @code{nil}. +@subsection Does @t{mu4e} support including all related messages in a thread, like Gmail does? +Yes --- see @ref{Including related messages}. +@subsection There seems to be a lot of duplicate messages --- how can I get rid of them? +See @ref{Skipping duplicates}. +@subsection Some messages are almost unreadable in emacs --- can I view them in an external web browser? +Indeed, airlines often send messages that heavily depend on html and +are hard to digest inside emacs. Fortunately, there's an @emph{action} +(@ref{Message view actions}) defined for this. Simply add to your +configuration: +@lisp +(add-to-list 'mu4e-view-actions + '("ViewInBrowser" . mu4e-action-view-in-browser) t) +@end lisp +Now, when viewing such a difficult message, type @kbd{aV}, and the +message opens inside a web browser. You can influence the browser to +use with @code{browse-url-generic-program}. +@subsection How can I read encrypted messages that I sent? +Since you do not own the recipient's key you typically cannot read +those mails --- so the trick is to encrypt outgoing mails with your +key, too. This can be automated by adding the following snippet to +your configuration (courtesy of user @t{kpachnis}): +@lisp +(require 'epg-config) +(setq mml2015-use 'epg + epg-user-id "gpg_key_id" + mml2015-encrypt-to-self t + mml2015-sign-with-sender t) +@end lisp + +@node Writing messages +@section Writing messages + +@subsection How can I automatically set the @t{From:}-address for a reply-message? +See @ref{Compose hooks}. + +@subsection How can I dynamically determine the folders for draft/sent/trashed messages? +See @ref{Dynamic folders}. + +@subsection How can I define aliases for (groups of) e-mail addresses? +See @ref{(emacs) Mail Aliases}. + +@subsection How can I automatically add some header to an outgoing message? +See @ref{Compose hooks}. + +@subsection How can I influence the way the original message looks when replying/forwarding? +Since @code{mu4e-compose-mode} derives from @xref{(message) Top}, you can re-use +many (though not @emph{all} of its facilities. + +@subsection How can I easily include attachments in the messages I write? +You can drag-and-drop from your desktop; alternatively, you can use @ref{(emacs) +Dired}. + +@subsection How can I start a new message-thread from a reply? +Remove the @t{In-Reply-To} header, and @t{mu4e} automatically removes +the (hidden) @t{References} header as well when sending it. This makes +the message show up as a top-level message rather than as a response. + +@subsection How can I attach an existing message? +Use @code{mu4e-action-capture-message} (i.e., @kbd{a c} in the headers + view) to `capture' the to-be-attached message, then when editing the + message, use @kbd{M-x mu4e-compose-attach-captured-message}. + +@subsection How can I sign or encrypt messages? +You can do so using Emacs' MIME-support --- check the +@t{Attachments}-menu while composing a message. Also see @ref{Signing +and encrypting}. + +@subsection Address auto-completion misses some addresses +If you have set @code{mu4e-compose-complete-only-personal} to non-nil, @t{mu4e} +only completes 'personal' addresses - so you tell it about your e-mail addresses +when setting up the database (@t{mu init}); @ref{Initializing the message +store}. + +If you cannot find specific addresses you'd expect to find, inspect the +values of @var{mu4e-compose-complete-only-personal}, +@var{mu4e-compose-complete-only-after} and +@var{mu4e-compose-complete-max}. + +@subsection How can I get rid of the message buffer after sending? +@lisp +(setq message-kill-buffer-on-exit t) +@end lisp + +@subsection Sending big messages is slow and blocks emacs --- what can I do about it? + +For this, there's @uref{https://github.com/jwiegley/emacs-async,emacs-async} +(also available from the Emacs package repository); add the following snippet to +your configuration: +@lisp +(require 'smtpmail-async) +(setq + send-mail-function 'async-smtpmail-send-it + message-send-mail-function 'async-smtpmail-send-it) +@end lisp +With this, messages are sent using a background Emacs instance. + +A word of warning though, this tends to not be as reliable as sending the +message in the normal, synchronous fashion, and people have reported silent +failures, where mail sending fails for some reason without any indication of +that. + +You can check the progress of the background delivery by checking the +@t{*Messages*}-buffer, which should show something like: +@verbatim +Delivering message to "William Shakespeare" <will@example.com>... +Mark set +Saving file /home/djcb/Maildir/sent/cur/20130706-044350-darklady:2,S... +Wrote /home/djcb/Maildir/sent/cur/20130706-044350-darklady:2,S +Sending...done +@end verbatim +The first and final messages are the most important, and there may be +considerable time between them, depending on the size of the message. + +@subsection Is it possible to view headers and messages, or compose new ones, in a separate frame or window? +Yes. There is built-in support for composing messages in a new frame or window. +Either use Emacs' standard @t{compose-mail-other-frame} (@kbd{C-x 5 m}) and +@t{compose-mail-other-window} (@kbd{C-x 4 m}) if you have set up @t{mu4e} as your Emacs +e-mailer. + +Additionally, there's the variable @code{mu4e-compose-switch} (see its +docstring) which you can customize to influence how @t{mu4e} creates new +messages. + +@subsection How can I apply format=flowed to my outgoing messages? +This enables receiving clients that support this feature to reflow +paragraphs. Plain text emails with @t{Content-Type: text/plain; +format=flowed} can be reflowed (i.e. line endings removed, paragraphs +refilled) by receiving clients that support this standard. Clients +that don't support this, show them as is, which means this feature is +truly non-invasive. + +Here's an explanatory blog post which also shows why this is a desirable +feature: @url{https://mathiasbynens.be/notes/gmail-plain-text} (if you don't +have it, your mails mostly look quite bad especially on mobile devices) and +here's the @uref{https://www.ietf.org/rfc/rfc2646.txt,RFC with all the details}. + +Since version 0.9.17, @t{mu4e} sends emails with @t{format=flowed} by setting +@lisp +(setq mu4e-compose-format-flowed t) +@end lisp + +@noindent +in your Emacs init file (@file{~/.emacs} or @file{~/.emacs.d/init.el}). The +transformation of your message into the proper format is done at the time of +sending. For this to happen properly, you should write each paragraph of your +message of as a long line (i.e. without carriage return). If you introduce +unwanted newlines in your paragraph, use @kbd{M-q} to reformat it as a single +line. + +If you want to send the message with paragraphs on single lines but +without @t{format=flowed} (because, say, the receiver does not +understand the latter as it is the case for Google or Github), use +@kbd{M-x use-hard-newlines} (to turn @code{use-hard-newlines} off) or +uncheck the box @t{format=flowed} in the @t{Text} menu when composing a +message. + +@subsection How can I force images to be shown at the end of my messages, regardless of where I insert them? +User Marcin Borkowski has a solution: +@lisp +(defun mml-attach-file--go-to-eob (orig-fun &rest args) + "Go to the end of buffer before attaching files." + (save-excursion + (save-restriction + (widen) + (goto-char (point-max)) + (apply orig-fun args)))) + +(advice-add 'mml-attach-file :around #'mml-attach-file--go-to-eob) +@end lisp + +@subsection How can I avoid Outlook display issues? + +Limited testing shows that certain Outlook clients do not work well with inline +replies, and the entire message including-and-below the first quoted section is +collapsed. This means recipients may not even notice important inline text, +especially if there is some top-posted content. This has been observed on OS X, +Windows, and Web-based Outlook clients accessing Office 365. + +It appears the bug is triggered by the standard reply regex "On ... +wrote:". Changing "On", or removing the trailing ":" appears to fix the +bug (in limited testing). Therefore, a simple work-around is to set +`message-citation-line-format` to something slightly non-standard, such +as: +@lisp +(setq message-citation-line-format "On %Y-%m-%d at %R %Z, %f wrote...") +@end lisp + +@node Known issues +@section Known issues + +Although they are not really @emph{questions}, we end this chapter with a list +of known issues and/or missing features in @t{mu4e}. Thus, users won't have to +search in vain for things that are not there (yet), and the author can use it as +a todo-list. + +@subsection UTF-8 language environment is required +@t{mu4e} does not work well if the Emacs language environment is not UTF-8; so, +if you encounter problems with encodings, be sure to have +@code{(set-language-environment "UTF-8")} in your @file{~/.emacs} (or its moral +equivalents in other places). + +@subsection Headers-buffer can get mis-aligned +Due to the way the headers buffer works, it can get misaligned. + +For the particular case where the header values are misaligned with the column +headings, you can try something like the following: +@lisp +(add-hook 'mu4e-headers-mode-hook #'my-mu4e-headers-mode-hook) +(defun my-mu4e-headers-mode-hook () + ;; Account for the fringe and other spacing in the header line. + (header-line-indent-mode 1) + (push (propertize " " 'display '(space :align-to header-line-indent-width)) + header-line-format) + ;; Ensure `text-scale-adjust' scales the header line with the headers themselves + ;; by ensuring the `default' face is in the inheritance hierarchy. + (face-remap-add-relative 'header-line '(:inherit (mu4e-header-face default))) +@end lisp + +This does not solve all possible issues; that would require a thorough rework of +the headers-view, which may happen at some time. + +@node Tips and Tricks +@appendix Tips and Tricks + +@menu +* Fancy characters:: Non-ascii characters in the UI +* Refiling messages:: Moving message to some archive folder +* Saving outgoing messages:: Automatically save sent messages +* Confirmation before sending:: Check messages before sending +@end menu + +@node Fancy characters +@section Fancy characters + +When using `fancy characters' (@code{mu4e-use-fancy-chars}) with the +@emph{Inconsolata}-font (and likely others as well), the display may be +slightly off; the reason for this issue is that Inconsolata does not +contain the glyphs for the `fancy' arrows and the glyphs that are used +as replacements are too high. + +To fix this, you can use something like the following workaround (in +your @t{.emacs}-file): +@lisp +(when (equal window-system 'x) + (set-fontset-font "fontset-default" 'unicode "Dejavu Sans Mono") + (set-face-font 'default "Inconsolata-10")) +@end lisp + +Other fonts with good support for Unicode are @t{unifont} and +@t{symbola}. + +For a more complete solution, but with greater overhead, you can also +try the @emph{unicode-fonts} package: +@lisp +(require 'unicode-fonts) +(require 'persistent-soft) ; To cache the fonts and reduce load time +(unicode-fonts-setup) +@end lisp + +It's possible to customize various header marks as well, with a ``fancy'' and +``non-fancy'' version (if you cannot see some the ``fancy'' characters, that is +an indication that the font you are using does not support those characters. + +@lisp + (setq + mu4e-headers-draft-mark '("D" . "💈") + mu4e-headers-flagged-mark '("F" . "📍") + mu4e-headers-new-mark '("N" . "🔥") + mu4e-headers-passed-mark '("P" . "❯") + mu4e-headers-replied-mark '("R" . "❮") + mu4e-headers-seen-mark '("S" . "☑") + mu4e-headers-trashed-mark '("T" . "💀") + mu4e-headers-attach-mark '("a" . "📎") + mu4e-headers-encrypted-mark '("x" . "🔒") + mu4e-headers-signed-mark '("s" . "🔑") + mu4e-headers-unread-mark '("u" . "⎕") + mu4e-headers-list-mark '("l" . "🔈") + mu4e-headers-personal-mark '("p" . "👨") + mu4e-headers-calendar-mark '("c" . "📅")) +@end lisp + +@node Refiling messages +@section Refiling messages + +By setting @code{mu4e-refile-folder} to a function, you can dynamically +determine where messages are to be refiled. If you want to do this based +on the subject of a message, you can use a function that matches the +subject against a list of regexes in the following way. First, set up a +variable @code{my-mu4e-subject-alist} containing regexes plus associated +mail folders: + +@lisp +(defvar my-mu4e-subject-alist '(("kolloqui\\(um\\|a\\)" . "/Kolloquium") + ("Calls" . "/Calls") + ("Lehr" . "/Lehre") + ("webseite\\|homepage\\|website" . "/Webseite")) + "List of subjects and their respective refile folders.") +@end lisp + +Now you can use the following function to automatically refile messages +based on their subject line: + +@lisp +(defun my-mu4e-refile-folder-function (msg) + "Set the refile folder for MSG." + (let ((subject (mu4e-message-field msg :subject)) + (folder (or (cdar (member* subject my-mu4e-subject-alist + :test #'(lambda (x y) + (string-match (car y) x)))) + "/General"))) + folder)) +@end lisp + +Note the @t{"/General"} folder: it is the default folder in case the +subject does not match any of the regexes in +@code{my-mu4e-subject-alist}. + +In order to make this work, you'll of course need to set +@code{mu4e-refile-folder} to this function: + +@lisp +(setq mu4e-refile-folder 'my-mu4e-refile-folder-function) +@end lisp + +If you have multiple accounts, you can accommodate them as well: + +@lisp +(defun my-mu4e-refile-folder-function (msg) + "Set the refile folder for MSG." + (let ((maildir (mu4e-message-field msg :maildir)) + (subject (mu4e-message-field msg :subject)) + folder) + (cond + ((string-match "Account1" maildir) + (setq folder (or (catch 'found + (dolist (mailing-list my-mu4e-mailing-lists) + (if (mu4e-message-contact-field-matches + msg :to (car mailing-list)) + (throw 'found (cdr mailing-list))))) + "/Account1/General"))) + ((string-match "Gmail" maildir) + (setq folder "/Gmail/All Mail")) + ((string-match "Account2" maildir) + (setq folder (or (cdar (member* subject my-mu4e-subject-alist + :test #'(lambda (x y) + (string-match + (car y) x)))) + "/Account2/General")))) + folder)) +@end lisp + +This function actually uses different methods to determine the refile +folder, depending on the account: for @emph{Account2}, it uses +@code{my-mu4e-subject-alist}, for the @emph{Gmail} account it simply uses the +folder ``All Mail''. For Account1, it uses another method: it files the +message based on the mailing list to which it was sent. This requires +another variable: + +@lisp +(defvar my-mu4e-mailing-lists + '(("mu-discuss@@googlegroups.com" . "/Account1/mu4e") + ("pandoc-discuss@@googlegroups.com" . "/Account1/Pandoc") + ("auctex@@gnu.org" . "/Account1/AUCTeX")) + "List of mailing list addresses and folders where + their messages are saved.") +@end lisp + +@node Saving outgoing messages +@section Saving outgoing messages + +Like @code{mu4e-refile-folder}, the variable @code{mu4e-sent-folder} can +also be set to a function, in order to dynamically determine the save +folder. One might, for example, wish to automatically put messages going +to mailing lists into the trash (because you'll receive them back from +the list anyway). If you have set up the variable +@code{my-mu4e-mailing-lists} as mentioned, you can use the following +function to determine a 'sent'-folder: + +@lisp +(defun my-mu4e-sent-folder-function (msg) + "Set the sent folder for the current message." + (let ((from-address (message-field-value "From")) + (to-address (message-field-value "To"))) + (cond + ((string-match "my.address@@account1.example.com" from-address) + (if (member* to-address my-mu4e-mailing-lists + :test #'(lambda (x y) + (string-match (car y) x))) + "/Trash" + "/Account1/Sent")) + ((string-match "my.address@@gmail.com" from-address) + "/Gmail/Sent Mail") + (t (mu4e-ask-maildir-check-exists "Save message to maildir: "))))) +@end lisp + +Note that this function doesn't use @code{(mu4e-message-field msg +:maildir)} to determine which account the message is being sent from. +The reason is that the function in @code{mu4e-sent-folder} is +called when you send the message, but before @t{mu4e} has created the +message struct from the compose buffer, so that +@code{mu4e-message-field} cannot be used. Instead, the function uses +@code{message-field-value}, which extracts the values of the headers in +the compose buffer. This means that it is not possible to extract the +account name from the message's maildir, so instead the from address is +used to determine the account. + +Again, the function shows three different possibilities: for the first +account (@t{my.address@@account1.example.com}) it uses +@code{my-mu4e-mailing-lists} again to determine if the message goes to a +mailing list. If so, the message is put in the trash folder, if not, it +is saved in @t{/Account1/Sent}. For the second (Gmail) account, sent +mail is simply saved in the Sent Mail folder. + +If the from address is not associated with Account1 or with the Gmail +account, the function uses @code{mu4e-ask-maildir-check-exists} to ask +the user for a maildir to save the message in. + +@node Confirmation before sending +@section Confirmation before sending + +To protect yourself from sending messages too hastily, you can add a +final confirmation, which you can of course make as elaborate as you +wish. + +@lisp +(defun confirm-empty-subject () + "Require confirmation before sending without subject." + (let ((sub (message-field-value "Subject"))) + (or (and sub (not (string-match "\\`[ \t]*\\'" sub))) + (yes-or-no-p "Really send without Subject? ") + (keyboard-quit)))) + +(add-hook 'message-send-hook #'confirm-empty-subject) +@end lisp + +If you @emph{always} want to be asked for for confirmation, set +@code{message-confirm-send} to non-@t{nil} so the question ``Send message?'' is +asked for confirmation. + +@node How it works +@appendix How it works + +While perhaps not interesting for all users of @t{mu4e}, some curious +souls may want to know how @t{mu4e} does its job. + +@menu +* High-level overview::How the pieces fit together +* mu server::The mu process running in the background +* Reading from the server::Processing responses from the server +* The message s-expression::What messages look like from the inside +@end menu + +@node High-level overview +@section High-level overview + +At a high level, we can summarize the structure of the @t{mu4e} system using +some ascii-art: + +@cartouche +@example + +---------+ + | emacs | + | +------+ + +----| mu4e | --> send mail (smtpmail) + +------+ + | A + V | ---/ search, view, move mail + +---------+ \ + | mu | + +---------+ + | A + V | + +---------+ + | Maildir | <--- receive mail (fetchmail, + +---------+ offlineimap, ...) +@end example +@end cartouche + +In words: +@itemize +@item Your e-mail messages are stored in a Maildir-directory +(typically, @file{~/Maildir} and its subdirectories), and new mail comes in +using tools like @t{fetchmail}, @t{offlineimap}, or through a local mail +server. +@item @t{mu} indexes these messages periodically, so you can quickly search for +them. @t{mu} can run in a special @t{server}-mode, where it provides services + to client software. +@item @t{mu4e}, which runs inside Emacs is + such a client; it communicates with @command{mu} (in its @t{server}-mode) to + search for messages, and manipulate them. +@item @t{mu4e} uses the facilities + offered by Emacs (the Gnus message editor and @t{smtpmail}) to send + messages. +@end itemize + +@node mu server +@section @t{mu server} + +@t{mu4e} is based on the @t{mu} e-mail searching/indexer. The latter +is a C++-program; there are different ways to communicate with a +client that is emacs-based. + +One way to implement this, would be to call the @t{mu} command-line +tool with some parameters and then parse the output. In fact, that was +the first approach --- @t{mu4e} would invoke e.g., @t{mu find} and +process the output in Emacs. + +However, with this approach, we need to load the entire e-mail +@emph{Xapian} database (in which the message is stored) for each +invocation. Wouldn't it be nicer to keep a running @t{mu} instance +around? Indeed, it would --- and thus, the @t{mu server} sub-command +was born. Running @t{mu server} starts a simple shell, in which you +can give commands to @command{mu}, which then spits out the +results/errors. @command{mu server} is not meant for humans, but it +can be used manually, which is great for debugging. + +@node Reading from the server +@section Reading from the server + +In the design, the next question was what format @t{mu} should use for its +output for @t{mu4e} (Emacs) to process. Some other programs use +@abbr{JSON} here, but it seemed easier (and possibly, more efficient) just to +talk to Emacs in its native language: @emph{s-expressions}, and +interpret those using the Emacs-function +@code{read-from-string}. See @ref{The message s-expression} for details on the +format. + +So, now let's look at how we process the data from @t{mu server} in +Emacs. We'll leave out a lot of details, @t{mu4e}-specifics, and look +at a bit more generic approach. + +The first thing to do is to create a process (for example, with +@code{start-process}), and then register a filter function for it, which is +invoked whenever the process has some data for us. Something like: + +@lisp + (let ((proc (start-process <arguments>))) + (set-process-filter proc 'my-process-filter) + (set-process-sentinel proc 'my-process-sentinel)) +@end lisp + +Note, the process sentinel is invoked when the process is terminated +--- so there you can clean things up. The function +@code{my-process-filter} is a user-defined function that takes the +process and the chunk of output as arguments; in @t{mu4e} it looks +something like (pseudo-lisp): + +@lisp +(defun my-process-filter (proc str) + ;; mu4e-buf: a global string variable to which data gets appended + ;; as we receive it + (setq mu4e-buf (concat mu4e-buf str)) + (when <we-have-received-a-full-expression> + <eat-expression-from mu4e-buf> + <evaluate-expression>)) +@end lisp + +@code{<evaluate-expression>} de-multiplexes the s-expression we got. +For example, if the s-expression looks like an e-mail message header, +it is processed by the header-handling function, which appends it to +the header list. If the s-expression looks like an error message, it +is reported to the user. And so on. + +The language between frontend and backend is documented partly in the +@t{mu-server} man-page and more completely in the output of @t{mu +server --commands}. + +@t{mu4e} can log these communications; you can use @kbd{M-x +mu4e-toggle-logging} to turn logging on and off, and you can view the +log using @kbd{M-x mu4e-show-log} (@key{$}). + +@node The message s-expression +@section The message s-expression + +As a word of warning, the details of the s-expression are internal to the mu4e - +mu communications, and are subject to change between versions. + +A typical message s-expression looks something like the following: + +@lisp +(:docid 32461 + :from ((:name "Nikola Tesla" :email "niko@@example.com")) + :to ((:name "Thomas Edison" :email "tom@@example.com")) + :cc ((:name "Rupert The Monkey" :email "rupert@@example.com")) + :subject "RE: what about the 50K?" + :date (20369 17624 0) + :size 4337 + :message-id "C8233AB82D81EE81AF0114E4E74@@123213.mail.example.com" + :path "/home/tom/Maildir/INBOX/cur/133443243973_1.10027.atlas:2,S" + :maildir "/INBOX" + :priority normal + :flags (seen attach) + .... +") +@end lisp + +This s-expression forms a property list (@t{plist}), and we can get +values from it using @t{plist-get}; for example @code{(plist-get msg +:subject)} would get you the message subject. However, it's better to +use the function @code{mu4e-message-field} to shield you from some of +the implementation details that are subject to change; and see the other +convenience functions in @file{mu4e-message.el}. + +Some notes on the format: +@itemize +@item The address fields are @emph{lists} of @t{plists} of the form +@code{(:name <name> :email <email>)}, where @t{name} can be @t{nil}. +@item The date is in format Emacs uses (for example in +@code{current-time}).@footnote{Emacs 32-bit integers have only 29 bits +available for the actual number; the other bits are use by Emacs for +internal purposes. Therefore, we need to split @t{time_t} in two +numbers.} +@end itemize + +@subsection Example: ping-pong + +As an example of the communication between @t{mu4e} and @command{mu}, +let's look at the @t{ping-pong}-sequence. When @t{mu4e} starts, it +sends a command @t{ping} to the @t{mu server} backend, to learn about +its version. @t{mu server} then responds with a @t{pong} s-expression +to provide this information (this is implemented in +@file{mu-cmd-server.c}). + +We start this sequence when @t{mu4e} is invoked (when the program is +started). It calls @t{mu4e--server-ping}, and registers a (lambda) +function for @t{mu4e-server-pong-func}, to handle the response. + +@verbatim +-> (ping) +<-<prefix>(:pong "mu" :props (:version "x.x.x" :doccount 78545)) +@end verbatim + +When we receive such a @t{pong} (in @file{mu4e-server.el}), the lambda +function we registered is called, and it compares the version we got +from the @t{pong} with the version we expected, and raises an error if +they differ. + +@node Debugging +@appendix Debugging + +As explained in @ref{How it works}, @t{mu4e} communicates with its +backend (@t{mu server}) by sending commands and receiving responses +(s-expressions). + +For debugging purposes, it can be very useful to see this data. For +this reason, @t{mu4e} can log all these messages. Note that the +`protocol' is documented to some extent in the @t{mu-server} manpage. + +You can enable (and disable) logging with @kbd{M-x +mu4e-toggle-logging}. The log-buffer is called @t{*mu4e-log*}, and in +the @ref{Main view}, @ref{Headers view} and @ref{Message view}, +there's a keybinding @key{$} that takes you there. You can quit it by +pressing @key{q}. + +Logging can be a bit resource-intensive, so you may not want to leave +it on all the time. By default, the log only maintains the most recent +1200 lines. @t{mu} itself keeps a log as well, you can find it in +@t{<MUHOME>/mu.log}, on Unix typically @t{~/.cache/mu/mu.log}. + +@node GNU Free Documentation License +@appendix GNU Free Documentation License + +@include fdl.texi + +@c @node Command Index +@c @unnumbered Command and Function Index +@c @printindex fn + +@c @node Variable Index +@c @unnumbered Variable Index +@c @printindex vr + +@node Concept Index +@unnumbered Concept Index +@printindex cp + +@bye + +@c Local Variables: +@c coding: utf-8 +@c End: diff --git a/mu4e/texinfo-klare.css b/mu4e/texinfo-klare.css new file mode 100644 index 0000000..e54a882 --- /dev/null +++ b/mu4e/texinfo-klare.css @@ -0,0 +1,228 @@ +/* + Custom CSS for HTML documents generated with Texinfo's makeinfo. + Public domain 2016 sirgazil. All rights waived. +*/ + + + +/* NATIVE ELEMENTS */ +a:link, +a:visited { + color: #245C8A; + text-decoration: none; +} + +a:active, +a:focus, +a:hover { + text-decoration: underline; +} + +abbr, +acronym { + cursor: help; +} + +blockquote { + color: #555753; + font-style: oblique; + margin: 30px 0px; + padding-left: 3em; +} + +body { + background-color: white; + box-shadow: 0 0 2px gray; + box-sizing: border-box; + color: #333; + font-family: sans-serif; + font-size: 16px; + margin: 50px auto; + max-width: 960px; + padding: 50px; +} + +code, +samp, +tt, +var { + color: purple; + font-size: 0.8em; +} + +div.example, +div.lisp { + margin: 0px; +} + +dl { + margin: 3em 0em; +} + +dl dl { + margin: 0em; +} + +dt { + background-color: #F5F5F5; + padding: 0.5em; +} + +h1, +h2, +h2.contents-heading, +h3, +h4 { + padding: 20px 0px 0px 0px; + font-weight: normal; +} + +h1 { + font-size: 2.4em; +} + +h2 { + font-size: 2.2em; + font-weight: bold; +} + +h3 { + font-size: 1.8em; +} + +h4 { + font-size: 1.4em; +} + +hr { + background-color: silver; + border-style: none; + height: 1px; + margin: 0px; +} + +html { + background-color: #F5F5F5; +} + +img { + max-width: 100%; +} + +li { + padding: 5px; +} + +pre.display, +pre.example, +pre.format, +pre.lisp, +pre.verbatim{ + overflow: auto; +} + +pre.example, +pre.lisp, +pre.verbatim { + background-color: #2D3743; + border-color: #000; + border-style: solid; + border-width: thin; + color: #E1E1E1; + font-size: smaller; + padding: 1em; +} + +pre.menu-comment { + border-color: #E4E4E4; + border-bottom-style: solid; + border-width: thin; + font-family: sans; +} + +table { + border-collapse: collapse; + margin: 40px 0px; +} + +table.index-cp *, +table.index-fn *, +table.index-ky *, +table.index-pg *, +table.index-tp *, +table.index-vr * { + background-color: inherit; + border-style: none; +} + +td, +th { + border-color: silver; + border-style: solid; + border-width: thin; + padding: 10px; +} + +th { + background-color: #F5F5F5; +} +/* END NATIVE ELEMENTS */ + + + +/* CLASSES */ +.contents { + margin-bottom: 4em; +} + +.float { + margin: 3em 0em; +} + +.float-caption { + font-size: smaller; + text-align: center; +} + +.float > img { + display: block; + margin: auto; +} + +.footnote { + font-size: smaller; + margin: 5em 0em; +} + +.footnote h3 { + display: inline; + font-size: small; +} + +.header { + background-color: #F2F2F2; + font-size: small; + padding: 0.2em 1em; +} + +.key { + color: purple; + font-size: 0.8em; +} + +.menu * { + border-style: none; +} + +.menu td { + padding: 0.5em 0em; +} + +.menu td:last-child { + width: 60%; +} + +.menu th { + background-color: inherit; +} +/* END CLASSES */ diff --git a/testdata/cjk/cur/test1 b/testdata/cjk/cur/test1 new file mode 100644 index 0000000..1538790 --- /dev/null +++ b/testdata/cjk/cur/test1 @@ -0,0 +1,10 @@ +From: "Bob" <bob@builder.com> +Subject: CJK 1 +To: "Chase" <chase@ppatrol.org> +Date: Thu, 18 Nov 2021 08:35:34 +0200 +Message-Id: 112342343e9dfo.fsf@builder.com +User-Agent: mu4e 1.7.5; emacs 29.0.50 + + サーバがダウンしました + +https://github.com/djcb/mu/issues/1428 diff --git a/testdata/cjk/cur/test2 b/testdata/cjk/cur/test2 new file mode 100644 index 0000000..875bff5 --- /dev/null +++ b/testdata/cjk/cur/test2 @@ -0,0 +1,10 @@ +From: "Bob" <bob@builder.com> +Subject: CJK 2 +To: "Chase" <chase@ppatrol.org> +Date: Thu, 18 Nov 2021 08:35:34 +0200 +Message-Id: 271r2342343e9dfo.fsf@builder.com +User-Agent: mu4e 1.7.5; emacs 29.0.50 + + スポンサーシップ募集 + +https://github.com/djcb/mu/issues/1428 diff --git a/testdata/cjk/cur/test3 b/testdata/cjk/cur/test3 new file mode 100644 index 0000000..f0efe71 --- /dev/null +++ b/testdata/cjk/cur/test3 @@ -0,0 +1,10 @@ +From: "Bob" <bob@builder.com> +Subject: CJK 3 +To: "Chase" <chase@ppatrol.org> +Date: Thu, 18 Nov 2021 08:35:34 +0200 +Message-Id: 3871r2342343e9dfo.fsf@builder.com +User-Agent: mu4e 1.7.5; emacs 29.0.50 + + サービス開始について + +https://github.com/djcb/mu/issues/1428 diff --git a/testdata/cjk/cur/test4 b/testdata/cjk/cur/test4 new file mode 100644 index 0000000..2bad399 --- /dev/null +++ b/testdata/cjk/cur/test4 @@ -0,0 +1,10 @@ +From: "Bob" <bob@builder.com> +Subject: CJK 4 +To: "Chase" <chase@ppatrol.org> +Date: Thu, 18 Nov 2021 08:35:34 +0200 +Message-Id: 4871r2342343e9dfo.fsf@builder.com +User-Agent: mu4e 1.7.5; emacs 29.0.50 + + ショルダーバック + +https://github.com/djcb/mu/issues/1428 diff --git a/testdata/testdir/cur/1220863042.12663_1.mindcrime!2,S b/testdata/testdir/cur/1220863042.12663_1.mindcrime!2,S new file mode 100644 index 0000000..ab1500f --- /dev/null +++ b/testdata/testdir/cur/1220863042.12663_1.mindcrime!2,S @@ -0,0 +1,146 @@ +Return-Path: <gcc-help-return-33661-xxxx.klub=gmail.com@gcc.gnu.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-4.9 required=3.0 tests=BAYES_00,DATE_IN_PAST_96_XX, + RCVD_IN_DNSWL_MED autolearn=ham version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 5123469CB3 + for <xxxx@localhost>; Thu, 7 Aug 2008 08:10:19 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [66.249.91.109] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Thu, 07 Aug 2008 08:10:19 +0300 (EEST) +Received: by 10.142.237.21 with SMTP id k21cs39272wfh; Wed, 6 Aug 2008 + 20:15:17 -0700 (PDT) +Received: by 10.65.133.8 with SMTP id k8mr2071878qbn.7.1218078916289; Wed, 06 + Aug 2008 20:15:16 -0700 (PDT) +Received: from sourceware.org (sourceware.org [209.132.176.174]) by + mx.google.com with SMTP id 28si7904461qbw.0.2008.08.06.20.15.15; Wed, 06 Aug + 2008 20:15:16 -0700 (PDT) +Received-SPF: neutral (google.com: 209.132.176.174 is neither permitted nor + denied by domain of gcc-help-return-33661-xxxx.klub=gmail.com@gcc.gnu.org) + client-ip=209.132.176.174; +Authentication-Results: mx.google.com; spf=neutral (google.com: + 209.132.176.174 is neither permitted nor denied by domain of + gcc-help-return-33661-xxxx.klub=gmail.com@gcc.gnu.org) + smtp.mail=gcc-help-return-33661-xxxx.klub=gmail.com@gcc.gnu.org +Received: (qmail 13493 invoked by alias); 7 Aug 2008 03:15:13 -0000 +Received: (qmail 13485 invoked by uid 22791); 7 Aug 2008 03:15:12 -0000 +Received: from mailgw1a.lmco.com (HELO mailgw1a.lmco.com) (192.31.106.7) + by sourceware.org (qpsmtpd/0.31) with ESMTP; Thu, 07 Aug 2008 03:14:27 +0000 +Received: from emss07g01.ems.lmco.com (relay5.ems.lmco.com [166.29.2.16])by + mailgw1a.lmco.com (LM-6) with ESMTP id m773EPZH014730for + <gcc-help@gcc.gnu.org>; Wed, 6 Aug 2008 21:14:25 -0600 (MDT) +Received: from CONVERSION2-DAEMON.lmco.com by lmco.com (PMDF V6.3-x14 #31428) + id <0K5700601NO18J@lmco.com> for gcc-help@gcc.gnu.org; Wed, 06 Aug 2008 + 21:14:25 -0600 (MDT) +Received: from EMSS04I00.us.lmco.com ([166.17.13.135]) by lmco.com (PMDF + V6.3-x14 #31428) with ESMTP id <0K5700H5MNNWGX@lmco.com> for + gcc-help@gcc.gnu.org; Wed, 06 Aug 2008 21:14:20 -0600 (MDT) +Received: from EMSS35M06.us.lmco.com ([158.187.107.143]) by + EMSS04I00.us.lmco.com with Microsoft SMTPSVC(5.0.2195.6713); Wed, 06 Aug + 2008 23:14:20 -0400 +Date: Thu, 31 Jul 2008 14:57:25 -0400 +From: "Mickey Mouse" <anon@example.com> +Subject: gcc include search order +To: "Donald Duck" <gcc-help@gcc.gnu.org> +Message-id: <3BE9E6535E3029448670913581E7A1A20D852173@emss35m06.us.lmco.com> +MIME-version: 1.0 +Content-type: text/plain; charset=us-ascii +Content-transfer-encoding: 7BIT +Content-class: urn:content-classes:message +Mailing-List: contact gcc-help-help@gcc.gnu.org; run by ezmlm +Precedence: klub +List-Id: <gcc-help.gcc.gnu.org> +List-Unsubscribe: <mailto:gcc-help-unsubscribe-xxxx.klub=gmail.com@gcc.gnu.org> +List-Archive: <http://gcc.gnu.org/ml/gcc-help/> +List-Post: <mailto:gcc-help@gcc.gnu.org> +List-Help: <mailto:gcc-help-help@gcc.gnu.org> +Sender: gcc-help-owner@gcc.gnu.org +Delivered-To: mailing list gcc-help@gcc.gnu.org +Content-Length: 3024 + + +Hi. +In my unit testing I need to change some header files (target is +vxWorks, which supports some things that the sun does not). +So, what I do is fetch the development tree, and then in a new unit test +directory I attempt to compile the unit under test. Since this is NOT +vxworks, I use sed to change some of the .h files and put them in a +./changed directory. + +When I try to compile the file, it is still using the .h file from the +original location, even though I have listed the include path for +./changed before the include path for the development tree. + +Here is a partial output from gcc using the -v option + +GNU CPP version 3.1 (cpplib) (sparc ELF) +GNU C++ version 3.1 (sparc-sun-solaris2.8) + compiled by GNU C version 3.1. +ignoring nonexistent directory "NONE/include" +#include "..." search starts here: +#include <...> search starts here: + . + changed + /export/home4/xxx/yyyy/builds/int_rel5_latest/src/mp/interface + /export/home4/xxx/yyyy/builds/int_rel5_latest/src/ap/app + /export/home4/xxx/yyyy/builds/int_rel5_latest/src/shared/common + /export/home4/xxx/yyyy/builds/int_rel5_latest/src/shared/interface + /usr/local/include/g++-v3 + /usr/local/include/g++-v3/sparc-sun-solaris2.8 + /usr/local/include/g++-v3/backward + /usr/local/include + /usr/local/lib/gcc-lib/sparc-sun-solaris2.8/3.1/include + /usr/local/sparc-sun-solaris2.8/include + /usr/include +End of search list. + +I know the changed file is correct and that the include is not working +as expected, because when I copy the file from ./changed, back into the +development tree, the compilation works as expected. + +One more bit of information. The source that I cam compiling is in +/export/home4/xxx/yyyy/builds/int_rel5_latest/src/ap/app +And it is including files from +/export/home4/xxx/yyyy/builds/int_rel5_latest/src/shared/common +These include files should be including the files from ./changed (when +they exist) but they are ignoring the .h files in the ./changed +directory and are instead using other, unchanged files in the +/export/home4/xxx/yyyy/builds/int_rel5_latest/src/shared/common +directory. + +The gcc command line is something like + + TEST_DIR="." + + CHANGED_DIR_NAME=changed + CHANGED_FILES_DIR=${TEST_DIR}/${CHANGED_DIR_NAME} + + CICU_HEADER_FILES="-I ${AP_INTERFACE_FILES} -I ${AP_APP_FILES} -I +${SHARED_COMMON_FILES} -I ${SHARED_INTERFACE_FILES}" + + HEADERS="-I ./ -I ${CHANGED_FILES_DIR} ${CICU_HEADER_FILES}" + DEFINES="-DSUNRUN -DA10_DEBUG -DJOETEST" + + CFLAGS="-v -c -g -O1 -pipe -Wformat -Wunused -Wuninitialized -Wshadow +-Wmissing-prototypes -Wmissing-declarations" + + printf "Compiling the UUT File\n" + gcc -fprofile-arcs -ftest-coverage ${CFLAGS} ${HEADERS} ${DEFINES} +${AP_APP_FILES}/unitUnderTest.cpp + + +I hope this explanation is clear. If anyone knows how to fix the command +line so that it gets the .h files in the "changed" directory are used +instead of files in the other include directories. + +Thanks +Joe + +---------------------------------------------------- +Time Flies like an Arrow. Fruit Flies like a Banana + + diff --git a/testdata/testdir/cur/1220863060.12663_3.mindcrime!2,S b/testdata/testdir/cur/1220863060.12663_3.mindcrime!2,S new file mode 100644 index 0000000..d0ff0d7 --- /dev/null +++ b/testdata/testdir/cur/1220863060.12663_3.mindcrime!2,S @@ -0,0 +1,230 @@ +Return-Path: <sqlite-dev-bounces@sqlite.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-2.6 required=3.0 tests=BAYES_00,HTML_MESSAGE + autolearn=ham version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id D724F6963B + for <xxxx@localhost>; Mon, 4 Aug 2008 21:49:27 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [72.14.221.111] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Mon, 04 Aug 2008 21:49:27 +0300 (EEST) +Received: by 10.142.51.12 with SMTP id y12cs86537wfy; Mon, 4 Aug 2008 00:38:51 + -0700 (PDT) +Received: by 10.151.113.5 with SMTP id q5mr272266ybm.37.1217835529913; Mon, 04 + Aug 2008 00:38:49 -0700 (PDT) +Received: from sqlite.org (sqlite.org [67.18.92.124]) by mx.google.com with + ESMTP id 5si5754915ywd.8.2008.08.04.00.38.30; Mon, 04 Aug 2008 00:38:50 -0700 + (PDT) +Received-SPF: pass (google.com: best guess record for domain of + sqlite-dev-bounces@sqlite.org designates 67.18.92.124 as permitted sender) + client-ip=67.18.92.124; +Authentication-Results: mx.google.com; spf=pass (google.com: best guess record + for domain of sqlite-dev-bounces@sqlite.org designates 67.18.92.124 as + permitted sender) smtp.mail=sqlite-dev-bounces@sqlite.org +Received: from sqlite.org (localhost [127.0.0.1]) by sqlite.org (Postfix) with + ESMTP id 765A511C46; Mon, 4 Aug 2008 03:38:27 -0400 (EDT) +X-Original-To: sqlite-dev@sqlite.org +Delivered-To: sqlite-dev@sqlite.org +Received: from ik-out-1112.google.com (ik-out-1112.google.com [66.249.90.176]) + by sqlite.org (Postfix) with ESMTP id 4C59511C41 for <sqlite-dev@sqlite.org>; + Mon, 4 Aug 2008 03:38:23 -0400 (EDT) +Received: by ik-out-1112.google.com with SMTP id b32so2163423ika.0 for + <sqlite-dev@sqlite.org>; Mon, 04 Aug 2008 00:38:23 -0700 (PDT) +Received: by 10.210.54.19 with SMTP id c19mr14589042eba.107.1217835502549; + Mon, 04 Aug 2008 00:38:22 -0700 (PDT) +Received: by 10.210.115.10 with HTTP; Mon, 4 Aug 2008 00:38:22 -0700 (PDT) +Message-ID: <477821040808040038s381bf382p7411451e3c1a2e4e@mail.gmail.com> +Date: Mon, 4 Aug 2008 10:38:22 +0300 +From: anon@example.com +To: sqlite-dev@sqlite.org +In-Reply-To: <73d4fc50808030747g303a170ieac567723c2d4f24@mail.gmail.com> +MIME-Version: 1.0 +References: <477821040808030533y41f1501dq32447b568b6e6ca5@mail.gmail.com> + <73d4fc50808030747g303a170ieac567723c2d4f24@mail.gmail.com> +Subject: Re: [sqlite-dev] SQLite exception A&B +X-BeenThere: sqlite-dev@sqlite.org +X-Mailman-Version: 2.1.9 +Priority: normal +Reply-To: sqlite-dev@sqlite.org +List-Id: <sqlite-dev.sqlite.org> +List-Unsubscribe: <http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev>, + <mailto:sqlite-dev-request@sqlite.org?subject=unsubscribe> +List-Archive: <http://sqlite.org:8080/cgi-bin/mailman/private/sqlite-dev> +List-Post: <mailto:sqlite-dev@sqlite.org> +List-Help: <mailto:sqlite-dev-request@sqlite.org?subject=help> +List-Subscribe: <http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev>, + <mailto:sqlite-dev-request@sqlite.org?subject=subscribe> +Content-Type: multipart/mixed; boundary="===============2123623832==" +Mime-version: 1.0 +Sender: sqlite-dev-bounces@sqlite.org +Errors-To: sqlite-dev-bounces@sqlite.org +Content-Length: 8475 + +--===============2123623832== +Content-Type: multipart/alternative; + boundary="----=_Part_29556_25702991.1217835502493" + +------=_Part_29556_25702991.1217835502493 +Content-Type: text/plain; charset=ISO-8859-1 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +Hi Grant, + +Thanks for your reply. +I am using a different session for each thread, whenever a thread wishes to +access the database it gets a session from the session pool and works with +that session until its work is done. + +Most of the actions the threads are doing on the database are quite +complicated and are required to be fully committed or completely ignored, so +yes, I am (most of the time) explicitly beginning and committing my +transactions. + +Regarding the SQLiteStatementImpl, I believe the Poco manual explains that +sessions and statements for that matter cannot be shared between threads, +therefore if you are using a session via one thread only it should work +fine. + +My first impression was that the problem was in the Poco infrastructure (I +have found several Poco related bugs in the past), but the problem ALWAYS +occurs when I perform the "BEGIN IMMEDIATE" action, if it were a Poco +related bug, I would expect to see it here and there without any relation to +this specific statement, but that is not the case. + +None the less, I will also post my question on the Poco forums. + +Nadav. + +On Sun, Aug 3, 2008 at 5:47 PM, Grant Gatchel <grant.gatchel@gmail.com>wrote: + +> Are you using the same Poco::Session for every thread or does each call +> create a new session/handle to the database? +> +> Are you explicitly BEGINning and COMMITting your transactions? +> +> In looking at the 1.3.2 branch of Poco::Data::SQLite, there appears to be a +> race condition in the SQLiteStatementImpl::next() method in which the member +> _nextResponse is being accessed before the SQLiteStatementImpl::hasNext() +> method has a chance to interpret that value and throw an exception. +> +> This question might be more suitable in the Poco forums or mailinglist. +> +> - Grant +> +> On Sun, Aug 3, 2008 at 8:33 AM, nadav g <nadav.gr@gmail.com> wrote: +> +>> Hi All, +>> +>> I have been using SQLite with Poco (www.appinf.com) as my infrastructure. +>> The program is running several threads that access this database very +>> often and are synchronized by SQLite itself. +>> Everything seems to work just fine most of time (usually days - weeks) but +>> I do get an occasional exception: +>> +>> Exception: SQL error or missing database: Iterator Error: trying to check +>> if there is a next value +>> +>> The backtrace leads to this statement: +>> *"BEGIN IMMEDIATE"* +>> +>> This specific code runs numerous times before an exception occurs (if +>> occurs at all) and I cannot think of any reason for it to fail later rather +>> than sooner. +>> It is pretty obvious that this situation occurs due to some rare thread +>> state, but I could not find any information that gives me any hint as to +>> what this state might be. +>> +>> So what I am asking is: +>> 1) Does anyone know why this sort of exception occurs? +>> 2) Can anyone think of a reason for such an exception to occur in the +>> situation I have described? +>> +>> Thanks in advance, +>> Nadav. +>> +>> +>> _______________________________________________ +>> sqlite-dev mailing list +>> sqlite-dev@sqlite.org +>> http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev +>> +>> +> +> _______________________________________________ +> sqlite-dev mailing list +> sqlite-dev@sqlite.org +> http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev +> +> + +------=_Part_29556_25702991.1217835502493 +Content-Type: text/html; charset=ISO-8859-1 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +<div dir="ltr">Hi Grant,<br><br>Thanks for your reply.<br>I am using a different session for each thread, whenever a thread wishes to access the database it gets a session from the session pool and works with that session until its work is done.<br> +<br>Most of the actions the threads are doing on the database are quite complicated and are required to be fully committed or completely ignored, so yes, I am (most of the time) explicitly beginning and committing my transactions.<br> +<br>Regarding the SQLiteStatementImpl, I believe the Poco manual explains that sessions and statements for that matter cannot be shared between threads, therefore if you are using a session via one thread only it should work fine.<br> +<br>My first impression was that the problem was in the Poco infrastructure (I have found several Poco related bugs in the past), but the problem ALWAYS occurs when I perform the "BEGIN IMMEDIATE" action, if it were a Poco related bug, I would expect to see it here and there without any relation to this specific statement, but that is not the case.<br> +<br>None the less, I will also post my question on the Poco forums.<br><br>Nadav.<br><br><div class="gmail_quote">On Sun, Aug 3, 2008 at 5:47 PM, Grant Gatchel <span dir="ltr"><<a href="mailto:grant.gatchel@gmail.com">grant.gatchel@gmail.com</a>></span> wrote:<br> +<blockquote class="gmail_quote" style="border-left: 1px solid rgb(204, 204, 204); margin: 0pt 0pt 0pt 0.8ex; padding-left: 1ex;"><div dir="ltr">Are you using the same Poco::Session for every thread or does each call create a new session/handle to the database?<br> +<br>Are you explicitly BEGINning and COMMITting your transactions?<br><br>In looking at the 1.3.2 branch of Poco::Data::SQLite, there appears to be a race condition in the SQLiteStatementImpl::next() method in which the member _nextResponse is being accessed before the SQLiteStatementImpl::hasNext() method has a chance to interpret that value and throw an exception.<br> + +<br>This question might be more suitable in the Poco forums or mailinglist.<br><br>- Grant<br> +<br><div class="gmail_quote"><div><div></div><div class="Wj3C7c"> +On Sun, Aug 3, 2008 at 8:33 AM, nadav g <span dir="ltr"><<a href="http://nadav.gr" target="_blank">nadav.gr</a>@<a href="http://gmail.com" target="_blank">gmail.com</a>></span> wrote:<br></div></div><blockquote class="gmail_quote" style="border-left: 1px solid rgb(204, 204, 204); margin: 0pt 0pt 0pt 0.8ex; padding-left: 1ex;"> +<div><div></div><div class="Wj3C7c"> + + +<div dir="ltr">Hi All,<br><br>I have been using SQLite with Poco (<a href="http://www.appinf.com" target="_blank">www.appinf.com</a>) as my infrastructure.<br>The program is running several threads that access this database very often and are synchronized by SQLite itself.<br> + + + + +Everything seems to work just fine most of time (usually days - weeks) but I do get an occasional exception:<br><br>Exception: SQL error or missing database: Iterator Error: trying to check if there is a next value<br><br> + + + + +The backtrace leads to this statement:<br><b>"BEGIN IMMEDIATE"</b><br><br>This specific code runs numerous times before an exception occurs (if occurs at all) and I cannot think of any reason for it to fail later rather than sooner.<br> + + + + +It is pretty obvious that this situation occurs due to some rare thread state, but I could not find any information that gives me any hint as to what this state might be.<br><br>So what I am asking is:<br>1) Does anyone know why this sort of exception occurs?<br> + + + + +2) Can anyone think of a reason for such an exception to occur in the situation I have described?<br><br>Thanks in advance,<br>Nadav.<br><br></div> +<br></div></div>_______________________________________________<br> +sqlite-dev mailing list<br> +<a href="mailto:sqlite-dev@sqlite.org" target="_blank">sqlite-dev@sqlite.org</a><br> +<a href="http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev" target="_blank">http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev</a><br> +<br></blockquote></div><br></div> +<br>_______________________________________________<br> +sqlite-dev mailing list<br> +<a href="mailto:sqlite-dev@sqlite.org">sqlite-dev@sqlite.org</a><br> +<a href="http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev" target="_blank">http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev</a><br> +<br></blockquote></div><br></div> + +------=_Part_29556_25702991.1217835502493-- + +--===============2123623832== +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +_______________________________________________ +sqlite-dev mailing list +sqlite-dev@sqlite.org +http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev + +--===============2123623832==-- + diff --git a/testdata/testdir/cur/1220863087.12663_15.mindcrime!2,PS b/testdata/testdir/cur/1220863087.12663_15.mindcrime!2,PS new file mode 100644 index 0000000..d6487c0 --- /dev/null +++ b/testdata/testdir/cur/1220863087.12663_15.mindcrime!2,PS @@ -0,0 +1,136 @@ +Return-Path: <help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-3.6 required=3.0 tests=BAYES_00,RCVD_IN_DNSWL_LOW, + SPF_PASS,WHOIS_NETSOLPR autolearn=ham version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 1A6CD69CB6 + for <xxxx@localhost>; Tue, 12 Aug 2008 21:42:38 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [72.14.221.109] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Tue, 12 Aug 2008 21:42:38 +0300 (EEST) +Received: by 10.142.237.21 with SMTP id k21cs123119wfh; Sun, 10 Aug 2008 + 22:06:31 -0700 (PDT) +Received: by 10.100.166.10 with SMTP id o10mr9327844ane.0.1218431190107; Sun, + 10 Aug 2008 22:06:30 -0700 (PDT) +Received: from lists.gnu.org (lists.gnu.org [199.232.76.165]) by mx.google.com + with ESMTP id c29si10110392anc.13.2008.08.10.22.06.29; Sun, 10 Aug 2008 + 22:06:30 -0700 (PDT) +Received-SPF: pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) client-ip=199.232.76.165; +Authentication-Results: mx.google.com; spf=pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) + smtp.mail=help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Received: from localhost ([127.0.0.1]:45637 helo=lists.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KSPbx-0006dj-96 for + xxxx.klub@gmail.com; Mon, 11 Aug 2008 01:06:29 -0400 +Received: from mailman by lists.gnu.org with tmda-scanned (Exim 4.43) id + 1KSPbE-0006cQ-Nd for help-gnu-emacs@gnu.org; Mon, 11 Aug 2008 01:05:44 -0400 +Received: from exim by lists.gnu.org with spam-scanned (Exim 4.43) id + 1KSPbD-0006bs-Px for help-gnu-emacs@gnu.org; Mon, 11 Aug 2008 01:05:44 -0400 +Received: from [199.232.76.173] (port=37426 helo=monty-python.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KSPbD-0006bk-HT for + help-gnu-emacs@gnu.org; Mon, 11 Aug 2008 01:05:43 -0400 +Received: from main.gmane.org ([80.91.229.2]:46446 helo=ciao.gmane.org) by + monty-python.gnu.org with esmtps (TLS-1.0:RSA_AES_256_CBC_SHA1:32) (Exim + 4.60) (envelope-from <geh-help-gnu-emacs@m.gmane.org>) id 1KSPbD-0003Kl-CA + for help-gnu-emacs@gnu.org; Mon, 11 Aug 2008 01:05:43 -0400 +Received: from list by ciao.gmane.org with local (Exim 4.43) id + 1KSPb9-00080r-CX for help-gnu-emacs@gnu.org; Mon, 11 Aug 2008 05:05:39 +0000 +Received: from bas2-toronto63-1088792724.dsl.bell.ca ([64.229.168.148]) by + main.gmane.org with esmtp (Gmexim 0.1 (Debian)) id 1AlnuQ-0007hv-00 for + <help-gnu-emacs@gnu.org>; Mon, 11 Aug 2008 05:05:39 +0000 +Received: from cpchan by bas2-toronto63-1088792724.dsl.bell.ca with local + (Gmexim 0.1 (Debian)) id 1AlnuQ-0007hv-00 for <help-gnu-emacs@gnu.org>; Mon, + 11 Aug 2008 05:05:39 +0000 +X-Injected-Via-Gmane: http://gmane.org/ +To: help-gnu-emacs@gnu.org +From: anon@example.com +Date: Mon, 11 Aug 2008 01:03:22 -0400 +Organization: Linux Private Site +Message-ID: <87bq00nnxh.fsf@MagnumOpus.Mercurius> +References: <877iav5s49.fsf@163.com> <86hc9yc5sj.fsf@timbral.net> + <877iat7udd.fsf@163.com> <87fxphcsxi.fsf@lion.rapttech.com.au> + <8504ddd4-5e3b-4ed5-bf77-aa9cce81b59a@1g2000pre.googlegroups.com> + <87k5es59we.fsf@lion.rapttech.com.au> + <63c824e3-62b1-4a93-8fa8-2813e1f9397f@v13g2000pro.googlegroups.com> + <874p5vsgg8.fsf@nonospaz.fatphil.org> + <8250972e-1886-4021-80bc-376e34881c80@v39g2000pro.googlegroups.com> + <87zlnnqvvs.fsf@nonospaz.fatphil.org> + <57add0e0-b39d-4c71-8d2c-d3b9ddfaa1a9@1g2000pre.googlegroups.com> + <87sktfnz5p.fsf@atthis.clsnet.nl> + <562e1111-d9e7-4b6a-b661-3f9af13fea17@b30g2000prf.googlegroups.com> + <87d4khoq97.fsf@atthis.clsnet.nl> + <0fe404c5-cab8-4692-8a27-532e737a7813@i24g2000prf.googlegroups.com> +Mime-Version: 1.0 +Content-Type: multipart/signed; boundary="=-=-="; micalg=pgp-sha1; + protocol="application/pgp-signature" +X-Complaints-To: usenet@ger.gmane.org +X-Gmane-NNTP-Posting-Host: bas2-toronto63-1088792724.dsl.bell.ca +X-Face: G; + Z,`sm>)4t4LB/GUrgH$W`!AmfHMj,LG)Z}X0ax@s9:0>0)B&@vcm{v-le)wng)?|o]D<V6&ay<F=H{M5?$T%p!dPdJeF,au\E@TA"v22K!Zl\\mzpU4]6$ZnAI3_L)h; + fpd}mn2py/7gv^|*85-D_f:07cT>\Z}0:6X +User-Agent: Gnus/5.110011 (No Gnus v0.11) Emacs/23.0.60 (gnu/linux) +Cancel-Lock: sha1:IKyfrl5drOw6HllHFSmWHAKEeC8= +X-detected-kernel: by monty-python.gnu.org: Linux 2.6, seldom 2.4 (older, 4) +Subject: Re: Can anybody tell me how to send HTML-format mail in gnus +X-BeenThere: help-gnu-emacs@gnu.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: Users list for the GNU Emacs text editor <help-gnu-emacs.gnu.org> +List-Unsubscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=unsubscribe> +List-Archive: <http://lists.gnu.org/pipermail/help-gnu-emacs> +List-Post: <mailto:help-gnu-emacs@gnu.org> +List-Help: <mailto:help-gnu-emacs-request@gnu.org?subject=help> +List-Subscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=subscribe> +Sender: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Errors-To: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Content-Length: 1229 +Lines: 36 + +--=-=-= +Content-Type: text/plain + +Xah <xahlee@gmail.com> writes: + +> So, i was reading about it in Wikipedia. Although i don't have a TV, +> and haven't had since 2000, but i still enjoyed the festive spirits +> anyhow. After all, i'm Chinese by blood. So, in my wandering, i ran +> into this welcome song on youtube: +> +> http://www.youtube.com/watch?v=1HEndNYVhZo + +What is your point? Your email is in plain text and I can click on the +link just fine- it is not exactly rocket science to implement parsing of +URL's to workable links in an Email program (a lot of programs does +that, including Gnus). Images can be included inline if you want. Also +mail markups such as *this*, **this** and _this_ have been around since +the Usenet days and displayed appropriately by a number of mailers. Like +others have said, most html messages that I have seen either contains +useless information, or are plain spam and can introduce a host of +security problems in some mailers. + +Charles + + +--=-=-= +Content-Type: application/pgp-signature + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v2.0.4-svn0 (GNU/Linux) + +iD8DBQFIn8gm3epPyyKbwPYRApbvAKDRirXwzMzI+NHV77+QcP3EgTPaCgCfb/6m +GtNVKdYAeftaYm1nwRVoCDA= +=ULo3 +-----END PGP SIGNATURE----- +--=-=-=-- + + + diff --git a/testdata/testdir/cur/1220863087.12663_19.mindcrime!2,S b/testdata/testdir/cur/1220863087.12663_19.mindcrime!2,S new file mode 100644 index 0000000..78efa2a --- /dev/null +++ b/testdata/testdir/cur/1220863087.12663_19.mindcrime!2,S @@ -0,0 +1,77 @@ +Return-Path: <help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-2.6 required=3.0 tests=BAYES_00 autolearn=ham + version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id C4D6569CB3 + for <xxxx@localhost>; Thu, 7 Aug 2008 08:10:08 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [66.249.91.109] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Thu, 07 Aug 2008 08:10:08 +0300 (EEST) +Received: by 10.142.237.21 with SMTP id k21cs34794wfh; Wed, 6 Aug 2008 + 13:40:29 -0700 (PDT) +Received: by 10.100.33.13 with SMTP id g13mr1093301ang.79.1218055228418; Wed, + 06 Aug 2008 13:40:28 -0700 (PDT) +Received: from lists.gnu.org (lists.gnu.org [199.232.76.165]) by mx.google.com + with ESMTP id d19si15908789and.17.2008.08.06.13.40.27; Wed, 06 Aug 2008 + 13:40:28 -0700 (PDT) +Received-SPF: pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) client-ip=199.232.76.165; +Authentication-Results: mx.google.com; spf=pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) + smtp.mail=help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Received: from localhost ([127.0.0.1]:56316 helo=lists.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KQpo3-0007Pc-Qk for + xxxx.klub@gmail.com; Wed, 06 Aug 2008 16:40:27 -0400 +From: anon@example.com +Newsgroups: gnu.emacs.help +Date: Wed, 6 Aug 2008 20:38:35 +0100 +Message-ID: <r6bpm5-6n6.ln1@news.ducksburg.com> +References: <55dbm5-qcl.ln1@news.ducksburg.com> + <mailman.15710.1217599959.18990.help-gnu-emacs@gnu.org> +Mime-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +X-Trace: individual.net bABVU1hcJwWAuRwe/097AAoOXnGGeYR8G1In635iFGIyfDLPUv +X-Orig-Path: news.ducksburg.com!news +Cancel-Lock: sha1:wK7dsPRpNiVxpL/SfvmNzlvUR94= + sha1:oepBoM0tJBLN52DotWmBBvW5wbg= +User-Agent: slrn/pre0.9.9-120/mm/ao (Ubuntu Hardy) +Path: news.stanford.edu!headwall.stanford.edu!newshub.sdsu.edu!feeder.erje.net!proxad.net!feeder1-2.proxad.net!feed.ac-versailles.fr!fu-berlin.de!uni-berlin.de!individual.net!not-for-mail +Xref: news.stanford.edu gnu.emacs.help:160868 +To: help-gnu-emacs@gnu.org +Subject: Re: Learning LISP; Scheme vs elisp. +X-BeenThere: help-gnu-emacs@gnu.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: Users list for the GNU Emacs text editor <help-gnu-emacs.gnu.org> +List-Unsubscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=unsubscribe> +List-Archive: <http://lists.gnu.org/pipermail/help-gnu-emacs> +List-Post: <mailto:help-gnu-emacs@gnu.org> +List-Help: <mailto:help-gnu-emacs-request@gnu.org?subject=help> +List-Subscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=subscribe> +Sender: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Errors-To: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Content-Length: 417 +Lines: 11 + +On 2008-08-01, Thien-Thi Nguyen wrote: + +> warriors attack, felling foe after foe, +> few growing old til they realize: to know +> what deceit is worth deflection; +> such receipt reversed rejection! +> then their heavy arms, e'er transformed to shields: +> balanced hooked charms, ploughed deep, rich yields. + +Aha: the exercise for the reader is to place the parens correctly. +Might take me a while to solve this puzzle. + diff --git a/testdata/testdir/cur/1220863087.12663_5.mindcrime!2,S b/testdata/testdir/cur/1220863087.12663_5.mindcrime!2,S new file mode 100644 index 0000000..de46cc8 --- /dev/null +++ b/testdata/testdir/cur/1220863087.12663_5.mindcrime!2,S @@ -0,0 +1,84 @@ +Return-Path: <sqlite-dev-bounces@sqlite.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-2.6 required=3.0 tests=BAYES_00 autolearn=ham + version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 32F276963F + for <xxxx@localhost>; Mon, 4 Aug 2008 21:49:34 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [72.14.221.111] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Mon, 04 Aug 2008 21:49:34 +0300 (EEST) +Received: by 10.142.51.12 with SMTP id y12cs89397wfy; Mon, 4 Aug 2008 02:41:16 + -0700 (PDT) +Received: by 10.150.156.20 with SMTP id d20mr963580ybe.104.1217842875596; Mon, + 04 Aug 2008 02:41:15 -0700 (PDT) +Received: from sqlite.org (sqlite.org [67.18.92.124]) by mx.google.com with + ESMTP id 6si3605185ywi.1.2008.08.04.02.40.57; Mon, 04 Aug 2008 02:41:15 -0700 + (PDT) +Received-SPF: pass (google.com: best guess record for domain of + sqlite-dev-bounces@sqlite.org designates 67.18.92.124 as permitted sender) + client-ip=67.18.92.124; +Authentication-Results: mx.google.com; spf=pass (google.com: best guess record + for domain of sqlite-dev-bounces@sqlite.org designates 67.18.92.124 as + permitted sender) smtp.mail=sqlite-dev-bounces@sqlite.org +Received: from sqlite.org (localhost [127.0.0.1]) by sqlite.org (Postfix) with + ESMTP id 7147F11C45; Mon, 4 Aug 2008 05:40:55 -0400 (EDT) +X-Original-To: sqlite-dev@sqlite.org +Delivered-To: sqlite-dev@sqlite.org +Received: from relay00.pair.com (relay00.pair.com [209.68.5.9]) by sqlite.org + (Postfix) with SMTP id B5F901192C for <sqlite-dev@sqlite.org>; Mon, 4 Aug + 2008 05:40:52 -0400 (EDT) +Received: (qmail 59961 invoked from network); 4 Aug 2008 09:40:50 -0000 +Received: from unknown (HELO ?192.168.0.17?) (unknown) by unknown with SMTP; 4 + Aug 2008 09:40:50 -0000 +X-pair-Authenticated: 87.13.75.164 +Message-Id: <83B5AF40-DBFA-4578-A043-04C80276E195@sqlabs.net> +From: anon@example.com +To: sqlite-dev@sqlite.org +Mime-Version: 1.0 (Apple Message framework v926) +Date: Mon, 4 Aug 2008 11:40:49 +0200 +X-Mailer: Apple Mail (2.926) +Subject: [sqlite-dev] VM optimization inside sqlite3VdbeExec +X-BeenThere: sqlite-dev@sqlite.org +X-Mailman-Version: 2.1.9 +Precedence: list +Reply-To: sqlite-dev@sqlite.org +List-Id: <sqlite-dev.sqlite.org> +List-Unsubscribe: <http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev>, + <mailto:sqlite-dev-request@sqlite.org?subject=unsubscribe> +List-Archive: <http://sqlite.org:8080/cgi-bin/mailman/private/sqlite-dev> +List-Post: <mailto:sqlite-dev@sqlite.org> +List-Help: <mailto:sqlite-dev-request@sqlite.org?subject=help> +List-Subscribe: <http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev>, + <mailto:sqlite-dev-request@sqlite.org?subject=subscribe> +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit +Sender: sqlite-dev-bounces@sqlite.org +Errors-To: sqlite-dev-bounces@sqlite.org +Content-Length: 639 + +Inside sqlite3VdbeExec there is a very big switch statement. +In order to increase performance with few modifications to the +original code, why not use this technique ? +http://docs.freebsd.org/info/gcc/gcc.info.Labels_as_Values.html + +With a properly defined "instructions" array, instead of the switch +statement you can use something like: +goto * instructions[pOp->opcode]; +--- +Marco Bambini +http://www.sqlabs.net +http://www.sqlabs.net/blog/ +http://www.sqlabs.net/realsqlserver/ + + + +_______________________________________________ +sqlite-dev mailing list +sqlite-dev@sqlite.org +http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev + diff --git a/testdata/testdir/cur/1220863087.12663_7.mindcrime!2,RS b/testdata/testdir/cur/1220863087.12663_7.mindcrime!2,RS new file mode 100644 index 0000000..b5c0651 --- /dev/null +++ b/testdata/testdir/cur/1220863087.12663_7.mindcrime!2,RS @@ -0,0 +1,138 @@ +Return-Path: <sqlite-dev-bounces@sqlite.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-2.6 required=3.0 tests=BAYES_00 autolearn=ham + version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 3EBAB6963B + for <xxxx@localhost>; Mon, 4 Aug 2008 21:49:35 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [72.14.221.111] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Mon, 04 Aug 2008 21:49:35 +0300 (EEST) +Received: by 10.142.51.12 with SMTP id y12cs89536wfy; Mon, 4 Aug 2008 02:48:56 + -0700 (PDT) +Received: by 10.150.134.21 with SMTP id h21mr7950048ybd.181.1217843335665; + Mon, 04 Aug 2008 02:48:55 -0700 (PDT) +Received: from sqlite.org (sqlite.org [67.18.92.124]) by mx.google.com with + ESMTP id 6si5897081ywi.1.2008.08.04.02.48.35; Mon, 04 Aug 2008 02:48:55 -0700 + (PDT) +Received-SPF: pass (google.com: best guess record for domain of + sqlite-dev-bounces@sqlite.org designates 67.18.92.124 as permitted sender) + client-ip=67.18.92.124; +Authentication-Results: mx.google.com; spf=pass (google.com: best guess record + for domain of sqlite-dev-bounces@sqlite.org designates 67.18.92.124 as + permitted sender) smtp.mail=sqlite-dev-bounces@sqlite.org +Received: from sqlite.org (localhost [127.0.0.1]) by sqlite.org (Postfix) with + ESMTP id ED01611C4E; Mon, 4 Aug 2008 05:48:31 -0400 (EDT) +X-Original-To: sqlite-dev@sqlite.org +Delivered-To: sqlite-dev@sqlite.org +Received: from mx0.security.ro (mx0.security.ro [80.96.72.194]) by sqlite.org + (Postfix) with ESMTP id EB3F51192C for <sqlite-dev@sqlite.org>; Mon, 4 Aug + 2008 05:48:28 -0400 (EDT) +Received: (qmail 348 invoked from network); 4 Aug 2008 12:48:03 +0300 +Received: from dev.security.ro (HELO ?192.168.1.70?) (192.168.1.70) by + mx0.security.ro with SMTP; 4 Aug 2008 12:48:03 +0300 +Message-ID: <4896D06A.8000901@security.ro> +Date: Mon, 04 Aug 2008 12:48:26 +0300 +From: anon@example.com +User-Agent: Thunderbird 2.0.0.16 (Windows/20080708) +MIME-Version: 1.0 +To: sqlite-dev@sqlite.org +References: <83B5AF40-DBFA-4578-A043-04C80276E195@sqlabs.net> +In-Reply-To: <83B5AF40-DBFA-4578-A043-04C80276E195@sqlabs.net> +Content-Type: multipart/mixed; boundary="------------000207070200050102060301" +X-BitDefender-Scanner: Clean, Agent: BitDefender qmail 2.0.0 on + mx0.security.ro +X-BitDefender-Spam: No (0) +X-BitDefender-SpamStamp: v1, whitelisted, total: 0 +Subject: Re: [sqlite-dev] VM optimization inside sqlite3VdbeExec +X-BeenThere: sqlite-dev@sqlite.org +X-Mailman-Version: 2.1.9 +Precedence: high +Reply-To: sqlite-dev@sqlite.org +List-Id: <sqlite-dev.sqlite.org> +List-Unsubscribe: <http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev>, + <mailto:sqlite-dev-request@sqlite.org?subject=unsubscribe> +List-Archive: <http://sqlite.org:8080/cgi-bin/mailman/private/sqlite-dev> +List-Post: <mailto:sqlite-dev@sqlite.org> +List-Help: <mailto:sqlite-dev-request@sqlite.org?subject=help> +List-Subscribe: <http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev>, + <mailto:sqlite-dev-request@sqlite.org?subject=subscribe> +Sender: sqlite-dev-bounces@sqlite.org +Errors-To: sqlite-dev-bounces@sqlite.org +Content-Length: 2212 + +This is a multi-part message in MIME format. +--------------000207070200050102060301 +Content-Type: text/plain; charset=ISO-8859-1; format=flowed +Content-Transfer-Encoding: 7bit + +Marco Bambini wrote: +> Inside sqlite3VdbeExec there is a very big switch statement. +> In order to increase performance with few modifications to the +> original code, why not use this technique ? +> http://docs.freebsd.org/info/gcc/gcc.info.Labels_as_Values.html +> +> With a properly defined "instructions" array, instead of the switch +> statement you can use something like: +> goto * instructions[pOp->opcode]; +> --- +> Marco Bambini +> http://www.sqlabs.net +> http://www.sqlabs.net/blog/ +> http://www.sqlabs.net/realsqlserver/ +> +> +> +> _______________________________________________ +> sqlite-dev mailing list +> sqlite-dev@sqlite.org +> http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev +> +All the world's not a VAX. This technique is GCC-specific. The SQLite +source must be as portable as possible thus tying it to a specific +compiler is out of the question. While one could conceivably use some +preprocessor magic to provide alternate implementations, that would be +impractical considering the sheer size of the code affected. +On the other hand - perhaps you could benchmark the change and provide +some data on whether this actually improves performance? + + +--------------000207070200050102060301 +Content-Type: text/x-vcard; charset=utf-8; + name="mihailim.vcf" +Content-Transfer-Encoding: 7bit +Content-Disposition: attachment; + filename="mihailim.vcf" + +begin:vcard +fn:Mihai Limbasan +n:Limbasan;Mihai +org:SC SECPRAL COM SRL +adr:;;str. Actorului nr. 9;Cluj-Napoca;Cluj;400441;Romania +email;internet:mihailim@security.ro +title:SoftwareDeveloper +tel;work:+40 264 449579 +tel;fax:+40 264 418594 +tel;cell:+40 729 038302 +url:http://secpral.ro/ +version:2.1 +end:vcard + + +--------------000207070200050102060301 +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +_______________________________________________ +sqlite-dev mailing list +sqlite-dev@sqlite.org +http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev + +--------------000207070200050102060301-- + diff --git a/testdata/testdir/cur/1252168370_3.14675.cthulhu!2,S b/testdata/testdir/cur/1252168370_3.14675.cthulhu!2,S new file mode 100644 index 0000000..4fad706 --- /dev/null +++ b/testdata/testdir/cur/1252168370_3.14675.cthulhu!2,S @@ -0,0 +1,21 @@ +Return-Path: <dfgh@floppydisk.nl> +X-Spam-Checker-Version: SpamAssassin 3.1.0 (2005-09-13) on mindcrime +X-Spam-Level: +Delivered-To: dfgh@floppydisk.nl +Message-ID: <43A09C49.9040902@euler.org> +Date: Wed, 14 Dec 2005 23:27:21 +0100 +From: Fred Flintstone <fred@euler.org> +User-Agent: Mozilla Thunderbird 1.0.7 (X11/20051010) +X-Accept-Language: nl-NL, nl, en +MIME-Version: 1.0 +To: dfgh@floppydisk.nl +Subject: Re: xyz +References: <439C1136.90504@euler.org> <4399DD94.5070309@euler.org> <20051209233303.GA13812@gauss.org> <439B41ED.2080402@euler.org> <4399DD94.5070309@euler.org> <20051209233303.GA13812@gauss.org> <439A1E03.3090604@euler.org> <20051211184308.GB13513@gauss.org> +In-Reply-To: <20051211184308.GB13513@gauss.org> +X-Enigmail-Version: 0.92.0.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 7bit +X-UIDL: T<?"!%LG"!cAK"!_j(#! +Content-Length: 1879 + +Test 123. diff --git a/testdata/testdir/cur/1283599333.1840_11.cthulhu!2, b/testdata/testdir/cur/1283599333.1840_11.cthulhu!2, new file mode 100644 index 0000000..25c7180 --- /dev/null +++ b/testdata/testdir/cur/1283599333.1840_11.cthulhu!2, @@ -0,0 +1,16 @@ +From: Frodo Baggins <frodo@example.com> +To: Bilbo Baggins <bilbo@anotherexample.com> +Subject: Greetings from =?UTF-8?B?TG90aGzDs3JpZW4=?= +User-Agent: Wanderlust/2.15.9 (Almost Unreal) Emacs/24.0 Mule/6.0 (HANACHIRUSATO) +Fcc: .sent +Organization: The Fellowship of the Ring +MIME-Version: 1.0 (generated by SEMI 1.14.6 - "Maruoka") +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit +Message-Id: <abcd$efgh@example.com> + + +Let's write some fünkÿ text +using umlauts. + +Foo. diff --git a/testdata/testdir/cur/1305664394.2171_402.cthulhu!2, b/testdata/testdir/cur/1305664394.2171_402.cthulhu!2, new file mode 100644 index 0000000..863f714 --- /dev/null +++ b/testdata/testdir/cur/1305664394.2171_402.cthulhu!2, @@ -0,0 +1,17 @@ +From: =?UTF-8?B?TcO8?= <testmu@testmu.xx> +To: Helmut =?UTF-8?B?S3LDtmdlcg==?= <hk@testmu.xxx> +Subject: =?UTF-8?B?TW90w7ZyaGVhZA==?= +User-Agent: Wanderlust/2.15.9 (Almost Unreal) Emacs/24.0 Mule/6.0 (HANACHIRUSATO) +References: <non-exist-01@msg.id> <non-exist-02@msg.id> <non-exist-03@msg.id> <non-exist-04@msg.id> +1n-Reply-To: <non-exist-04@msg.id> +MIME-Version: 1.0 (generated by SEMI 1.14.6 - "Maruoka") +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + + +Test for issue #38, where apparently searching for accented words in subject, +to etc. fails. + +What about here? Queensrÿche. Mötley Crüe. + + diff --git a/testdata/testdir/cur/encrypted!2,S b/testdata/testdir/cur/encrypted!2,S new file mode 100644 index 0000000..f75fd40 --- /dev/null +++ b/testdata/testdir/cur/encrypted!2,S @@ -0,0 +1,56 @@ +Return-path: <> +Envelope-to: peter@example.com +Delivery-date: Fri, 11 May 2012 16:22:03 +0300 +Received: from localhost.example.com ([127.0.0.1] helo=borealis) + by borealis with esmtp (Exim 4.77) + id 1SSpnB-00038a-Ux + for djcb@localhost; Fri, 11 May 2012 16:21:58 +0300 +Delivered-To: peter@example.com +From: Brian <brian@example.com> +To: Peter <peter@example.com> +Subject: encrypted +User-agent: mu4e 0.9.8.5-dev1; emacs 24.1.50.8 +Date: Fri, 11 May 2012 16:21:42 +0300 +Message-ID: <!&!AAAAAAAAAYAAAAAAAAAOH1+8mkk+lLn7Gg5fke7FbCgAAAEAAAAJ7eBDgcactKhXL6r8cEnJ8BAAAAAA==@example.com> +MIME-Version: 1.0 +Content-Type: multipart/encrypted; boundary="=-=-="; + protocol="application/pgp-encrypted" + +--=-=-= +Content-Type: application/pgp-encrypted + +Version: 1 + +--=-=-= +Content-Type: application/octet-stream + +-----BEGIN PGP MESSAGE----- +Version: GnuPG v1.4.12 (GNU/Linux) + +hQQOA1T38TPQrHD6EA//YXkUB4Dy09ngCRyHWbXmV3XBjuKTr8xrak5ML1kwurav +gyagOHKLMU+5CKvObChiKtXhtgU0od7IC8o+ALlHevQ0XXcqNYA2KUfX8R7akq7d +Xx9mA6D8P7Y/P8juUCLBpfrCi2GC42DtvPZSUu3bL/ctUJ3InPHIfHibKF2HMm7/ +gUHAKY8VPJF39dLP8GLcfki6qFdeWbxgtzmuyzHfCBCLnDL0J9vpEQBpGDFMcc4v +cCbmMJaiPOmRb6U4WOuRVnuXuTztLiIn0jMslzOSFDcLTVBAsrC01r71O+XZKfN4 +mIfcpcWJYKM2NQW8Jwf+8Hr84uznBqs8uTTlrmppjkAHZGqGMjiQDxLhDVaCQzMy +O8PSV4xT6HPlKXOwV1OLc+vm0A0RAdSBctgZg40oFn4XdB1ur8edwAkLvc0hJKaz +gyTQiPaXm2Uh2cDeEx4xNgXmwCKasqc9jAlnDC2QwA33+pw3OqgZT5h1obn0fAeR +mgB+iW1503DIi/96p8HLZcr2EswLEH9ViHIEaFj/vlR5BaOncsLB0SsNV/MHRvym +Xg5GUjzPIiyBZ3KaR9OIBiZ5eXw+bSrPAo/CAs0Zwxag7W3CH//oK39Qo1GnkYpc +4IQxhx4IwkzqtCnripltV/kfpGu0yA/OdK8lOjkUqCwvL97o73utXIxm21Zd3mEP +/iLNrduZjMCq+goz1pDAQa9Dez6VjwRuRPTqeAac8Fx/nzrVzIoIEAt36hpuaH1l +KpbmHpKgsUWcrE5iYT0RRlRRtRF4PfJg8PUmP1hvw8TaEmNfT+0HgzcJB/gRsVdy +gTzkzUDzGZLhRcpmM5eW4BkuUmIO7625pM6Jd3HOGyfCGSXyEZGYYeVKzv8xbzYf +QM6YYKooRN9Ya2jdcWguW0sCSJO/RZ9eaORpTeOba2+Fp6w5L7lga+XM9GLfgref +Cf39XX1RsmRBsrJTw0z5COf4bT8G3/IfQP0QyKWIFITiFjGmpZhLsKQ3KT4vSe/d +gTY1xViVhkjvMFn3cgSOSrvktQpAhsXx0IRazN0T7pTU33a5K0SrZajY9ynFDIw9 +we7XYyVwZzYEXjGih5mTH1PhWYK5fZZEKKqaz5TyYv9SeWJ+8FrHeXUKD38SQEHM +qkpl9Iv17RF4Qy9uASWwRoobhKO+GykTaBSTyw8R8ctG/hfAlnaZxQ3TwNyHWyvU +9SVJsp27ulv/W9MLZtGpEMK0ckAR164Vyou1KOn200BqxbC2tJpegNeD2TP5ZtdY +HIcxkgKr0haYcDnVEf1ulSxv23pZWIexbgvVCG7dRL0eB+6O28f9CWehle10MDyM +0AYyw8Da2cu7PONMovqt4nayScyGTacFBp7c2KXR9DGZ0mcBwOjL/mGRKcVWN3MG +2auCrwn2KVWmKZI3Jp0T8KhfGBnFs9lUElpDTOiED1/2bKz6Yoc385QtWx99DFMZ +IWiH5wMxkWFpzjE+GHiJ09vSbTTL4JY9eu2n5nxQmtjYMBVxQm7S7qwH +=0Paa +-----END PGP MESSAGE----- +--=-=-=-- diff --git a/testdata/testdir/cur/multimime!2,FS b/testdata/testdir/cur/multimime!2,FS new file mode 100644 index 0000000..84f85aa --- /dev/null +++ b/testdata/testdir/cur/multimime!2,FS @@ -0,0 +1,27 @@ +Return-path: <> +Envelope-to: djcb@localhost +Delivery-date: Sun, 20 May 2012 09:59:51 +0300 +From: Steve Jobs <jobs@example.com> +To: Bill Gates <bg@example.com> +Subject: multimime +User-agent: mu4e 0.9.8.4; emacs 23.3.1 +Date: Sat, 19 May 2012 20:57:56 +0100 +Message-ID: <m2fwaw2baz.fsf@example.com> +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="=-=-=" + +--=-=-= +Content-Type: text/plain + +abc +--=-=-= +Content-Type: application/octet-stream +Content-Disposition: attachment; filename="test1.C" +Content-Transfer-Encoding: base64 + +aGVyZSBpcyBhIHNpbXBsZSB0ZXN0IGZpbGUuCg== +--=-=-= +Content-Type: text/plain + +def +--=-=-=-- diff --git a/testdata/testdir/cur/multirecip!2,S b/testdata/testdir/cur/multirecip!2,S new file mode 100644 index 0000000..c997503 --- /dev/null +++ b/testdata/testdir/cur/multirecip!2,S @@ -0,0 +1,11 @@ +Date: Thu, 15 May 2016 14:57:25 -0200 +From: +To: a@example.com,b@example.com,c@example.com +Cc: d@example.com,e@example.com +Subject: test with multi to and cc +Message-id: <3BE9E652343245@emss35m06.us.lmco.com> + +Message with multi cc and to. + + + diff --git a/testdata/testdir/cur/signed!2,S b/testdata/testdir/cur/signed!2,S new file mode 100644 index 0000000..a2e7e21 --- /dev/null +++ b/testdata/testdir/cur/signed!2,S @@ -0,0 +1,36 @@ +Return-path: <> +Envelope-to: skipio@localhost +Delivery-date: Fri, 11 May 2012 16:21:57 +0300 +Received: from localhost.roma.net([127.0.0.1] helo=borealis) + by borealis with esmtp (Exim 4.77) + id 1SSpnB-00038a-55 + for djcb@localhost; Fri, 11 May 2012 16:21:57 +0300 +Delivered-To: diggler@gmail.com +From: Skipio <skipio@roma.net> +To: Hannibal <hanni@carthago.net> +Subject: signed +User-agent: mu4e 0.9.8.5-dev1; emacs 24.1.50.8 +Date: Fri, 11 May 2012 16:20:45 +0300 +Message-ID: <878vgy97ma.fsf@roma.net> +MIME-Version: 1.0 +Content-Type: multipart/signed; boundary="=-=-="; micalg=pgp-sha1; + protocol="application/pgp-signature" + +--=-=-= +Content-Type: text/plain + + +I am signed! + +--=-=-= +Content-Type: application/pgp-signature + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.12 (GNU/Linux) + +iEYEARECAAYFAk+tEi0ACgkQ6WrHoQF92jxTzACeKd/XxY+P7bpymWL3JBRHaW9p +DpwAoKw7PDW4z/lNTkWjndVTjoO9jGhs +=blXz +-----END PGP SIGNATURE----- +--=-=-=-- + diff --git a/testdata/testdir/cur/signed-encrypted!2,S b/testdata/testdir/cur/signed-encrypted!2,S new file mode 100644 index 0000000..a3910e6 --- /dev/null +++ b/testdata/testdir/cur/signed-encrypted!2,S @@ -0,0 +1,54 @@ +Return-path: <> +Envelope-to: karjala@localhost +Delivery-date: Fri, 11 May 2012 16:37:57 +0300 +From: karjala@example.com +To: lapinkulta@example.com +Subject: signed + encrypted +User-agent: mu4e 0.9.8.5-dev1; emacs 24.1.50.8 +Date: Fri, 11 May 2012 16:36:08 +0300 +Message-ID: <874nrm96wn.fsf@example.com> +MIME-Version: 1.0 +Content-Type: multipart/encrypted; boundary="=-=-="; + protocol="application/pgp-encrypted" + +--=-=-= +Content-Type: application/pgp-encrypted + +Version: 1 + +--=-=-= +Content-Type: application/octet-stream + +-----BEGIN PGP MESSAGE----- +Version: GnuPG v1.4.12 (GNU/Linux) + +hQQOA1T38TPQrHD6EA/+K4kSpMa7zk+qihUkQnHSq28xYxisNQx6X5DVNjA/Qx16 +uZj/40ae+PoSMTVfklP+B2S/IomuTW6dwVqS7aQ3u4MTzi+YOi11k1lEMD7hR0Wb +L0i48o3/iCPuCTpnOsaLZvRL06g+oTi0BF2pgz/YdsgsBTGrTb3pkDGSlLIhvh/J +P8eE3OuzkXS6d8ymJKx2S2wQJrc1AFf1BgJfgc5T0iAvcV+zIMG+PIYcVd04zVpj +cORFEfvGgfxWkeX+Ks3tu/l5PA1EesnoqFdNFZm+RKBg3RFsOm8tBlJ46xJjfeHg +zLgifeSLy3tOX7CvWYs9torrx7s7UOI2gV8kzBqz+a7diyCMezceeQ9l0nIRybwW +C9Egp8Bpfb02iXTOGdE/vRiNItQH14GKmXf4nCSwdtQUm3yzaqY9yL3xBxAlW53e +YOFfPMESt+E7IlPn0c7llWGrcdrhJbUEoGOIPezES7kdeNPzi8G1lLtvT04/SSZJ +QxPH5FNzSFaYFAQSdI7TR69P7L7vtLL8ndkjY49HfLFXochQQzsqrzVxzRCruHxA +zbZSRptNf9SuXEaX9buO1vlFHheGvrCKzEWa6O7JD/DiyrE/zqy4jdlh9abMCouQ +GWGSbn8jk6SMTQQ2Yv/VOyFqifHZp0UJD59tyIdenpxoYu5M0lwHLNVDlRjLEwUQ +AIDz1tbLoM7lxs2FOKGr8QqbKIeMfL+NUmbvVIDc4mJrOlRnHh+cZYm4Z49iTl1v +bYNMYgR5nY7W6rqh0ae7ZOW0h2NzpkAwTzuf1YrSjNavd9KBwOCFtAoZhRwfwFVx +ju+ByHFNnf7g/R6DekHS0pSiatM0cPDJT05atEZb+13CRHHznonmLHi+VahXjrpg +cIUA8Lhjdfm6Fsabo7gNZnTTRxNBqUXKK2vJF/XLbNrH5K2BH2dCCmUNtm3yFWiM +DOzaw3665Y3S6MvZdyKpatbNrVoJdBpRgPxJ1YCSEituFUqHJBStay+aRb5fVkQR +w3+9hWw+Ob0+2EumKbgfQ7iMwTZBCZP4VOxkoqdHvs9aWm4N7wHtXsyCew3icbJx +lyUWsDx/FI+HlQRfOqeAMxmp8kKybmHNw8oGiw+uPPUHSD1NFYVm2DtwhYll3Fvs +YY7r5s3yP1ZnwxMqWI3OsExVUXs8MS4UTAgO+cggO7YidPcANbBDihBFP8mTXtni +Oo5n5v+/eRoLfHmnsGcaK8EkKsfFHpbqn4gxXGcBuHaTTJ/ZhbW6bi1WWZA9ExaJ +IeTDtp5Bks1pJvTjCDacvgwl3rEBM6yaeIvB7575Y/GPMTOZhawhfOxV1smMmTKI +JOWYb3+PuN2cvWetkjFgH8re4sRXq22DKBZHJEWYU8sH0sACAePnIr+pkrOtGeJB +t1zBqZUnrupH6ptk9n/AjbQ+XSMTEKu55gSjYLAYx1EHApx52QLkdh+ej5xCIVeY +6wS1Iipkoc6/r6F7CKctupXurNY2AlD4uQIOfD6kQgkqK4PY3hsRHQA+Zqj6oRfr +kxysFJZvhgt26IeBVapFs10WuYt9iHfpbPUBQUIZCLyPAh08UdVW64Uc2DvUPy+I +C+3RrmTHQPP/YNKgDQaZ3ySVEDkqjaDPmXr5K0Ibaib2dtPCLcA= +=pv03 +-----END PGP MESSAGE----- +--=-=-=-- + diff --git a/testdata/testdir/cur/special!2,Sabc b/testdata/testdir/cur/special!2,Sabc new file mode 100644 index 0000000..7f1de8e --- /dev/null +++ b/testdata/testdir/cur/special!2,Sabc @@ -0,0 +1,10 @@ +Date: Thu, 1 Jun 2012 14:57:25 -0200 +From: "Rocky Balboa" <rocky@example.com> +To: "Ivan Drago" <ivan@example.com> +Subject: currying and tail optimization +Message-id: <3BE9E653ef345@emss35m06.us.lmco.com> +MIME-version: 1.0 +Content-type: text/plain; charset=us-ascii +Content-transfer-encoding: 7BIT + +Test 123. I'm a special message with special flags. diff --git a/testdata/testdir/new/1220863087.12663_21.mindcrime b/testdata/testdir/new/1220863087.12663_21.mindcrime new file mode 100644 index 0000000..4101716 --- /dev/null +++ b/testdata/testdir/new/1220863087.12663_21.mindcrime @@ -0,0 +1,111 @@ +Return-Path: <help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-2.6 required=3.0 tests=BAYES_00 autolearn=ham + version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 6389969CB2 + for <xxxx@localhost>; Thu, 7 Aug 2008 08:10:07 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [66.249.91.109] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Thu, 07 Aug 2008 08:10:07 +0300 (EEST) +Received: by 10.142.237.21 with SMTP id k21cs34769wfh; Wed, 6 Aug 2008 + 13:38:53 -0700 (PDT) +Received: by 10.100.6.13 with SMTP id 13mr4103508anf.83.1218055131215; Wed, 06 + Aug 2008 13:38:51 -0700 (PDT) +Received: from lists.gnu.org (lists.gnu.org [199.232.76.165]) by mx.google.com + with ESMTP id b32si10199298ana.34.2008.08.06.13.38.49; Wed, 06 Aug 2008 + 13:38:51 -0700 (PDT) +Received-SPF: pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) client-ip=199.232.76.165; +DomainKey-Status: good (test mode) +Authentication-Results: mx.google.com; spf=pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) + smtp.mail=help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org; domainkeys=pass + (test mode) header.From=juanma_bellon@yahoo.es +Received: from localhost ([127.0.0.1]:55648 helo=lists.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KQpmT-0005W9-AQ for + xxxx.klub@gmail.com; Wed, 06 Aug 2008 16:38:49 -0400 +Received: from mailman by lists.gnu.org with tmda-scanned (Exim 4.43) id + 1KQplz-0005U5-Pk for help-gnu-emacs@gnu.org; Wed, 06 Aug 2008 16:38:19 -0400 +Received: from exim by lists.gnu.org with spam-scanned (Exim 4.43) id + 1KQplw-0005Nw-OG for help-gnu-emacs@gnu.org; Wed, 06 Aug 2008 16:38:19 -0400 +Received: from [199.232.76.173] (port=45465 helo=monty-python.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KQplw-0005NX-I6 for + help-gnu-emacs@gnu.org; Wed, 06 Aug 2008 16:38:16 -0400 +Received: from n74a.bullet.mail.sp1.yahoo.com ([98.136.45.21]:29868) by + monty-python.gnu.org with smtp (Exim 4.60) (envelope-from + <juanma_bellon@yahoo.es>) id 1KQplw-0007EF-7Z for help-gnu-emacs@gnu.org; + Wed, 06 Aug 2008 16:38:16 -0400 +Received: from [216.252.122.216] by n74.bullet.mail.sp1.yahoo.com with NNFMP; + 06 Aug 2008 20:38:14 -0000 +Received: from [68.142.237.89] by t1.bullet.sp1.yahoo.com with NNFMP; 06 Aug + 2008 20:38:14 -0000 +Received: from [69.147.75.180] by t5.bullet.re3.yahoo.com with NNFMP; 06 Aug + 2008 20:38:14 -0000 +Received: from [127.0.0.1] by omp101.mail.re1.yahoo.com with NNFMP; 06 Aug + 2008 20:38:14 -0000 +X-Yahoo-Newman-Id: 778995.62909.bm@omp101.mail.re1.yahoo.com +Received: (qmail 43643 invoked from network); 6 Aug 2008 20:38:14 -0000 +DomainKey-Signature: a=rsa-sha1; q=dns; c=nofws; s=s1024; d=yahoo.es; + h=Received:X-YMail-OSG:X-Yahoo-Newman-Property:From:To:Subject:Date:User-Agent:References:In-Reply-To:MIME-Version:Content-Type:Content-Transfer-Encoding:Content-Disposition:Message-Id; + b=ThdHlND5CNUsLPGuk+XhCWkdUA9w7lg4hiAgx8F8egsmQteMpwUlV/Y5tfe6K3O2jzHjtsklkzWqm7WY3VAcxxD/QgxLnianK5ZQHoelDAiGaFRqu8Y42XMZso2ccCBFWUQaKo9C+KIfa3e3ci73qehVxTtmr7bxLjurcSYEBPo= + ; +Received: from unknown (HELO 212251170160.customer.cdi.no) + (juanma_bellon@212.251.170.160 with plain) by smtp109.plus.mail.re1.yahoo.com + with SMTP; 6 Aug 2008 20:38:14 -0000 +X-YMail-OSG: k86L54kVM1kiZbUlYx7gayoBrCLYMFIRDL.KJLBKetNucAbwU4RjeeE1vhjw33hREaUig0CCjG7BTwIfbeZZpRmUcHbxl6gR0z6Sd3lYqA-- +X-Yahoo-Newman-Property: ymail-3 +From: anon@example.com +To: help-gnu-emacs@gnu.org +Date: Wed, 6 Aug 2008 22:38:15 +0200 +User-Agent: KMail/1.9.6 (enterprise 0.20070907.709405) +References: <mailman.15123.1216681940.18990.help-gnu-emacs@gnu.org> + <mailman.15143.1216715014.18990.help-gnu-emacs@gnu.org> + <9bc17528-8ea9-49f7-8e9d-07f5ede91415@p31g2000prf.googlegroups.com> +In-Reply-To: <9bc17528-8ea9-49f7-8e9d-07f5ede91415@p31g2000prf.googlegroups.com> +MIME-Version: 1.0 +Content-Type: text/plain; charset="utf-8" +Content-Transfer-Encoding: quoted-printable +Content-Disposition: inline +Message-Id: <200808062238.15634.juanma_bellon@yahoo.es> +X-detected-kernel: by monty-python.gnu.org: FreeBSD 6.x (1) +Subject: Re: basic question: going back to dired +X-BeenThere: help-gnu-emacs@gnu.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: Users list for the GNU Emacs text editor <help-gnu-emacs.gnu.org> +List-Unsubscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=unsubscribe> +List-Archive: <http://lists.gnu.org/pipermail/help-gnu-emacs> +List-Post: <mailto:help-gnu-emacs@gnu.org> +List-Help: <mailto:help-gnu-emacs-request@gnu.org?subject=help> +List-Subscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=subscribe> +Sender: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Errors-To: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Content-Length: 361 + +On Thursday 31 July 2008, Xah wrote: +> what's the logic of =E2=80=9COK=E2=80=9D? + +=46or all I know, it comes from "0 Knock-outs" (from USA civil war times, +IIRC), i.e., all went really well. + +But this is really off-topic. +=2D-=20 +Juanma + +"Having a smoking section in a restaurant is like + having a peeing section in a swimming pool." + -- Edward Burr + + + + + diff --git a/testdata/testdir/new/1220863087.12663_23.mindcrime b/testdata/testdir/new/1220863087.12663_23.mindcrime new file mode 100644 index 0000000..ca46f2b --- /dev/null +++ b/testdata/testdir/new/1220863087.12663_23.mindcrime @@ -0,0 +1,105 @@ +Return-Path: <help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-2.6 required=3.0 tests=BAYES_00 autolearn=ham + version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id C3EF069CB3 + for <xxxx@localhost>; Thu, 7 Aug 2008 08:10:10 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [66.249.91.109] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Thu, 07 Aug 2008 08:10:10 +0300 (EEST) +Received: by 10.142.237.21 with SMTP id k21cs35153wfh; Wed, 6 Aug 2008 + 13:58:17 -0700 (PDT) +Received: by 10.100.166.10 with SMTP id o10mr4182182ane.0.1218056296101; Wed, + 06 Aug 2008 13:58:16 -0700 (PDT) +Received: from lists.gnu.org (lists.gnu.org [199.232.76.165]) by mx.google.com + with ESMTP id d34si13875743and.3.2008.08.06.13.58.14; Wed, 06 Aug 2008 + 13:58:16 -0700 (PDT) +Received-SPF: pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) client-ip=199.232.76.165; +Authentication-Results: mx.google.com; spf=pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) + smtp.mail=help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org; dkim=pass (test + mode) header.i=@gmail.com +Received: from localhost ([127.0.0.1]:33418 helo=lists.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KQq5G-0001aY-Cr for + xxxx.klub@gmail.com; Wed, 06 Aug 2008 16:58:14 -0400 +Received: from mailman by lists.gnu.org with tmda-scanned (Exim 4.43) id + 1KQq4n-0001Z9-06 for help-gnu-emacs@gnu.org; Wed, 06 Aug 2008 16:57:45 -0400 +Received: from exim by lists.gnu.org with spam-scanned (Exim 4.43) id + 1KQq4l-0001V8-6c for help-gnu-emacs@gnu.org; Wed, 06 Aug 2008 16:57:44 -0400 +Received: from [199.232.76.173] (port=46438 helo=monty-python.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KQq4k-0001Un-V2 for + help-gnu-emacs@gnu.org; Wed, 06 Aug 2008 16:57:42 -0400 +Received: from ik-out-1112.google.com ([66.249.90.180]:17562) by + monty-python.gnu.org with esmtp (Exim 4.60) (envelope-from + <lekktu@gmail.com>) id 1KQq4k-0001fk-OW for help-gnu-emacs@gnu.org; Wed, 06 + Aug 2008 16:57:42 -0400 +Received: by ik-out-1112.google.com with SMTP id c21so94956ika.2 for + <help-gnu-emacs@gnu.org>; Wed, 06 Aug 2008 13:57:41 -0700 (PDT) +DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=gamma; + h=domainkey-signature:received:received:message-id:date:from:to + :subject:cc:in-reply-to:mime-version:content-type + :content-transfer-encoding:content-disposition:references; + bh=TTNY9749hpg1+TXOwdaCr+zbQGhBUt3IvsjLWp+pxp0=; + b=BOfudUT/SiW9V4e9+k3dXDzwm+ogdrq4m5OlO+f1H+oE6OAYGIm8dbdqDAOwUewBoS + jRpfZo07YamP9rkko79SeFdQnf7UAPFAw9x7DFCm3x6muSlCcJBR7vYs1rgHOSINAn2B + vQx2//lKR4fXfKNURNu+B30KrvoEmw6m2C8dI= +DomainKey-Signature: a=rsa-sha1; c=nofws; d=gmail.com; s=gamma; + h=message-id:date:from:to:subject:cc:in-reply-to:mime-version + :content-type:content-transfer-encoding:content-disposition :references; + b=UMDBulH/LwxDywEH0pfK3DbJ4u2kIZCVDLIM++PqrdcR82HjcS/O3Jhf5OFrf7Fnyj + GH76xmc7zkTG/3aQy2WY6DeWCJaFarEItmhxy3h/xS+kUKeDARzNox0OzK6lIv/u9bdy + f2LnFlYRJ7Q5vy3lxpxAWB4v0qCwtF9LjWFg4= +Received: by 10.210.47.7 with SMTP id u7mr3100239ebu.30.1218056261587; Wed, 06 + Aug 2008 13:57:41 -0700 (PDT) +Received: by 10.210.71.14 with HTTP; Wed, 6 Aug 2008 13:57:41 -0700 (PDT) +Message-ID: <f7ccd24b0808061357t453f5962w8b61f9a453b684d0@mail.gmail.com> +Date: Wed, 6 Aug 2008 22:57:41 +0200 +From: anon@example.com +To: Juanma <juanma_bellon@yahoo.es> +In-Reply-To: <200808062238.15634.juanma_bellon@yahoo.es> +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline +References: <mailman.15123.1216681940.18990.help-gnu-emacs@gnu.org> + <mailman.15143.1216715014.18990.help-gnu-emacs@gnu.org> + <9bc17528-8ea9-49f7-8e9d-07f5ede91415@p31g2000prf.googlegroups.com> + <200808062238.15634.juanma_bellon@yahoo.es> +X-detected-kernel: by monty-python.gnu.org: Linux 2.6 (newer, 2) +Cc: help-gnu-emacs@gnu.org +Subject: Re: basic question: going back to dired +X-BeenThere: help-gnu-emacs@gnu.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: Users list for the GNU Emacs text editor <help-gnu-emacs.gnu.org> +List-Unsubscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=unsubscribe> +List-Archive: <http://lists.gnu.org/pipermail/help-gnu-emacs> +List-Post: <mailto:help-gnu-emacs@gnu.org> +List-Help: <mailto:help-gnu-emacs-request@gnu.org?subject=help> +List-Subscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=subscribe> +Sender: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Errors-To: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Content-Length: 309 + +On Wed, Aug 6, 2008 at 22:38, Juanma <juanma_bellon@yahoo.es> wrote: + +> For all I know, it comes from "0 Knock-outs" (from USA civil war times, +> IIRC), i.e., all went really well. + +See http://en.wikipedia.org/wiki/Okay#Etymology + +"0 knock-outs" is among the "Improbable or refuted etymologies". + + Juanma + + diff --git a/testdata/testdir/new/1220863087.12663_25.mindcrime b/testdata/testdir/new/1220863087.12663_25.mindcrime new file mode 100644 index 0000000..588ace1 --- /dev/null +++ b/testdata/testdir/new/1220863087.12663_25.mindcrime @@ -0,0 +1,98 @@ +Return-Path: <help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-3.6 required=3.0 tests=BAYES_00,RCVD_IN_DNSWL_LOW, + SPF_PASS autolearn=ham version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id D68E769CB5 + for <xxxx@localhost>; Fri, 8 Aug 2008 20:56:25 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [72.14.221.111] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Fri, 08 Aug 2008 20:56:25 +0300 (EEST) +Received: by 10.142.237.21 with SMTP id k21cs71287wfh; Fri, 8 Aug 2008 + 07:40:46 -0700 (PDT) +Received: by 10.100.122.8 with SMTP id u8mr3824321anc.77.1218206446062; Fri, + 08 Aug 2008 07:40:46 -0700 (PDT) +Received: from lists.gnu.org (lists.gnu.org [199.232.76.165]) by mx.google.com + with ESMTP id d35si2718351and.38.2008.08.08.07.40.45; Fri, 08 Aug 2008 + 07:40:46 -0700 (PDT) +Received-SPF: pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) client-ip=199.232.76.165; +Authentication-Results: mx.google.com; spf=pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) + smtp.mail=help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Received: from localhost ([127.0.0.1]:47349 helo=lists.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KRT93-0006Po-A3 for + xxxx.klub@gmail.com; Fri, 08 Aug 2008 10:40:45 -0400 +Path: news.stanford.edu!headwall.stanford.edu!newshub.sdsu.edu!news-out.readnews.com!news-xxxfer.readnews.com!panix!not-for-mail +From: anon@example.com +Newsgroups: gnu.emacs.help +Date: Fri, 08 Aug 2008 10:07:30 -0400 +Organization: PANIX Public Access Internet and UNIX, NYC +Message-ID: <uwsireh25.fsf@one.dot.net> +References: <mailman.15123.1216681940.18990.help-gnu-emacs@gnu.org> + <mailman.15143.1216715014.18990.help-gnu-emacs@gnu.org> + <9bc17528-8ea9-49f7-8e9d-07f5ede91415@p31g2000prf.googlegroups.com> + <200808062238.15634.juanma_bellon@yahoo.es> + <mailman.15958.1218056266.18990.help-gnu-emacs@gnu.org> +NNTP-Posting-Host: panix5.panix.com +Mime-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +X-Trace: reader1.panix.com 1218204439 22850 166.84.1.5 (8 Aug 2008 14:07:19 + GMT) +X-Complaints-To: abuse@panix.com +NNTP-Posting-Date: Fri, 8 Aug 2008 14:07:19 +0000 (UTC) +User-Agent: Gnus/5.11 (Gnus v5.11) Emacs/22.2 (windows-nt) +Cancel-Lock: sha1:Ckkp5oJPIMuAVgEHGnS/9MkZsEs= +Xref: news.stanford.edu gnu.emacs.help:160963 +To: help-gnu-emacs@gnu.org +Subject: Re: basic question: going back to dired +X-BeenThere: help-gnu-emacs@gnu.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: Users list for the GNU Emacs text editor <help-gnu-emacs.gnu.org> +List-Unsubscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=unsubscribe> +List-Archive: <http://lists.gnu.org/pipermail/help-gnu-emacs> +List-Post: <mailto:help-gnu-emacs@gnu.org> +List-Help: <mailto:help-gnu-emacs-request@gnu.org?subject=help> +List-Subscribe: <http://lists.gnu.org/mailman/listinfo/help-gnu-emacs>, + <mailto:help-gnu-emacs-request@gnu.org?subject=subscribe> +Sender: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Errors-To: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Content-Length: 710 +Lines: 27 + +I seem to remember from my early school days it was a campaign slogan +for someone nick-named Kinderhook that went something like + +Old Kinderhook is OK + +- Chris + +"Juanma Barranquero" <lekktu@gmail.com> writes: + +> On Wed, Aug 6, 2008 at 22:38, Juanma <juanma_bellon@yahoo.es> wrote: +> +>> For all I know, it comes from "0 Knock-outs" (from USA civil war times, +>> IIRC), i.e., all went really well. +> +> See http://en.wikipedia.org/wiki/Okay#Etymology +> +> "0 knock-outs" is among the "Improbable or refuted etymologies". +> +> Juanma +> +> + +-- + (. .) + =ooO=(_)=Ooo===================================== + Chris McMahan | first_initiallastname@one.dot.net + ================================================= + diff --git a/testdata/testdir/new/1220863087.12663_9.mindcrime b/testdata/testdir/new/1220863087.12663_9.mindcrime new file mode 100644 index 0000000..734ee35 --- /dev/null +++ b/testdata/testdir/new/1220863087.12663_9.mindcrime @@ -0,0 +1,209 @@ +Return-Path: <sqlite-dev-bounces@sqlite.org> +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-1.2 required=3.0 tests=BAYES_00,HTML_MESSAGE, + MIME_QP_LONG_LINE autolearn=no version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 4E3CF6963B + for <xxxx@localhost>; Mon, 4 Aug 2008 21:49:37 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [72.14.221.111] + by mindcrime with IMAP (fetchmail-6.3.8) + for <xxxx@localhost> (single-drop); Mon, 04 Aug 2008 21:49:37 +0300 (EEST) +Received: by 10.142.51.12 with SMTP id y12cs94317wfy; Mon, 4 Aug 2008 05:48:28 + -0700 (PDT) +Received: by 10.150.152.17 with SMTP id z17mr1245909ybd.194.1217854107583; + Mon, 04 Aug 2008 05:48:27 -0700 (PDT) +Received: from sqlite.org (sqlite.org [67.18.92.124]) by mx.google.com with + ESMTP id 9si6334793yws.5.2008.08.04.05.47.57; Mon, 04 Aug 2008 05:48:27 -0700 + (PDT) +Received-SPF: pass (google.com: best guess record for domain of + sqlite-dev-bounces@sqlite.org designates 67.18.92.124 as permitted sender) + client-ip=67.18.92.124; +Authentication-Results: mx.google.com; spf=pass (google.com: best guess record + for domain of sqlite-dev-bounces@sqlite.org designates 67.18.92.124 as + permitted sender) smtp.mail=sqlite-dev-bounces@sqlite.org +Received: from sqlite.org (localhost [127.0.0.1]) by sqlite.org (Postfix) with + ESMTP id 4FBC111C6F; Mon, 4 Aug 2008 08:47:54 -0400 (EDT) +X-Original-To: sqlite-dev@sqlite.org +Delivered-To: sqlite-dev@sqlite.org +Received: from cpsmtpo-eml02.kpnxchange.com (cpsmtpo-eml02.kpnxchange.com + [213.75.38.151]) by sqlite.org (Postfix) with ESMTP id AA4F111C10 for + <sqlite-dev@sqlite.org>; Mon, 4 Aug 2008 08:47:51 -0400 (EDT) +Received: from hpsmtp-eml21.kpnxchange.com ([213.75.38.121]) by + cpsmtpo-eml02.kpnxchange.com with Microsoft SMTPSVC(6.0.3790.1830); Mon, 4 + Aug 2008 14:47:50 +0200 +Received: from cpbrm-eml13.kpnsp.local ([195.121.247.250]) by + hpsmtp-eml21.kpnxchange.com with Microsoft SMTPSVC(6.0.3790.1830); Mon, 4 + Aug 2008 14:47:50 +0200 +Received: from hpsmtp-eml30.kpnxchange.com ([10.94.53.250]) by + cpbrm-eml13.kpnsp.local with Microsoft SMTPSVC(6.0.3790.1830); Mon, 4 Aug + 2008 14:47:50 +0200 +Received: from localhost ([10.94.53.250]) by hpsmtp-eml30.kpnxchange.com with + Microsoft SMTPSVC(6.0.3790.1830); Mon, 4 Aug 2008 14:47:49 +0200 +Content-class: urn:content-classes:message +MIME-Version: 1.0 +X-MimeOLE: Produced By Microsoft Exchange V6.5 +Date: Mon, 4 Aug 2008 14:46:06 +0200 +Message-ID: <F687EC042917A94E8BB4B0902946453AE17D6C@CPEXBE-EML18.kpnsp.local> +X-MS-Has-Attach: +X-MS-TNEF-Correlator: +Thread-Topic: [sqlite-dev] VM optimization inside sqlite3VdbeExec +Thread-Index: Acj2FjkWvteFtLHTTYeVz4ES7E2ggAAGRxeI +References: <83B5AF40-DBFA-4578-A043-04C80276E195@sqlabs.net> +From: anon@example.com +To: <sqlite-dev@sqlite.org> +X-OriginalArrivalTime: 04 Aug 2008 12:47:49.0650 (UTC) + FILETIME=[4D577720:01C8F630] +Subject: Re: [sqlite-dev] VM optimization inside sqlite3VdbeExec +X-BeenThere: sqlite-dev@sqlite.org +X-Mailman-Version: 2.1.9 +Precedence: list +Reply-To: sqlite-dev@sqlite.org +List-Id: <sqlite-dev.sqlite.org> +List-Unsubscribe: <http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev>, + <mailto:sqlite-dev-request@sqlite.org?subject=unsubscribe> +List-Archive: <http://sqlite.org:8080/cgi-bin/mailman/private/sqlite-dev> +List-Post: <mailto:sqlite-dev@sqlite.org> +List-Help: <mailto:sqlite-dev-request@sqlite.org?subject=help> +List-Subscribe: <http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev>, + <mailto:sqlite-dev-request@sqlite.org?subject=subscribe> +Content-Type: multipart/mixed; boundary="===============1911358387==" +Mime-version: 1.0 +Sender: sqlite-dev-bounces@sqlite.org +Errors-To: sqlite-dev-bounces@sqlite.org +Content-Length: 5318 + +This is a multi-part message in MIME format. + +--===============1911358387== +Content-class: urn:content-classes:message +Content-Type: multipart/alternative; + boundary="----_=_NextPart_001_01C8F630.0FC2EC1E" + +This is a multi-part message in MIME format. + +------_=_NextPart_001_01C8F630.0FC2EC1E +Content-Type: text/plain; + charset="iso-8859-1" +Content-Transfer-Encoding: quoted-printable + +Actually, almost every C compiler will already do what you suggest: if = +the range of case labels is compact, the switch will be compiled using a = +jump table. Only if the range is limited and/or sparse other techniques = +will be used, such as linear search and binary search. +=20 +I'm pretty sure, if you perform the tests suggested by Mihai, that you = +will find zero performance difference, neither better, nor worse. +=20 +Paul +=20 +________________________________ + +From: anon@example.com +Sent: Mon 8/4/2008 11:40 AM +To: sqlite-dev@sqlite.org +Subject: [sqlite-dev] VM optimization inside sqlite3VdbeExec + + + +Inside sqlite3VdbeExec there is a very big switch statement. +In order to increase performance with few modifications to the=20 +original code, why not use this technique ? +http://docs.freebsd.org/info/gcc/gcc.info.Labels_as_Values.html = +<http://docs.freebsd.org/info/gcc/gcc.info.Labels_as_Values.html>=20 + +With a properly defined "instructions" array, instead of the switch=20 +statement you can use something like: +goto * instructions[pOp->opcode]; +--- +Marco Bambini +http://www.sqlabs.net <http://www.sqlabs.net/>=20 +http://www.sqlabs.net/blog/ <http://www.sqlabs.net/blog/>=20 +http://www.sqlabs.net/realsqlserver/ = +<http://www.sqlabs.net/realsqlserver/>=20 + + + +_______________________________________________ +sqlite-dev mailing list +sqlite-dev@sqlite.org +http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev = +<http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev>=20 + + + +------_=_NextPart_001_01C8F630.0FC2EC1E +Content-Type: text/html; + charset="iso-8859-1" +Content-Transfer-Encoding: quoted-printable + +<HTML dir=3Dltr><HEAD><TITLE>[sqlite-dev] VM optimization inside = +sqlite3VdbeExec=0A= +=0A= +=0A= +=0A= +
=0A= +
Actually, = +almost every C compiler will already do what you suggest: if the range = +of case labels is compact, the switch will be compiled using a jump = +table. Only if the range is limited and/or sparse other techniques will = +be used, such as linear search and binary search.
=0A= +
 
=0A= +
I'm pretty sure, if you = +perform the tests suggested by Mihai, that you will find zero = +performance difference, neither better, nor worse.
=0A= +
 
=0A= +
Paul
=0A= +
 
=0A= +
=0A= +
=0A= +
=0A= +
From: = +sqlite-dev-bounces@sqlite.org on behalf of Marco Bambini
Sent: = +Mon 8/4/2008 11:40 AM
To: = +sqlite-dev@sqlite.org
Subject: [sqlite-dev] VM optimization = +inside sqlite3VdbeExec

=0A= +
=0A= +

Inside sqlite3VdbeExec there is a very = +big switch statement.
In order to increase performance with few = +modifications to the 
original code, why not use this technique = +?
= +http://docs.freebsd.org/info/gcc/gcc.info.Labels_as_Values.html<= +/FONT>

With a properly defined = +"instructions" array, instead of the switch 
statement you can = +use something like:
goto * = +instructions[pOp->opcode];
---
Marco Bambini
http://www.sqlabs.net
http://www.sqlabs.net/blog/
http://www.sqlabs.net/realsqlserver/



<= +FONT face=3DArial = +size=3D2>_______________________________________________
sqlite-dev = +mailing list
sqlite-dev@sqlite.org
http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev

+------_=_NextPart_001_01C8F630.0FC2EC1E-- + + +--===============1911358387== +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Content-Disposition: inline + +_______________________________________________ +sqlite-dev mailing list +sqlite-dev@sqlite.org +http://sqlite.org:8080/cgi-bin/mailman/listinfo/sqlite-dev + +--===============1911358387==-- + diff --git a/testdata/testdir/tmp/1220863087.12663.ignore b/testdata/testdir/tmp/1220863087.12663.ignore new file mode 100644 index 0000000..588ace1 --- /dev/null +++ b/testdata/testdir/tmp/1220863087.12663.ignore @@ -0,0 +1,98 @@ +Return-Path: +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-3.6 required=3.0 tests=BAYES_00,RCVD_IN_DNSWL_LOW, + SPF_PASS autolearn=ham version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id D68E769CB5 + for ; Fri, 8 Aug 2008 20:56:25 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [72.14.221.111] + by mindcrime with IMAP (fetchmail-6.3.8) + for (single-drop); Fri, 08 Aug 2008 20:56:25 +0300 (EEST) +Received: by 10.142.237.21 with SMTP id k21cs71287wfh; Fri, 8 Aug 2008 + 07:40:46 -0700 (PDT) +Received: by 10.100.122.8 with SMTP id u8mr3824321anc.77.1218206446062; Fri, + 08 Aug 2008 07:40:46 -0700 (PDT) +Received: from lists.gnu.org (lists.gnu.org [199.232.76.165]) by mx.google.com + with ESMTP id d35si2718351and.38.2008.08.08.07.40.45; Fri, 08 Aug 2008 + 07:40:46 -0700 (PDT) +Received-SPF: pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) client-ip=199.232.76.165; +Authentication-Results: mx.google.com; spf=pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) + smtp.mail=help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Received: from localhost ([127.0.0.1]:47349 helo=lists.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KRT93-0006Po-A3 for + xxxx.klub@gmail.com; Fri, 08 Aug 2008 10:40:45 -0400 +Path: news.stanford.edu!headwall.stanford.edu!newshub.sdsu.edu!news-out.readnews.com!news-xxxfer.readnews.com!panix!not-for-mail +From: anon@example.com +Newsgroups: gnu.emacs.help +Date: Fri, 08 Aug 2008 10:07:30 -0400 +Organization: PANIX Public Access Internet and UNIX, NYC +Message-ID: +References: + + <9bc17528-8ea9-49f7-8e9d-07f5ede91415@p31g2000prf.googlegroups.com> + <200808062238.15634.juanma_bellon@yahoo.es> + +NNTP-Posting-Host: panix5.panix.com +Mime-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +X-Trace: reader1.panix.com 1218204439 22850 166.84.1.5 (8 Aug 2008 14:07:19 + GMT) +X-Complaints-To: abuse@panix.com +NNTP-Posting-Date: Fri, 8 Aug 2008 14:07:19 +0000 (UTC) +User-Agent: Gnus/5.11 (Gnus v5.11) Emacs/22.2 (windows-nt) +Cancel-Lock: sha1:Ckkp5oJPIMuAVgEHGnS/9MkZsEs= +Xref: news.stanford.edu gnu.emacs.help:160963 +To: help-gnu-emacs@gnu.org +Subject: Re: basic question: going back to dired +X-BeenThere: help-gnu-emacs@gnu.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: Users list for the GNU Emacs text editor +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +Sender: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Errors-To: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Content-Length: 710 +Lines: 27 + +I seem to remember from my early school days it was a campaign slogan +for someone nick-named Kinderhook that went something like + +Old Kinderhook is OK + +- Chris + +"Juanma Barranquero" writes: + +> On Wed, Aug 6, 2008 at 22:38, Juanma wrote: +> +>> For all I know, it comes from "0 Knock-outs" (from USA civil war times, +>> IIRC), i.e., all went really well. +> +> See http://en.wikipedia.org/wiki/Okay#Etymology +> +> "0 knock-outs" is among the "Improbable or refuted etymologies". +> +> Juanma +> +> + +-- + (. .) + =ooO=(_)=Ooo===================================== + Chris McMahan | first_initiallastname@one.dot.net + ================================================= + diff --git a/testdata/testdir2/Foo/cur/arto.eml b/testdata/testdir2/Foo/cur/arto.eml new file mode 100644 index 0000000..ffa0526 --- /dev/null +++ b/testdata/testdir2/Foo/cur/arto.eml @@ -0,0 +1,448 @@ +Return-Path: <> +X-Original-To: f00f@localhost +Delivered-To: f00f@localhost +Received: from puppet (puppet [127.0.0.1]) + by f00fmachines.nl (Postfix) with ESMTP id A534D39C7F1 + for ; Mon, 23 May 2011 20:30:05 +0300 (EEST) +Delivered-To: diggler@gmail.com +Received: from ew-in-f109.1e100.net [174.15.27.101] + by puppet with POP3 (fetchmail-6.3.18) + for (single-drop); Mon, 23 May 2011 20:30:05 +0300 (EEST) +Received: by 10.142.147.13 with SMTP id u13cs87252wfd; + Mon, 23 May 2011 01:54:10 -0700 (PDT) +Received: by 10.204.7.74 with SMTP id c10mr1984197bkc.104.1306140849326; + Mon, 23 May 2011 01:54:09 -0700 (PDT) +Received: from MTX4.mbn1.net (mtx4.mbn1.net [213.188.129.252]) + by mx.google.com with ESMTP id e6si18117551bkw.39.2011.05.23.01.54.07; + Mon, 23 May 2011 01:54:08 -0700 (PDT) +Received-SPF: pass (google.com: best guess record for domain of MTX4.mbn1.net designates 213.188.129.252 as permitted sender) client-ip=213.188.129.252; +Authentication-Results: mx.google.com; spf=pass (google.com: best guess record for domain of MTX4.mbn1.net designates 213.188.129.252 as permitted sender) smtp.mail= +Resent-From: +X-Default-Received-SPF: pass (skip=forwardok (res=PASS)) x-ip-name=192.168.10.123; +From: ArtOlive +To: "f00f@f00fmachines.nl" +Reply-To: +Date: Mon, 23 May 2011 10:53:45 +0200 +Subject: NIEUWSBRIEF ART OLIVE | juni exposite in galerie ArtOlive +MIME-Version: 1.0 +Content-Type: multipart/alternative; + boundary="_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d" +X-Mailer: aspNetEmail ver 3.5.2.10 +X-Sender: 87.93.13.24 +X-RemoteIP: 87.93.13.24 +Originating-IP: 87.93.13.24 +X-MAILINGLIJST-ID: <10374608.109906.11909.2011523105345.MSGID@mailinglijst.nl> +Message-ID: <10374608.109906.11909.2011523105345.MSGID@mailinglijst.nl> +X-Authenticated-User: guest@mailinglijst.eu +X-STA-Metric: 0 (engine=030) +X-STA-NotSpam: geinformeerd vormen spec:usig:3.8.2 twee samen +X-STA-Spam: 2011 &bull • e-mailing subject:juni +X-BTI-AntiSpam: score:0,sta:0/030,dnsbl:passed,sw:passed,bsn:10/passed,spf:off,bsctr:passed/1,dk:off,pbmf:none,ipr:0/3,trusted:no,ts:no,bs:no,ubl:passed +X-Auto-Response-Suppress: DR, RN, NRN, OOF, AutoReply +Resent-Message-Id: <19740414233016.EB6835A132F5FCF4@MTX4.mbn1.net> +Resent-Date: Mon, 23 May 2011 10:54:07 +0200 (CEST) + +--_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d +Content-Type: text/plain; charset="iso-8859-15" +Content-Transfer-Encoding: quoted-printable + +ART-O-NEWS; juni 2011 Westergasfabriekterrein Polonceaukade 17 10= +14 DA Amsterdam tel: 020-6758504 info@artolive.nl www.artolive.nlJuni= + expositie bij ArtOlive: Peter van den Akker en Marinel Vieleers + + +Zondag 5 juni + Elke maand vindt er in de galerie van ArtOlive een expositie plaats. We = +lichten enkele kunstenaars uit (die je misschien al kent van onze website= +), waarbij we een spannende mix van materiaal en techniek presenteren. Ti= +jdens de expositie staan we klaar om elke vraag over ons kunstaanbod te b= +eantwoorden.=20 + +De exposities zijn te bezoeken van maandag t/m vrijdag, tussen 10:00 en 1= +7:00 uur. De opening is altijd op de eerste zondag van de maand. Dit valt= + samen met de Sunday Market die elke maand op het Cultuurpark Westergasfa= +briek georganiseerd wordt. De Sunday Market is gratis te bezoeken en staa= +t vol met kunst, design, mode en heerlijke hapjes, en er hangt altijd een= + vrolijke sfeer. Een ideaal moment dus om in te haken en deze maand twee = +kunstenaars te presenteren: Peter van den Akker en Marinel Vieleers. + +We verwelkomen je graag op zondag 5 juni 2011, van 12:00 t/m 17:00 uur op= + de Polonceaukade 17 van het Cultuurpark Westergasfabriek in Amsterdam!=20= + + + + bekijk meer werk op www.artolive.nl... Peter van den Akker + + +"In mijn beelden en schilderijen staat het mensbeeld centraal; niet als i= +ndividu maar als universele gestalte, waarbij ik op transparante wijze ti= +jdsbeelden en gelaagdheid in het menselijke handelen naar voren breng. Ve= +rhoudingen tussen mensen, verschuivingen in wereldculturen en verandering= +en in techniek, architectuur, natuur en mensbeeld vormen mijn inspiratieb= +ronnen. Het zijn allemaal beelden en sferen die naast en met elkaar besta= +an. Mijn werkwijze omvat vele technieken in verschillende materialen: sch= +ilderijen, gemengde technieken op papier/collages, zeefdrukken, beelden i= +n cortenstaal, keramische objecten." + +Peter van den Akker exposeert regelmatig in binnen- en buitenland bij gal= +erie=EBn en musea en is in verschillende kunstinstellingen en bedrijfscol= +lecties opgenomen. + + + lees meer over Peter... Marinel Vieleers + + +Marinel Vieleers probeert het menselijke in de bouwwerken - en ook vaak i= +ets van het karakter van de bouwer of bewoner - te laten zien. Het zijn m= +aar subtiele details die dat alles weergeven. + +De 'tand des tijds' of invloed van mensen op de gebouwen spelen vaak mee = +in het werk. Koper, cement, lood en andere materialen worden in haar nieu= +we werk gebruikt. Op deze manier kan ze gemakkelijker improviseren en nog= + directer op haar gevoel afgaan. + +Marinel is gefascineerd door de schoonheid van het imperfecte. De gelaagd= +heid van ouderdom, het verval. De imperfectie die ontstaat door toevallig= +e omstandigheden maakt een huis, een muur, een schutting, hout of steen b= +oeiend. Het is doorleefd en het toont een stukje van zijn geschiedenis. + + + lees meer over Marinel... =20 +ZONDAG 5 MEI - Juni expositie in de galerie van ArtOlive met Marinel Viel= +eers en Peter van den Akker + +Opening op zondag 5 mei, tijdens de Sunday Market op het Cultuurpark West= +ergasfabriek in Amsterdam. Je bent van harte uitgenodigd om tussen 12:00 = +en 17:00 uur langs te komen! + +Daarna is de expositie te zien op werkdagen (ma - vrij) tussen 10:00 en 1= +7:00. De expositie duurt tot 24 juni 2011. + wil je niet langer door artolive ge=EFnformeerd worden? Klik dan hier om= + je af te melden.=20 + kreeg je dit mailtje doorgestuurd en wil je voortaan zelf ook graag de n= +ieuwsbrief ontvangen?=20 + klik dan hier om je aan te melden.=20 + +Deze e-mailing is verzorgd met MailingLijst + +--_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d +Content-Type: text/html; charset="iso-8859-15" +Content-Transfer-Encoding: quoted-printable + + + + + + Artolive + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
3D"artolive"
ART-O-NEWS • juni 2011 +
Westergasfabriekterrein   Polonceau= +kade 17 1014 DA Amsterdam   tel: 020-6758504  info@artolive.nl<= +/a>  www.artolive.nl
+ + + + + + + +
Juni= + expositie bij ArtOlive: Peter van den Akker en Marinel Vieleers +

Zondag 5 juni
+ Elke maand vindt er in de galerie van Art= +Olive een expositie plaats. We lichten enkele kunstenaars uit (die je mis= +schien al kent van onze website), waarbij we een spannende mix van materi= +aal en techniek presenteren. Tijdens de expositie staan we klaar om elke = +vraag over ons kunstaanbod te beantwoorden.

+

De exposities zijn te bezoeken van maa= +ndag t/m vrijdag, tussen 10:00 en 17:00 uur. De opening is altijd op de e= +erste zondag van de maand. Dit valt samen met de Sunday Market die elke m= +aand op het Cultuurpark Westergasfabriek georganiseerd wordt. De Sunday M= +arket is gratis te bezoeken en staat vol met kunst, design, mode en heerl= +ijke hapjes, en er hangt altijd een vrolijke sfeer. Een ideaal moment dus= + om in te haken en deze maand twee kunstenaars te presenteren: Peter van = +den Akker en Marinel Vieleers.

+

We verwelkomen je graag op zondag 5 ju= +ni 2011, van 12:00 t/m 17:00 uur op de Polonceaukade 17 van het Cultuurpa= +rk Westergasfabriek in Amsterdam!

+
+

3D""  = +bekijk= + meer werk op www.artolive.nl...   

+
+
+ + + + = + + + + + + +
3D""
+ + + + + + + + + + +
<= +a target=3D"_blank" href=3D"http://www.mailinglijst.eu/redirect.aspx?l=3D= +154043&a=3D10374608&t=3DH"> <= +/td> + Peter van den Akker
+

"In mijn beelden en schild= +erijen staat het mensbeeld centraal; niet als individu maar als universel= +e gestalte, waarbij ik op transparante wijze tijdsbeelden en gelaagdheid = +in het menselijke handelen naar voren breng. Verhoudingen tussen mensen, = +verschuivingen in wereldculturen en veranderingen in techniek, architectu= +ur, natuur en mensbeeld vormen mijn inspiratiebronnen. Het zijn allemaal = +beelden en sferen die naast en met elkaar bestaan. Mijn werkwijze omvat v= +ele technieken in verschillende materialen: schilderijen, gemengde techni= +eken op papier/collages, zeefdrukken, beelden in cortenstaal, keramische = +objecten.”

+

Peter van den Akker expose= +ert regelmatig in binnen- en buitenland bij galerieën en musea en is= + in verschillende kunstinstellingen en bedrijfscollecties opgenomen.

+
+

3D""  = +lees meer over Peter...   

+
= + Marinel Vieleers
+

Marinel Vieleers probeert = +het menselijke in de bouwwerken - en ook vaak iets van het karakter van d= +e bouwer of bewoner - te laten zien. Het zijn maar subtiele details die d= +at alles weergeven.

+

De ‘tand des tijds&r= +squo; of invloed van mensen op de gebouwen spelen vaak mee in het werk. K= +oper, cement, lood en andere materialen worden in haar nieuwe werk gebrui= +kt. Op deze manier kan ze gemakkelijker improviseren en nog directer op h= +aar gevoel afgaan.

+

Marinel is gefascineerd do= +or de schoonheid van het imperfecte. De gelaagdheid van ouderdom, het ver= +val. De imperfectie die ontstaat door toevallige omstandigheden maakt een= + huis, een muur, een schutting, hout of steen boeiend. Het is doorleefd e= +n het toont een stukje van zijn geschiedenis.

+
+

3D""  = +lees meer ov= +er Marinel...   

+
+
+
+ + + + + + + + + +
3D""
+ + + + + + + + + + + +
+
+
+ + + + + + + + + +
3D""
+ + + + + + + + + + + + + + + +

+ 		ZONDAG 5 MEI - Juni expositie in de galerie van ArtOlive met = +Marinel Vieleers en Peter van den Akker
+

+ 		Opening op zondag 5 mei, = +tijdens de Sunday Market op het Cultuurpark Westergasfabriek in Amsterdam= +. Je bent van harte uitgenodigd om tussen 12:00 en 17:00 uur langs te kom= +en!
+

+ 		Daarna is de expositie te zien op werkdagen (ma - vrij= +) tussen 10:00 en 17:00. De expositie duurt tot 24 juni 2011.
+
+
+
3D"Kunst wil je niet langer door artolive geïnformeerd worden? Klik= + dan hier om je af te melden.
+ kreeg je dit mailtje doorgestuurd en wil je voortaan = +zelf ook graag de nieuwsbrief ontvangen?
+ klik dan hier om= + je aan te melden.
+

<= +HR SIZE=3D1 STYLE=3D"COLOR:#d3d3d3" SIZE=3D1>Deze e-mailing is verzorgd m= +et MailingLijst

+ + +--_=aspNetEmail=_5ed4592191214c7a99bd7f6a3a0f077d-- diff --git a/testdata/testdir2/Foo/cur/fraiche.eml b/testdata/testdir2/Foo/cur/fraiche.eml new file mode 100644 index 0000000..c0bf442 --- /dev/null +++ b/testdata/testdir2/Foo/cur/fraiche.eml @@ -0,0 +1,10 @@ +From: Sender +To: Recip +Subject: search accents +Date: 2012-12-08 00:48 +MIME-Version: 1.0 +Content-Type: text/plain; charset=utf-8 +Content-Transfer-Encoding: 8bit + +line 1: Глокая куздра штеко будланула бокра и курдячит бокрёнка +line 2: crème fraîche diff --git a/testdata/testdir2/Foo/cur/mail5 b/testdata/testdir2/Foo/cur/mail5 new file mode 100644 index 0000000..b72195d --- /dev/null +++ b/testdata/testdir2/Foo/cur/mail5 @@ -0,0 +1,625 @@ +From: Sitting Bull +To: George Custer +Subject: pics for you +Mail-Reply-To: djcb@djcbsoftware.nl +User-Agent: Hunkpapa/2.15.9 (Almost Unreal) +Message-Id: CAHSaMxZ9rk5ASjqsbXizjTQuSk583=M6TORHz=bfogtmbGGs5A@mail.gmail.com +Fcc: .sent +MIME-Version: 1.0 (generated by SEMI 1.14.6 - "Maruoka") +Content-Type: multipart/mixed; + boundary="Multipart_Sun_Oct_17_10:37:40_2010-1" + +--Multipart_Sun_Oct_17_10:37:40_2010-1 +Content-Type: text/plain; charset=US-ASCII + +Dude! Here are some pics! + + +--Multipart_Sun_Oct_17_10:37:40_2010-1 +Content-Type: image/jpeg +Content-Disposition: inline; filename="sittingbull.jpg" +Content-Transfer-Encoding: base64 + +/9j/4AAQSkZJRgABAQAAAQABAAD/4QvoRXhpZgAASUkqAAgAAAAIABIBCQABAAAAAQAAABoBCQAB +AAAASAAAABsBCQABAAAASAAAACgBCQABAAAAAgAAADEBAgAOAAAAbgAAADIBAgAUAAAAfAAAABMC +CQABAAAAAQAAAGmHBAABAAAAkAAAAN4AAABndGh1bWIgMi4xMS4zADIwMTA6MTA6MTcgMTA6MzM6 +MzcABgAAkAcABAAAADAyMjEBkQcABAAAAAECAwAAoAcABAAAADAxMDABoAkAAQAAAAEAAAACoAkA +AQAAAMgAAAADoAkAAQAAAGsBAAAAAAAABgADAQMAAQAAAAYAAAAaAQkAAQAAAEgAAAAbAQkAAQAA +AEgAAAAoAQkAAQAAAAIAAAABAgQAAQAAACwBAAACAgQAAQAAALMKAAAAAAAA/9j/4AAQSkZJRgAB +AQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwc +KDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIy +MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCACAAEcDASIAAhEBAxEB/8QAHwAA +AQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIh +MUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpT +VFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5 +usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAA +AAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEI +FEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVm +Z2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK +0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDq77xdrX/CQ6laRXjRxQTF +ECovA/EUg8Sa6W/5CUuP9xP8K5yWQnxjrw9Lwj9BWjkgZHFAG6mu6yV51OXP+4n/AMTUq61rBB/4 +mU2f9xP/AImsJJTuAJFW0YDnfmgCTUPFGqWFq882p3G1eyqmT/47VfRfGGpawkgGp3CyIeg2cj1+ +7XK+O7zybCGMNjzHyR6gD/69ZvgG8zqU67vvRZH4EUAesJe6m/XVLv8ANf8A4mpf7Qvl/wCX+6b6 +uP8ACs+ObKdaeh3Hg9aANTw/4gurjxTLpU7tIv2cTKzHpgkH+n5UVheHGI+KWzJwdNP/AKFRQBzD +7f8AhMfEDEHH24j/AMdWrs0oCkDrVKJs+NfEsZ+79u/9kWrd5GqKTmgCstwwkyT0p5uzu61mOzbj +zSFn3DmgDB8ePLPe2MEQZyykhRzk9/5Va8D6Vd2Mz3d3CYxJHiPd16+la0hhMybkUvxhj1HWr5uM +uB0wMYoA3YJARjvV+DBPasC2lYsOuK3LVunWgCLQRj4sIPXTGP8A4/RSaF/yV2P30tv/AEOigDmY +QD408UE9ftw/9AFXpv3iFT9Kgs4t3jXxV6C+H/oAq5cxkMcCgDFltXVyMVVv7iGwtzNcNsQfiT+F +a8jbAxdgFAzmuTZZfEV81vG+xTyX/uIPT3P9aAIBr9vdNHcQI/lxk5DDBrfsLuK+jE0MqupPOOx9 +KzNY8L6fbaaYrdGXb3BOcnuT+FcpodzN4c8RRRyylrW4IDE9MdM/UUAes2wbOAK27PhRms6CJlwc +VrWowRkUAV9CP/F3YffSm/8AQ6Kfo+P+FuWp9dLf/wBDooAxrH/kd/Ff/X8P/Ra1evUOcgVW01Qf +G/izIz/py/8Aota2LqPK4xQBxniWc2mi3MxBGFA/Mgf1rmtEF/Z6HNqMNuzvPnY+7G1V6Hoe+T0r +qfGmnT3Xhm8WNWJVQ+B/skH+lUPBt3d3PhuzXyBM6xBY0YfKDnALewxmgDE1BfEDaPaXNzMRJPIQ ++TjCgDHb69u9ZGt2Us2lrdNDtMLAgq27Kng84Fd74qnaMwWB8qWRTnzUcfePGSOx4ziuf1kzT6S9 +tuRHlVUG5sDJOMA+lAHofh5/tvh3T7k4ZnhXcfcDB/UVuRQEdqzvDelPo/hywsJGDSRRjeR0yeTj +2ya3I8/3aAMXSU2/FmzJ/wCgbJ/6FRUunf8AJV7H/sGy/wDoQooAyNJXf448XYPS+X/0Wtb8ynyj +0rm/DIll8W+KDKQ0pvF3FehPlr0rvINMzbfN8rsc7upH0oA5ie3mktZSI1ICn5W43e1ec6ZrDwax +facIj9liUNtUcgE8j0IzXrHiqS20rQJbiadoyBsWQjc2T2HvXnvhbREuzeXTbvMlfILcsF6D6jFA +GJr+pWE1ymFkwFzhlwo+i1xevazLd3Fva2+UiQhh7kdPyr0jVfA8t0BeXNybe35UK2EJAJwST/QG +uS1Pw7HYalbKHUIxYxyDd8wHUnNAHsnhXVBrGhWkrBlmEYVww6sAATXQInA5rn/AOZtIa3mQHZI+ +xwfvAnJ6d8n9a6yazEKhlzgUAc1YAr8WbH302X/0IUU6xBPxYsSe2my/+hUUAV/Bdj5fi3xWJJDJ +JHeopY8bj5a5OK9AUArwARXEeFjjxh4xbub5f/RYrsIZgJhGTjcuQMGgDnfHiwnw1KJoVkUuB8yg +hfeuZ+HemTLpjx3OCZNzKUbPy54/Sut8Z263OlJE1wYgzkkjvgH86yfBb+XYWuIGiEithWzn9aAN +loTcO0ctuGjV9oMg5JGCSOOnp9K8/wDH1qH1iERrukRAqqB3Jzj9BXpsk6F+oyCuRjJ54rhNcg+3 +Ge5XiUSL5ZGc87sdPagDQ+HlvJHoAdo9h85mUY7dK7WSRCoB6HiuV8IiW10JYs7yszDJ7fN/k1tG +Rpb4xj7qpnj3Iwfx5oAwLMgfF+1UHI/suTH/AH3RTLJNnxltx2Olvj/vuigB3hgf8Vp4vH/T8v8A +6AK6aRWFk2CA2CPSua8M4/4T3xcp/wCftD/45XR6q32e1JjUySCRdqA4J3HH9aAKHiJTceH4mliK +r5e5lDfMpx2Iqp4eQR6Zp75Y4jX7xyfTn8q29djjbS/LMqxYGFdugNZWlskOh2pKgYj2AqO4OB/M +0AW7+NLQ3Fwi/O6hsk5yRwOO3WuS1qGJtNuvN3iNJkX5e+EIxn8f1re1e4ubq8jSOMiBArZJ/wBY +xOcfQcZ+tVNTsYh4dnjmG9PMJIP8XYUAQ20z2Hg6OeJGTYQzd+N3Le+RzXQ6TGwtjLLkuxAy3XAH +f8Saw9Mlt7vwsI4yZI9m07xtyM/y5rqodqxIFAIx1oA5iDj4w2ZHfS3/APQjRSw8/GGzx20x/wD0 +I0UAee+I/GV/4S+IXiAWlvFKJ7gM28njC+1Ubn4v6xclC1hbAq6vwzdjn+lXviB4X1O88b6lPBYX +EkUkgZXWJiDwPQVzH/CH61zjSbwj1EDf4UAbV78YdZvYPJbT7UA+7HP61HbfFXXLW3SFdOtSqZ67 +v8fesg+Ddbzn+yL3P/XBv8Kcvg3Xc5Oj3x/7YP8A4UAaY+KuvIQP7PtM5JXKt/jUF78Udcu7F7WS +ytEVv4grZHPB61VPg/Ws/wDIGvs9v3T/AOFMPg7XcHOk32P+uD/4UAWLb4l6vb2zQJZ2m1gP4WGC +FAz19q17f4va0sSobS04GB8rf41z3/CIayOuk3g/7d2/wqRfCWr8f8S27/78P/hQB33w78Q3fib4 +jR3l3HHG6WTxgR5xjOe/1oq78JvCmo6dq8+qXUBhhETQqJAVYsSDkA8496KAP//ZAP/bAEMABQME +BAQDBQQEBAUFBQYHDAgHBwcHDwsLCQwRDxISEQ8RERMWHBcTFBoVEREYIRgaHR0fHx8TFyIkIh4k +HB4fHv/bAEMBBQUFBwYHDggIDh4UERQeHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4e +Hh4eHh4eHh4eHh4eHh4eHv/AABEIAWsAyAMBIgACEQEDEQH/xAAdAAABBAMBAQAAAAAAAAAAAAAH +AwQFBgACCAEJ/8QAVhAAAQMCBAIHAwYHCwoGAgMAAQIDEQAEBQYSITFBBwgTIlFhcRSBkRUjMqGx +0RYXQoKSssElJjNDRFJicpOi8CQ0NmNzo7PC0uEnNVNUVYMJRWR08f/EABQBAQAAAAAAAAAAAAAA +AAAAAAD/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwDqjNudMEyw6lvE3HEqUnUS +kDSmeEkkCTB241Aq6X8mgSL5J/PT99DPrX3S232GwohPbp5/6v8A70PMsrSuwYKgkgtjiONB0Ovp +lykFQLhJ8+0Fefjoyh/6/wDfFBctsJWkpt0wRzSK9ft7cwVMNmf6IoDOOmjJ5/lB/SFejpmyeT/n +I/TFBdFqwACGGR6IFOWbRhZ3Ya349wUBgT0y5QP8qT+n/wBq3T0x5OPG8A/OFCy3tbZCRpt2j+YK +cotLMjvWjM/1BQEs9MWTR/LUn0VWfjiyb/72hki1sw5q9kaHL6ApZDFspQHsze3DuCgI6umLJqUg ++2Eg+AJ/ZSR6aMmj+UOfon7qozjTACUhlsCP5opo402AQWkeumgIK+mrKAA0uuK8e6R+ykj03ZU5 +JePx+6h6gaTGlPHmkUwzDjlrhDKVvtIWXVaUIjj4+6gKKOm7KZUQrtU+YCj/AMteHpuyvvpbeV7l +f9NAHEc3Ls8ed9iYZLTZIAI4+vpRAy1i7WKYS3dhtIDghQHIjiKC9npwy3O1rcH3K/6a1V034Ef4 +OyfPqlf/AE1WG1KKxAAHhFO0LgTCfhQTf47cGPCwuJ/qr/6a9R004as7YXcn0bX/ANNRLbqhsAOF +OWnVgEnh4UEm30wWbhhvBL5e3Jpz/ppwOlMLSC3lzElT4suD/kqHN642YT9tIu3T7h2UYHnQWW36 +S9YleXcQA8Qhf7U05b6SMPXiFpYGwfZfu19myl9XZ6leEkR9dVBy7XEFXLcUN+mPEn7VnCrplZQ6 +zclaFeCgAQfqoOqbO4L6VBxvsnURrRqConcbisqo9FOYE5mw1zGENdkLllpZQSCUkApO/wCbWUAY +63i9N0x3v5SkHy+aqj5QBXhFqRzbFWrrjrWm5a1ER7Unh/saqOSXP3Bst/4pNBbWhLY1nvCt1NyZ +mRWtuoKESCOdKjSOB99BiEjYHYU4ZQpKpTwpArTxpVt8g7RQPmlQkpP2UoFE8VkU0buEkK4A+NbN +rCzHDagfQCmN69TtsPjTRLoBA1b0oHFaZPwoH6IPODFJPwOG9asklJJ228a8egp9aBspcHhvQf6T +b55eZHWVu6m2kgISDsnaT76Kt46hlpx1WyUJKj7q59xzEFXV9dXC1krccK9/M0CybtYUTrMnj50R +uiPGLl69OHBTaWG2isgjdRkffQgFyoqO441eOiW4WM1W8D6aVBXpFAfGCVGQmnDZO07jmaYW74G0 +yKfNuJUBB50D0ISAIAiNyK3UUkQIpDtgUhAAIApNToAKQACedAotaZgCfOvQrUCAIpsVAma31kEE +bbUG60Dwn1oW9O6S3h2HqnYvK291E1SlEkzvQr6f1qRg1gVHc3B4+lAaeq2rXkFhcz3NPwcXWU06 +pzqnchtQ4CkBQjT/AKxVZQDXrpOEXLJO0XYj+xFVDI6lHAbEeLSfsq1dd2UPWyxMG7HHhPZCqtkX +/wAhw9Svo9gj7KC4Wx2MCKdt7ncTTaxbGmBz3p4hISNzHhQJrQSYnjXiUkQOYpZzupJJpFK5g7n3 +0ChQSkbxSluSFDfam5WACokj1NetPAqA29KCXb0HdQ9TSwQlSduVR6XBpJnetkXWgCTQSTQCRpMk +Uk8djzpsm9ExNaOP6pOqBQRWcblNtly/eI3SwoTPiIrnbEXAl7j4UaulrEHLbJ9z2ZEuKS2fQneg +Fevy62ASQYoHCHkqXsqrl0U3aWs52mpelLgU3v5jah20/C9zO/hVmyc8Bj+HjfvXCB9dB09bq0mD +zqQZUkAVDtriOdO7ZzSYO8UEoHBw3r0KB3mm6XUxERWdpO0UDhJBVvSulKh9kU2bUdzvvSzKu9tI +mgXShOjfaTQk6xZ/cOw4GLk7/m0XlqlGxEcCDzoP9Y1KhgNhI/lR/VNAWuqGpJyG3pBHdVPn84qs +rTqgqJyKyCI+bXHp2qqygH/XoQEW9i5M6r0CJ/1VVnJKP3Aw8QP4BvYf1RVh69K1lm0BMgXyQPL5 +moPJ50YDY+TCP1RQW+yRoaC44HhSygNXe99NbW7lCUKA48K3cdgbk8KDa5IAMcKj3XiklIMTSrj0 +zCppi8STwNAo5cEo4RXtq53pn1pke8NIUR5UoydWoA+lBMdrCdjSLr5G8yfCmJdWE6ZpNx2NiZoH +4uZjlThKitJ35TUOl4DntUlZOgoiaAe9Nl+tnDbS1QAULdlzxHgftoMLdm4VzAmKKvTsvsrywUqS +y8y42seBBBB9xoSWTa7hVwocGmyrh7qDZpUKFXzoptG77NdoFkwiXD7uFUvCMOu7+5SxbNLccO8A +TRd6Nsq3eB5mbVezq9l7ThsCTEUBcQvhtuKdWy0qVJ40wS4NqcNOTvA+NBKpMokRSiFKnx8opo07 +KAAYNOGwrko0DlB/JO005Y0gTNNW9pkzTm3SDJJoHIQhIlKZ50IesgFDLdgYgC74/mmjC03KaEXW +WBTlmykH/Ox+qqgJnVAV+8S3hMbOA78fnVVlZ1Oio5BbnkHI/tVVlAOOvSIbtlSN8QSD/Y0yyayh +3Ldg4BBNs37+6KedfERaWqo2+UE/8E1FdGdwF5Yw0kgzatmPzRQWdhnRClCTWPAEHeKkENodbSpO +xpBy3IJJjegjFwjh6U1eWSoxw50+uGQHPE86j7qEqiY9KBstYSe6B61gfLapB5QaaPOQs77V4twR +wNA8TcFyPKsdd25UyS6RBAivXXJSDQOm1jyinlq/CoSY22qDDh1AVI22mUqmNqCpdI+GXOZb9Ng1 +I9jZL+qNiVSAPiKiMhZQFixcHGGgkvJPaJJ4JBolsKaTdO9zdTcKV4CdqalLSrtevcQEkeVA7ytl +/CcOm7sLVKFugb+VSt06k3WgfSSO9HKss30NsagAkAd0GminCXSr8onegfNKSDO/nJp22tCSDUYF +xEJpVDkkwDPrQTTCoGxBmnjDp4KG9QVu4rVxjepS3WY1KVPpQSbSiT508tBvBImo9hJICt96f2yS +TwIigkWht6UHes8ojK9mJE+2Dj/UVRhaEpn7aDnWhH71LP8A/uj9VVATupmZyCiePzn/ABVVledT +OfxfszG6XYj/AGyqygHfX3A+TrPmTiKZ3/1JqvdFX+h+EqPO2TU91+NrK1Mf/sER/YmoLopSpWR8 +II/9sPtNARbNSdAKdzHCtnY0Eq4zypnbFaEAqImnRUFt6p93OgjbyEEn6qhrxQUo+FTGIp7m23rU +HeGKBi6E6orRcRtWyiSoxWp4ERQeoAI3NeLSnnW7Q2341s5skcOMUCSEpMk08to0gA/Gmu4HCvW1 +qoPMQWtOJIDR/I3HiKyycacfU4o86Y4qpxBcutiUphIpLLy1LGtSN1RsTwoLS0SQpx3uoG4HlSHa +gq1JMSa9ullVulJAA5+dN0wYiKCQbcJ+lNLoVzmmjKoEmndudSoVAHpQOrbVq3mIqWtYME7Co9oE +xHAeFPWVqCQY250EsyuBsSPWn7C1CI4+tRdq4F7gjzFSduoCBHoKCRYUYAG1CHrQJjKFqqB/nqf1 +FUWbck96NO3jNCXrPmMm23Ha9R+oqgJPUvM9H6PLtR/vTWVt1MkoT0eMaQAVpdUYPPtSPurKAb9f +ZQNkwnf/AMwR/wAA1GdD7QOQMHUeduPtNSfX3H7n2y4mcSSPT5g1G9D69XR9gxCeFvHHzNBczGkR +tWqXIBjlW4HdO0U1XqQsGOe9ApcAKTHEGoe7tyJqWWSBv61roQ6nvCgrD7cEyPqpNCYn0qavrMhW +w5U1NsYAAoGLAEmaUcAinItSZ8R4VjjJ0Qo+tBHK+NeoI0kVXs3ZptMIS42wk3FwB9EGAPU0KMWz +zma6uCtm67BoHZLSQEx6nc0BdzRfBq0UhKu8BwFI4BchttK1EyrlPCqjhmJOXlrbquHUPOrbBJJ5 +8+FaXC32NTguLi2MmCYU2fLjPxoCel0vDUFFQPCnSAQaH+Vc625fRhuKpbtX+CHAruL+6iC0QtKV +JUFA8CKByyFE8KeI2CTzpG2E8h8acpQSRCRH20DxlRMGPqp2zOkJB2prZtlRIg/Gn7bSUwNW9A+t +BAO4B8Kk7UeJmo20SUkmeHlUnbK2G1A+aiZjfxoS9Z8k5Ltwrleo4f1VUV0d6IG/OhR1mkg5HZkz +/lqP1VUBI6l5H4vmRJOzvHl86aykOpOf3iEb/Sdj+0rKCgdfUj5NtxzGJI9/zBqP6Gkj8XuDDn2H +/Malev6lHybakQD8oNg7f6hVMehVIV0dYMqJhj/mNBcm29Q24U3u2gDtUohvS2dtzTO6H+IoI93d +PmBTcq0xBmnDoM7b7U1KZUQCBvQLKcQtuFRSCwAmBFIvBxJ2MGaVAUUhQIngaBNUg7kVV81YoUhV +rbuweCiOPpUxmC9Rh9it5Su+dkDmTQ9Nz2q1PrO5JCd5JNAxxyztBaOOXyktsNp1rUefqaH7dpc5 +hvlDCLJabVJgOLG5H2D0q7YhhV5nDFrfBLYrTbA9reOAwAkHYep/YKL+X8qWGHYe1aWjCUBtMDQn +c+cmgDtvlq7scEQ2grTdtnVqKdhVXx3EscYTpumoIkdogbKHmDXRWJ2NotlxpbjRUOYVJ9N6G+bc +Mt3EdghaFx9IAUAXfxB25bIe0haOHd+qiR0QZ1dRdIwXFXdbayEsOqP0T/NNU/MuXH7Qret0KKUn +dIE/CoCxUW3UnfcyCOINB2BaiRI8Nop+0gxP21R+iPHF45gCUPq1XVsezd347bGr8wlQERQLWp0r +Gw41IoAKp2k0xQkg/Qnwp9bSYJSZFA7YSfjUhbN6uMAimTBIUCakbZW44UD1hkJAoUdZ9pKMgNqH +H25H6qqLTR7oPOhZ1ngD0dSOIvGz9SqC4dSZJGQyeRU7H6YrK36lBH4vk7flvQfzxWUFE6/oBwm1 +Vz+UWh7+wVTboPSR0cYKkCf8nk/pGpHr/o/e9aLjf5SbH+4XTPoOUk9G2CyCT7PwA/pGgvaR3eAN +M71E8qkkJBRsDvTa7QNBFBCuoHGKaqbGuZEU/udIERvTbUmSNO9A0ukjSTG9Nm1BC5PDnUg8pIHC +ozEn0tMLcCeCSfqoB9nrElXOLi2bJ0IOkeE86rVzcBDKnAoBLRgCNyfL/HjXmO4koPaoPaOEmf5s +86aYHbPY7mmywdk6mmyHbjwKZkz60Bl6NcGRhWX0XDrWm6vPnnIHeAPAe4VZx2iEFSVrKYgpI0/X +UJeYuzhrae0Qt9SANmwdIH7ajh0j4WrW204dYEEE8NqCXW6+7dP2yUMNnTKdZJ24cao2ZGCVO9ol +tzcmUnh/j7qXTm3tbC8xdcKQnsWRJ7pIUoq+qKqeMZ3YUp9xxxsJcUCQngOJ2+NAmptDxKHEqSCN +vGhnnCxOHY8sISEpe76YECecfb76J2GXDV8wLj2a4DDhlDikEA+h8Kr/AEk4St7Chdtd4sHUI4xz +Hw391At0JY38mZntmnHdLV38y4nlP5J/x4102ywVAEbiuLMDfcYeauGjpUlQUDHMGu1MpXPyll+x +vkbh5hKifOKBZtuCRoiKWbQoGYpx2KkmYO/lSiGjI2EeYoMaQSSDTplJA2rxtJG1OmUgD/tQKM6h +50MOsv3ujlwgzF02ftopJMAAxt5ULeslKujq5I/9w2frNBb+pM6Dkfsuep8n9NH31lJ9SUfvOBgb +F/cf10VlBVuv7q/B61g7fKTW3/0Lpn0IjT0d4IN/82HPzNO+v8Vfg/apkR8osn/crpr0HEno5wUE +cLeJnzNARGSCmCD8ab3IBkAcqctjbhwrV1oqTMAUEBdpABnemRAHvqYu7WASQDNMOw7xAKfHjQMn +YjvDaq/m64RaYFevcNLSj9VWp+3IEyKo3S8U2+RcSXIkthI9SQKAKXV4u5ue21BLY3JngImrb0S2 +TrmCXuYVuKb9pfUkr0nZpA4Ty3maHDSXsQLOHW/8PdOJZbHLvH7AAfjXUOW8rWWGZbssMTZtutMt +BGpbQMniVRE7mTQDzMfSJctYXcpwbLq30MoSFP3KinXO3cQBJ9SRQpu/lvE8WS+jCQXXVAyylaUu +T68TXQubsNNpZK1YwLVsDV3mSSPSFCoPJeTH3rv5dvO2UiYtu1RpKxH0o4xvzNBHXuX0WnRpcGzQ +supWlSyqZMjccfGN6FGG2+Iv4u5cW+ALxNLSoaZUfmwoc1Abq9Jrq3GsPtrbI12gcFNHaI3oJW9q +7Z9qbEpSpapg+NA2ezHnJ4rZxLCrZLVshHYWzdutBXPECCoAjzqRtlKxNpxp22U0tSTLS4kCPXen ++XLW5xR2EN3iXE/SCmwUj3wD9tTSsHLPdUtK1cYiaDnq1bXa4jcWLuy23FAe7b7q616v1yb/ACFa +JMyySg78INc19IViLHNAuG0hPa94jzTxo79Vq/DtjiWHDcJWHk+iuXxFAXrls7xO1etJkCCaevNy +NXOvGGxG6QDQaIbM8AactohMRWyWyD605aSInwoGwTIgCD40MOsa3/4cXh8Hm/1qLSkhMqNDHrHN +T0Y38Dg43+sKCd6k4H4EHxDj36yKyvOpTIyZpJPF79dFZQVLr/T+Dtt4DEmf+Cuk+g1uejfAzEk2 +wn4ml+v8EjK9sYknE2Y8vmXKT6BQ4OjTAyT/ACfh+caAiBoDYCaxxOlAGnjSralQJg7V4+sdmBtQ +Rd2gaZM7nwqLdShCzE1NXS2wmJmot5CSdSQregY3KthE+kUNOnjtF5Hf7PVHatlfoFCim+hOkwDI +86p3SRhyr/Kt/apaKipklIHGRuKDmjKV81ZZuwi6cWSlNzpE8BtE/EiuqLbMPtFi2WVDVA2rjXEy +5bX1skApW2AQPA6ia6awJQNvavpSZWgKI5EEUBLwmwbxFSHMQYZd098FSQQD761GMWuK45eYPhYT +cKsUJNw4ncIKphI84BJqs5hzKcPy4+q0lVypISNIkk8gPEkmKnsuZCvMHyEi3sMQFnmC6V7Re3Rb +16nFcUnyTwHp50Epmxm3ssnrYeUe0uEHSIoIYg0cKS9cOW5Uw1ClqHEJ5n1q59JWKYhh7tthN0+u +7et2NetKY1pHExyoft4lfYnir1u6sPWL+mUqT9HxE+HxoLhaXS2sND9k5rbdROpJ2INa212OzK1L +nn3qhMtMPYU67hThKmEElon+YeHw4e6nFxPblQBKPyY/x5UAt6VLwPZoba3DYbn3knf6qLXVGbcV +e4u8QezS02kEjmSTFDHNuXcRxnNDKrG3U4SnRJMSZ/711B0I5OcyhktuxvA37a4suPqRuJ5CecCg +ujoEQfCtmOI2rZ0d2vEpEbcKBy2AVcvfS6UgbDnTNJPnSrRK5UmRQKqC9ESDvyoY9Y4q/FlfiQfn +G/1hRPWogaYihh1joPRhfEj+Mb/WFBMdSon8DgDyL8emtFZWvUnCRlFYSRIL0+utP7IrKCr9f3/R +q2PH90mR/uV170EKKejLAxAP+TbfpGvOv+CMsWpPPEmY/sV1r0Bk/izwPcx7Od/zjQEhlRUNk+6l +XEymY5UmyDp4x50uggpHemgj3mZSSRA8aadmEqAVFS9yfmzHhUU8UzvyNB4tpBG4NMLy2YeZWFN8 +QeNPA4QdMkT9VIvOHSUhQ3oORulXJl7a5tvbq2tyq3U4FIQBvBAO310bMBsQcKw+4ZIIS2kKSR5V +KZ9sPaWm1WqW37i2lSxH0gBBFUTLeamLW+Tg97cBt9StSEcAB/N9aAws4LgLCbTEbrSi3aWLo61d +1Kk7ifQ7+6vXeke2uLdCsvYLf46tThSk2zKuzBH85wjSBVeTeu3rCMPNom/tnFauyUdKfzvETyqb +fubpFs2wthtCUCENsSAPACKCk50xPMvt72L4nkt4Xa2SygtuIUkIIjeFHkfKh5huMWdk+W73Bb2y +UDu6WSpER4iYq45rwS9duV3Nz7Ssap+ceJ5cAJquM9o2QEIJ7wkKPCgnsHxCzxa1F0w6haSkplJ5 +UjdH5tdwe4gCRJj/ABzpo0lq3X2rbKGVLBBKe7J9OdUnpQzULWx+R7R3/KHk6VkH6CefvNBMdEmc +13PSMm1u3W12C3VBgFIlJ8Z84+uuvLIzaoKtyRXzpy7fXFjcpWypTa5ACk7Eedd99HmJt4zlHDbx +D6XlKYSFrTwKgN/roJ1xHdFY2J3mt3EmIrUCO7FAqhAVsRtThIAiCD5UigGCZO1bpSqNiaBR3SpH +CCKFvWNI/Fne7TLjY/vUTl6tJoXdY0KPRnfGP41sf3qCb6lP+h7h0gDW8J/PTWVp1Ilzk59E8HXf +tRWUFX6/5P4NWoKhHyizt/8AS5SnV+bDnRpgaU7xb8PzjSH/AOQNUZfsk+OINH/dOUr1eFhHRlgg +3nsJHl3lUBONvGxTBitUthPGZpVLx3kztz51p2xJ2igTuWu6ajLhvjvUs8XltEttqWI3IrZGFJWl +K7l4hSk6uz578KCtLQrXEz4RUknAHT2a1uaFkayI29KnWcIsrZ/tN1rSJgqmo1u5vLpt1p1pTUPE +srk7jjvQBvH04hhuarxzELlaGn0Q2hHBJ3gTXOefrwt5oadUl1p9h4HQvYxqkGuoOmF+1U0bJ25Z +aeWsKMqEq8QJrm7F8PcxfPTLKy12LSzD6mtaFJBkBQHHwoDBaYld4c004SpbRAKVDiAeVWFrPNi1 +aAugrlOgEHcbc6jsv263sBZtbhse0sICXE6eGw5elRSsJtA4/rag8Y+6gaZqzheXlyvsSWrdHdSh +KREcKgm8bt0IlxYTGxkQaZZvtW14u0D80yGwVRtJBqCxRy2ClOaU7bJkzvQSmI5mff2YgNjYeJNC +/MTy38auHnFd7VzNWW9uDDVtbgLed2QmOE8SagMesfZr0NKJU5pBVvxJoNMLt3HvyoI70zw3/wC9 +dP8AQRnVzD7K3wm5uGlWSTpSeaT4ek0CMm4GL68asw65LpCNCB4kcffRz6JMAbX0iXOEBxsJsWUO +IhI0qBH+PfQdDsrQ4RpcSraYml9AUkq2EVDW2GO2OKe0hsr1JIUUq2I9KmEPs6ezSpJWRtNAow2C +jma2TqA7vCvGFOCQpPpW5PmRHGg8UCUgn30LusgI6Mr8CZ7Rs/3hRWOyBznwoXdZRMdGV/wMuNfr +UDvqQz+CNwTw7Z0Ae9FZXvUiH70LiOAedjb/AGdZQC/riY9cZgybaXdx2QWnEGUKDaFIAUGV6tlC +eYq1dX1BPRrgRHO3/wCY1WuubhNnhOUrC1sX3LhtF61LjiYUo9k5JMVaur4kDoxwLf8Ak5P95VAT +A0rmB8a2w+1L9ylomATvHhSrTetMJBJPKp7B8P7BsFMdqrdRPKg8xK0Qzh6bdlYbkiSBvE71AnCc +TxHGRcqAtrdteyie84ANtuQq6KZSmCoaz4mtVpII22oIe2wq2t2tMOOL3lajJ3qDzc37Jhb2lSkF +YgL/AJm3GrhpUkn3VSOljMWDZawBy5xd4gOgpbQlGpSj5Cg59zRaP67w35beeZAUwp8gl1JO59QK +p/RVgS2XL/GL5am+3LyWg2jVI4QBB47/AAqbxvNb2a1vMYQ+i11KSEQ0QVeEq4/DzqwdGjLlvlkL +unHEPquH9bkAqRGqVAcyDvHlQXHEMETbu219aJKA+yhtxB5EDYn7KrGYLY21yEOoLayCFBSYNF+8 +w4sXFu+EqubNTRVp4qVpbBA8NzJk1TukZeMu4fdN4ZgaMTcaLIDIZDim9X0zMgwPf6UHP2e03Tt/ +bN2rS3HFpACW06lKPkBxppYdGmasQR7RiCRhtvpK4cV84QP6PL311ecm2jFs0xgybSxu1sIcDxZk +lJ4jYgmD9tVvFsoWiQ45mDHVvW8QvfsEQDChtufpJI35Gg57Tl/C7ZDjdtcsWz7SE9s65K3EhXPh +z4eFVvNtrg15b9ph63W32nEMtoKCS8Oa1K4DyFdE57s8k4060zqQbi2SG0sNgw547jYgcffQr6RL +Sywu6ZFtbN2zCXO0LaR/CECdz/jlQVnLWH3RxRoWoLEaVLUtUwf5225H30XeifGlYJnO0tn0ds3f +/MG6UgJgkyD6HhVUyW3b3l+wEpSxcusjSDJC522+HD0qy4Thq8VzrY2QcKmRqaeCO4oECAoBQmAR +QdTWTYCAVLCieFe3WFWV4pKnWh2iFSlSTBFQfR85eHD3MOxLvXlkoNqWf4xP5KvfVwZbIIOxmgi1 +4elgSkrIPDypqtlQE1ZnW0qbg7yKjLxlTKVAJGmOJoItepKR3gDFC/rJKJ6Mr2AdJcb3P9aig4VE +bDj4UMOsekp6ML2SILrUfpUEj1JmyjJrytiFOOnj5oH7KyvepPIyfcTzccP95IrKAb9dN9p7J1gb +a+urtkXjPefIKpDTgIMbE93iKtHV1LKujjL5fC+z9mIMeSlVS+tZh2L2HRrZt47Zi1vTiTQKO11w +OzcgzJnh4mip1YcNb/FTgDrqQrVbE7jh31UBOsLcdhrbZ0JiEDnUvbM9miIk86SsdK3VqaVqbTtE +bTzp4J5UGq0kxArRxsngDTkkgbcBWizwmRQNXGjvFBHrLZYxXHUYabJhbzDetC0gx3jEb8uHOjq4 +ElMzNUHpcfScEFmq5RbofVC1qXpgDfY+M0HI/YYtl1a3bmyt7C3LnZ26zHarIMaoEmOfhRKyM8nE +7Fm4tihLaFKWHNMBKtCiVkb89yKHXSo7a2+IlouqKWkKTDjsjURsB5wZnzq+9ByG2cCtmlKI1rB4 +mSSOBI5GfhQG51SMLwhd9d3wWtNqhaU6YTAG+mPEmn2AN2tvZm9cCW37sh1ZPiYAHwgU6Yw9rEcv +27ToAOgBJT4D9hpZGHNuhDK4T2YiOQoEsVtnlLtvZ7QOuIUdLmqNCSNxHP0oZ490dXl5jryPlJ9F +m6kOOslZWFKPE7kxwG1EZnGGnMSvLNKoatEEvOzEE8BNKNOYa7iSk2zrbtwWwpxSSVGOUnnQU+wy +Rg+HIQpNqhSkc1JGxoE9OeErv82Iw9huO9Ko2CUwkn7a6rv7VTiDpMeBP21zn0j26lZuvLy3e1Ia +CVSonhI24eCaCjWtk0xcW5xC8RbtWoCmm0nvEJMgGOG9WHo4dxRzOthdXGHutoQ+oF7V3QF8IJ3P +ED31EvM/InbEqZccvHNRURqCWwIjf3mpfJ79xdZywtCriXC+lxSNR7yAdyYMAbbCg6WbZXa4na4m +2SEqhm4HLSfon3GPjVub2G1QVtbC6wxbau6FoInwqTwl0uWqAsy433HPUUEnJg7CKbvoC0qB4U4k +cfqpFwlSjHKgg7pstrIKNPhAoSdZhSm+jW4kDvXLSfrP3Uar5jtGpmKB3WkXp6O3GuZu2p+ugn+p +aQrJTp0gaXHUkxx7yT+2sr3qWQMjL8S69z/pJrKCqdfhR/B21bIgG8ZIMcfm3Pvq79WMA9DeXdWw +9nV8A4qqf19Qn8F7NYBKxetD3FDn3Vaeq68X+iDA0L/ikKSNuI1E/toDCNKW+7tXjbknTMKB3pNR +BRusSRwpG1bQlZdTqle5M0D8pIRIMzWi0k7kVs26VARuOHpWy9kydtqBvyAoR9PDdy6W0NBOlLJV +KuAMnf3UXlQTx40JOmrHLe1vfYLllLluGPnSeRPn8KAA4sxhGY0IwXEbdAcaKSxfJRpVpJAjVzTx +MHwFGPLGXLXA+yw6zh1lgobSswZKYG9B3GUWzCEXdmXHGFPoShKR/Bq8FetHnJTbjlip1/6Yuz9L +YjvJP7aC9ZYYPyUyoOiQkApSQUp8hG23Davc2JxC3wi7fwlkvXvZHs0AgFR8idp8JpbKaAjCGwmS +EqKQeaoP0jtxPH31LPJ7VlSRsY29aClZTwFdnkq3axBhab19IuL8FYUpxziQo8Dvy4VL4Szh7XaN +WduGlIjVCCBvy4fZUi2T2imjsJmDSVuy6h59bjoUCrupH5I8KBtjTybTCLl8qCSlohJP84jb6656 +zk0FX+IS0pbhW2lKUnc7EzRw6QHpwL2clZ7d1KShIOpQG8SOA2Ek+nOg3iuFrxBV+Ge0S4HHXAAd ++4Dtz5GgodlhTuLOXV9blt+3t1qTcIKtK2FA8DOxnl76neizALh3NTd4ppRJCVIIJ+jJ+HCvcq4h +8nZdRamwLpecU4tLg1BaCd9W3Ab8TPCp/oVxC3s80Iw95d2lVwSlhLoEJgcoHODQdA2Dei1QmDJG +9LJZ7N4vN7H8pP8AOrGNSedOAQU7+NB6l1KyIO8SKb3L4QstEwTBn1P214NFvreIJK+fgPCoyzWi +/eXfPhaQh5SG0qVACdoMefH30Ek+4lCi2dwQTx50Detesfi6J4TeND6lUYbgqf7VsK0KSe799BDr +VOLPR8pCjwvWgY9FUFr6k8fgSuP/AFHj/eR91ZXvUm/0GV/Xe/XTWUFc695P4NtJO6ResHj/AKty +rb1YRo6IMvAAd5lZ9/aKqqdexKjgCIE/5Vbn+45Vs6ti9PRLl0Dj7Or/AIi6AqO6QtC1qjkBHE1o +HCEKCdlQSJ4DwmtL3UGwtKgkpM1s22OzCn1pJMcOFBtgq1rtRJKlT3jBAJ8p4ipIokAz7qq+FYnj +D+OutrtbVOFto2eQ4dQVJATEQdoPvqypdA3kEcaDCxJ5jehL0k4JbX+aF2D4Kn7tslsaZ2jaD4yf +qovJeBA9aGGbcYUxmS4XcLW2GwopMDToHL12nagDeMWrOQ8Ut38Te+bfeShTDYSrWUwZUOW870UM +MxBD2EX17bgFKrp4twdjCEkfZQn6VMHxDMuZrB/CXbd3tnA0u2dX88gkzqG26efjtRiwWy7DLhY0 +gr9ofQTp3nSdx+jQXTLLxUxdJVGpL6jtMQrcRPkR75qQQ+kKUkq3mq1ktSoeSlBQkttqEj6RKASf +MST7wamXwWX+2EQdlDwoFApS7wmBoB2ptbG6TdvOOvsG3OzTSEFOnfz4+tN7fEW3seXZNmS0NawP +yR51o9d3Vzj6mBZhq3aSfnCoFThkbwOAoIHON2HsXtbALWgBQVISd1KJgTwiEmR6VV8rgqexG9AB +0WjzgJG0mpa5Wm4zViN8hQcRaoXJ0gAdmgAAEHeFFcnjO3KkujlTSbK+uX/4MMLCpH5Ox4e80Alv +7O89rtLK0tlXK1EgISqEK27yp2HAbDyqU6MsLXhucEqWyHHGnNTbhSJIVPCCY9Kl8CtsYvbxzFuz +bu7dLmi3t1PFGpO41pjhyEceNPMpl+zzkwzeKLr2IK+ilOlCAlJIgevPnQGFNwrsCofSArXA7x29 +Sokd1KiJ9K2QzpSNUwRxpLB22MPs3xqVHaqWeZMkmBQa4u+3fONWdqVrLdylL+gwWiBqTPiOHuNM +ct3D14L9zswi1RcuIb1cVadp9Nqe4241h6HMQbMLQytao4KhOxPjwpnktl1rKNqHge2cRrcP9JW5 ++2gmrRsItgTBJ8KA3WzRoyUFJ4KvG5+CqP6GyGUgAcKBnWzQT0epIH8ub+xVBYepbIyQ2AQR8/Pl +84Kyk+pPq/A10HcBbo9O8jasoILr1ScBRBgh63j4O1YerOo/ilwHURs0sbf7VdV/r0j9wkqn8u3+ +1yrB1Z5X0TYFwjslgf2q6Aq3X+bSPCeNIYchNywQ5JTBETtTi4SVWy4HLamuEhSElJM78qB1CGG2 +7VpCUpnvnkkePrUQ/cXjOYmbFAUbd4FSf6McR9dSmINLUJCCptB1KTzX4D0mqhmPH3MvZgZvbxtR +sy+1bKITMLcCt/iAKAjhsJQIiYoW9Ilp+6V44yg3DhQIQ2qTJ2I08440TWHw/bJdSkiUzvQ2xxdy +xib7rDalvF8nYctXCaAMYfjWJp6Q8Js8SY1D2xBCwkoIIOwE8eKZo34OV+zW0pK+1vS4QFcAttdD +XGMYwu66SLG4xKzadvWLhDbLikFKgeB9QJPwotYez2YwxI4FDK9/zx+2gTynCb20WVhRfsQAmBKA +hRmY33Kvqqw4mlCLRxxwEpSJgbk1XsHhl7D5b0Bu5fZU6Y4EylBnfcmdvDzq4ONpXG21AN8i4ZmH +D845gfxLsBZ3LiV2znaa3FbniD9EAbRw22qw2bTFniV865dXLj4BUpK1Hs+E90UviDqm8UJgCFAH +zBH3iorPVwm1wdy5ceLZdSGUKClJAUo7GR/jhQVxntG8v4ze3LTSHnWtB0DYFaiYnme9x58aQwNt +bWT8XdSrQPZwkq8CZn6op1i/zOT20JVPbuBSTxkAEj7BS+VOweyhiDlwvTbuFxKlEcEgQf20AJuF +Y1eW9vb2jtxpYK3FqZdG0ERPDxiiV0WYWu8xNGP3ilpdb7gSsGCqIkVBXgwDCMFQ40t5Srx5QZd1 +hSVRAkDkB4bVa+iS+YcuHMMR2kBHtACySdzvx5cKAj3t8pi1WUtqWoJkAVC/L9naWKbh1Sbi8WUM +9i2QopWQYkDcDjvUvi94zZYY/dOphttsqJAnYChDlN22ds14xiS/ZXrm+cuWme00rUClPZpUBvJA +n86gvrz1zieBui4T2NzeOIY7IHZvUdwPQA1dLVkW7DbKYAQkCqlhlstzMOCWOgDskru3gTMHTpAJ +9VH4VeFp+cMAbeNBouNEBRG24oJdbFH/AIcpM8b5refJVG90d0bbmgp1sEk9Hze8RfN7fmqoJTqU +6fwKfE97tndp80VlJdSrbLFyJnvu/aisoIrrzpP4OoVy7S3+1ypjqvuT0SYIkk7B0f71dRnXlSDl +hKo4Lt/f3nKfdV0a+inByPyQ6D69qugM6BLShHEUyw7a4dQDMHantuklBHKmjbfY3ijBAO9BviT7 +bPdeMJcQQY/Z9dQV6S4n2e+CXgp5tbIUmSUpgyfMGneeGFuYCt5DikKZUlyQJ2Bk+u01st1tWYbS +zCNZuGlKkD+DSIG/hMigstuEhlO0gjaqRmppJvXvY3GW7pSiYdMJkDjPjV8Q2EtgDkIqi52tmlG4 +L3dE90zxJHDyoAnjDdwzm21uMX7uKXVwA2yIKUJTxWFDZRO3186NzjqWbXCVE8EIT8Ck/fQlunm3 +834azcWVtqt3VJQtLkqEjf7d/Si9i7KU4HaPJ4MupmPAgj7qBtdtqRbX51hPs16h8o2GuTATJ9QR +5gVamHSS2VRCk+FRl9bj2m7HY9qLi2JCAY1KA2G+wM86c2F12+CW7y161pQNStpkcZjnQNcetwhZ +dA7y1JHHjVTz5cLdcw2wYQHpd1OTuECISvbeZkDx34xVhz1iAscKNyYKUd4jVHI0PsOujfZgaxC5 +uVIbZSFFtPNOkOBRj1IA8vSgks4OMpNtZtAFthWohJHArA4egVTjCbFf4txag6XHWCtfA7rOoj6z +UFfF66XcXUavablQSTP0Q2qOJ/pD3ir4y2G8Hbti1rBbCVDgIAoOfsaxFOG481hC3LZ21QrtWm1t +hayrgrSeRIMRwgUR+ibCWkdpj3cC32g2hCTPZpHI+e31VRs2ZYtHLxePKJUwpxTSWUcdYPDUOCSI +38iKu/RG4ub+xDehhhSVISeI1DfhtxFARnG27izW28gKQRBBHEUEsFwkXnTJcYeo6rSwWl9CI4bC +J8d/so3gAtQKpOG4a3adIuL4i0nvOWaJ24qkxQWzKzYdxbEsVIOnULdonwTx+s/VU8rvKK5nwApD +CrX2TDmrfYECVkCJJ3P106aSJ8qBEqIAEDhQZ61xB6PmduF83+qujY62nwoJ9a8j8XzQif8ALkfq +roH3UsP72LoT/GO7e9FZWvUtIGWn0wZK3if0kVlAy68IJyxO8Tb/AK6/vpfqquBPRfhQJjvuiDz+ +cVSXXg3yuE+HYH++uvOqopA6M7DWB3XHY/TNAemBKCBSdy2lDqSY350lZ3KdxIn7a0xJ0ltCkmCn +eg0x1CXMJfbIiUECee1UToGzLe461jFtibRVcWF4WUPkfwiOQnxEVaMRxGbRYWZEGo7oktLS0yym +5YA1XTzjyz4kqNBeytU+NVvMFlbYip5t5TqVEBIUhUaTyPhU4t4p4c6qGbHL9DinLI/OIlQSTHLj +QDfOeCsYNmBDlpZEvKR2i7kypSjqg78Ad55UUFLL2WlIUkmWwRtPDehPjeM3t04W7nuBtxsFA2JK +l/ZANF7C1JVh7bSu8kpjc0Gl1iLbVrh12tQRKkoJJ2hXdH1kVtgbgFvdWyne1LbihqkHUPHbn4+d +QePYbcYng7Fqw5p7J4FSvNCpA+IFLNXTeFIxS+dSUKTal5QJ2UUpMke/9njQD7NmNX+dM7KyvhyU +2+HWw7R+4c37VKFaVCBwEggTE0/dbfQWbewabsy6pWs9nJDadjII2Jnlt4VZzY2eH4cvFbawtkYl +fNN+0PBAG8TJ99Nra0Q7jTSyhLa3GktrK3ZEDcpTzJkn4cqDW8swi5wu23AbaU85pHFS1AD6pq2L +HzACRAAqKdQh7FiscilA9E//AO1KuL1IMcAKATZrFthgdwnEb1NjhhJdU+133DqV3U6dJj135VNZ +CxrK7PZ4Xg945erMBb3YmSTJ75gR68KpvSFdtXV3ibz60e0W1ou5LSRsEAqS2k+oMnwIqMyLauYM +3heI3lk9bKfulWt20pWkgLIgn3EEUHQEd07kCq1lwPO5qxRTgGhCkBJ8QBP2mpe2eXZdnaP6ltaQ +lp8qmT/NV4Hz5+tQVveLtMcu9AkuOoB9KAgNqUobEA1oHFa9JFJ27mod0+tK7SFzvzoN1lWmZ4UE ++taf3gsTzvkAforo0uujSQN6B/WqcKskWyRzvk/qroJfqYT+DTuw2U9+sisrfqZD97D3kp2fXUis +oGHXc3y3Eb6WD/vF1TOr7mvCMJyFbWt3i1nbPJdc1IceCSJVI2NXTrrtzgIUebbH1OL++uNnEkHa +RvQd42PSFlkAFWP4bI//AJKPvp09n/La298cw0+l0j764C7yTxPrXhKualUHcGN51wJVo6pvGcOJ +0HhcI++k+jDOGCYZlJi1uscw8LC1qANygEBSiY4+dcSHUocTvXg17gK4UH0Jts/ZXUolzMOGJA8b +pG/11Vcx59y6rGJbx2xcbKIMXSAkHj41xBqXEaj8a1UTO5NB1FjucsuqddGH3tj3HWe0V24AUe/M +c1RIPGN6I+HdImVm7VtCswYYCEj+Up++uFIJ8ayFfzj4bUHdFp0j5TQ++hWYcNSgr1g+0p58ai84 +dIeUVWi1s47h90lbKmXbcXAhaVbevjw8fSuLAFR9I153juSZoO2nek3J5Nuk5isAlpIIT2oIGxHv +2Net9KOUWFBacdwwlcqWQ6JnhwiuJAFExJisg8N6Dta16UMn9sFuZhsATqO7nifup4elTJn5OZLD ++1rhzSrzrIJTtsaDoPpMz1gb+NG7sL21uBd4Y7au6FA7laon0mansdz9lrEsh2S14tZ+2pQytbYW +NWtIAPv2rl7SRsTNbBJ8aDtW36UMn3Fg2HcespU2nUC5wMVBPdIWWk4s1+7to4nWklwrHAHnXJAC +toJrfccKDuWz6UsohO+YLEf/AGU5b6UMnkE/hDYE/wC0rhi1WG7hC3UFxCFAlMxqHhNSPttq46Sq +1S2kuqXAPAEyEjyHCg7Vd6T8omNOYLDf/WULOsDmzBMeyqzbYdidtdOi7SvQ2qSBpUJ+uufFLK3V +rQAlKjIA5DwpdqeZJoOu+pjtlh8Rtqd/WTWUr1N2+zyo54rLiuP9NI/ZWUDLrop1YAgaTu21B5fw +iv8AtXHrzMmNJ25iuzuuIytzLgUB3UtN/Hta5DW1J+jQRBZAMwTWBid441KlqTFZ2IkkpE0EZ7OJ +3FeKY7xhMVK9l4CvOy24cPKgiCx5D41qWoMQRUuWSREDjNeFkapIoEEi0Fro1Q4EJAIRzBnf69/S +lXrlpSrwtdnp1ksgtjgVSeXhtXpYT/NE16GeGw28qDxm4bLlmXez0hep8dn4K25eG1aBVsm00KIL +obUmQjiZOx90QeVO2WmFFKFMjUTx1QPftTj2FggkBgDURBe3+ygi7z2Zxp/sVJSlTiS03o3QnfaY +9PXjSeG+zNsuJfSky82RIJOkTq5c5G3OpP2JorCAlkd0GS5sd/t+6tnbW3aX2iW0KSFboS7Mj4UE +Y8m2DKezUndrTu3BCtczw8K2U1a+03L6XG9Dpc7NGg93+by293hSzrLanCW29KTwSTMV4liOVBHP +WiUL0ocS5wkpBiffSaWNztvUr2G86a9DMDYR40EX2BE6RNbJYPGNvGpMMA8q9LJ8B8KBghnfhSoa +QdwmDTzsiTukVuGjEQAaBu22EwIpw0gTsK3S0rbu07tmVTPCg606nrShk5bihA1OJHn3xWVKdUtC +R0bJUEgEPLB24nWr9kfCsoNOtVaOXeUEttNqWsplISkknStBP1Sa5NXhS9KdVncSAdXcO/hX0PxG +wssRt/Z7+0ZuWpnS6gKE+O9RRyblgmfka2HpI/bQcDDDGAynVZ3Xac1Rtz/7Ui9hqVbtMOpE7yJr +6ADKGWxwwlj4q++tGcmZYZWtbWEMoKzKgFqgnxiYoPn/APJrv/oufCvBhywf4Je3ka+hP4L4BEfJ +jUep++vBlfAAZGGNT6q++g+e3yavj2S/gaw4a4Y+Zc/RNfQo5XwAmThjU+p++vPwWy/M/JjXxV99 +B89Pkp4meycj+qa2GEvT/Aun8019ChlfAASRhrUnzV99bHLeBnjhrJ+P30Hz2ThL+/zDv6Ne/JFw +Zlh39E19Ck5dwVPDDmR8aw5dwX/49r6/voPnl8kvD+Idn+qa1+S3Z/gnI80mvoactYESScOaJPmf +vrwZZwEGRhjIPv8AvoPnmMLdH8W5+ia9+TF7y0ufSvoYct4EeOGMH1BNe/g7gn/xzP1/fQfPA4cs +H6Cx5RXhw9QH0VV9DXcsYAv6WGMn4/fSasqZdJk4Wz8VffQfPgWPPQfSsNlvOk719Al5Qy2RvhLP +xV99aLydlk8cIY+KvvoOChZ4fKtSbgbbcONYLO030h7yMV3j+BuWNR/ce34+f316jKGWwSBhLIB2 +O6vvoOE7e0swiXRcSBtoAiYP7Yq69FHRxdZ9xt2zs1qtLW3QHLi5WmQkEwEj+keXoTXXyMpZcSNK +cJYA4wCfvp5Z4DhNqoqtrQNEiDoWoftoGeRMrYdlDBk4PhfaezoMhThBUSSSSSAPGsqfSAlISOAr +KD//2Q== + + + +--Multipart_Sun_Oct_17_10:37:40_2010-1 +Content-Type: image/jpeg +Content-Disposition: inline; filename="custer.jpg" +Content-Transfer-Encoding: base64 + +/9j/4AAQSkZJRgABAQAAAQABAAD/4Q1kRXhpZgAASUkqAAgAAAAIABIBCQABAAAAAQAAABoBCQAB +AAAAyAAAABsBBQABAAAAbgAAACgBAwABAAAAAgAAADEBAgAOAAAAdgAAADIBAgAUAAAAhAAAABMC +CQABAAAAAQAAAGmHBAABAAAAmAAAAOYAAADIAAAAAQAAAGd0aHVtYiAyLjExLjMAMjAwNTowMTox +MCAwMDo1NzowMwAGAACQBwAEAAAAMDIyMQGRBwAEAAAAAQIDAACgBwAEAAAAMDEwMAGgAwABAAAA +//8AAAKgCQABAAAAyAAAAAOgCQABAAAA9gAAAAAAAAAGAAMBAwABAAAABgAAABoBCQABAAAASAAA +ABsBCQABAAAASAAAACgBCQABAAAAAgAAAAECBAABAAAANAEAAAICBAABAAAAJwwAAAAAAAD/2P/g +ABBKRklGAAEBAAABAAEAAP/bAEMACAYGBwYFCAcHBwkJCAoMFA0MCwsMGRITDxQdGh8eHRocHCAk +LicgIiwjHBwoNyksMDE0NDQfJzk9ODI8LjM0Mv/bAEMBCQkJDAsMGA0NGDIhHCEyMjIyMjIyMjIy +MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMv/AABEIAIAAaAMBIgACEQED +EQH/xAAfAAABBQEBAQEBAQAAAAAAAAAAAQIDBAUGBwgJCgv/xAC1EAACAQMDAgQDBQUEBAAAAX0B +AgMABBEFEiExQQYTUWEHInEUMoGRoQgjQrHBFVLR8CQzYnKCCQoWFxgZGiUmJygpKjQ1Njc4OTpD +REVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmq +srO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4eLj5OXm5+jp6vHy8/T19vf4+fr/xAAfAQADAQEB +AQEBAQEBAAAAAAAAAQIDBAUGBwgJCgv/xAC1EQACAQIEBAMEBwUEBAABAncAAQIDEQQFITEGEkFR +B2FxEyIygQgUQpGhscEJIzNS8BVictEKFiQ04SXxFxgZGiYnKCkqNTY3ODk6Q0RFRkdISUpTVFVW +V1hZWmNkZWZnaGlqc3R1dnd4eXqCg4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrC +w8TFxsfIycrS09TV1tfY2dri4+Tl5ufo6ery8/T19vf4+fr/2gAMAwEAAhEDEQA/ANNRzUiIfSlR +DmrUUfrXOajEiz2qZYPap0jqlrOr2+i2RkkZDMR+7jLYJPqfanYCd4OOlQGe2ibbLcQo/ozgGvPL +vxRqE6kSXsj+yfu1/JeT+dYsl55m7JUZ6kD+vWiwrnsAlgfG2eI/RxUhjrxlLhgMLKcdua0YPE2q +2mDHfSMB2c7h+tFgueqeVimmPiuO034hKWEepW4APHmw9vqK7G3ube9tlntZUliboymiwXECCgpU +oSnGPNIZEqgUVKIz3opAVUXDVajFV4/vVT8R6yNC0SW6THnsfLhB/vHv+AyaaApeKPF0WjK1nZ7Z +L0jk9RF9ff2rzC71Ge7naaeZpZWOSzHNVZriSaRndyzsdzMTkk1GFLHABP0qhEhnc5xWjpmg32q5 +8oY9M96hs7DzHG4MBjJJGK9j8HW1ta6ckqqHVunHQ0BY8wl8H6hDGWOcjttNYtxbzWxIcHivoXWN +X0nS7YzXh5I+WMDJNeY+KTb3umHUotOlhgkfYjkfqaLhY4ESZrb8Oa/caLqCMrFrZyBLH2I9fqKx +Y4mkcIilmJwABkk16R4V8BmDy9Q1dfnGGjtj292/woCx2qjIB9aftp+MUvFSMYFopxopDKCDDdK5 +L4i2V9dWdlJb28kkEJcylBnaTjBP68116ctWjanFUhHzpit7SLa0t1W7u5lCk4AxmvZbzwvoGoOZ +rnS7dpDyXUbCfrjFeZ3WkW0OuXEcMMbQK52RljtA9jTESWNvDqExntAJDu2YEf3R6/413Xhq7itb +IW88MgKMct2zmvN47h9C1WCS1uzGGbbIqnPynsexFelLMloUMjF4pVBEh75oAvX50q7+RIXknkwG +Pt6c9q0bu00e/wBEk0m/kht4pE2KrOAwPYjPcVmWlza2TSXTbWwMgep9K5zWdavNSmISzS4bOMjA +x6AUhm3pHhfRNFVZLG3EkmOLiQ72P0PQfhWnITUVjJNLYQPOmyUoN6+hxUjZzQAwr7UgWpKCMUgG +7RRRzmikxlGNavwDiqiDB6VegGQOKpCJygkhaM9GBBrzHxFp50+5la8tnO4DbIg6LnqP0r0S/v5L +K1MtvaS3bZKbYcHDDqD7+3WvONT1u+1W/jbUSVtY3yYEGAB396YjCtLQXt1mOBhAjZDPyT7V3GkX +bJbtp92pe1I+V8ZMf19qzta8T6dounx6dplhbySkbi/U4PIJP9K4y88Q6reYMk2yPOQiKFU/h3oA +73VLC6trFrmCVJ7dTn5X6fhWLo3ih31i2N3Bi1iYqWjHTPc+uKg0zxCb+2NrNaHj75hOFI9wKs6j +FF9hf7CotmjAZWTj659aAPUQVdA6EMpGQR0IpjCuJ8GeKVNtJYXshd4xuiKLyw7jH9K7SK4guGdI +nBePG9Dwy5GRkHkUDFA5HWnMBikIwaKkBo64FFAPNFIZVQVi+IvEk+mSJa2OVnxueQKDt9AM1uov +NedeIJkufENyySEAEIMHjgY/mKpCZaOri/hjg1G3jnhRtylB5LofVWXHP1zWpqF1pt1pg8+4jmlS +PbA6o32lj/02ydpGOMjr+lcsXliHOJF7jHNQSyAYmiPyA5I9DTEQnTVimMwywHOD6VoafJHp13Hc +pZW92gJIhnGUORiljcSJnqDUMTBQ0fdf5UAaFhANDj1STUbP7PJN+9EQHHlnJUKPTJNSW4G4jqCB +1qKKzu0EeoXV0k0N0CqiSXc4C5HI9OaSNsXCIvCg4oAzNbj1O01iDVXhS3km+dPKUKpHTOB61b0j +xNLYayl3JGBE6COWNOAQOhFMv7QGCa7N5E489kFuZCXTHOcdhWMGBJJ6CgD3BJEliSVTlXAZT7Gn +AA1xvgnXJrxX0+5fd5SAxEjnb0wf0rshjFSMTABopwxRSGVgDn615TNCTNMrctHIwP516xzuGK8z +1FBa+I7yJgNrynH48/1qkJleOZo0GMstMlDy5MKxsx6g8GpGUxSnb9084p5RZUyuFb06UxGfazPE +WicFWB4B9Kk3fv8Ad68VDOri4BaNt443D+tEchKsfegDQsVQy3DjJcR/d7dR+Va9lDGLR/NkLOzE +gAenTn6iudsJtmoMeoaMg+3IrbtpJVjXa+O/NAGK4Q3M5bG7ewz+NZqAZcdlY5q3bxXOrasba0QN +LNIxUFgo7nqa1PCPhttZ1a6S7Vha2rnz9p5Yj+EGgDLsZ7i0nS6t5WikU5BHp6V61o2pxatp8c6M +vmYxIgP3W9KI/Bukyu0stmgZkwkSsVWMepx1NcOss/h3Wp47K4WRUfa+4cSD6UmM9IGM0VQ0zVrf +VLbzImxIB88Z6r/9aipAn3ZNcB41tGj1gzKMGVFcH3HH9K7pDzXJeNCrTRuzHem1AueMNuOfzX9K +pAznLa7juUCSEJMvr3qZwydRx6isi4ty3zJw1TWksjrs3kOOoJpiL0twwXlCwHoKxri6WFj8pG/k +A1fe6vIODtKmu1+Gc9vcazNHeRxyZh+RXjVuc8nJ5oA4zwxp91rmpSW1nGGnZMZbgAZ5JP5V6Cvw +213Jb7TYgbcKPMbr/wB816pGsESARpGi+iqAKT7VCPlMi5+tAHisPwc8SLJk39hH/tpI+R/47XWe +FvBeqeGtHnhlENxO8hlYRSH5vQDIHP1xXoSzow4ZT9DSvIO1AHiepeMNUWaWCBPsgDFWDrl8jsc9 +PpXMajqdxe3BuLt90zAAsABkDp0rq/ixpTWeuwanbDC3kZDr/trwT+IIrziS6uUcJcqGWgDpPDOo +pb+ILctJxITGQffp+uKKyNFtjc67YxqdyNMrfgDk/wAqKljR64vB5rK8Q6DaapCtxI0qXC7I0Kth +eXAGR3+8fzrTDYNQavKRol465DpEZFI6gryP1FJMbPMEcqxRhytNkwrLInDDriqST/ay3z4lB3A+ +tSrIWQq3DjqDVEl9h9oiDIC2e1V4H1KxvEuIGe2KHPmZxtqkbuazYtC5FVJ767vG/eyEr6dKAPQr +P4k6tZRHcy3KKeGlXJI98VuQ+OdM1EB3nubN3IASWM4Y+gIzXksN4YcB49yjsank1L7Q4ZvlVfuj +3oC57To2vQX8jJBdJujO0hnCnP0ODXWxC6dBuLL6ZFeP2/izwp4hsIodejmtNREflvdRICjnszY5 +OepGOtcvc6vqeh3Ij0jXbnyifl8mZth+maLDPU/i9Cf+EOt53YB4rtcHvypBx+leILdgDbICwrqL +7TvHPiVo7fUTdTxp8y+dIAg9+uM/rXS6D8PbDTQs+pst5cjkJ/yzU/T+L8fyp3AyfAmiXHn/ANrz +I0duqkQhurk8Z+mM0V6BI+FCqAFHAA6CiobA/9kA/+EMRWh0dHA6Ly9ucy5hZG9iZS5jb20veGFw +LzEuMC8APD94cGFja2V0IGJlZ2luPSLvu78iIGlkPSJXNU0wTXBDZWhpSHpyZVN6TlRjemtjOWQi +Pz4KPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUg +NC40LjAtRXhpdjIiPgogPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5 +LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgog +ICAgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIgogICAgeG1sbnM6dGlm +Zj0iaHR0cDovL25zLmFkb2JlLmNvbS90aWZmLzEuMC8iCiAgICB4bWxuczpleGlmPSJodHRwOi8v +bnMuYWRvYmUuY29tL2V4aWYvMS4wLyIKICAgIHhtbG5zOnhtcE1NPSJodHRwOi8vbnMuYWRvYmUu +Y29tL3hhcC8xLjAvbW0vIgogICAgeG1sbnM6ZGM9Imh0dHA6Ly9wdXJsLm9yZy9kYy9lbGVtZW50 +cy8xLjEvIgogICB4bXA6Q3JlYXRlRGF0ZT0iMjAwNS0wMS0xMFQwMDowNzoyNyswMTowMCIKICAg +eG1wOk1vZGlmeURhdGU9IjIwMDUtMDEtMTBUMDA6NTc6MDMrMDE6MDAiCiAgIHhtcDpNZXRhZGF0 +YURhdGU9IjIwMDUtMDEtMTBUMDA6NTc6MDMrMDE6MDAiCiAgIHhtcDpDcmVhdG9yVG9vbD0iQWRv +YmUgUGhvdG9zaG9wIENTIFdpbmRvd3MiCiAgIHRpZmY6T3JpZW50YXRpb249IjEiCiAgIHRpZmY6 +WFJlc29sdXRpb249IjIwMC8xIgogICB0aWZmOllSZXNvbHV0aW9uPSIyMDAvMSIKICAgdGlmZjpS +ZXNvbHV0aW9uVW5pdD0iMiIKICAgZXhpZjpDb2xvclNwYWNlPSI0Mjk0OTY3Mjk1IgogICBleGlm +OlBpeGVsWERpbWVuc2lvbj0iNzU1IgogICBleGlmOlBpeGVsWURpbWVuc2lvbj0iOTMwIgogICB4 +bXBNTTpEb2N1bWVudElEPSJhZG9iZTpkb2NpZDpwaG90b3Nob3A6Zjg2ZTcwZTQtNjI5OC0xMWQ5 +LTllM2YtZDQyZjM0NjM5ZGJiIgogICB4bXBNTTpJbnN0YW5jZUlEPSJ1dWlkOmY4NmU3MGU1LTYy +OTgtMTFkOS05ZTNmLWQ0MmYzNDYzOWRiYiIKICAgZGM6Zm9ybWF0PSJpbWFnZS9qcGVnIi8+CiA8 +L3JkZjpSREY+CjwveDp4bXBtZXRhPgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +IAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAog +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAg +ICAgICAgICAgICAgICAgICAgCjw/eHBhY2tldCBlbmQ9InciPz7/2wBDAAUDBAQEAwUEBAQFBQUG +BwwIBwcHBw8LCwkMEQ8SEhEPERETFhwXExQaFRERGCEYGh0dHx8fExciJCIeJBweHx7/2wBDAQUF +BQcGBw4ICA4eFBEUHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4e +Hh4eHh7/wAARCAD2AMgDASIAAhEBAxEB/8QAHQAAAAcBAQEAAAAAAAAAAAAAAAIDBAUGBwEICf/E +AEIQAAIBAwIEBAQDBgUBCAMBAAECAwAEEQUhBhIxQQcTUWEicYGRFDKhI0JSscHRCBVicoLwFiQl +M5KissJDU9Lh/8QAGQEBAQEBAQEAAAAAAAAAAAAAAAECAwQF/8QAGhEBAQEBAQEBAAAAAAAAAAAA +AAERAjEhEv/aAAwDAQACEQMRAD8Am8Hl60dEGN+tAbnalEBIr5+PUIoGcmllXPSgkW9LxxntVBET +2pVY/Y0tFEe+KcpFRDYRDHSjrD/pp4sdHWIUwMHhwOlJcpXr0qVeEEd6aXCxxRtLK6xou7MxwB8z +TDSSpjelAtQFxxvwjaSmGXW7dnH/AOtWkH3UEUeLjjhGXATV0z7wyD/61cNTTDbG1AKcdKjBxXww +RznWbUA9Mkj+lSem3+n6lE0mnXtvdopwzQyBgp9DjpTDXGQ0mY6fFD6Vzy8DpTDTMR7UOT3p0U9q +KF3oaasm1F5PSnTr3pMAb1FJrGwGRRuX1o64xvRtqYG5X2oyp7UpgV1agCKAQSKFKZHLQozUEoNL +Rj3oBd+lLxJsKrTsaEmnccW+d65Ehz0pzGp9KuI5HHv0pdU7Yrsa+1LqmSBitSAsabdKrXiLxfb8 +IabG6xR3V/O4EVsXweXuxx2/maX4w410PhmJknlWe7A2gRhkf7j2/nWCaxxK9/qs+ohea6mbJuJd +2A7BR0UDoKuJaud54gcYX0Jk/wC46Lbno5T4sf8ALJ/Sqlq+sx3j/wDiOoahq7A5HmSFIwfYf2qA +uLxpn55ZGkf1Y5ptLPynAxVxEsdQXcwWNrCvuvM33phM8jS+Z5rZBz1pn+JxuWxRGvxzbDI9aYbE +rHfXgGFuX/T+1OrPXdZs3LWt/NCx6lTjP2xUGl3EO9HN3Gds/ah8XnTvEjiiywZLsXKj92UBs/ff +9at+heL9jNiPV7FoG7vCdvsf71izzAocGkWkyD1phuPVuh63o+uRc+mX8NwQMlAcOPmp3p8UA7V5 +Gs725sp0mtLiSGRDlWRiCD7EVq3Afi5KrR2HFA8yPoLxB8S/7x+8Pcb/ADqYNfkA3pLlpWKWC6t4 +7m1lSaGReZJEbKsD3BoKtYWEwvSjhQe1GA9aOoFA2K4JoAb9Kdcq5zQEagnYVKpALkHrQpyAB0Az +6UKioTlIpxCMikyMiloU2qwLRfMU6iGe4puin0p3Ah6kYHf2rUQsgVFLuyqoGSScACsj8TfFURPJ +pXDEo2+GW8Hf2T29/t61D+MHiM+pTy6BoMpFhGeWedT/AOeR2B/h/n8qyiSQjuc961ImndzdzTyN +NPK0sjHJLHNNZLk5601klY96SZqrOnRuipJApKWdn70iuTTqztWnkUAEg+gqp9JRrJIQEVmPsM0s +LG9dsC2lyenwmtO4D4PZ3jmKeYp64GQPnWoDw8WeBZIrZUbqMj+dTVx5l/yvUACfw0gx1yKQaOaM +kNGy/MV6UuuALhIyGRDj2qj8W8E3do3mCDIJ3wOgppjICzD2owmPepnWdKaAtlSCO2KgGBBwe1VM +OBJnYUA2BTdTR1zQaD4UceXHDd+thfyNJpE74dTv5JP76+3qO/zr0PHiQB42DKwyrA5BHrXjtc16 +l8Lo75OBNIF+GE34cYDdeTJ5P/bisdRqLGq980cLjpXQpB60dQc1lonymuBd8UsFocveoogXahSm +2+1CsqglI6E05twu1NU69KdW/XFaiHcYXOKznx44xOi6QvD2ny8t7fITOyneOE7Y+bbj5A+orRlK +Rq0kjBUQFmJ6ADqa8l8ba5LxDxTf6vITi4lJjH8KDZR9ABW+WUY03KMCkC5cnJorHJoA4rQ6elBI +2kOFGTjNGUcx32qZsdAubiESoC+eyjcUQwgspVTmZod+ilxk1a+EtC8/UIeZ+WOVSSmNw3/WKjG0 +60tpbeCZJTLIQVKHIO/Tb5Vb7Vbhbxr+1jES82Vjkckuo/d6bHHr7VBtXhrp8ttaxhreGKFfzcw/ +X1rTbdomQBQCpGxA61ROFW/EaBDcsQrcgyo3GcdKnLNTHmWV0VTjdTjb5f8A+UaieFjHK/mHBQZJ +J6Yqr8XXPD8UDie9tkC5BzKPsKrfGnFGu6jM2k8OlxynlaY/EgPYbd/sKyjTOFeK+JeJmtdUSRgk +gDM/wDrjI9fuaiadcY22k3EL3MbRgcpCuB136VjOoIou2ZNlJyK9NeKfhUR4fwS6aZDLYEtcIDu4 +I3b6YrzJdczTNzdtvtViEQoNKwxs7qiKWZjgADJJqS4b4e1fiC/Wy0ewmu5m68i/Co9WPQD3NejP +C7wtseEuTU9UaK+1gj4SBmO3/wBuerf6vtVXFO8K/CR08nXOKoimCHgsGG59DJ//AD9/Sti5eu1O +5zzHem/b3rPqyCgHNGUUXau5qK6a4a4DXcZqApOxwDQoxWhWK0gsdDinEGAelJMgxS0CgVqMmHHM +7wcDa3LHs4sJsEdvgIryU29ez0tobmNre4iSWGVSkiMMhlIwQazXjHwGtLsyXfCt8LWQ7i0uCSny +V+o+ufnXSMvPGN66BVl4m4H4p4clZdW0W7hQHaUJzxn5OuR+tV4Ieb8pG/pVE5w/odxcSiaWNhGm +CB6g9/lWn6XprWmnk5jifPQ9MHPX5AVQtK1trTTrlDkyOnlg+m2/9BTF77W7iN3Ms5UAFiW6elRF +6gsNQZ31aOS3gnjCrIzDnJU9cdh9KXKZto52hYWkcgAxj2OGx03qF4D1qOyv4oNSvVCznq+8cfpz +fP8AStj/AOxtla6VJDaXLXMDp580gwU5zvzAg7/agsXBN7pqWqWYYK4QOQTsexq4wjTrpArMvKw+ +1ZXw1oyxW8byOfPyc4bOKmJWvLRSPMbH5gcYxmoq5axacP2ulSAzx28e55om5Wyfl3qJ4Jt9Gtbt +ruCGeaRsBGlkJyM5O9VaOG8v7oJzFhnY9Rg1oGh6BIlp5WQVC9em+KKstmq3isvLEI5MgqNwfXtW +O6j4EcH2/Fd7qF811PBNKZorQNyRIDuVyNzv7jtU5xBxbxhoDfh7Hh6wEEPWV7gF3/2ov9a5oPE2 +rcUomoXUFvBaRgoAHLSM/cEY+ED077VUSem2NhpNmLLS7K3srdOkcKBR+nU+9CQk96OxBpN8YqVT +dzSecjpSjgE7UFUBTtvQJMoAz61xRntSpAoBRisqIF36Cjcu3SukUBt1oOcu9CjKAaFYq6r2AcZI +pxEvTem4SnMKYrUZPrUbipa2bG9RNqN6koelbglElBXBGR6Go/VLezNlPKmnW0kgQkc0S7n7UvET +iofjLVrjS9PDWyKZHzueigdT+o+9VHmziHSiuu3d6UjJmdZ1B/KBJvnHrnI9iDUqAlxZLplxp6lF +AdnXClhk7j164ovF/Pa6v5v4ZppJCWl5JCNic8uOg3Pak5+JntoRaS2DRyQrkCQbgEk9vcn70ZV/ +XtFktLCS4ihcRscDKbjpt/Or34IW8eraeLS4vrkJbyeYsAlbkLDcZXoaoGtahf3/ADT80nk8u4J+ +EbdKmPCrU7jh/VI71gTaSEeZj933+VFbglytnfGKdVUMxOMGpaSS3uYEBQZGQDmm2o2FvxBpy3tn +MAWXmSRNwfqKqY1HUdKumt7wFcbFlG30qYrQOHohBdBlwA24wOm+f+vrSfGfF8nOdN0+8MKQjMrq +uSx9B/aqmeJkS2llRsyKhKgnHNjNQcU97qzNGzRKZy3MScg/P5bUDbiHjp7GZFt9OlupWBBeQkkD +tnHqMnFO/Crij8fxBLbRwGOKaIyEZ3LZzzH7kU2nsI9Jljt5ktnuZRhZMZ39fUbZH2rQ+H+GtN0x +zeW9nDFdSKFkdR1wP0+lBNk5FEkyRSpGBik2HzqBHFGA964QAcZoZHTNWtDADHWuMpxkGuLgnFK4 +GOlZoR5cbmisTncUs4xtSRHM1AF+VCjqu2TQrNVAxg9KdQj1pJBvTmMHpWoyc2y5bYU+jUimtuu1 +PIx9a0HMewFR/E1g2oaXJChxJjCt6Zp+uwBpxFvjeqjzxfF9O1r8XeIwYebHKh/ccMcbf7Sp981A +Xer8P3Mt3eXrPJMIisUUSgZboC3etw8W4OFE0sXWs31tYXiDML8vNJJ/p5Ruw/lXn6/01dXuGvtO +tMO8uJQgPIpYZH3wftREUbt7pvwFkk0dtLgSK+Pi9/b9auOkaekdqEIxt27Va+A/CfWbuwTU4tPE +kL5BmdwoGOu27Y+QNPf8hcSyxxhAiNy87HkUn5tj+9Ax4M1274duxDcRmfT2OSq9Y/cf2q7cQaVb +a1apcWEsTLIAUYbA59ao2qXXDehoX1PU43cDaKE4P1JGfsPrTXhjiXUtUMknDn+W21ssn/kTO3O5 +9cbkfeinXEXDuraYhafSJJ0XOJIGz+lUK/4ku9PZ1tw6GPJIdcEe1bXJxWLfTHg1WExl15SVBIBP +p9axbibSdT4g1aX/ACyyldJGCmRl5VC+pJoENB41upOIrK/1W1W8tYHBeHJGR6/Mdd69P6Ve2Wp6 +bBfafKJLaZOZGH9fevMs3DtnoFsJNTW6ZCcGeJQVQ+46/fFX3wr4ls9Im/Cxail1pc7ZYdHgb+Ir +6euD70I2Vl+HNJkUvsyBlIKkZBBpJhRSLYwaKuD1FKkY61zAA6Vm0EQKW2pZeXuKSU4pQnbaoorh +TSZ5RR96IwqDqnOcChRosAdaFZqocYB6UrG2/Sk1BzS8Y3rcQ7tj7U9i3ppboaF1r3D+l+bFf6xY +296q/s4ZSTgnozKu+PatodalfWmm2Zu76dIYV/ebv7AdSfYVmPFnirO4ktOHoDD+7+JlALf8V7fX +7VfOEtW1K0sNSng4j0Xi7VrlwbeGS9NmkSY/IkfKwXt8+5rJ9S401nT9TOleLPAyzo7HlvIIBDOo +/iR1+GQD2OKuMqNftPqd1JPqM81xNJ+aSVizH61evCS+s40l0q5jhWYDy/2oGCMgxsc9gwAPtUzx +F4ZTDQouJeGmm1TRp4xMvNHyzRoRnJHRh7j7Vmmr+ZYTW+q2p/aQNhwD+de6mojWPETxMvdE0KG1 +1LmFxFzJDaKRGWYE/E6rsFHQD296wHiDjbiTXJna71KZUb/8UR5Fx6bdfrTXiC9u9b1iW/uA3NId +lySFHTApyugO9mLiHJI3xVVXzzN8TMSfc0406/u9OnE9pO8TjuO9WCLR47q3DACGdR3GVb2P96tX +iN4K8Q8G8LR8QXV5YzxYX8RBEx5oS3pnZgDtQJaD4j3z2fk6laC5jGzSR/mX3q26JxClzah47kTI +35SVwe+29UfwY0NNQur2/uA4ihVY0IPVzv8AI7Dv61aNStX03U5YT5fKeZ4+VcdOQ7j1xn7VApqL +/wCZWNzbSr+zmjZcHsazC202WwnVortlukJ/J0BHbPetK06QvEzEHIkI+hFI3fh1+O4e1Pia31uO +G5t5ARZsm7DAyQc5/Sg0Dwd4ztdU4Y/C6jMyX1rJycnLn4NsfYnGPf2q6aVq2m6tC02n3ccyI5jf +BwUYdQwO4Psa8kcLa3LpGssWeTyXfEgU4Jwa2vWLxdQ4e1oWGnabc6Zcwfio721YJcQuF5v2i7MQ +G2BGwzvRZWsOuKTIIqgeC/FjaxpB0q+maS+tF+FnOWkj9fmOn2rQGO9ZsUl0augkVwjfOKAB9qiu +52pNmzttRiDRGU5xQGQ9xQrqKcbihWaIxF70snWklB74paJcmtQRXHHEq8NaA90ih7uU8lup9cbs +fYf2rDLvWJ9QvGur6WWWd2y7Mc5q9eOV60Op6XbqhYLA8jHtu2P6VQYruAkeZFsepxW2KfQTI35G +Gau/DvG99aWo0zW401vR2xz211livujHdSO3p7VSLdrGX9/lPTI9KXaCXk/ZOjemaDW9M02NpbTi +LT/FjVrPhaxbzf8AL7mTnmgYDaH4iQVxtuDt69aoviLrXD+u67LcaLpE1pBJnzWcgLMf4gmAVPr2 +/nVMnlBcRXCBJ13QkbA06065FxCWcYdG5WT0IoGI0eISMgUe2RR9OJt5Ws3xjtt1qUB518zG6HtU +frScskVygIwd8UHZ7copUCjaxe8bcR6Yuj3V7NeW5OUiCc0jY3+ZG2aXibmUSb4YdRU3wbxqvA2q +zak1i9z5kBjDkcxjOf5f2FTQXwqnWz4eOmvGY5orhxICuG5tuvv2ol9rcOs69d+UcpZOsRYfvFld +f54qNtOJ5+Jtf1XVpMwySzI/L3/Ly/8A1FHg0aLS9aklslKw3flM8Z3AbzR09tzVEjYcoeULsGAc +CmvEOnarqt1aWelreSySK3PBbgnnUYO4HYUdJkSWAKd+XlOPepJ+J9S4VhttZ01Od1k5JEJ/OpBy +PuB9hQZLxHpk+n6sySQvE2fiR1wVYdQQad2F5eWkb/h5XiJjaNgDsVbZhj3qa4o1W64nu5tVvIDF +M7c2CcmohlBJ2xlaB9wzrl1w/r8Go2ZUyxDHK35WB6g16R4T1634i0SHUYF8st8MkZOeRh1FeW/J +WRw2enU1qvgnxTp+nR3Gj6lMtuZ5Q8EjbKSRjlJ7dBUpK2QkD0rg3zRGIJ2O1GSstlFwAc0VsA7V +35UXBzUo6p2oV0KaFZVGKm9LxKM9M0T0NdklENvLLjPIhYD5CukRjfi1qCXHF88eQY7ZFgH2yf1J ++1VyGOORMYGKbNK97JPLcMWkkcyOT3J60haieGQqmeXO2a0wkJ7FGHwkKexBpBLqeycJOTy9m7U/ +tJo5sebGMinM8VpLGedMqN996COvF/Hw+ZFyyOo6A9fl71H2NwEvg2cGQcrjpuOmf+u1PxZRK7SW +NyFx+5UfqyK3/eEBW4jPM4/jH96Im7d/jOGG/auXo57RlO/ptTHS7xJ4lIOT0p60mQ4GxopppdwG +QwOfy7UtIshlFuInmD4AULnOegqOGI7wMo2JwfSn8mXQgNysR8LZ70Fp4h8ML/hWK01C51Czb8eq +q0ETEtGx3HsR7im96wjWJFlVmjKq3fmxlz/8ahtK1TXdc4jjt9cvGljit2EQXbmIAAz9KXnYWVq8 +apjCSSDfJGwTr/yNAiMlFy/7oORvjepDihUk0CGFZFjDToDI/wCUZOMn7060vT7OXQVvpLadCwZY +wzBEkPL8JJPQBs5OcHp61E8Xs3+TwW867mQcy47gGgl+P+CtL4d0qyv9L1w6glwgLq6gEZHUY/lv +Wb3MmG5c4PKR9akYbaRinPczyRpuiO2QufTNRuqxNHdKvYsD/wBfagPFtEAegFFkbzcKq7A9aDDK +igx8qHb8zHCiiPQfhJrX+bcJQiWQyT2rGFyeu3Q/aropX0rzp4e8YS8JXqxPCJ7O5I89R+ZcfvCv +QOnXttqFlFeWkgkglUMjDuKzY1DvPegGoLRWqNFFbI7UK4g2O1CshgAcelGRFkUxt0YEH610jAo0 +LKD71qDzaI1g1Ke1JB5XZM/I4o1xGY2Db4FLcWRfgOML9OgS8kH0LH+4p75AmgDcwwa2wYYLplNi +BtiixzzRH4gWA7EVyM/h5miz8NPIRHMOUAADvnFENpYrO7X87wSnoU/tTC80bUoR5kEgvUG+xww+ +h/pUxJawr8cec02FxJE27HlHrQQOkTG2lliZCuGyA2xHtUxFKzfER9qYaxBLPci9tD5g5cSRj8w9 +x6ijW8+YsZHTegPLvuMDBp2JAIlbbcY6U2tpPMV8/LOKAceXjB2oJ3hCeN9Z3UMywvjbp0rt9bmS +OeeQZeQQrGN8AkFjt26ioLhed4uKIU5iOdXBH/GrLqDfEkaDc3BJ/wCKqv8ASirgbmxtLENZ6W7x +ae37cMQFYc+Qq8wOSAASfXvvVP8AEdndhLJzeZ+LfmBbJyc7VJPdW0ziW6hilkAzzFFBHyqJ8RLt +brTLec8gkacFiFwT8JoIK2lXlBYbVE8RMFaKT19/ejG5wAq0x1uVnhiB3IbagdwqHVc9MUSP9vdF +v3E+FaPptvdX7Q2NhDJPdzEJHGgySTWq2/hOukcMvd6teSC/EZZUhwY1OM4JxknttjrQZqYlDGRx +sOgNaX4M8Sw2Usmi6hOscUp5rcucAN3X60Xh3wf4k1e1ju7ua105GHMsUuWk/wCQH5flnNUnUrFo +NVms5MB7VzGeX+JTg4+1QemVwRkYxRGJzWd+H3HtrJDBo+sytDdKOSOdz8MnoCexrRRg4IIIPTFZ +aHRtutCiEkChWappJkLjJrkaEnJo8rCgjYxvW4MN8XrNYeNr3GR5ipJ09VFRWg3XnQlMnmTYg1dP +G+z/APGbO8HSa25D81J/oRWXQztYXvm78j7PVYqw6hFGSJOXp1OKbpzKOZExipGIxXduGT4sjam3 +lmJzG+du4NULW84YBZCPfaiXtosykxt13xXFCEEDOR39aNDKUk/KceuelEVq/gu4JcRrICD13okr +XBhE8x+IHDEdSOxNW55oHT4lXPqKjr+2je0kjHLlxjtRURZNiPYjfvXWbAIYCkbZuVeQnp1HvSU8 +yDIBG+2KIW055DxDp7W7AP5uCcA/Dj4v0zVnhdpZIWOxMZkP/Ni230IqrcMRo2tyTqCTDbyMBnbJ +HKP/AJVaLGRBcXDkhkiYRqADsFGB/KosSyxQOg50JPuKqvH7CJrG2RsjDyN+gFS9veTT3W4+HPrV +X48uObWljBwI4gMDsTRTbhvTbjXtfstHtZooprqURh5Gwq+5qa8ZeCLjgi6s7d9VttRSUkiSIcpV +h1BXP61VrW4ltikttIYZo2DI69Qw70e9utS1/VrSPULlp5ZZVXJwOpAJrSN7/wAO3B6Wul/9qL5R ++JuU5bcMu6Reo92/lWo2GmS6hqBv78AJGSLe3JyFH8Te/f2+dNOHEkt+FtOtRcRmNY1AePBGw2pT +U9RFoHSW9htI+txcu3KIl74z1JOwHr8qjUTkY/Fo0Vs7RW6bM46k98e9Y54+No6alYWtjbRw3MMZ +82RVwFQ9Ax9Sd9/ep3WvFzQ7G3FhollcXap8IkJ8tSPXfcn3xVP408QdP4k4Yk0M6GLbmkEvmGfn +JcHOTsPehqhpb2kjiTl53H72dqv/AADxs9hyaZquTaDaObmyY/Y+1Z000iAsRGF6DkJOPnSE1xME +ODzKR2rNiPTqyxTQrLDIskbDKspyCKFZ14IX7XGgXFm0vP5MpIBOcA9PpQrNaXlmwBtXRkkGkQwz +SqZyKSireMVh+I4TivAPitJ1JP8ApbY/ry1h+pW3OTgnHbevQPHsN3qOjwaDYcguNTnEILdAoBcn +/wBtYMRzKFIww2atxmmGl6nPps3lOeaPO4qz2t5aXyghwD71WLy18zJC71FwzSW8uCSKrK+XFpyn +nifJPXemp8xdgjZ+RqOtLucqG5+YEbUs2pyoCWJoHEqyn8sbkkdlNRV4uoKwPkuvtykVIwcQRK2G +yKPNrEEzpySFGXpk/Cw9DRVT1qeW1lWQoyCUZx03HWo1LtpG3JrdvBySz1fxAtbDUeG7PUomVjI8 +8SukAxs45ts5GPXrXp+z4b4chAaHQ9MjI6FbVB/SkMeG/CzTdS1jiRNOtdPu5TduiGZISyRKrcxZ +j6bV6l4a8FuDLSxWO7hvb+U7vJNM0ZLHr8KYx+taqkcUIVI41VewUYArsigSZGKuKoKeEHAMJDpo +rhvX8VKf5tTW78DfDW+na5udDleV8Zb8ZMOnyatLyCu/pQQgDqKDNbPwJ8M7W6juU4e8xozkLLcy +un1Utg/WlX8EfDsasurW+hm3u0bmUxXEiqp9lzj9K0gkAda7nK7VRVZeDdIeJwqzwzH8s6vzPGfU +Bsr+lecPGPh/ifh/iBbLW9Sl1GxkzJY3LfCGGdwR0DDv8/evWb7HOdqpvjLwsvGHh9qGnxIDf26G +4sX7iVRnl/5br9amFeQiOQ4yCPbrTeSLLl4zv1371DQX8yA+YCGHX2NBuJCshR0UgbdKiHdxd8jl +dwfSm348xnm5Oai/ibe/bflRj0P96ZXHNBLyMpA96GtG8G9cS34rWDIWO9Qxuv8AqG6n+n1oVV/D +S3a7490qNCwVZfNbHooJ/pQrnZjUejBjPSl12IG1Nhkt0pxGD1NSVcQ3F17HpeqcOalNJyRRaj5b +tnAAdGWsIlYJql7CSMpcOvX0Y1vvG3DZ4q4Yn0qO4W3nLLJDKwyEZTnt7ZFYxxfwJq3DV9LNNfR6 +j+xWeeRFK8nM3IOvXfG9bjNRbJzHAxnGMUwurJJVbb608t5gFywJJ9KcZXBBGRVREabM1rP5E26n +cGp0Q288PMNjTC/tkki5kwCB2613SblsCF2APT50AvNMj5TIuB3z3qNSMBvi7Hap65TbP9dqirhV +QFsYOelNMW7wn1ltC4uspjPy28kgSTmOwB6H23r2Fpl+rwhieY8ucj+9fP1794mJBAxWhcEeJHF1 +vYCCDU3Fra/xgNk9lyRnHtRXtA3S8iuCppvd38ceDg7HtWD6B40W7Wwe9tQZlHxRBsBv9p7VcND4 +80ni+ynbTGkt7222ntJtnXPQjsw9xV1V8XX7U8o5XO5H8v70dNcsOflZ+VvQmszvG1Yc7RJzxk5y +N+tIQX8sbjmkhBO551II+9T9GNfTUbaTHJMjfI04S4UjGay+zvp5CpJBP+kbCp2y1GRByFmPfc9K +foxb2lDE7/ShDIomUZ2OxHzquG8nZQyinNpdyNgEYYGmmPH3jLoFpo3iVrulEGBBcmaEqNuST4wP +pzY+lUu60i2EBImVpPbvWwf4z7Q23iBpeoRLgXenAE+pR2/owrCo7iUOAT9KrIvJJbyfDkYNSkFx +Hd24jlxzr+U0QeVdDHMFIG5NMJk8p/gl+1X0aj4Eaf5vFNzeFdrW2IB92IA/QGhVn8BdNntOF59U +uYyn46UeVnqyLtn5Ek/ahXLq/Wo0LlAbY0dAc9TSJOD70ojjY1iNU7iyM4yKqfF8IvNa1KyfcT8N +3DKD3aN1dftirVDKp9Kz/wAbLq60pdO1ywmWN+SazfftIvp8s1uJWNluQq/NsRnrTtJkbBUjGKj7 +OSOeE2rN8QGVNNj51jNljzRnvW2U7JKrKQoyR1phM/lSK64z1NKQOsq8/MObrmkJjzty8h9M1BOW +E63UIQMOalZdBkul+J8DtvVTE01pLzwyMuPQU4bi3UUQJkEjbJFMNSl1wzbwjnurxQvcLTC61OGO +FdO08IkS+nf5moDUtbvr5irynHtSNoxiIbO9WRFnVWEBfm3A9abadJxE16l3pN7cWk8eyyxyFDj0 +yO3tSNpqSD4ZcYPWp231e0t7blgYA43FBYLbxG474Zt+eW/ttULYDrOhJP1BGftVguOK+MNV4cte +I7LheG4sp8q01rO+Y5FPxKy9j36dMVlVzfSX1zsMhdzWm/4fePLPhniSTh7XJUj0bWCFLufggnH5 +XPoD+Un5HtRZVq4J8Q+GDpbza/FrMFzAc3Kxwq/kL/ER1K57gbbVpPDHFPAuuMo0jjbT5XbcQzv5 +T/8Apf8AtTTirwj0niC9N7a6oulXirmKSABiT/qGd1x2715q8SOAZ9D4gm0y4jWw1INlMHFtdL2a +Mn8pP8JqZFe1raxkKjlaGZT3RhinSWLKdosZr532mu8UcPXLQ2uralp8sZwViuHT+Rqaj8XPEeKE +xJxfqnKRjeXJ+53rWJrYv8cd9YC94ask5Wv44pXch90jJUAEe5B+1eaDKSMgkGlNV1HUNVvZL3Ur +ye7uZDl5ZnLMfqaluDODeIeLLvydHsHkjU4kuH+GKP5sf5DJ9qvgghI4YkM2/vWreEXhhfcQSxaz +r8Ulto6kMkbfC917DuF9+/b1GkcB+EPDvDix3mphdX1Jfi5pF/Yxn/Snf5tn5Cr9PP22A6Vi9Lhp +NHFEkcEEaxRRqFRFGAoHQAUKTnkHPmhXOqTcnNBGNChWI0VjbvgVnX+IiMtwbZzKQDHeA/dWoUK6 +c+s1g0N2/mrMuzKas0YS/tQ7LguvN8qFCt1kwV2tJ/KU5XNO2cqSQBkj0oUKBheFmzvudyah51Oe +tChViVyOFQfenCxKRvQoVUJSRgHrSTMQ2ATQoVYsOIZ3gBAJpve3Ek5BJ70KFIJXh/i/ibQZEl0r +Wbu2KkEKJCV/9J2q73Pi3xPxRZ/5JqENhNPe8tsLqSLLICcbD60KFLCM+4gDw63cWUkjTpaSNApb +qQpIrWPD/wAMeG+IuDLW/vWvYbqUNl4ZRjrtsQaFCs9XIs9XHhvwa4M04CS9hudVl5sg3EnKg/4r +jP1zWhW8dvY2kdpZW8NtbxjCRRIFVR7AUKFc9tawSSViDTSRsnehQqKbyjLbUKFCs0j/2Q== + +Cheerio! + +--Multipart_Sun_Oct_17_10:37:40_2010-1-- diff --git a/testdata/testdir2/Foo/new/.noindex b/testdata/testdir2/Foo/new/.noindex new file mode 100644 index 0000000..e69de29 diff --git a/testdata/testdir2/Foo/tmp/.noindex b/testdata/testdir2/Foo/tmp/.noindex new file mode 100644 index 0000000..e69de29 diff --git a/testdata/testdir2/bar/.noupdate b/testdata/testdir2/bar/.noupdate new file mode 100644 index 0000000..e69de29 diff --git a/testdata/testdir2/bar/cur/181736.eml b/testdata/testdir2/bar/cur/181736.eml new file mode 100644 index 0000000..56255c4 --- /dev/null +++ b/testdata/testdir2/bar/cur/181736.eml @@ -0,0 +1,42 @@ +Path: uutiset.elisa.fi!feeder2.news.elisa.fi!feeder.erje.net!newsfeed.kamp.net!newsfeed0.kamp.net!nx02.iad01.newshosting.com!newshosting.com!post01.iad!not-for-mail +X-newsreader: xrn 9.03-beta-14-64bit +Sender: jimbo@lews (Jimbo Foobarcuux) +From: jimbo@slp53.sl.home (Jimbo Foobarcuux) +Reply-To: slp53@pacbell.net +Subject: Re: Are writes "atomic" to readers of the file? +Newsgroups: comp.unix.programmer +References: <87hbblwelr.fsf@sapphire.mobileactivedefense.com> <8762s0jreh.fsf@sapphire.mobileactivedefense.com> <87hbbjc5jt.fsf@sapphire.mobileactivedefense.com> <8ioh48-8mu.ln1@leafnode-msgid.gclare.org.uk> +Organization: UseNetServer - www.usenetserver.com +X-Complaints-To: abuse@usenetserver.com +Message-ID: +Date: 08 Mar 2011 17:04:20 GMT +Lines: 27 +Xref: uutiset.elisa.fi comp.unix.programmer:181736 + +John Denver writes: +>Eric the Red wrote: +> +>>> There _IS_ a requirement that all reads and writes to regular files +>>> be atomic. There is also an ordering guarantee. Any implementation +>>> that doesn't provide both atomicity and ordering guarantees is broken. +>> +>> But where is it specified? +> +>The place where it is stated most explicitly is in XSH7 2.9.7 +>Thread Interactions with Regular File Operations: +> +> All of the following functions shall be atomic with respect to each +> other in the effects specified in POSIX.1-2008 when they operate on +> regular files or symbolic links: +> +> [List of functions that includes read() and write()] +> +> If two threads each call one of these functions, each call shall +> either see all of the specified effects of the other call, or none +> of them. +> + +And, for the purposes of this paragraph, the two threads need not be +part of the same process. + +jimbo diff --git a/testdata/testdir2/bar/cur/mail1 b/testdata/testdir2/bar/cur/mail1 new file mode 100644 index 0000000..56808c6 --- /dev/null +++ b/testdata/testdir2/bar/cur/mail1 @@ -0,0 +1,38 @@ +Date: Thu, 31 Jul 2008 14:57:25 -0400 +From: "John Milton" +Subject: Fere libenter homines id quod volunt credunt +To: "Julius Caesar" +Message-id: <3BE9E6535E3029448670913581E7A1A20D852173@emss35m06.us.lmco.com> +MIME-version: 1.0 +x-label: Paradise losT +X-Keywords: milton,john +Content-type: text/plain; charset=us-ascii +Content-transfer-encoding: 7BIT +Precedence: high + +OF Mans First Disobedience, and the Fruit +Of that Forbidden Tree, whose mortal tast +Brought Death into the World, and all our woe, +With loss of Eden, till one greater Man +Restore us, and regain the blissful Seat, [ 5 ] +Sing Heav'nly Muse,that on the secret top +Of Oreb, or of Sinai, didst inspire +That Shepherd, who first taught the chosen Seed, +In the Beginning how the Heav'ns and Earth +Rose out of Chaos: Or if Sion Hill [ 10 ] +Delight thee more, and Siloa's Brook that flow'd +Fast by the Oracle of God; I thence +Invoke thy aid to my adventrous Song, +That with no middle flight intends to soar +Above th' Aonian Mount, while it pursues [ 15 ] +Things unattempted yet in Prose or Rhime. +And chiefly Thou O Spirit, that dost prefer +Before all Temples th' upright heart and pure, +Instruct me, for Thou know'st; Thou from the first +Wast present, and with mighty wings outspread [ 20 ] +Dove-like satst brooding on the vast Abyss +And mad'st it pregnant: What in me is dark +Illumin, what is low raise and support; +That to the highth of this great Argument +I may assert Eternal Providence, [ 25 ] +And justifie the wayes of God to men. diff --git a/testdata/testdir2/bar/cur/mail2 b/testdata/testdir2/bar/cur/mail2 new file mode 100644 index 0000000..3799f30 --- /dev/null +++ b/testdata/testdir2/bar/cur/mail2 @@ -0,0 +1,14 @@ +Date: Thu, 31 Jul 2008 14:57:25 -0400 +From: "Socrates" +Subject: cool stuff +To: "Alcibiades" +Message-id: <3BE9E6535E0D852173@emss35m06.us.lmco.com> +MIME-version: 1.0 +Content-type: text/plain; charset=us-ascii +Content-transfer-encoding: 7BIT +Precedence: high + +The hour of departure has arrived, and we go our ways—I to die, and you to +live. Which is better God only knows. + +http-emacs diff --git a/testdata/testdir2/bar/cur/mail3 b/testdata/testdir2/bar/cur/mail3 new file mode 100644 index 0000000..646365e --- /dev/null +++ b/testdata/testdir2/bar/cur/mail3 @@ -0,0 +1,34 @@ +From: Napoleon Bonaparte +To: Edmond =?UTF-8?B?RGFudMOocw==?= +Subject: rock on dude +User-Agent: Wanderlust/2.15.9 (Almost Unreal) Emacs/24.0 Mule/6.0 (HANACHIRUSATO) +Fcc: .sent +MIME-Version: 1.0 (generated by SEMI 1.14.6 - "Maruoka") +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Le 24 février 1815, la vigie de Notre-Dame de la Garde signala le trois-mâts +le Pharaon, venant de Smyrne, Trieste et Naples. + +Comme d'habitude, un pilote côtier partit aussitôt du port, rasa le château +d'If, et alla aborder le navire entre le cap de Morgion et l'île de Rion. + +Aussitôt, comme d'habitude encore, la plate-forme du fort Saint-Jean s'était +couverte de curieux; car c'est toujours une grande affaire à Marseille que +l'arrivée d'un bâtiment, surtout quand ce bâtiment, comme le Pharaon, a été +construit, gréé, arrimé sur les chantiers de la vieille Phocée, et appartient +à un armateur de la ville. + +Cependant ce bâtiment s'avançait; il avait heureusement franchi le détroit que +quelque secousse volcanique a creusé entre l'île de Calasareigne et l'île de +Jaros; il avait doublé Pomègue, et il s'avançait sous ses trois huniers, son +grand foc et sa brigantine, mais si lentement et d'une allure si triste, que +les curieux, avec cet instinct qui pressent un malheur, se demandaient quel +accident pouvait être arrivé à bord. Néanmoins les experts en navigation +reconnaissaient que si un accident était arrivé, ce ne pouvait être au +bâtiment lui-même; car il s'avançait dans toutes les conditions d'un navire +parfaitement gouverné: son ancre était en mouillage, ses haubans de beaupré +décrochés; et près du pilote, qui s'apprêtait à diriger le Pharaon par +l'étroite entrée du port de Marseille, était un jeune homme au geste rapide et +à l'œil actif, qui surveillait chaque mouvement du navire et répétait chaque +ordre du pilote. diff --git a/testdata/testdir2/bar/cur/mail4 b/testdata/testdir2/bar/cur/mail4 new file mode 100644 index 0000000..4d21a48 --- /dev/null +++ b/testdata/testdir2/bar/cur/mail4 @@ -0,0 +1,29 @@ +Return-Path: +Delivered-To: foo@example.com +Received: from [128.88.204.56] by freemailng0304.web.de with HTTP; + Mon, 07 May 2005 00:27:52 +0200 +Date: Mon, 07 May 2005 00:27:52 +0200 +Message-Id: <293847329847@web.de> +MIME-Version: 1.0 +From: =?iso-8859-1?Q? "=F6tzi" ?= +To: foo@example.com +Subject: =?iso-8859-1?Q?Re:=20der=20b=E4r=20und=20das=20m=E4dchen?= +Precedence: fm-user +Organization: http://freemail.web.de/ +Content-Type: text/plain; charset="iso-8859-1" +Content-Transfer-Encoding: 8bit +X-MIME-Autoconverted: from quoted-printable to 8bit by mailhost6.ladot.com id j48MScQ30791 +X-Label: \backslash +X-UIDL: 93h!!\i + +123 diff --git a/testdata/testdir2/bar/cur/mail6 b/testdata/testdir2/bar/cur/mail6 new file mode 100644 index 0000000..c9b799b --- /dev/null +++ b/testdata/testdir2/bar/cur/mail6 @@ -0,0 +1,18 @@ +Date: Thu, 31 Jul 2008 14:57:25 -0400 +From: "Geoff Tate" +Subject: eyes of a stranger +To: "Enrico Fermi" +Message-id: <3BE9E6535E302944823E7A1A20D852173@msg.id> +MIME-version: 1.0 +X-label: @NextActions operation:mindcrime Queensrÿche +Content-type: text/plain; charset=us-ascii +Content-transfer-encoding: 7BIT +Precedence: high + +And I raise my head and stare +Into the eyes of a stranger +I've always known that the mirror never lies +People always turn away +From the eyes of a stranger +Afraid to know what +Lies behind the stare diff --git a/testdata/testdir2/bar/cur/mail7 b/testdata/testdir2/bar/cur/mail7 new file mode 100644 index 0000000..48660f3 --- /dev/null +++ b/testdata/testdir2/bar/cur/mail7 @@ -0,0 +1,16 @@ +Date: Mon, 11 Sep 2023 19:57:25 -0400 +From: "Tommy" +Subject: Hide and seek +To: "Andreas" +Message-id: <3BE9E65sdfklsajdfl3E7A1A20D852173@msg.id> +MIME-version: 1.0 + +Behind the polished barrier +The pending storm draws near +Although it's an inferno +You can not step back for your fears +Hurry son I need to rest +finish the puzzle you do it best +I'll do what I can +But I am telling you +It can't be done without you! diff --git a/testdata/testdir2/bar/new/.noindex b/testdata/testdir2/bar/new/.noindex new file mode 100644 index 0000000..e69de29 diff --git a/testdata/testdir2/bar/tmp/.noindex b/testdata/testdir2/bar/tmp/.noindex new file mode 100644 index 0000000..e69de29 diff --git a/testdata/testdir2/wom_bat/cur/atomic b/testdata/testdir2/wom_bat/cur/atomic new file mode 100644 index 0000000..c3c6792 --- /dev/null +++ b/testdata/testdir2/wom_bat/cur/atomic @@ -0,0 +1,20 @@ +Date: Sat, 12 Nov 2011 12:06:23 -0400 +From: "Richard P. Feynman" +Subject: atoms +To: "Democritus" +Message-id: <3BE9E6535E302944823E7A1A20D852173@msg.id> +MIME-version: 1.0 +Content-type: text/plain; charset=us-ascii +Content-transfer-encoding: 7BIT +Precedence: high + +If, in some cataclysm, all scientific knowledge were to be destroyed, +and only one sentence passed on to the next generation of creatures, +what statement would contain the most information in the fewest words? +I believe it is the atomic hypothesis (or atomic fact, or whatever you +wish to call it) that all things are made of atoms — little particles +that move around in perpetual motion, attracting each other when they +are a little distance apart, but repelling upon being squeezed into +one another. In that one sentence you will see an enormous amount of +information about the world, if just a little imagination and thinking +are applied. diff --git a/testdata/testdir2/wom_bat/cur/rfc822.1 b/testdata/testdir2/wom_bat/cur/rfc822.1 new file mode 100644 index 0000000..71c3107 --- /dev/null +++ b/testdata/testdir2/wom_bat/cur/rfc822.1 @@ -0,0 +1,44 @@ +Return-Path: +Subject: Fwd: rfc822 +From: foobar +To: martin +Content-Type: multipart/mixed; boundary="=-XHhVx/BCC6tJB87HLPqF" +Message-Id: <1077300332.871.27.camel@example.com> +Mime-Version: 1.0 +X-Mailer: Ximian Evolution 1.4.5 +Date: Fri, 20 Feb 2004 19:05:33 +0100 + +--=-XHhVx/BCC6tJB87HLPqF +Content-Type: text/plain +Content-Transfer-Encoding: 7bit + +Hello world, forwarding some RFC822 message + +--=-XHhVx/BCC6tJB87HLPqF +Content-Disposition: inline +Content-Type: message/rfc822 + +Return-Path: +Message-ID: <9A01B19D0D605D478E8B72E1367C66340141B9C5@example.com> +From: frob@example.com +To: foo@example.com +Subject: hopjesvla +Date: Sat, 13 Dec 2003 19:35:56 +0100 +MIME-Version: 1.0 +Content-Type: text/plain; charset=iso-8859-1 +Content-Transfer-Encoding: 7bit + +The ship drew on and had safely passed the strait, which some volcanic shock +has made between the Calasareigne and Jaros islands; had doubled Pomegue, and +approached the harbor under topsails, jib, and spanker, but so slowly and +sedately that the idlers, with that instinct which is the forerunner of evil, +asked one another what misfortune could have happened on board. However, those +experienced in navigation saw plainly that if any accident had occurred, it was +not to the vessel herself, for she bore down with all the evidence of being +skilfully handled, the anchor a-cockbill, the jib-boom guys already eased off, +and standing by the side of the pilot, who was steering the Pharaon towards the +narrow entrance of the inner port, was a young man, who, with activity and +vigilant eye, watched every motion of the ship, and repeated each direction of +the pilot. + +--=-XHhVx/BCC6tJB87HLPqF-- diff --git a/testdata/testdir2/wom_bat/cur/rfc822.2 b/testdata/testdir2/wom_bat/cur/rfc822.2 new file mode 100644 index 0000000..316fa3f --- /dev/null +++ b/testdata/testdir2/wom_bat/cur/rfc822.2 @@ -0,0 +1,44 @@ +From: dwarf@siblings.net +To: root@eruditorum.org +Subject: Fwd: test abc +References: <8639ddr9wu.fsf@cthulhu.djcbsoftware> +User-agent: mu 0.98pre; emacs 24.0.91.9 +Date: Thu, 24 Nov 2011 14:24:00 +0200 +Message-ID: <861usxr9nj.fsf@cthulhu.djcbsoftware> +Content-Type: multipart/mixed; boundary="=-=-=" +MIME-Version: 1.0 + +--=-=-= +Content-Type: text/plain + +Saw the website. Am willing to stipulate that you are not RIST 9E03. Suspect +that you are the Dentist, who yearns for honest exchange of views. Anonymous, +digitally signed e-mail is the only safe vehicle for same. + +If you want me to believe you are not the Dentist, provide plausible +explanation for your question regarding why we are building the Crypt. + +Yours truly, + +--=-=-= +Content-Type: message/rfc822 +Content-Disposition: inline; filename= + "1322137188_3.11919.foo:2,S" +Content-Description: rfc822 + +From: dwarf@siblings.net +To: root@eruditorum.org +Subject: test abc +User-agent: mu 0.98pre; emacs 24.0.91.9 +Date: Thu, 24 Nov 2011 14:18:25 +0200 +Message-ID: <8639ddr9wu.fsf@cthulhu.djcbsoftware> +Content-Type: text/plain +MIME-Version: 1.0 + +As I stepped on this unknown middle-aged Filipina's feet during an ill-advised +ballroom dancing foray, she leaned close to me and uttered some latitude and +longitude figures with a conspicuously large number of significant digits of +precision, implying a maximum positional error on the order of the size of a +dinner plate. Gosh, was I ever curious! + +--=-=-=-- diff --git a/testdata/testdir4/1220863042.12663_1.mindcrime!2,S b/testdata/testdir4/1220863042.12663_1.mindcrime!2,S new file mode 100644 index 0000000..ab1500f --- /dev/null +++ b/testdata/testdir4/1220863042.12663_1.mindcrime!2,S @@ -0,0 +1,146 @@ +Return-Path: +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-4.9 required=3.0 tests=BAYES_00,DATE_IN_PAST_96_XX, + RCVD_IN_DNSWL_MED autolearn=ham version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id 5123469CB3 + for ; Thu, 7 Aug 2008 08:10:19 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [66.249.91.109] + by mindcrime with IMAP (fetchmail-6.3.8) + for (single-drop); Thu, 07 Aug 2008 08:10:19 +0300 (EEST) +Received: by 10.142.237.21 with SMTP id k21cs39272wfh; Wed, 6 Aug 2008 + 20:15:17 -0700 (PDT) +Received: by 10.65.133.8 with SMTP id k8mr2071878qbn.7.1218078916289; Wed, 06 + Aug 2008 20:15:16 -0700 (PDT) +Received: from sourceware.org (sourceware.org [209.132.176.174]) by + mx.google.com with SMTP id 28si7904461qbw.0.2008.08.06.20.15.15; Wed, 06 Aug + 2008 20:15:16 -0700 (PDT) +Received-SPF: neutral (google.com: 209.132.176.174 is neither permitted nor + denied by domain of gcc-help-return-33661-xxxx.klub=gmail.com@gcc.gnu.org) + client-ip=209.132.176.174; +Authentication-Results: mx.google.com; spf=neutral (google.com: + 209.132.176.174 is neither permitted nor denied by domain of + gcc-help-return-33661-xxxx.klub=gmail.com@gcc.gnu.org) + smtp.mail=gcc-help-return-33661-xxxx.klub=gmail.com@gcc.gnu.org +Received: (qmail 13493 invoked by alias); 7 Aug 2008 03:15:13 -0000 +Received: (qmail 13485 invoked by uid 22791); 7 Aug 2008 03:15:12 -0000 +Received: from mailgw1a.lmco.com (HELO mailgw1a.lmco.com) (192.31.106.7) + by sourceware.org (qpsmtpd/0.31) with ESMTP; Thu, 07 Aug 2008 03:14:27 +0000 +Received: from emss07g01.ems.lmco.com (relay5.ems.lmco.com [166.29.2.16])by + mailgw1a.lmco.com (LM-6) with ESMTP id m773EPZH014730for + ; Wed, 6 Aug 2008 21:14:25 -0600 (MDT) +Received: from CONVERSION2-DAEMON.lmco.com by lmco.com (PMDF V6.3-x14 #31428) + id <0K5700601NO18J@lmco.com> for gcc-help@gcc.gnu.org; Wed, 06 Aug 2008 + 21:14:25 -0600 (MDT) +Received: from EMSS04I00.us.lmco.com ([166.17.13.135]) by lmco.com (PMDF + V6.3-x14 #31428) with ESMTP id <0K5700H5MNNWGX@lmco.com> for + gcc-help@gcc.gnu.org; Wed, 06 Aug 2008 21:14:20 -0600 (MDT) +Received: from EMSS35M06.us.lmco.com ([158.187.107.143]) by + EMSS04I00.us.lmco.com with Microsoft SMTPSVC(5.0.2195.6713); Wed, 06 Aug + 2008 23:14:20 -0400 +Date: Thu, 31 Jul 2008 14:57:25 -0400 +From: "Mickey Mouse" +Subject: gcc include search order +To: "Donald Duck" +Message-id: <3BE9E6535E3029448670913581E7A1A20D852173@emss35m06.us.lmco.com> +MIME-version: 1.0 +Content-type: text/plain; charset=us-ascii +Content-transfer-encoding: 7BIT +Content-class: urn:content-classes:message +Mailing-List: contact gcc-help-help@gcc.gnu.org; run by ezmlm +Precedence: klub +List-Id: +List-Unsubscribe: +List-Archive: +List-Post: +List-Help: +Sender: gcc-help-owner@gcc.gnu.org +Delivered-To: mailing list gcc-help@gcc.gnu.org +Content-Length: 3024 + + +Hi. +In my unit testing I need to change some header files (target is +vxWorks, which supports some things that the sun does not). +So, what I do is fetch the development tree, and then in a new unit test +directory I attempt to compile the unit under test. Since this is NOT +vxworks, I use sed to change some of the .h files and put them in a +./changed directory. + +When I try to compile the file, it is still using the .h file from the +original location, even though I have listed the include path for +./changed before the include path for the development tree. + +Here is a partial output from gcc using the -v option + +GNU CPP version 3.1 (cpplib) (sparc ELF) +GNU C++ version 3.1 (sparc-sun-solaris2.8) + compiled by GNU C version 3.1. +ignoring nonexistent directory "NONE/include" +#include "..." search starts here: +#include <...> search starts here: + . + changed + /export/home4/xxx/yyyy/builds/int_rel5_latest/src/mp/interface + /export/home4/xxx/yyyy/builds/int_rel5_latest/src/ap/app + /export/home4/xxx/yyyy/builds/int_rel5_latest/src/shared/common + /export/home4/xxx/yyyy/builds/int_rel5_latest/src/shared/interface + /usr/local/include/g++-v3 + /usr/local/include/g++-v3/sparc-sun-solaris2.8 + /usr/local/include/g++-v3/backward + /usr/local/include + /usr/local/lib/gcc-lib/sparc-sun-solaris2.8/3.1/include + /usr/local/sparc-sun-solaris2.8/include + /usr/include +End of search list. + +I know the changed file is correct and that the include is not working +as expected, because when I copy the file from ./changed, back into the +development tree, the compilation works as expected. + +One more bit of information. The source that I cam compiling is in +/export/home4/xxx/yyyy/builds/int_rel5_latest/src/ap/app +And it is including files from +/export/home4/xxx/yyyy/builds/int_rel5_latest/src/shared/common +These include files should be including the files from ./changed (when +they exist) but they are ignoring the .h files in the ./changed +directory and are instead using other, unchanged files in the +/export/home4/xxx/yyyy/builds/int_rel5_latest/src/shared/common +directory. + +The gcc command line is something like + + TEST_DIR="." + + CHANGED_DIR_NAME=changed + CHANGED_FILES_DIR=${TEST_DIR}/${CHANGED_DIR_NAME} + + CICU_HEADER_FILES="-I ${AP_INTERFACE_FILES} -I ${AP_APP_FILES} -I +${SHARED_COMMON_FILES} -I ${SHARED_INTERFACE_FILES}" + + HEADERS="-I ./ -I ${CHANGED_FILES_DIR} ${CICU_HEADER_FILES}" + DEFINES="-DSUNRUN -DA10_DEBUG -DJOETEST" + + CFLAGS="-v -c -g -O1 -pipe -Wformat -Wunused -Wuninitialized -Wshadow +-Wmissing-prototypes -Wmissing-declarations" + + printf "Compiling the UUT File\n" + gcc -fprofile-arcs -ftest-coverage ${CFLAGS} ${HEADERS} ${DEFINES} +${AP_APP_FILES}/unitUnderTest.cpp + + +I hope this explanation is clear. If anyone knows how to fix the command +line so that it gets the .h files in the "changed" directory are used +instead of files in the other include directories. + +Thanks +Joe + +---------------------------------------------------- +Time Flies like an Arrow. Fruit Flies like a Banana + + diff --git a/testdata/testdir4/1220863087.12663_19.mindcrime!2,S b/testdata/testdir4/1220863087.12663_19.mindcrime!2,S new file mode 100644 index 0000000..78efa2a --- /dev/null +++ b/testdata/testdir4/1220863087.12663_19.mindcrime!2,S @@ -0,0 +1,77 @@ +Return-Path: +X-Spam-Checker-Version: SpamAssassin 3.2.5 (2008-06-10) on mindcrime +X-Spam-Level: +X-Spam-Status: No, score=-2.6 required=3.0 tests=BAYES_00 autolearn=ham + version=3.2.5 +X-Original-To: xxxx@localhost +Delivered-To: xxxx@localhost +Received: from mindcrime (localhost [127.0.0.1]) + by mail.xxxxsoftware.nl (Postfix) with ESMTP id C4D6569CB3 + for ; Thu, 7 Aug 2008 08:10:08 +0300 (EEST) +Delivered-To: xxxx.klub@gmail.com +Received: from gmail-imap.l.google.com [66.249.91.109] + by mindcrime with IMAP (fetchmail-6.3.8) + for (single-drop); Thu, 07 Aug 2008 08:10:08 +0300 (EEST) +Received: by 10.142.237.21 with SMTP id k21cs34794wfh; Wed, 6 Aug 2008 + 13:40:29 -0700 (PDT) +Received: by 10.100.33.13 with SMTP id g13mr1093301ang.79.1218055228418; Wed, + 06 Aug 2008 13:40:28 -0700 (PDT) +Received: from lists.gnu.org (lists.gnu.org [199.232.76.165]) by mx.google.com + with ESMTP id d19si15908789and.17.2008.08.06.13.40.27; Wed, 06 Aug 2008 + 13:40:28 -0700 (PDT) +Received-SPF: pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) client-ip=199.232.76.165; +Authentication-Results: mx.google.com; spf=pass (google.com: domain of + help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org designates 199.232.76.165 + as permitted sender) + smtp.mail=help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Received: from localhost ([127.0.0.1]:56316 helo=lists.gnu.org) by + lists.gnu.org with esmtp (Exim 4.43) id 1KQpo3-0007Pc-Qk for + xxxx.klub@gmail.com; Wed, 06 Aug 2008 16:40:27 -0400 +From: anon@example.com +Newsgroups: gnu.emacs.help +Date: Wed, 6 Aug 2008 20:38:35 +0100 +Message-ID: +References: <55dbm5-qcl.ln1@news.ducksburg.com> + +Mime-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +X-Trace: individual.net bABVU1hcJwWAuRwe/097AAoOXnGGeYR8G1In635iFGIyfDLPUv +X-Orig-Path: news.ducksburg.com!news +Cancel-Lock: sha1:wK7dsPRpNiVxpL/SfvmNzlvUR94= + sha1:oepBoM0tJBLN52DotWmBBvW5wbg= +User-Agent: slrn/pre0.9.9-120/mm/ao (Ubuntu Hardy) +Path: news.stanford.edu!headwall.stanford.edu!newshub.sdsu.edu!feeder.erje.net!proxad.net!feeder1-2.proxad.net!feed.ac-versailles.fr!fu-berlin.de!uni-berlin.de!individual.net!not-for-mail +Xref: news.stanford.edu gnu.emacs.help:160868 +To: help-gnu-emacs@gnu.org +Subject: Re: Learning LISP; Scheme vs elisp. +X-BeenThere: help-gnu-emacs@gnu.org +X-Mailman-Version: 2.1.5 +Precedence: list +List-Id: Users list for the GNU Emacs text editor +List-Unsubscribe: , + +List-Archive: +List-Post: +List-Help: +List-Subscribe: , + +Sender: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Errors-To: help-gnu-emacs-bounces+xxxx.klub=gmail.com@gnu.org +Content-Length: 417 +Lines: 11 + +On 2008-08-01, Thien-Thi Nguyen wrote: + +> warriors attack, felling foe after foe, +> few growing old til they realize: to know +> what deceit is worth deflection; +> such receipt reversed rejection! +> then their heavy arms, e'er transformed to shields: +> balanced hooked charms, ploughed deep, rich yields. + +Aha: the exercise for the reader is to place the parens correctly. +Might take me a while to solve this puzzle. + diff --git a/testdata/testdir4/1252168370_3.14675.cthulhu!2,S b/testdata/testdir4/1252168370_3.14675.cthulhu!2,S new file mode 100644 index 0000000..1e69622 --- /dev/null +++ b/testdata/testdir4/1252168370_3.14675.cthulhu!2,S @@ -0,0 +1,22 @@ +Return-Path: +X-Spam-Checker-Version: SpamAssassin 3.1.0 (2005-09-13) on mindcrime +X-Spam-Level: +Delivered-To: dfgh@floppydisk.nl +Message-ID: <43A09C49.9040902@euler.org> +Date: Wed, 14 Dec 2005 23:27:21 +0100 +From: Fred Flintstone +User-Agent: Mozilla Thunderbird 1.0.7 (X11/20051010) +X-Accept-Language: nl-NL, nl, en +MIME-Version: 1.0 +To: dfgh@floppydisk.nl +List-Id: =?utf-8?q?Example_of_List_Id?= +Subject: Re: xyz +References: <439C1136.90504@euler.org> <4399DD94.5070309@euler.org> <20051209233303.GA13812@gauss.org> <439B41ED.2080402@euler.org> <4399DD94.5070309@euler.org> <20051209233303.GA13812@gauss.org> <439A1E03.3090604@euler.org> <20051211184308.GB13513@gauss.org> +In-Reply-To: <20051211184308.GB13513@gauss.org> +X-Enigmail-Version: 0.92.0.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 7bit +X-UIDL: T +To: Bilbo Baggins +Subject: Greetings from =?UTF-8?B?TG90aGzDs3JpZW4=?= +User-Agent: Wanderlust/2.15.9 (Almost Unreal) Emacs/24.0 Mule/6.0 (HANACHIRUSATO) +Fcc: .sent +Organization: The Fellowship of the Ring +MIME-Version: 1.0 (generated by SEMI 1.14.6 - "Maruoka") +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + + +Let's write some fünkÿ text +using umlauts. + +Foo. diff --git a/testdata/testdir4/1305664394.2171_402.cthulhu!2, b/testdata/testdir4/1305664394.2171_402.cthulhu!2, new file mode 100644 index 0000000..863f714 --- /dev/null +++ b/testdata/testdir4/1305664394.2171_402.cthulhu!2, @@ -0,0 +1,17 @@ +From: =?UTF-8?B?TcO8?= +To: Helmut =?UTF-8?B?S3LDtmdlcg==?= +Subject: =?UTF-8?B?TW90w7ZyaGVhZA==?= +User-Agent: Wanderlust/2.15.9 (Almost Unreal) Emacs/24.0 Mule/6.0 (HANACHIRUSATO) +References: +1n-Reply-To: +MIME-Version: 1.0 (generated by SEMI 1.14.6 - "Maruoka") +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + + +Test for issue #38, where apparently searching for accented words in subject, +to etc. fails. + +What about here? Queensrÿche. Mötley Crüe. + + diff --git a/testdata/testdir4/181736.eml b/testdata/testdir4/181736.eml new file mode 100644 index 0000000..56255c4 --- /dev/null +++ b/testdata/testdir4/181736.eml @@ -0,0 +1,42 @@ +Path: uutiset.elisa.fi!feeder2.news.elisa.fi!feeder.erje.net!newsfeed.kamp.net!newsfeed0.kamp.net!nx02.iad01.newshosting.com!newshosting.com!post01.iad!not-for-mail +X-newsreader: xrn 9.03-beta-14-64bit +Sender: jimbo@lews (Jimbo Foobarcuux) +From: jimbo@slp53.sl.home (Jimbo Foobarcuux) +Reply-To: slp53@pacbell.net +Subject: Re: Are writes "atomic" to readers of the file? +Newsgroups: comp.unix.programmer +References: <87hbblwelr.fsf@sapphire.mobileactivedefense.com> <8762s0jreh.fsf@sapphire.mobileactivedefense.com> <87hbbjc5jt.fsf@sapphire.mobileactivedefense.com> <8ioh48-8mu.ln1@leafnode-msgid.gclare.org.uk> +Organization: UseNetServer - www.usenetserver.com +X-Complaints-To: abuse@usenetserver.com +Message-ID: +Date: 08 Mar 2011 17:04:20 GMT +Lines: 27 +Xref: uutiset.elisa.fi comp.unix.programmer:181736 + +John Denver writes: +>Eric the Red wrote: +> +>>> There _IS_ a requirement that all reads and writes to regular files +>>> be atomic. There is also an ordering guarantee. Any implementation +>>> that doesn't provide both atomicity and ordering guarantees is broken. +>> +>> But where is it specified? +> +>The place where it is stated most explicitly is in XSH7 2.9.7 +>Thread Interactions with Regular File Operations: +> +> All of the following functions shall be atomic with respect to each +> other in the effects specified in POSIX.1-2008 when they operate on +> regular files or symbolic links: +> +> [List of functions that includes read() and write()] +> +> If two threads each call one of these functions, each call shall +> either see all of the specified effects of the other call, or none +> of them. +> + +And, for the purposes of this paragraph, the two threads need not be +part of the same process. + +jimbo diff --git a/testdata/testdir4/encrypted!2,S b/testdata/testdir4/encrypted!2,S new file mode 100644 index 0000000..b6470e7 --- /dev/null +++ b/testdata/testdir4/encrypted!2,S @@ -0,0 +1,57 @@ +Return-path: <> +Envelope-to: peter@example.com +Delivery-date: Fri, 11 May 2012 16:22:03 +0300 +Received: from localhost.example.com ([127.0.0.1] helo=borealis) + by borealis with esmtp (Exim 4.77) + id 1SSpnB-00038a-Ux + for djcb@localhost; Fri, 11 May 2012 16:21:58 +0300 +Delivered-To: peter@example.com +From: Brian +To: Peter +Subject: encrypted +User-agent: mu4e 0.9.8.5-dev1; emacs 24.1.50.8 +Date: Fri, 11 May 2012 16:21:42 +0300 +Message-ID: <877gwi97kp.fsf@example.com> +MIME-Version: 1.0 +Content-Type: multipart/encrypted; boundary="=-=-="; + protocol="application/pgp-encrypted" + +--=-=-= +Content-Type: application/pgp-encrypted + +Version: 1 + +--=-=-= +Content-Type: application/octet-stream + +-----BEGIN PGP MESSAGE----- +Version: GnuPG v1.4.12 (GNU/Linux) + +hQQOA1T38TPQrHD6EA//YXkUB4Dy09ngCRyHWbXmV3XBjuKTr8xrak5ML1kwurav +gyagOHKLMU+5CKvObChiKtXhtgU0od7IC8o+ALlHevQ0XXcqNYA2KUfX8R7akq7d +Xx9mA6D8P7Y/P8juUCLBpfrCi2GC42DtvPZSUu3bL/ctUJ3InPHIfHibKF2HMm7/ +gUHAKY8VPJF39dLP8GLcfki6qFdeWbxgtzmuyzHfCBCLnDL0J9vpEQBpGDFMcc4v +cCbmMJaiPOmRb6U4WOuRVnuXuTztLiIn0jMslzOSFDcLTVBAsrC01r71O+XZKfN4 +mIfcpcWJYKM2NQW8Jwf+8Hr84uznBqs8uTTlrmppjkAHZGqGMjiQDxLhDVaCQzMy +O8PSV4xT6HPlKXOwV1OLc+vm0A0RAdSBctgZg40oFn4XdB1ur8edwAkLvc0hJKaz +gyTQiPaXm2Uh2cDeEx4xNgXmwCKasqc9jAlnDC2QwA33+pw3OqgZT5h1obn0fAeR +mgB+iW1503DIi/96p8HLZcr2EswLEH9ViHIEaFj/vlR5BaOncsLB0SsNV/MHRvym +Xg5GUjzPIiyBZ3KaR9OIBiZ5eXw+bSrPAo/CAs0Zwxag7W3CH//oK39Qo1GnkYpc +4IQxhx4IwkzqtCnripltV/kfpGu0yA/OdK8lOjkUqCwvL97o73utXIxm21Zd3mEP +/iLNrduZjMCq+goz1pDAQa9Dez6VjwRuRPTqeAac8Fx/nzrVzIoIEAt36hpuaH1l +KpbmHpKgsUWcrE5iYT0RRlRRtRF4PfJg8PUmP1hvw8TaEmNfT+0HgzcJB/gRsVdy +gTzkzUDzGZLhRcpmM5eW4BkuUmIO7625pM6Jd3HOGyfCGSXyEZGYYeVKzv8xbzYf +QM6YYKooRN9Ya2jdcWguW0sCSJO/RZ9eaORpTeOba2+Fp6w5L7lga+XM9GLfgref +Cf39XX1RsmRBsrJTw0z5COf4bT8G3/IfQP0QyKWIFITiFjGmpZhLsKQ3KT4vSe/d +gTY1xViVhkjvMFn3cgSOSrvktQpAhsXx0IRazN0T7pTU33a5K0SrZajY9ynFDIw9 +we7XYyVwZzYEXjGih5mTH1PhWYK5fZZEKKqaz5TyYv9SeWJ+8FrHeXUKD38SQEHM +qkpl9Iv17RF4Qy9uASWwRoobhKO+GykTaBSTyw8R8ctG/hfAlnaZxQ3TwNyHWyvU +9SVJsp27ulv/W9MLZtGpEMK0ckAR164Vyou1KOn200BqxbC2tJpegNeD2TP5ZtdY +HIcxkgKr0haYcDnVEf1ulSxv23pZWIexbgvVCG7dRL0eB+6O28f9CWehle10MDyM +0AYyw8Da2cu7PONMovqt4nayScyGTacFBp7c2KXR9DGZ0mcBwOjL/mGRKcVWN3MG +2auCrwn2KVWmKZI3Jp0T8KhfGBnFs9lUElpDTOiED1/2bKz6Yoc385QtWx99DFMZ +IWiH5wMxkWFpzjE+GHiJ09vSbTTL4JY9eu2n5nxQmtjYMBVxQm7S7qwH +=0Paa +-----END PGP MESSAGE----- +--=-=-=-- + diff --git a/testdata/testdir4/mail1 b/testdata/testdir4/mail1 new file mode 100644 index 0000000..a4e19c1 --- /dev/null +++ b/testdata/testdir4/mail1 @@ -0,0 +1,38 @@ +Date: Thu, 31 Jul 2008 14:57:25 -0400 +From: "John Milton" +Subject: Fere libenter homines id quod volunt credunt +To: "Julius Caesar" +Message-id: <3BE9E6535E3029448670913581E7A1A20D852173@emss35m06.us.lmco.com> +MIME-version: 1.0 +x-label: Paradise losT +X-keywords: john, milton +Content-type: text/plain; charset=us-ascii +Content-transfer-encoding: 7BIT +Precedence: high + +OF Mans First Disobedience, and the Fruit +Of that Forbidden Tree, whose mortal tast +Brought Death into the World, and all our woe, +With loss of Eden, till one greater Man +Restore us, and regain the blissful Seat, [ 5 ] +Sing Heav'nly Muse,that on the secret top +Of Oreb, or of Sinai, didst inspire +That Shepherd, who first taught the chosen Seed, +In the Beginning how the Heav'ns and Earth +Rose out of Chaos: Or if Sion Hill [ 10 ] +Delight thee more, and Siloa's Brook that flow'd +Fast by the Oracle of God; I thence +Invoke thy aid to my adventrous Song, +That with no middle flight intends to soar +Above th' Aonian Mount, while it pursues [ 15 ] +Things unattempted yet in Prose or Rhime. +And chiefly Thou O Spirit, that dost prefer +Before all Temples th' upright heart and pure, +Instruct me, for Thou know'st; Thou from the first +Wast present, and with mighty wings outspread [ 20 ] +Dove-like satst brooding on the vast Abyss +And mad'st it pregnant: What in me is dark +Illumin, what is low raise and support; +That to the highth of this great Argument +I may assert Eternal Providence, [ 25 ] +And justifie the wayes of God to men. diff --git a/testdata/testdir4/mail5 b/testdata/testdir4/mail5 new file mode 100644 index 0000000..b12387a --- /dev/null +++ b/testdata/testdir4/mail5 @@ -0,0 +1,624 @@ +From: Sitting Bull +To: George Custer +Subject: pics for you +Mail-Reply-To: djcb@djcbsoftware.nl +User-Agent: Hunkpapa/2.15.9 (Almost Unreal) +Fcc: .sent +MIME-Version: 1.0 (generated by SEMI 1.14.6 - "Maruoka") +Content-Type: multipart/mixed; + boundary="Multipart_Sun_Oct_17_10:37:40_2010-1" + +--Multipart_Sun_Oct_17_10:37:40_2010-1 +Content-Type: text/plain; charset=US-ASCII + +Dude! Here are some pics! + + +--Multipart_Sun_Oct_17_10:37:40_2010-1 +Content-Type: image/jpeg +Content-Disposition: inline; filename="sittingbull.jpg" +Content-Transfer-Encoding: base64 + +/9j/4AAQSkZJRgABAQAAAQABAAD/4QvoRXhpZgAASUkqAAgAAAAIABIBCQABAAAAAQAAABoBCQAB +AAAASAAAABsBCQABAAAASAAAACgBCQABAAAAAgAAADEBAgAOAAAAbgAAADIBAgAUAAAAfAAAABMC +CQABAAAAAQAAAGmHBAABAAAAkAAAAN4AAABndGh1bWIgMi4xMS4zADIwMTA6MTA6MTcgMTA6MzM6 +MzcABgAAkAcABAAAADAyMjEBkQcABAAAAAECAwAAoAcABAAAADAxMDABoAkAAQAAAAEAAAACoAkA +AQAAAMgAAAADoAkAAQAAAGsBAAAAAAAABgADAQMAAQAAAAYAAAAaAQkAAQAAAEgAAAAbAQkAAQAA +AEgAAAAoAQkAAQAAAAIAAAABAgQAAQAAACwBAAACAgQAAQAAALMKAAAAAAAA/9j/4AAQSkZJRgAB +AQAAAQABAAD/2wBDAAgGBgcGBQgHBwcJCQgKDBQNDAsLDBkSEw8UHRofHh0aHBwgJC4nICIsIxwc +KDcpLDAxNDQ0Hyc5PTgyPC4zNDL/2wBDAQkJCQwLDBgNDRgyIRwhMjIyMjIyMjIyMjIyMjIyMjIy +MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjL/wAARCACAAEcDASIAAhEBAxEB/8QAHwAA +AQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIh +MUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpT +VFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5 +usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAA +AAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEI +FEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVm +Z2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK +0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwDq77xdrX/CQ6laRXjRxQTF +ECovA/EUg8Sa6W/5CUuP9xP8K5yWQnxjrw9Lwj9BWjkgZHFAG6mu6yV51OXP+4n/AMTUq61rBB/4 +mU2f9xP/AImsJJTuAJFW0YDnfmgCTUPFGqWFq882p3G1eyqmT/47VfRfGGpawkgGp3CyIeg2cj1+ +7XK+O7zybCGMNjzHyR6gD/69ZvgG8zqU67vvRZH4EUAesJe6m/XVLv8ANf8A4mpf7Qvl/wCX+6b6 +uP8ACs+ObKdaeh3Hg9aANTw/4gurjxTLpU7tIv2cTKzHpgkH+n5UVheHGI+KWzJwdNP/AKFRQBzD +7f8AhMfEDEHH24j/AMdWrs0oCkDrVKJs+NfEsZ+79u/9kWrd5GqKTmgCstwwkyT0p5uzu61mOzbj +zSFn3DmgDB8ePLPe2MEQZyykhRzk9/5Va8D6Vd2Mz3d3CYxJHiPd16+la0hhMybkUvxhj1HWr5uM +uB0wMYoA3YJARjvV+DBPasC2lYsOuK3LVunWgCLQRj4sIPXTGP8A4/RSaF/yV2P30tv/AEOigDmY +QD408UE9ftw/9AFXpv3iFT9Kgs4t3jXxV6C+H/oAq5cxkMcCgDFltXVyMVVv7iGwtzNcNsQfiT+F +a8jbAxdgFAzmuTZZfEV81vG+xTyX/uIPT3P9aAIBr9vdNHcQI/lxk5DDBrfsLuK+jE0MqupPOOx9 +KzNY8L6fbaaYrdGXb3BOcnuT+FcpodzN4c8RRRyylrW4IDE9MdM/UUAes2wbOAK27PhRms6CJlwc +VrWowRkUAV9CP/F3YffSm/8AQ6Kfo+P+FuWp9dLf/wBDooAxrH/kd/Ff/X8P/Ra1evUOcgVW01Qf +G/izIz/py/8Aota2LqPK4xQBxniWc2mi3MxBGFA/Mgf1rmtEF/Z6HNqMNuzvPnY+7G1V6Hoe+T0r +qfGmnT3Xhm8WNWJVQ+B/skH+lUPBt3d3PhuzXyBM6xBY0YfKDnALewxmgDE1BfEDaPaXNzMRJPIQ ++TjCgDHb69u9ZGt2Us2lrdNDtMLAgq27Kng84Fd74qnaMwWB8qWRTnzUcfePGSOx4ziuf1kzT6S9 +tuRHlVUG5sDJOMA+lAHofh5/tvh3T7k4ZnhXcfcDB/UVuRQEdqzvDelPo/hywsJGDSRRjeR0yeTj +2ya3I8/3aAMXSU2/FmzJ/wCgbJ/6FRUunf8AJV7H/sGy/wDoQooAyNJXf448XYPS+X/0Wtb8ynyj +0rm/DIll8W+KDKQ0pvF3FehPlr0rvINMzbfN8rsc7upH0oA5ie3mktZSI1ICn5W43e1ec6ZrDwax +facIj9liUNtUcgE8j0IzXrHiqS20rQJbiadoyBsWQjc2T2HvXnvhbREuzeXTbvMlfILcsF6D6jFA +GJr+pWE1ymFkwFzhlwo+i1xevazLd3Fva2+UiQhh7kdPyr0jVfA8t0BeXNybe35UK2EJAJwST/QG +uS1Pw7HYalbKHUIxYxyDd8wHUnNAHsnhXVBrGhWkrBlmEYVww6sAATXQInA5rn/AOZtIa3mQHZI+ +xwfvAnJ6d8n9a6yazEKhlzgUAc1YAr8WbH302X/0IUU6xBPxYsSe2my/+hUUAV/Bdj5fi3xWJJDJ +JHeopY8bj5a5OK9AUArwARXEeFjjxh4xbub5f/RYrsIZgJhGTjcuQMGgDnfHiwnw1KJoVkUuB8yg +hfeuZ+HemTLpjx3OCZNzKUbPy54/Sut8Z263OlJE1wYgzkkjvgH86yfBb+XYWuIGiEithWzn9aAN +loTcO0ctuGjV9oMg5JGCSOOnp9K8/wDH1qH1iERrukRAqqB3Jzj9BXpsk6F+oyCuRjJ54rhNcg+3 +Ge5XiUSL5ZGc87sdPagDQ+HlvJHoAdo9h85mUY7dK7WSRCoB6HiuV8IiW10JYs7yszDJ7fN/k1tG +Rpb4xj7qpnj3Iwfx5oAwLMgfF+1UHI/suTH/AH3RTLJNnxltx2Olvj/vuigB3hgf8Vp4vH/T8v8A +6AK6aRWFk2CA2CPSua8M4/4T3xcp/wCftD/45XR6q32e1JjUySCRdqA4J3HH9aAKHiJTceH4mliK +r5e5lDfMpx2Iqp4eQR6Zp75Y4jX7xyfTn8q29djjbS/LMqxYGFdugNZWlskOh2pKgYj2AqO4OB/M +0AW7+NLQ3Fwi/O6hsk5yRwOO3WuS1qGJtNuvN3iNJkX5e+EIxn8f1re1e4ubq8jSOMiBArZJ/wBY +xOcfQcZ+tVNTsYh4dnjmG9PMJIP8XYUAQ20z2Hg6OeJGTYQzd+N3Le+RzXQ6TGwtjLLkuxAy3XAH +f8Saw9Mlt7vwsI4yZI9m07xtyM/y5rqodqxIFAIx1oA5iDj4w2ZHfS3/APQjRSw8/GGzx20x/wD0 +I0UAee+I/GV/4S+IXiAWlvFKJ7gM28njC+1Ubn4v6xclC1hbAq6vwzdjn+lXviB4X1O88b6lPBYX +EkUkgZXWJiDwPQVzH/CH61zjSbwj1EDf4UAbV78YdZvYPJbT7UA+7HP61HbfFXXLW3SFdOtSqZ67 +v8fesg+Ddbzn+yL3P/XBv8Kcvg3Xc5Oj3x/7YP8A4UAaY+KuvIQP7PtM5JXKt/jUF78Udcu7F7WS +ytEVv4grZHPB61VPg/Ws/wDIGvs9v3T/AOFMPg7XcHOk32P+uD/4UAWLb4l6vb2zQJZ2m1gP4WGC +FAz19q17f4va0sSobS04GB8rf41z3/CIayOuk3g/7d2/wqRfCWr8f8S27/78P/hQB33w78Q3fib4 +jR3l3HHG6WTxgR5xjOe/1oq78JvCmo6dq8+qXUBhhETQqJAVYsSDkA8496KAP//ZAP/bAEMABQME +BAQDBQQEBAUFBQYHDAgHBwcHDwsLCQwRDxISEQ8RERMWHBcTFBoVEREYIRgaHR0fHx8TFyIkIh4k +HB4fHv/bAEMBBQUFBwYHDggIDh4UERQeHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4e +Hh4eHh4eHh4eHh4eHh4eHv/AABEIAWsAyAMBIgACEQEDEQH/xAAdAAABBAMBAQAAAAAAAAAAAAAH +AwQFBgACCAEJ/8QAVhAAAQMCBAIHAwYHCwoGAgMAAQIDEQAEBQYSITFBBwgTIlFhcRSBkRUjMqGx +0RYXQoKSssElJjNDRFJicpOi8CQ0NmNzo7PC0uEnNVNUVYMJRWR08f/EABQBAQAAAAAAAAAAAAAA +AAAAAAD/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwDqjNudMEyw6lvE3HEqUnUS +kDSmeEkkCTB241Aq6X8mgSL5J/PT99DPrX3S232GwohPbp5/6v8A70PMsrSuwYKgkgtjiONB0Ovp +lykFQLhJ8+0Fefjoyh/6/wDfFBctsJWkpt0wRzSK9ft7cwVMNmf6IoDOOmjJ5/lB/SFejpmyeT/n +I/TFBdFqwACGGR6IFOWbRhZ3Ya349wUBgT0y5QP8qT+n/wBq3T0x5OPG8A/OFCy3tbZCRpt2j+YK +cotLMjvWjM/1BQEs9MWTR/LUn0VWfjiyb/72hki1sw5q9kaHL6ApZDFspQHsze3DuCgI6umLJqUg ++2Eg+AJ/ZSR6aMmj+UOfon7qozjTACUhlsCP5opo402AQWkeumgIK+mrKAA0uuK8e6R+ykj03ZU5 +JePx+6h6gaTGlPHmkUwzDjlrhDKVvtIWXVaUIjj4+6gKKOm7KZUQrtU+YCj/AMteHpuyvvpbeV7l +f9NAHEc3Ls8ed9iYZLTZIAI4+vpRAy1i7WKYS3dhtIDghQHIjiKC9npwy3O1rcH3K/6a1V034Ef4 +OyfPqlf/AE1WG1KKxAAHhFO0LgTCfhQTf47cGPCwuJ/qr/6a9R004as7YXcn0bX/ANNRLbqhsAOF +OWnVgEnh4UEm30wWbhhvBL5e3Jpz/ppwOlMLSC3lzElT4suD/kqHN642YT9tIu3T7h2UYHnQWW36 +S9YleXcQA8Qhf7U05b6SMPXiFpYGwfZfu19myl9XZ6leEkR9dVBy7XEFXLcUN+mPEn7VnCrplZQ6 +zclaFeCgAQfqoOqbO4L6VBxvsnURrRqConcbisqo9FOYE5mw1zGENdkLllpZQSCUkApO/wCbWUAY +63i9N0x3v5SkHy+aqj5QBXhFqRzbFWrrjrWm5a1ER7Unh/saqOSXP3Bst/4pNBbWhLY1nvCt1NyZ +mRWtuoKESCOdKjSOB99BiEjYHYU4ZQpKpTwpArTxpVt8g7RQPmlQkpP2UoFE8VkU0buEkK4A+NbN +rCzHDagfQCmN69TtsPjTRLoBA1b0oHFaZPwoH6IPODFJPwOG9asklJJ228a8egp9aBspcHhvQf6T +b55eZHWVu6m2kgISDsnaT76Kt46hlpx1WyUJKj7q59xzEFXV9dXC1krccK9/M0CybtYUTrMnj50R +uiPGLl69OHBTaWG2isgjdRkffQgFyoqO441eOiW4WM1W8D6aVBXpFAfGCVGQmnDZO07jmaYW74G0 +yKfNuJUBB50D0ISAIAiNyK3UUkQIpDtgUhAAIApNToAKQACedAotaZgCfOvQrUCAIpsVAma31kEE +bbUG60Dwn1oW9O6S3h2HqnYvK291E1SlEkzvQr6f1qRg1gVHc3B4+lAaeq2rXkFhcz3NPwcXWU06 +pzqnchtQ4CkBQjT/AKxVZQDXrpOEXLJO0XYj+xFVDI6lHAbEeLSfsq1dd2UPWyxMG7HHhPZCqtkX +/wAhw9Svo9gj7KC4Wx2MCKdt7ncTTaxbGmBz3p4hISNzHhQJrQSYnjXiUkQOYpZzupJJpFK5g7n3 +0ChQSkbxSluSFDfam5WACokj1NetPAqA29KCXb0HdQ9TSwQlSduVR6XBpJnetkXWgCTQSTQCRpMk +Uk8djzpsm9ExNaOP6pOqBQRWcblNtly/eI3SwoTPiIrnbEXAl7j4UaulrEHLbJ9z2ZEuKS2fQneg +Fevy62ASQYoHCHkqXsqrl0U3aWs52mpelLgU3v5jah20/C9zO/hVmyc8Bj+HjfvXCB9dB09bq0mD +zqQZUkAVDtriOdO7ZzSYO8UEoHBw3r0KB3mm6XUxERWdpO0UDhJBVvSulKh9kU2bUdzvvSzKu9tI +mgXShOjfaTQk6xZ/cOw4GLk7/m0XlqlGxEcCDzoP9Y1KhgNhI/lR/VNAWuqGpJyG3pBHdVPn84qs +rTqgqJyKyCI+bXHp2qqygH/XoQEW9i5M6r0CJ/1VVnJKP3Aw8QP4BvYf1RVh69K1lm0BMgXyQPL5 +moPJ50YDY+TCP1RQW+yRoaC44HhSygNXe99NbW7lCUKA48K3cdgbk8KDa5IAMcKj3XiklIMTSrj0 +zCppi8STwNAo5cEo4RXtq53pn1pke8NIUR5UoydWoA+lBMdrCdjSLr5G8yfCmJdWE6ZpNx2NiZoH +4uZjlThKitJ35TUOl4DntUlZOgoiaAe9Nl+tnDbS1QAULdlzxHgftoMLdm4VzAmKKvTsvsrywUqS +y8y42seBBBB9xoSWTa7hVwocGmyrh7qDZpUKFXzoptG77NdoFkwiXD7uFUvCMOu7+5SxbNLccO8A +TRd6Nsq3eB5mbVezq9l7ThsCTEUBcQvhtuKdWy0qVJ40wS4NqcNOTvA+NBKpMokRSiFKnx8opo07 +KAAYNOGwrko0DlB/JO005Y0gTNNW9pkzTm3SDJJoHIQhIlKZ50IesgFDLdgYgC74/mmjC03KaEXW +WBTlmykH/Ox+qqgJnVAV+8S3hMbOA78fnVVlZ1Oio5BbnkHI/tVVlAOOvSIbtlSN8QSD/Y0yyayh +3Ldg4BBNs37+6KedfERaWqo2+UE/8E1FdGdwF5Yw0kgzatmPzRQWdhnRClCTWPAEHeKkENodbSpO +xpBy3IJJjegjFwjh6U1eWSoxw50+uGQHPE86j7qEqiY9KBstYSe6B61gfLapB5QaaPOQs77V4twR +wNA8TcFyPKsdd25UyS6RBAivXXJSDQOm1jyinlq/CoSY22qDDh1AVI22mUqmNqCpdI+GXOZb9Ng1 +I9jZL+qNiVSAPiKiMhZQFixcHGGgkvJPaJJ4JBolsKaTdO9zdTcKV4CdqalLSrtevcQEkeVA7ytl +/CcOm7sLVKFugb+VSt06k3WgfSSO9HKss30NsagAkAd0GminCXSr8onegfNKSDO/nJp22tCSDUYF +xEJpVDkkwDPrQTTCoGxBmnjDp4KG9QVu4rVxjepS3WY1KVPpQSbSiT508tBvBImo9hJICt96f2yS +TwIigkWht6UHes8ojK9mJE+2Dj/UVRhaEpn7aDnWhH71LP8A/uj9VVATupmZyCiePzn/ABVVledT +OfxfszG6XYj/AGyqygHfX3A+TrPmTiKZ3/1JqvdFX+h+EqPO2TU91+NrK1Mf/sER/YmoLopSpWR8 +II/9sPtNARbNSdAKdzHCtnY0Eq4zypnbFaEAqImnRUFt6p93OgjbyEEn6qhrxQUo+FTGIp7m23rU +HeGKBi6E6orRcRtWyiSoxWp4ERQeoAI3NeLSnnW7Q2341s5skcOMUCSEpMk08to0gA/Gmu4HCvW1 +qoPMQWtOJIDR/I3HiKyycacfU4o86Y4qpxBcutiUphIpLLy1LGtSN1RsTwoLS0SQpx3uoG4HlSHa +gq1JMSa9ullVulJAA5+dN0wYiKCQbcJ+lNLoVzmmjKoEmndudSoVAHpQOrbVq3mIqWtYME7Co9oE +xHAeFPWVqCQY250EsyuBsSPWn7C1CI4+tRdq4F7gjzFSduoCBHoKCRYUYAG1CHrQJjKFqqB/nqf1 +FUWbck96NO3jNCXrPmMm23Ha9R+oqgJPUvM9H6PLtR/vTWVt1MkoT0eMaQAVpdUYPPtSPurKAb9f +ZQNkwnf/AMwR/wAA1GdD7QOQMHUeduPtNSfX3H7n2y4mcSSPT5g1G9D69XR9gxCeFvHHzNBczGkR +tWqXIBjlW4HdO0U1XqQsGOe9ApcAKTHEGoe7tyJqWWSBv61roQ6nvCgrD7cEyPqpNCYn0qavrMhW +w5U1NsYAAoGLAEmaUcAinItSZ8R4VjjJ0Qo+tBHK+NeoI0kVXs3ZptMIS42wk3FwB9EGAPU0KMWz +zma6uCtm67BoHZLSQEx6nc0BdzRfBq0UhKu8BwFI4BchttK1EyrlPCqjhmJOXlrbquHUPOrbBJJ5 +8+FaXC32NTguLi2MmCYU2fLjPxoCel0vDUFFQPCnSAQaH+Vc625fRhuKpbtX+CHAruL+6iC0QtKV +JUFA8CKByyFE8KeI2CTzpG2E8h8acpQSRCRH20DxlRMGPqp2zOkJB2prZtlRIg/Gn7bSUwNW9A+t +BAO4B8Kk7UeJmo20SUkmeHlUnbK2G1A+aiZjfxoS9Z8k5Ltwrleo4f1VUV0d6IG/OhR1mkg5HZkz +/lqP1VUBI6l5H4vmRJOzvHl86aykOpOf3iEb/Sdj+0rKCgdfUj5NtxzGJI9/zBqP6Gkj8XuDDn2H +/Malev6lHybakQD8oNg7f6hVMehVIV0dYMqJhj/mNBcm29Q24U3u2gDtUohvS2dtzTO6H+IoI93d +PmBTcq0xBmnDoM7b7U1KZUQCBvQLKcQtuFRSCwAmBFIvBxJ2MGaVAUUhQIngaBNUg7kVV81YoUhV +rbuweCiOPpUxmC9Rh9it5Su+dkDmTQ9Nz2q1PrO5JCd5JNAxxyztBaOOXyktsNp1rUefqaH7dpc5 +hvlDCLJabVJgOLG5H2D0q7YhhV5nDFrfBLYrTbA9reOAwAkHYep/YKL+X8qWGHYe1aWjCUBtMDQn +c+cmgDtvlq7scEQ2grTdtnVqKdhVXx3EscYTpumoIkdogbKHmDXRWJ2NotlxpbjRUOYVJ9N6G+bc +Mt3EdghaFx9IAUAXfxB25bIe0haOHd+qiR0QZ1dRdIwXFXdbayEsOqP0T/NNU/MuXH7Qret0KKUn +dIE/CoCxUW3UnfcyCOINB2BaiRI8Nop+0gxP21R+iPHF45gCUPq1XVsezd347bGr8wlQERQLWp0r +Gw41IoAKp2k0xQkg/Qnwp9bSYJSZFA7YSfjUhbN6uMAimTBIUCakbZW44UD1hkJAoUdZ9pKMgNqH +H25H6qqLTR7oPOhZ1ngD0dSOIvGz9SqC4dSZJGQyeRU7H6YrK36lBH4vk7flvQfzxWUFE6/oBwm1 +Vz+UWh7+wVTboPSR0cYKkCf8nk/pGpHr/o/e9aLjf5SbH+4XTPoOUk9G2CyCT7PwA/pGgvaR3eAN +M71E8qkkJBRsDvTa7QNBFBCuoHGKaqbGuZEU/udIERvTbUmSNO9A0ukjSTG9Nm1BC5PDnUg8pIHC +ozEn0tMLcCeCSfqoB9nrElXOLi2bJ0IOkeE86rVzcBDKnAoBLRgCNyfL/HjXmO4koPaoPaOEmf5s +86aYHbPY7mmywdk6mmyHbjwKZkz60Bl6NcGRhWX0XDrWm6vPnnIHeAPAe4VZx2iEFSVrKYgpI0/X +UJeYuzhrae0Qt9SANmwdIH7ajh0j4WrW204dYEEE8NqCXW6+7dP2yUMNnTKdZJ24cao2ZGCVO9ol +tzcmUnh/j7qXTm3tbC8xdcKQnsWRJ7pIUoq+qKqeMZ3YUp9xxxsJcUCQngOJ2+NAmptDxKHEqSCN +vGhnnCxOHY8sISEpe76YECecfb76J2GXDV8wLj2a4DDhlDikEA+h8Kr/AEk4St7Chdtd4sHUI4xz +Hw391At0JY38mZntmnHdLV38y4nlP5J/x4102ywVAEbiuLMDfcYeauGjpUlQUDHMGu1MpXPyll+x +vkbh5hKifOKBZtuCRoiKWbQoGYpx2KkmYO/lSiGjI2EeYoMaQSSDTplJA2rxtJG1OmUgD/tQKM6h +50MOsv3ujlwgzF02ftopJMAAxt5ULeslKujq5I/9w2frNBb+pM6Dkfsuep8n9NH31lJ9SUfvOBgb +F/cf10VlBVuv7q/B61g7fKTW3/0Lpn0IjT0d4IN/82HPzNO+v8Vfg/apkR8osn/crpr0HEno5wUE +cLeJnzNARGSCmCD8ab3IBkAcqctjbhwrV1oqTMAUEBdpABnemRAHvqYu7WASQDNMOw7xAKfHjQMn +YjvDaq/m64RaYFevcNLSj9VWp+3IEyKo3S8U2+RcSXIkthI9SQKAKXV4u5ue21BLY3JngImrb0S2 +TrmCXuYVuKb9pfUkr0nZpA4Ty3maHDSXsQLOHW/8PdOJZbHLvH7AAfjXUOW8rWWGZbssMTZtutMt +BGpbQMniVRE7mTQDzMfSJctYXcpwbLq30MoSFP3KinXO3cQBJ9SRQpu/lvE8WS+jCQXXVAyylaUu +T68TXQubsNNpZK1YwLVsDV3mSSPSFCoPJeTH3rv5dvO2UiYtu1RpKxH0o4xvzNBHXuX0WnRpcGzQ +supWlSyqZMjccfGN6FGG2+Iv4u5cW+ALxNLSoaZUfmwoc1Abq9Jrq3GsPtrbI12gcFNHaI3oJW9q +7Z9qbEpSpapg+NA2ezHnJ4rZxLCrZLVshHYWzdutBXPECCoAjzqRtlKxNpxp22U0tSTLS4kCPXen ++XLW5xR2EN3iXE/SCmwUj3wD9tTSsHLPdUtK1cYiaDnq1bXa4jcWLuy23FAe7b7q616v1yb/ACFa +JMyySg78INc19IViLHNAuG0hPa94jzTxo79Vq/DtjiWHDcJWHk+iuXxFAXrls7xO1etJkCCaevNy +NXOvGGxG6QDQaIbM8AactohMRWyWyD605aSInwoGwTIgCD40MOsa3/4cXh8Hm/1qLSkhMqNDHrHN +T0Y38Dg43+sKCd6k4H4EHxDj36yKyvOpTIyZpJPF79dFZQVLr/T+Dtt4DEmf+Cuk+g1uejfAzEk2 +wn4ml+v8EjK9sYknE2Y8vmXKT6BQ4OjTAyT/ACfh+caAiBoDYCaxxOlAGnjSralQJg7V4+sdmBtQ +Rd2gaZM7nwqLdShCzE1NXS2wmJmot5CSdSQregY3KthE+kUNOnjtF5Hf7PVHatlfoFCim+hOkwDI +86p3SRhyr/Kt/apaKipklIHGRuKDmjKV81ZZuwi6cWSlNzpE8BtE/EiuqLbMPtFi2WVDVA2rjXEy +5bX1skApW2AQPA6ia6awJQNvavpSZWgKI5EEUBLwmwbxFSHMQYZd098FSQQD761GMWuK45eYPhYT +cKsUJNw4ncIKphI84BJqs5hzKcPy4+q0lVypISNIkk8gPEkmKnsuZCvMHyEi3sMQFnmC6V7Re3Rb +16nFcUnyTwHp50Epmxm3ssnrYeUe0uEHSIoIYg0cKS9cOW5Uw1ClqHEJ5n1q59JWKYhh7tthN0+u +7et2NetKY1pHExyoft4lfYnir1u6sPWL+mUqT9HxE+HxoLhaXS2sND9k5rbdROpJ2INa212OzK1L +nn3qhMtMPYU67hThKmEElon+YeHw4e6nFxPblQBKPyY/x5UAt6VLwPZoba3DYbn3knf6qLXVGbcV +e4u8QezS02kEjmSTFDHNuXcRxnNDKrG3U4SnRJMSZ/711B0I5OcyhktuxvA37a4suPqRuJ5CecCg +ujoEQfCtmOI2rZ0d2vEpEbcKBy2AVcvfS6UgbDnTNJPnSrRK5UmRQKqC9ESDvyoY9Y4q/FlfiQfn +G/1hRPWogaYihh1joPRhfEj+Mb/WFBMdSon8DgDyL8emtFZWvUnCRlFYSRIL0+utP7IrKCr9f3/R +q2PH90mR/uV170EKKejLAxAP+TbfpGvOv+CMsWpPPEmY/sV1r0Bk/izwPcx7Od/zjQEhlRUNk+6l +XEymY5UmyDp4x50uggpHemgj3mZSSRA8aadmEqAVFS9yfmzHhUU8UzvyNB4tpBG4NMLy2YeZWFN8 +QeNPA4QdMkT9VIvOHSUhQ3oORulXJl7a5tvbq2tyq3U4FIQBvBAO310bMBsQcKw+4ZIIS2kKSR5V +KZ9sPaWm1WqW37i2lSxH0gBBFUTLeamLW+Tg97cBt9StSEcAB/N9aAws4LgLCbTEbrSi3aWLo61d +1Kk7ifQ7+6vXeke2uLdCsvYLf46tThSk2zKuzBH85wjSBVeTeu3rCMPNom/tnFauyUdKfzvETyqb +fubpFs2wthtCUCENsSAPACKCk50xPMvt72L4nkt4Xa2SygtuIUkIIjeFHkfKh5huMWdk+W73Bb2y +UDu6WSpER4iYq45rwS9duV3Nz7Ssap+ceJ5cAJquM9o2QEIJ7wkKPCgnsHxCzxa1F0w6haSkplJ5 +UjdH5tdwe4gCRJj/ABzpo0lq3X2rbKGVLBBKe7J9OdUnpQzULWx+R7R3/KHk6VkH6CefvNBMdEmc +13PSMm1u3W12C3VBgFIlJ8Z84+uuvLIzaoKtyRXzpy7fXFjcpWypTa5ACk7Eedd99HmJt4zlHDbx +D6XlKYSFrTwKgN/roJ1xHdFY2J3mt3EmIrUCO7FAqhAVsRtThIAiCD5UigGCZO1bpSqNiaBR3SpH +CCKFvWNI/Fne7TLjY/vUTl6tJoXdY0KPRnfGP41sf3qCb6lP+h7h0gDW8J/PTWVp1Ilzk59E8HXf +tRWUFX6/5P4NWoKhHyizt/8AS5SnV+bDnRpgaU7xb8PzjSH/AOQNUZfsk+OINH/dOUr1eFhHRlgg +3nsJHl3lUBONvGxTBitUthPGZpVLx3kztz51p2xJ2igTuWu6ajLhvjvUs8XltEttqWI3IrZGFJWl +K7l4hSk6uz578KCtLQrXEz4RUknAHT2a1uaFkayI29KnWcIsrZ/tN1rSJgqmo1u5vLpt1p1pTUPE +srk7jjvQBvH04hhuarxzELlaGn0Q2hHBJ3gTXOefrwt5oadUl1p9h4HQvYxqkGuoOmF+1U0bJ25Z +aeWsKMqEq8QJrm7F8PcxfPTLKy12LSzD6mtaFJBkBQHHwoDBaYld4c004SpbRAKVDiAeVWFrPNi1 +aAugrlOgEHcbc6jsv263sBZtbhse0sICXE6eGw5elRSsJtA4/rag8Y+6gaZqzheXlyvsSWrdHdSh +KREcKgm8bt0IlxYTGxkQaZZvtW14u0D80yGwVRtJBqCxRy2ClOaU7bJkzvQSmI5mff2YgNjYeJNC +/MTy38auHnFd7VzNWW9uDDVtbgLed2QmOE8SagMesfZr0NKJU5pBVvxJoNMLt3HvyoI70zw3/wC9 +dP8AQRnVzD7K3wm5uGlWSTpSeaT4ek0CMm4GL68asw65LpCNCB4kcffRz6JMAbX0iXOEBxsJsWUO +IhI0qBH+PfQdDsrQ4RpcSraYml9AUkq2EVDW2GO2OKe0hsr1JIUUq2I9KmEPs6ezSpJWRtNAow2C +jma2TqA7vCvGFOCQpPpW5PmRHGg8UCUgn30LusgI6Mr8CZ7Rs/3hRWOyBznwoXdZRMdGV/wMuNfr +UDvqQz+CNwTw7Z0Ae9FZXvUiH70LiOAedjb/AGdZQC/riY9cZgybaXdx2QWnEGUKDaFIAUGV6tlC +eYq1dX1BPRrgRHO3/wCY1WuubhNnhOUrC1sX3LhtF61LjiYUo9k5JMVaur4kDoxwLf8Ak5P95VAT +A0rmB8a2w+1L9ylomATvHhSrTetMJBJPKp7B8P7BsFMdqrdRPKg8xK0Qzh6bdlYbkiSBvE71AnCc +TxHGRcqAtrdteyie84ANtuQq6KZSmCoaz4mtVpII22oIe2wq2t2tMOOL3lajJ3qDzc37Jhb2lSkF +YgL/AJm3GrhpUkn3VSOljMWDZawBy5xd4gOgpbQlGpSj5Cg59zRaP67w35beeZAUwp8gl1JO59QK +p/RVgS2XL/GL5am+3LyWg2jVI4QBB47/AAqbxvNb2a1vMYQ+i11KSEQ0QVeEq4/DzqwdGjLlvlkL +unHEPquH9bkAqRGqVAcyDvHlQXHEMETbu219aJKA+yhtxB5EDYn7KrGYLY21yEOoLayCFBSYNF+8 +w4sXFu+EqubNTRVp4qVpbBA8NzJk1TukZeMu4fdN4ZgaMTcaLIDIZDim9X0zMgwPf6UHP2e03Tt/ +bN2rS3HFpACW06lKPkBxppYdGmasQR7RiCRhtvpK4cV84QP6PL311ecm2jFs0xgybSxu1sIcDxZk +lJ4jYgmD9tVvFsoWiQ45mDHVvW8QvfsEQDChtufpJI35Gg57Tl/C7ZDjdtcsWz7SE9s65K3EhXPh +z4eFVvNtrg15b9ph63W32nEMtoKCS8Oa1K4DyFdE57s8k4060zqQbi2SG0sNgw547jYgcffQr6RL +Sywu6ZFtbN2zCXO0LaR/CECdz/jlQVnLWH3RxRoWoLEaVLUtUwf5225H30XeifGlYJnO0tn0ds3f +/MG6UgJgkyD6HhVUyW3b3l+wEpSxcusjSDJC522+HD0qy4Thq8VzrY2QcKmRqaeCO4oECAoBQmAR +QdTWTYCAVLCieFe3WFWV4pKnWh2iFSlSTBFQfR85eHD3MOxLvXlkoNqWf4xP5KvfVwZbIIOxmgi1 +4elgSkrIPDypqtlQE1ZnW0qbg7yKjLxlTKVAJGmOJoItepKR3gDFC/rJKJ6Mr2AdJcb3P9aig4VE +bDj4UMOsekp6ML2SILrUfpUEj1JmyjJrytiFOOnj5oH7KyvepPIyfcTzccP95IrKAb9dN9p7J1gb +a+urtkXjPefIKpDTgIMbE93iKtHV1LKujjL5fC+z9mIMeSlVS+tZh2L2HRrZt47Zi1vTiTQKO11w +OzcgzJnh4mip1YcNb/FTgDrqQrVbE7jh31UBOsLcdhrbZ0JiEDnUvbM9miIk86SsdK3VqaVqbTtE +bTzp4J5UGq0kxArRxsngDTkkgbcBWizwmRQNXGjvFBHrLZYxXHUYabJhbzDetC0gx3jEb8uHOjq4 +ElMzNUHpcfScEFmq5RbofVC1qXpgDfY+M0HI/YYtl1a3bmyt7C3LnZ26zHarIMaoEmOfhRKyM8nE +7Fm4tihLaFKWHNMBKtCiVkb89yKHXSo7a2+IlouqKWkKTDjsjURsB5wZnzq+9ByG2cCtmlKI1rB4 +mSSOBI5GfhQG51SMLwhd9d3wWtNqhaU6YTAG+mPEmn2AN2tvZm9cCW37sh1ZPiYAHwgU6Yw9rEcv +27ToAOgBJT4D9hpZGHNuhDK4T2YiOQoEsVtnlLtvZ7QOuIUdLmqNCSNxHP0oZ490dXl5jryPlJ9F +m6kOOslZWFKPE7kxwG1EZnGGnMSvLNKoatEEvOzEE8BNKNOYa7iSk2zrbtwWwpxSSVGOUnnQU+wy +Rg+HIQpNqhSkc1JGxoE9OeErv82Iw9huO9Ko2CUwkn7a6rv7VTiDpMeBP21zn0j26lZuvLy3e1Ia +CVSonhI24eCaCjWtk0xcW5xC8RbtWoCmm0nvEJMgGOG9WHo4dxRzOthdXGHutoQ+oF7V3QF8IJ3P +ED31EvM/InbEqZccvHNRURqCWwIjf3mpfJ79xdZywtCriXC+lxSNR7yAdyYMAbbCg6WbZXa4na4m +2SEqhm4HLSfon3GPjVub2G1QVtbC6wxbau6FoInwqTwl0uWqAsy433HPUUEnJg7CKbvoC0qB4U4k +cfqpFwlSjHKgg7pstrIKNPhAoSdZhSm+jW4kDvXLSfrP3Uar5jtGpmKB3WkXp6O3GuZu2p+ugn+p +aQrJTp0gaXHUkxx7yT+2sr3qWQMjL8S69z/pJrKCqdfhR/B21bIgG8ZIMcfm3Pvq79WMA9DeXdWw +9nV8A4qqf19Qn8F7NYBKxetD3FDn3Vaeq68X+iDA0L/ikKSNuI1E/toDCNKW+7tXjbknTMKB3pNR +BRusSRwpG1bQlZdTqle5M0D8pIRIMzWi0k7kVs26VARuOHpWy9kydtqBvyAoR9PDdy6W0NBOlLJV +KuAMnf3UXlQTx40JOmrHLe1vfYLllLluGPnSeRPn8KAA4sxhGY0IwXEbdAcaKSxfJRpVpJAjVzTx +MHwFGPLGXLXA+yw6zh1lgobSswZKYG9B3GUWzCEXdmXHGFPoShKR/Bq8FetHnJTbjlip1/6Yuz9L +YjvJP7aC9ZYYPyUyoOiQkApSQUp8hG23Davc2JxC3wi7fwlkvXvZHs0AgFR8idp8JpbKaAjCGwmS +EqKQeaoP0jtxPH31LPJ7VlSRsY29aClZTwFdnkq3axBhab19IuL8FYUpxziQo8Dvy4VL4Szh7XaN +WduGlIjVCCBvy4fZUi2T2imjsJmDSVuy6h59bjoUCrupH5I8KBtjTybTCLl8qCSlohJP84jb6656 +zk0FX+IS0pbhW2lKUnc7EzRw6QHpwL2clZ7d1KShIOpQG8SOA2Ek+nOg3iuFrxBV+Ge0S4HHXAAd ++4Dtz5GgodlhTuLOXV9blt+3t1qTcIKtK2FA8DOxnl76neizALh3NTd4ppRJCVIIJ+jJ+HCvcq4h +8nZdRamwLpecU4tLg1BaCd9W3Ab8TPCp/oVxC3s80Iw95d2lVwSlhLoEJgcoHODQdA2Dei1QmDJG +9LJZ7N4vN7H8pP8AOrGNSedOAQU7+NB6l1KyIO8SKb3L4QstEwTBn1P214NFvreIJK+fgPCoyzWi +/eXfPhaQh5SG0qVACdoMefH30Ek+4lCi2dwQTx50Detesfi6J4TeND6lUYbgqf7VsK0KSe799BDr +VOLPR8pCjwvWgY9FUFr6k8fgSuP/AFHj/eR91ZXvUm/0GV/Xe/XTWUFc695P4NtJO6ResHj/AKty +rb1YRo6IMvAAd5lZ9/aKqqdexKjgCIE/5Vbn+45Vs6ti9PRLl0Dj7Or/AIi6AqO6QtC1qjkBHE1o +HCEKCdlQSJ4DwmtL3UGwtKgkpM1s22OzCn1pJMcOFBtgq1rtRJKlT3jBAJ8p4ipIokAz7qq+FYnj +D+OutrtbVOFto2eQ4dQVJATEQdoPvqypdA3kEcaDCxJ5jehL0k4JbX+aF2D4Kn7tslsaZ2jaD4yf +qovJeBA9aGGbcYUxmS4XcLW2GwopMDToHL12nagDeMWrOQ8Ut38Te+bfeShTDYSrWUwZUOW870UM +MxBD2EX17bgFKrp4twdjCEkfZQn6VMHxDMuZrB/CXbd3tnA0u2dX88gkzqG26efjtRiwWy7DLhY0 +gr9ofQTp3nSdx+jQXTLLxUxdJVGpL6jtMQrcRPkR75qQQ+kKUkq3mq1ktSoeSlBQkttqEj6RKASf +MST7wamXwWX+2EQdlDwoFApS7wmBoB2ptbG6TdvOOvsG3OzTSEFOnfz4+tN7fEW3seXZNmS0NawP +yR51o9d3Vzj6mBZhq3aSfnCoFThkbwOAoIHON2HsXtbALWgBQVISd1KJgTwiEmR6VV8rgqexG9AB +0WjzgJG0mpa5Wm4zViN8hQcRaoXJ0gAdmgAAEHeFFcnjO3KkujlTSbK+uX/4MMLCpH5Ox4e80Alv +7O89rtLK0tlXK1EgISqEK27yp2HAbDyqU6MsLXhucEqWyHHGnNTbhSJIVPCCY9Kl8CtsYvbxzFuz +bu7dLmi3t1PFGpO41pjhyEceNPMpl+zzkwzeKLr2IK+ilOlCAlJIgevPnQGFNwrsCofSArXA7x29 +Sokd1KiJ9K2QzpSNUwRxpLB22MPs3xqVHaqWeZMkmBQa4u+3fONWdqVrLdylL+gwWiBqTPiOHuNM +ct3D14L9zswi1RcuIb1cVadp9Nqe4241h6HMQbMLQytao4KhOxPjwpnktl1rKNqHge2cRrcP9JW5 ++2gmrRsItgTBJ8KA3WzRoyUFJ4KvG5+CqP6GyGUgAcKBnWzQT0epIH8ub+xVBYepbIyQ2AQR8/Pl +84Kyk+pPq/A10HcBbo9O8jasoILr1ScBRBgh63j4O1YerOo/ilwHURs0sbf7VdV/r0j9wkqn8u3+ +1yrB1Z5X0TYFwjslgf2q6Aq3X+bSPCeNIYchNywQ5JTBETtTi4SVWy4HLamuEhSElJM78qB1CGG2 +7VpCUpnvnkkePrUQ/cXjOYmbFAUbd4FSf6McR9dSmINLUJCCptB1KTzX4D0mqhmPH3MvZgZvbxtR +sy+1bKITMLcCt/iAKAjhsJQIiYoW9Ilp+6V44yg3DhQIQ2qTJ2I08440TWHw/bJdSkiUzvQ2xxdy +xib7rDalvF8nYctXCaAMYfjWJp6Q8Js8SY1D2xBCwkoIIOwE8eKZo34OV+zW0pK+1vS4QFcAttdD +XGMYwu66SLG4xKzadvWLhDbLikFKgeB9QJPwotYez2YwxI4FDK9/zx+2gTynCb20WVhRfsQAmBKA +hRmY33Kvqqw4mlCLRxxwEpSJgbk1XsHhl7D5b0Bu5fZU6Y4EylBnfcmdvDzq4ONpXG21AN8i4ZmH +D845gfxLsBZ3LiV2znaa3FbniD9EAbRw22qw2bTFniV865dXLj4BUpK1Hs+E90UviDqm8UJgCFAH +zBH3iorPVwm1wdy5ceLZdSGUKClJAUo7GR/jhQVxntG8v4ze3LTSHnWtB0DYFaiYnme9x58aQwNt +bWT8XdSrQPZwkq8CZn6op1i/zOT20JVPbuBSTxkAEj7BS+VOweyhiDlwvTbuFxKlEcEgQf20AJuF +Y1eW9vb2jtxpYK3FqZdG0ERPDxiiV0WYWu8xNGP3ilpdb7gSsGCqIkVBXgwDCMFQ40t5Srx5QZd1 +hSVRAkDkB4bVa+iS+YcuHMMR2kBHtACySdzvx5cKAj3t8pi1WUtqWoJkAVC/L9naWKbh1Sbi8WUM +9i2QopWQYkDcDjvUvi94zZYY/dOphttsqJAnYChDlN22ds14xiS/ZXrm+cuWme00rUClPZpUBvJA +n86gvrz1zieBui4T2NzeOIY7IHZvUdwPQA1dLVkW7DbKYAQkCqlhlstzMOCWOgDskru3gTMHTpAJ +9VH4VeFp+cMAbeNBouNEBRG24oJdbFH/AIcpM8b5refJVG90d0bbmgp1sEk9Hze8RfN7fmqoJTqU +6fwKfE97tndp80VlJdSrbLFyJnvu/aisoIrrzpP4OoVy7S3+1ypjqvuT0SYIkk7B0f71dRnXlSDl +hKo4Lt/f3nKfdV0a+inByPyQ6D69qugM6BLShHEUyw7a4dQDMHantuklBHKmjbfY3ijBAO9BviT7 +bPdeMJcQQY/Z9dQV6S4n2e+CXgp5tbIUmSUpgyfMGneeGFuYCt5DikKZUlyQJ2Bk+u01st1tWYbS +zCNZuGlKkD+DSIG/hMigstuEhlO0gjaqRmppJvXvY3GW7pSiYdMJkDjPjV8Q2EtgDkIqi52tmlG4 +L3dE90zxJHDyoAnjDdwzm21uMX7uKXVwA2yIKUJTxWFDZRO3186NzjqWbXCVE8EIT8Ck/fQlunm3 +834azcWVtqt3VJQtLkqEjf7d/Si9i7KU4HaPJ4MupmPAgj7qBtdtqRbX51hPs16h8o2GuTATJ9QR +5gVamHSS2VRCk+FRl9bj2m7HY9qLi2JCAY1KA2G+wM86c2F12+CW7y161pQNStpkcZjnQNcetwhZ +dA7y1JHHjVTz5cLdcw2wYQHpd1OTuECISvbeZkDx34xVhz1iAscKNyYKUd4jVHI0PsOujfZgaxC5 +uVIbZSFFtPNOkOBRj1IA8vSgks4OMpNtZtAFthWohJHArA4egVTjCbFf4txag6XHWCtfA7rOoj6z +UFfF66XcXUavablQSTP0Q2qOJ/pD3ir4y2G8Hbti1rBbCVDgIAoOfsaxFOG481hC3LZ21QrtWm1t +hayrgrSeRIMRwgUR+ibCWkdpj3cC32g2hCTPZpHI+e31VRs2ZYtHLxePKJUwpxTSWUcdYPDUOCSI +38iKu/RG4ub+xDehhhSVISeI1DfhtxFARnG27izW28gKQRBBHEUEsFwkXnTJcYeo6rSwWl9CI4bC +J8d/so3gAtQKpOG4a3adIuL4i0nvOWaJ24qkxQWzKzYdxbEsVIOnULdonwTx+s/VU8rvKK5nwApD +CrX2TDmrfYECVkCJJ3P106aSJ8qBEqIAEDhQZ61xB6PmduF83+qujY62nwoJ9a8j8XzQif8ALkfq +roH3UsP72LoT/GO7e9FZWvUtIGWn0wZK3if0kVlAy68IJyxO8Tb/AK6/vpfqquBPRfhQJjvuiDz+ +cVSXXg3yuE+HYH++uvOqopA6M7DWB3XHY/TNAemBKCBSdy2lDqSY350lZ3KdxIn7a0xJ0ltCkmCn +eg0x1CXMJfbIiUECee1UToGzLe461jFtibRVcWF4WUPkfwiOQnxEVaMRxGbRYWZEGo7oktLS0yym +5YA1XTzjyz4kqNBeytU+NVvMFlbYip5t5TqVEBIUhUaTyPhU4t4p4c6qGbHL9DinLI/OIlQSTHLj +QDfOeCsYNmBDlpZEvKR2i7kypSjqg78Ad55UUFLL2WlIUkmWwRtPDehPjeM3t04W7nuBtxsFA2JK +l/ZANF7C1JVh7bSu8kpjc0Gl1iLbVrh12tQRKkoJJ2hXdH1kVtgbgFvdWyne1LbihqkHUPHbn4+d +QePYbcYng7Fqw5p7J4FSvNCpA+IFLNXTeFIxS+dSUKTal5QJ2UUpMke/9njQD7NmNX+dM7KyvhyU +2+HWw7R+4c37VKFaVCBwEggTE0/dbfQWbewabsy6pWs9nJDadjII2Jnlt4VZzY2eH4cvFbawtkYl +fNN+0PBAG8TJ99Nra0Q7jTSyhLa3GktrK3ZEDcpTzJkn4cqDW8swi5wu23AbaU85pHFS1AD6pq2L +HzACRAAqKdQh7FiscilA9E//AO1KuL1IMcAKATZrFthgdwnEb1NjhhJdU+133DqV3U6dJj135VNZ +CxrK7PZ4Xg945erMBb3YmSTJ75gR68KpvSFdtXV3ibz60e0W1ou5LSRsEAqS2k+oMnwIqMyLauYM +3heI3lk9bKfulWt20pWkgLIgn3EEUHQEd07kCq1lwPO5qxRTgGhCkBJ8QBP2mpe2eXZdnaP6ltaQ +lp8qmT/NV4Hz5+tQVveLtMcu9AkuOoB9KAgNqUobEA1oHFa9JFJ27mod0+tK7SFzvzoN1lWmZ4UE ++taf3gsTzvkAforo0uujSQN6B/WqcKskWyRzvk/qroJfqYT+DTuw2U9+sisrfqZD97D3kp2fXUis +oGHXc3y3Eb6WD/vF1TOr7mvCMJyFbWt3i1nbPJdc1IceCSJVI2NXTrrtzgIUebbH1OL++uNnEkHa +RvQd42PSFlkAFWP4bI//AJKPvp09n/La298cw0+l0j764C7yTxPrXhKualUHcGN51wJVo6pvGcOJ +0HhcI++k+jDOGCYZlJi1uscw8LC1qANygEBSiY4+dcSHUocTvXg17gK4UH0Jts/ZXUolzMOGJA8b +pG/11Vcx59y6rGJbx2xcbKIMXSAkHj41xBqXEaj8a1UTO5NB1FjucsuqddGH3tj3HWe0V24AUe/M +c1RIPGN6I+HdImVm7VtCswYYCEj+Up++uFIJ8ayFfzj4bUHdFp0j5TQ++hWYcNSgr1g+0p58ai84 +dIeUVWi1s47h90lbKmXbcXAhaVbevjw8fSuLAFR9I153juSZoO2nek3J5Nuk5isAlpIIT2oIGxHv +2Net9KOUWFBacdwwlcqWQ6JnhwiuJAFExJisg8N6Dta16UMn9sFuZhsATqO7nifup4elTJn5OZLD ++1rhzSrzrIJTtsaDoPpMz1gb+NG7sL21uBd4Y7au6FA7laon0mansdz9lrEsh2S14tZ+2pQytbYW +NWtIAPv2rl7SRsTNbBJ8aDtW36UMn3Fg2HcespU2nUC5wMVBPdIWWk4s1+7to4nWklwrHAHnXJAC +toJrfccKDuWz6UsohO+YLEf/AGU5b6UMnkE/hDYE/wC0rhi1WG7hC3UFxCFAlMxqHhNSPttq46Sq +1S2kuqXAPAEyEjyHCg7Vd6T8omNOYLDf/WULOsDmzBMeyqzbYdidtdOi7SvQ2qSBpUJ+uufFLK3V +rQAlKjIA5DwpdqeZJoOu+pjtlh8Rtqd/WTWUr1N2+zyo54rLiuP9NI/ZWUDLrop1YAgaTu21B5fw +iv8AtXHrzMmNJ25iuzuuIytzLgUB3UtN/Hta5DW1J+jQRBZAMwTWBid441KlqTFZ2IkkpE0EZ7OJ +3FeKY7xhMVK9l4CvOy24cPKgiCx5D41qWoMQRUuWSREDjNeFkapIoEEi0Fro1Q4EJAIRzBnf69/S +lXrlpSrwtdnp1ksgtjgVSeXhtXpYT/NE16GeGw28qDxm4bLlmXez0hep8dn4K25eG1aBVsm00KIL +obUmQjiZOx90QeVO2WmFFKFMjUTx1QPftTj2FggkBgDURBe3+ygi7z2Zxp/sVJSlTiS03o3QnfaY +9PXjSeG+zNsuJfSky82RIJOkTq5c5G3OpP2JorCAlkd0GS5sd/t+6tnbW3aX2iW0KSFboS7Mj4UE +Y8m2DKezUndrTu3BCtczw8K2U1a+03L6XG9Dpc7NGg93+by293hSzrLanCW29KTwSTMV4liOVBHP +WiUL0ocS5wkpBiffSaWNztvUr2G86a9DMDYR40EX2BE6RNbJYPGNvGpMMA8q9LJ8B8KBghnfhSoa +QdwmDTzsiTukVuGjEQAaBu22EwIpw0gTsK3S0rbu07tmVTPCg606nrShk5bihA1OJHn3xWVKdUtC +R0bJUEgEPLB24nWr9kfCsoNOtVaOXeUEttNqWsplISkknStBP1Sa5NXhS9KdVncSAdXcO/hX0PxG +wssRt/Z7+0ZuWpnS6gKE+O9RRyblgmfka2HpI/bQcDDDGAynVZ3Xac1Rtz/7Ui9hqVbtMOpE7yJr +6ADKGWxwwlj4q++tGcmZYZWtbWEMoKzKgFqgnxiYoPn/APJrv/oufCvBhywf4Je3ka+hP4L4BEfJ +jUep++vBlfAAZGGNT6q++g+e3yavj2S/gaw4a4Y+Zc/RNfQo5XwAmThjU+p++vPwWy/M/JjXxV99 +B89Pkp4meycj+qa2GEvT/Aun8019ChlfAASRhrUnzV99bHLeBnjhrJ+P30Hz2ThL+/zDv6Ne/JFw +Zlh39E19Ck5dwVPDDmR8aw5dwX/49r6/voPnl8kvD+Idn+qa1+S3Z/gnI80mvoactYESScOaJPmf +vrwZZwEGRhjIPv8AvoPnmMLdH8W5+ia9+TF7y0ufSvoYct4EeOGMH1BNe/g7gn/xzP1/fQfPA4cs +H6Cx5RXhw9QH0VV9DXcsYAv6WGMn4/fSasqZdJk4Wz8VffQfPgWPPQfSsNlvOk719Al5Qy2RvhLP +xV99aLydlk8cIY+KvvoOChZ4fKtSbgbbcONYLO030h7yMV3j+BuWNR/ce34+f316jKGWwSBhLIB2 +O6vvoOE7e0swiXRcSBtoAiYP7Yq69FHRxdZ9xt2zs1qtLW3QHLi5WmQkEwEj+keXoTXXyMpZcSNK +cJYA4wCfvp5Z4DhNqoqtrQNEiDoWoftoGeRMrYdlDBk4PhfaezoMhThBUSSSSSAPGsqfSAlISOAr +KD//2Q== + + + +--Multipart_Sun_Oct_17_10:37:40_2010-1 +Content-Type: image/jpeg +Content-Disposition: inline; filename="custer.jpg" +Content-Transfer-Encoding: base64 + +/9j/4AAQSkZJRgABAQAAAQABAAD/4Q1kRXhpZgAASUkqAAgAAAAIABIBCQABAAAAAQAAABoBCQAB +AAAAyAAAABsBBQABAAAAbgAAACgBAwABAAAAAgAAADEBAgAOAAAAdgAAADIBAgAUAAAAhAAAABMC +CQABAAAAAQAAAGmHBAABAAAAmAAAAOYAAADIAAAAAQAAAGd0aHVtYiAyLjExLjMAMjAwNTowMTox +MCAwMDo1NzowMwAGAACQBwAEAAAAMDIyMQGRBwAEAAAAAQIDAACgBwAEAAAAMDEwMAGgAwABAAAA +//8AAAKgCQABAAAAyAAAAAOgCQABAAAA9gAAAAAAAAAGAAMBAwABAAAABgAAABoBCQABAAAASAAA +ABsBCQABAAAASAAAACgBCQABAAAAAgAAAAECBAABAAAANAEAAAICBAABAAAAJwwAAAAAAAD/2P/g +ABBKRklGAAEBAAABAAEAAP/bAEMACAYGBwYFCAcHBwkJCAoMFA0MCwsMGRITDxQdGh8eHRocHCAk +LicgIiwjHBwoNyksMDE0NDQfJzk9ODI8LjM0Mv/bAEMBCQkJDAsMGA0NGDIhHCEyMjIyMjIyMjIy +MjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMjIyMv/AABEIAIAAaAMBIgACEQED +EQH/xAAfAAABBQEBAQEBAQAAAAAAAAAAAQIDBAUGBwgJCgv/xAC1EAACAQMDAgQDBQUEBAAAAX0B +AgMABBEFEiExQQYTUWEHInEUMoGRoQgjQrHBFVLR8CQzYnKCCQoWFxgZGiUmJygpKjQ1Njc4OTpD +REVGR0hJSlNUVVZXWFlaY2RlZmdoaWpzdHV2d3h5eoOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmq +srO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4eLj5OXm5+jp6vHy8/T19vf4+fr/xAAfAQADAQEB +AQEBAQEBAAAAAAAAAQIDBAUGBwgJCgv/xAC1EQACAQIEBAMEBwUEBAABAncAAQIDEQQFITEGEkFR +B2FxEyIygQgUQpGhscEJIzNS8BVictEKFiQ04SXxFxgZGiYnKCkqNTY3ODk6Q0RFRkdISUpTVFVW +V1hZWmNkZWZnaGlqc3R1dnd4eXqCg4SFhoeIiYqSk5SVlpeYmZqio6Slpqeoqaqys7S1tre4ubrC +w8TFxsfIycrS09TV1tfY2dri4+Tl5ufo6ery8/T19vf4+fr/2gAMAwEAAhEDEQA/ANNRzUiIfSlR +DmrUUfrXOajEiz2qZYPap0jqlrOr2+i2RkkZDMR+7jLYJPqfanYCd4OOlQGe2ibbLcQo/ozgGvPL +vxRqE6kSXsj+yfu1/JeT+dYsl55m7JUZ6kD+vWiwrnsAlgfG2eI/RxUhjrxlLhgMLKcdua0YPE2q +2mDHfSMB2c7h+tFgueqeVimmPiuO034hKWEepW4APHmw9vqK7G3ube9tlntZUliboymiwXECCgpU +oSnGPNIZEqgUVKIz3opAVUXDVajFV4/vVT8R6yNC0SW6THnsfLhB/vHv+AyaaApeKPF0WjK1nZ7Z +L0jk9RF9ff2rzC71Ge7naaeZpZWOSzHNVZriSaRndyzsdzMTkk1GFLHABP0qhEhnc5xWjpmg32q5 +8oY9M96hs7DzHG4MBjJJGK9j8HW1ta6ckqqHVunHQ0BY8wl8H6hDGWOcjttNYtxbzWxIcHivoXWN +X0nS7YzXh5I+WMDJNeY+KTb3umHUotOlhgkfYjkfqaLhY4ESZrb8Oa/caLqCMrFrZyBLH2I9fqKx +Y4mkcIilmJwABkk16R4V8BmDy9Q1dfnGGjtj292/woCx2qjIB9aftp+MUvFSMYFopxopDKCDDdK5 +L4i2V9dWdlJb28kkEJcylBnaTjBP68116ctWjanFUhHzpit7SLa0t1W7u5lCk4AxmvZbzwvoGoOZ +rnS7dpDyXUbCfrjFeZ3WkW0OuXEcMMbQK52RljtA9jTESWNvDqExntAJDu2YEf3R6/413Xhq7itb +IW88MgKMct2zmvN47h9C1WCS1uzGGbbIqnPynsexFelLMloUMjF4pVBEh75oAvX50q7+RIXknkwG +Pt6c9q0bu00e/wBEk0m/kht4pE2KrOAwPYjPcVmWlza2TSXTbWwMgep9K5zWdavNSmISzS4bOMjA +x6AUhm3pHhfRNFVZLG3EkmOLiQ72P0PQfhWnITUVjJNLYQPOmyUoN6+hxUjZzQAwr7UgWpKCMUgG +7RRRzmikxlGNavwDiqiDB6VegGQOKpCJygkhaM9GBBrzHxFp50+5la8tnO4DbIg6LnqP0r0S/v5L +K1MtvaS3bZKbYcHDDqD7+3WvONT1u+1W/jbUSVtY3yYEGAB396YjCtLQXt1mOBhAjZDPyT7V3GkX +bJbtp92pe1I+V8ZMf19qzta8T6dounx6dplhbySkbi/U4PIJP9K4y88Q6reYMk2yPOQiKFU/h3oA +73VLC6trFrmCVJ7dTn5X6fhWLo3ih31i2N3Bi1iYqWjHTPc+uKg0zxCb+2NrNaHj75hOFI9wKs6j +FF9hf7CotmjAZWTj659aAPUQVdA6EMpGQR0IpjCuJ8GeKVNtJYXshd4xuiKLyw7jH9K7SK4guGdI +nBePG9Dwy5GRkHkUDFA5HWnMBikIwaKkBo64FFAPNFIZVQVi+IvEk+mSJa2OVnxueQKDt9AM1uov +NedeIJkufENyySEAEIMHjgY/mKpCZaOri/hjg1G3jnhRtylB5LofVWXHP1zWpqF1pt1pg8+4jmlS +PbA6o32lj/02ydpGOMjr+lcsXliHOJF7jHNQSyAYmiPyA5I9DTEQnTVimMwywHOD6VoafJHp13Hc +pZW92gJIhnGUORiljcSJnqDUMTBQ0fdf5UAaFhANDj1STUbP7PJN+9EQHHlnJUKPTJNSW4G4jqCB +1qKKzu0EeoXV0k0N0CqiSXc4C5HI9OaSNsXCIvCg4oAzNbj1O01iDVXhS3km+dPKUKpHTOB61b0j +xNLYayl3JGBE6COWNOAQOhFMv7QGCa7N5E489kFuZCXTHOcdhWMGBJJ6CgD3BJEliSVTlXAZT7Gn +AA1xvgnXJrxX0+5fd5SAxEjnb0wf0rshjFSMTABopwxRSGVgDn615TNCTNMrctHIwP516xzuGK8z +1FBa+I7yJgNrynH48/1qkJleOZo0GMstMlDy5MKxsx6g8GpGUxSnb9084p5RZUyuFb06UxGfazPE +WicFWB4B9Kk3fv8Ad68VDOri4BaNt443D+tEchKsfegDQsVQy3DjJcR/d7dR+Va9lDGLR/NkLOzE +gAenTn6iudsJtmoMeoaMg+3IrbtpJVjXa+O/NAGK4Q3M5bG7ewz+NZqAZcdlY5q3bxXOrasba0QN +LNIxUFgo7nqa1PCPhttZ1a6S7Vha2rnz9p5Yj+EGgDLsZ7i0nS6t5WikU5BHp6V61o2pxatp8c6M +vmYxIgP3W9KI/Bukyu0stmgZkwkSsVWMepx1NcOss/h3Wp47K4WRUfa+4cSD6UmM9IGM0VQ0zVrf +VLbzImxIB88Z6r/9aipAn3ZNcB41tGj1gzKMGVFcH3HH9K7pDzXJeNCrTRuzHem1AueMNuOfzX9K +pAznLa7juUCSEJMvr3qZwydRx6isi4ty3zJw1TWksjrs3kOOoJpiL0twwXlCwHoKxri6WFj8pG/k +A1fe6vIODtKmu1+Gc9vcazNHeRxyZh+RXjVuc8nJ5oA4zwxp91rmpSW1nGGnZMZbgAZ5JP5V6Cvw +213Jb7TYgbcKPMbr/wB816pGsESARpGi+iqAKT7VCPlMi5+tAHisPwc8SLJk39hH/tpI+R/47XWe +FvBeqeGtHnhlENxO8hlYRSH5vQDIHP1xXoSzow4ZT9DSvIO1AHiepeMNUWaWCBPsgDFWDrl8jsc9 +PpXMajqdxe3BuLt90zAAsABkDp0rq/ixpTWeuwanbDC3kZDr/trwT+IIrziS6uUcJcqGWgDpPDOo +pb+ILctJxITGQffp+uKKyNFtjc67YxqdyNMrfgDk/wAqKljR64vB5rK8Q6DaapCtxI0qXC7I0Kth +eXAGR3+8fzrTDYNQavKRol465DpEZFI6gryP1FJMbPMEcqxRhytNkwrLInDDriqST/ay3z4lB3A+ +tSrIWQq3DjqDVEl9h9oiDIC2e1V4H1KxvEuIGe2KHPmZxtqkbuazYtC5FVJ767vG/eyEr6dKAPQr +P4k6tZRHcy3KKeGlXJI98VuQ+OdM1EB3nubN3IASWM4Y+gIzXksN4YcB49yjsank1L7Q4ZvlVfuj +3oC57To2vQX8jJBdJujO0hnCnP0ODXWxC6dBuLL6ZFeP2/izwp4hsIodejmtNREflvdRICjnszY5 +OepGOtcvc6vqeh3Ij0jXbnyifl8mZth+maLDPU/i9Cf+EOt53YB4rtcHvypBx+leILdgDbICwrqL +7TvHPiVo7fUTdTxp8y+dIAg9+uM/rXS6D8PbDTQs+pst5cjkJ/yzU/T+L8fyp3AyfAmiXHn/ANrz +I0duqkQhurk8Z+mM0V6BI+FCqAFHAA6CiobA/9kA/+EMRWh0dHA6Ly9ucy5hZG9iZS5jb20veGFw +LzEuMC8APD94cGFja2V0IGJlZ2luPSLvu78iIGlkPSJXNU0wTXBDZWhpSHpyZVN6TlRjemtjOWQi +Pz4KPHg6eG1wbWV0YSB4bWxuczp4PSJhZG9iZTpuczptZXRhLyIgeDp4bXB0az0iWE1QIENvcmUg +NC40LjAtRXhpdjIiPgogPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5 +LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4KICA8cmRmOkRlc2NyaXB0aW9uIHJkZjphYm91dD0iIgog +ICAgeG1sbnM6eG1wPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvIgogICAgeG1sbnM6dGlm +Zj0iaHR0cDovL25zLmFkb2JlLmNvbS90aWZmLzEuMC8iCiAgICB4bWxuczpleGlmPSJodHRwOi8v +bnMuYWRvYmUuY29tL2V4aWYvMS4wLyIKICAgIHhtbG5zOnhtcE1NPSJodHRwOi8vbnMuYWRvYmUu +Y29tL3hhcC8xLjAvbW0vIgogICAgeG1sbnM6ZGM9Imh0dHA6Ly9wdXJsLm9yZy9kYy9lbGVtZW50 +cy8xLjEvIgogICB4bXA6Q3JlYXRlRGF0ZT0iMjAwNS0wMS0xMFQwMDowNzoyNyswMTowMCIKICAg +eG1wOk1vZGlmeURhdGU9IjIwMDUtMDEtMTBUMDA6NTc6MDMrMDE6MDAiCiAgIHhtcDpNZXRhZGF0 +YURhdGU9IjIwMDUtMDEtMTBUMDA6NTc6MDMrMDE6MDAiCiAgIHhtcDpDcmVhdG9yVG9vbD0iQWRv +YmUgUGhvdG9zaG9wIENTIFdpbmRvd3MiCiAgIHRpZmY6T3JpZW50YXRpb249IjEiCiAgIHRpZmY6 +WFJlc29sdXRpb249IjIwMC8xIgogICB0aWZmOllSZXNvbHV0aW9uPSIyMDAvMSIKICAgdGlmZjpS +ZXNvbHV0aW9uVW5pdD0iMiIKICAgZXhpZjpDb2xvclNwYWNlPSI0Mjk0OTY3Mjk1IgogICBleGlm +OlBpeGVsWERpbWVuc2lvbj0iNzU1IgogICBleGlmOlBpeGVsWURpbWVuc2lvbj0iOTMwIgogICB4 +bXBNTTpEb2N1bWVudElEPSJhZG9iZTpkb2NpZDpwaG90b3Nob3A6Zjg2ZTcwZTQtNjI5OC0xMWQ5 +LTllM2YtZDQyZjM0NjM5ZGJiIgogICB4bXBNTTpJbnN0YW5jZUlEPSJ1dWlkOmY4NmU3MGU1LTYy +OTgtMTFkOS05ZTNmLWQ0MmYzNDYzOWRiYiIKICAgZGM6Zm9ybWF0PSJpbWFnZS9qcGVnIi8+CiA8 +L3JkZjpSREY+CjwveDp4bXBtZXRhPgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +IAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgIAog +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgCiAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgIAogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgCiAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAg +ICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAKICAgICAgICAg +ICAgICAgICAgICAgICAgICAgCjw/eHBhY2tldCBlbmQ9InciPz7/2wBDAAUDBAQEAwUEBAQFBQUG +BwwIBwcHBw8LCwkMEQ8SEhEPERETFhwXExQaFRERGCEYGh0dHx8fExciJCIeJBweHx7/2wBDAQUF +BQcGBw4ICA4eFBEUHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4eHh4e +Hh4eHh7/wAARCAD2AMgDASIAAhEBAxEB/8QAHQAAAAcBAQEAAAAAAAAAAAAAAAIDBAUGBwEICf/E +AEIQAAIBAwIEBAQDBgUBCAMBAAECAwAEEQUhBhIxQQcTUWEicYGRFDKhI0JSscHRCBVicoLwFiQl +M5KissJDU9Lh/8QAGQEBAQEBAQEAAAAAAAAAAAAAAAECAwQF/8QAGhEBAQEBAQEBAAAAAAAAAAAA +AAERAjEhEv/aAAwDAQACEQMRAD8Am8Hl60dEGN+tAbnalEBIr5+PUIoGcmllXPSgkW9LxxntVBET +2pVY/Y0tFEe+KcpFRDYRDHSjrD/pp4sdHWIUwMHhwOlJcpXr0qVeEEd6aXCxxRtLK6xou7MxwB8z +TDSSpjelAtQFxxvwjaSmGXW7dnH/AOtWkH3UEUeLjjhGXATV0z7wyD/61cNTTDbG1AKcdKjBxXww +RznWbUA9Mkj+lSem3+n6lE0mnXtvdopwzQyBgp9DjpTDXGQ0mY6fFD6Vzy8DpTDTMR7UOT3p0U9q +KF3oaasm1F5PSnTr3pMAb1FJrGwGRRuX1o64xvRtqYG5X2oyp7UpgV1agCKAQSKFKZHLQozUEoNL +Rj3oBd+lLxJsKrTsaEmnccW+d65Ehz0pzGp9KuI5HHv0pdU7Yrsa+1LqmSBitSAsabdKrXiLxfb8 +IabG6xR3V/O4EVsXweXuxx2/maX4w410PhmJknlWe7A2gRhkf7j2/nWCaxxK9/qs+ohea6mbJuJd +2A7BR0UDoKuJaud54gcYX0Jk/wC46Lbno5T4sf8ALJ/Sqlq+sx3j/wDiOoahq7A5HmSFIwfYf2qA +uLxpn55ZGkf1Y5ptLPynAxVxEsdQXcwWNrCvuvM33phM8jS+Z5rZBz1pn+JxuWxRGvxzbDI9aYbE +rHfXgGFuX/T+1OrPXdZs3LWt/NCx6lTjP2xUGl3EO9HN3Gds/ah8XnTvEjiiywZLsXKj92UBs/ff +9at+heL9jNiPV7FoG7vCdvsf71izzAocGkWkyD1phuPVuh63o+uRc+mX8NwQMlAcOPmp3p8UA7V5 +Gs725sp0mtLiSGRDlWRiCD7EVq3Afi5KrR2HFA8yPoLxB8S/7x+8Pcb/ADqYNfkA3pLlpWKWC6t4 +7m1lSaGReZJEbKsD3BoKtYWEwvSjhQe1GA9aOoFA2K4JoAb9Kdcq5zQEagnYVKpALkHrQpyAB0Az +6UKioTlIpxCMikyMiloU2qwLRfMU6iGe4puin0p3Ah6kYHf2rUQsgVFLuyqoGSScACsj8TfFURPJ +pXDEo2+GW8Hf2T29/t61D+MHiM+pTy6BoMpFhGeWedT/AOeR2B/h/n8qyiSQjuc961ImndzdzTyN +NPK0sjHJLHNNZLk5601klY96SZqrOnRuipJApKWdn70iuTTqztWnkUAEg+gqp9JRrJIQEVmPsM0s +LG9dsC2lyenwmtO4D4PZ3jmKeYp64GQPnWoDw8WeBZIrZUbqMj+dTVx5l/yvUACfw0gx1yKQaOaM +kNGy/MV6UuuALhIyGRDj2qj8W8E3do3mCDIJ3wOgppjICzD2owmPepnWdKaAtlSCO2KgGBBwe1VM +OBJnYUA2BTdTR1zQaD4UceXHDd+thfyNJpE74dTv5JP76+3qO/zr0PHiQB42DKwyrA5BHrXjtc16 +l8Lo75OBNIF+GE34cYDdeTJ5P/bisdRqLGq980cLjpXQpB60dQc1lonymuBd8UsFocveoogXahSm +2+1CsqglI6E05twu1NU69KdW/XFaiHcYXOKznx44xOi6QvD2ny8t7fITOyneOE7Y+bbj5A+orRlK +Rq0kjBUQFmJ6ADqa8l8ba5LxDxTf6vITi4lJjH8KDZR9ABW+WUY03KMCkC5cnJorHJoA4rQ6elBI +2kOFGTjNGUcx32qZsdAubiESoC+eyjcUQwgspVTmZod+ilxk1a+EtC8/UIeZ+WOVSSmNw3/WKjG0 +60tpbeCZJTLIQVKHIO/Tb5Vb7Vbhbxr+1jES82Vjkckuo/d6bHHr7VBtXhrp8ttaxhreGKFfzcw/ +X1rTbdomQBQCpGxA61ROFW/EaBDcsQrcgyo3GcdKnLNTHmWV0VTjdTjb5f8A+UaieFjHK/mHBQZJ +J6Yqr8XXPD8UDie9tkC5BzKPsKrfGnFGu6jM2k8OlxynlaY/EgPYbd/sKyjTOFeK+JeJmtdUSRgk +gDM/wDrjI9fuaiadcY22k3EL3MbRgcpCuB136VjOoIou2ZNlJyK9NeKfhUR4fwS6aZDLYEtcIDu4 +I3b6YrzJdczTNzdtvtViEQoNKwxs7qiKWZjgADJJqS4b4e1fiC/Wy0ewmu5m68i/Co9WPQD3NejP +C7wtseEuTU9UaK+1gj4SBmO3/wBuerf6vtVXFO8K/CR08nXOKoimCHgsGG59DJ//AD9/Sti5eu1O +5zzHem/b3rPqyCgHNGUUXau5qK6a4a4DXcZqApOxwDQoxWhWK0gsdDinEGAelJMgxS0CgVqMmHHM +7wcDa3LHs4sJsEdvgIryU29ez0tobmNre4iSWGVSkiMMhlIwQazXjHwGtLsyXfCt8LWQ7i0uCSny +V+o+ufnXSMvPGN66BVl4m4H4p4clZdW0W7hQHaUJzxn5OuR+tV4Ieb8pG/pVE5w/odxcSiaWNhGm +CB6g9/lWn6XprWmnk5jifPQ9MHPX5AVQtK1trTTrlDkyOnlg+m2/9BTF77W7iN3Ms5UAFiW6elRF +6gsNQZ31aOS3gnjCrIzDnJU9cdh9KXKZto52hYWkcgAxj2OGx03qF4D1qOyv4oNSvVCznq+8cfpz +fP8AStj/AOxtla6VJDaXLXMDp580gwU5zvzAg7/agsXBN7pqWqWYYK4QOQTsexq4wjTrpArMvKw+ +1ZXw1oyxW8byOfPyc4bOKmJWvLRSPMbH5gcYxmoq5axacP2ulSAzx28e55om5Wyfl3qJ4Jt9Gtbt +ruCGeaRsBGlkJyM5O9VaOG8v7oJzFhnY9Rg1oGh6BIlp5WQVC9em+KKstmq3isvLEI5MgqNwfXtW +O6j4EcH2/Fd7qF811PBNKZorQNyRIDuVyNzv7jtU5xBxbxhoDfh7Hh6wEEPWV7gF3/2ov9a5oPE2 +rcUomoXUFvBaRgoAHLSM/cEY+ED077VUSem2NhpNmLLS7K3srdOkcKBR+nU+9CQk96OxBpN8YqVT +dzSecjpSjgE7UFUBTtvQJMoAz61xRntSpAoBRisqIF36Cjcu3SukUBt1oOcu9CjKAaFYq6r2AcZI +pxEvTem4SnMKYrUZPrUbipa2bG9RNqN6koelbglElBXBGR6Go/VLezNlPKmnW0kgQkc0S7n7UvET +iofjLVrjS9PDWyKZHzueigdT+o+9VHmziHSiuu3d6UjJmdZ1B/KBJvnHrnI9iDUqAlxZLplxp6lF +AdnXClhk7j164ovF/Pa6v5v4ZppJCWl5JCNic8uOg3Pak5+JntoRaS2DRyQrkCQbgEk9vcn70ZV/ +XtFktLCS4ihcRscDKbjpt/Or34IW8eraeLS4vrkJbyeYsAlbkLDcZXoaoGtahf3/ADT80nk8u4J+ +EbdKmPCrU7jh/VI71gTaSEeZj933+VFbglytnfGKdVUMxOMGpaSS3uYEBQZGQDmm2o2FvxBpy3tn +MAWXmSRNwfqKqY1HUdKumt7wFcbFlG30qYrQOHohBdBlwA24wOm+f+vrSfGfF8nOdN0+8MKQjMrq +uSx9B/aqmeJkS2llRsyKhKgnHNjNQcU97qzNGzRKZy3MScg/P5bUDbiHjp7GZFt9OlupWBBeQkkD +tnHqMnFO/Crij8fxBLbRwGOKaIyEZ3LZzzH7kU2nsI9Jljt5ktnuZRhZMZ39fUbZH2rQ+H+GtN0x +zeW9nDFdSKFkdR1wP0+lBNk5FEkyRSpGBik2HzqBHFGA964QAcZoZHTNWtDADHWuMpxkGuLgnFK4 +GOlZoR5cbmisTncUs4xtSRHM1AF+VCjqu2TQrNVAxg9KdQj1pJBvTmMHpWoyc2y5bYU+jUimtuu1 +PIx9a0HMewFR/E1g2oaXJChxJjCt6Zp+uwBpxFvjeqjzxfF9O1r8XeIwYebHKh/ccMcbf7Sp981A +Xer8P3Mt3eXrPJMIisUUSgZboC3etw8W4OFE0sXWs31tYXiDML8vNJJ/p5Ruw/lXn6/01dXuGvtO +tMO8uJQgPIpYZH3wftREUbt7pvwFkk0dtLgSK+Pi9/b9auOkaekdqEIxt27Va+A/CfWbuwTU4tPE +kL5BmdwoGOu27Y+QNPf8hcSyxxhAiNy87HkUn5tj+9Ax4M1274duxDcRmfT2OSq9Y/cf2q7cQaVb +a1apcWEsTLIAUYbA59ao2qXXDehoX1PU43cDaKE4P1JGfsPrTXhjiXUtUMknDn+W21ssn/kTO3O5 +9cbkfeinXEXDuraYhafSJJ0XOJIGz+lUK/4ku9PZ1tw6GPJIdcEe1bXJxWLfTHg1WExl15SVBIBP +p9axbibSdT4g1aX/ACyyldJGCmRl5VC+pJoENB41upOIrK/1W1W8tYHBeHJGR6/Mdd69P6Ve2Wp6 +bBfafKJLaZOZGH9fevMs3DtnoFsJNTW6ZCcGeJQVQ+46/fFX3wr4ls9Im/Cxail1pc7ZYdHgb+Ir +6euD70I2Vl+HNJkUvsyBlIKkZBBpJhRSLYwaKuD1FKkY61zAA6Vm0EQKW2pZeXuKSU4pQnbaoorh +TSZ5RR96IwqDqnOcChRosAdaFZqocYB6UrG2/Sk1BzS8Y3rcQ7tj7U9i3ppboaF1r3D+l+bFf6xY +296q/s4ZSTgnozKu+PatodalfWmm2Zu76dIYV/ebv7AdSfYVmPFnirO4ktOHoDD+7+JlALf8V7fX +7VfOEtW1K0sNSng4j0Xi7VrlwbeGS9NmkSY/IkfKwXt8+5rJ9S401nT9TOleLPAyzo7HlvIIBDOo +/iR1+GQD2OKuMqNftPqd1JPqM81xNJ+aSVizH61evCS+s40l0q5jhWYDy/2oGCMgxsc9gwAPtUzx +F4ZTDQouJeGmm1TRp4xMvNHyzRoRnJHRh7j7Vmmr+ZYTW+q2p/aQNhwD+de6mojWPETxMvdE0KG1 +1LmFxFzJDaKRGWYE/E6rsFHQD296wHiDjbiTXJna71KZUb/8UR5Fx6bdfrTXiC9u9b1iW/uA3NId +lySFHTApyugO9mLiHJI3xVVXzzN8TMSfc0406/u9OnE9pO8TjuO9WCLR47q3DACGdR3GVb2P96tX +iN4K8Q8G8LR8QXV5YzxYX8RBEx5oS3pnZgDtQJaD4j3z2fk6laC5jGzSR/mX3q26JxClzah47kTI +35SVwe+29UfwY0NNQur2/uA4ihVY0IPVzv8AI7Dv61aNStX03U5YT5fKeZ4+VcdOQ7j1xn7VApqL +/wCZWNzbSr+zmjZcHsazC202WwnVortlukJ/J0BHbPetK06QvEzEHIkI+hFI3fh1+O4e1Pia31uO +G5t5ARZsm7DAyQc5/Sg0Dwd4ztdU4Y/C6jMyX1rJycnLn4NsfYnGPf2q6aVq2m6tC02n3ccyI5jf +BwUYdQwO4Psa8kcLa3LpGssWeTyXfEgU4Jwa2vWLxdQ4e1oWGnabc6Zcwfio721YJcQuF5v2i7MQ +G2BGwzvRZWsOuKTIIqgeC/FjaxpB0q+maS+tF+FnOWkj9fmOn2rQGO9ZsUl0augkVwjfOKAB9qiu +52pNmzttRiDRGU5xQGQ9xQrqKcbihWaIxF70snWklB74paJcmtQRXHHEq8NaA90ih7uU8lup9cbs +fYf2rDLvWJ9QvGur6WWWd2y7Mc5q9eOV60Op6XbqhYLA8jHtu2P6VQYruAkeZFsepxW2KfQTI35G +Gau/DvG99aWo0zW401vR2xz211livujHdSO3p7VSLdrGX9/lPTI9KXaCXk/ZOjemaDW9M02NpbTi +LT/FjVrPhaxbzf8AL7mTnmgYDaH4iQVxtuDt69aoviLrXD+u67LcaLpE1pBJnzWcgLMf4gmAVPr2 +/nVMnlBcRXCBJ13QkbA06065FxCWcYdG5WT0IoGI0eISMgUe2RR9OJt5Ws3xjtt1qUB518zG6HtU +frScskVygIwd8UHZ7copUCjaxe8bcR6Yuj3V7NeW5OUiCc0jY3+ZG2aXibmUSb4YdRU3wbxqvA2q +zak1i9z5kBjDkcxjOf5f2FTQXwqnWz4eOmvGY5orhxICuG5tuvv2ol9rcOs69d+UcpZOsRYfvFld +f54qNtOJ5+Jtf1XVpMwySzI/L3/Ly/8A1FHg0aLS9aklslKw3flM8Z3AbzR09tzVEjYcoeULsGAc +CmvEOnarqt1aWelreSySK3PBbgnnUYO4HYUdJkSWAKd+XlOPepJ+J9S4VhttZ01Od1k5JEJ/OpBy +PuB9hQZLxHpk+n6sySQvE2fiR1wVYdQQad2F5eWkb/h5XiJjaNgDsVbZhj3qa4o1W64nu5tVvIDF +M7c2CcmohlBJ2xlaB9wzrl1w/r8Go2ZUyxDHK35WB6g16R4T1634i0SHUYF8st8MkZOeRh1FeW/J +WRw2enU1qvgnxTp+nR3Gj6lMtuZ5Q8EjbKSRjlJ7dBUpK2QkD0rg3zRGIJ2O1GSstlFwAc0VsA7V +35UXBzUo6p2oV0KaFZVGKm9LxKM9M0T0NdklENvLLjPIhYD5CukRjfi1qCXHF88eQY7ZFgH2yf1J ++1VyGOORMYGKbNK97JPLcMWkkcyOT3J60haieGQqmeXO2a0wkJ7FGHwkKexBpBLqeycJOTy9m7U/ +tJo5sebGMinM8VpLGedMqN996COvF/Hw+ZFyyOo6A9fl71H2NwEvg2cGQcrjpuOmf+u1PxZRK7SW +NyFx+5UfqyK3/eEBW4jPM4/jH96Im7d/jOGG/auXo57RlO/ptTHS7xJ4lIOT0p60mQ4GxopppdwG +QwOfy7UtIshlFuInmD4AULnOegqOGI7wMo2JwfSn8mXQgNysR8LZ70Fp4h8ML/hWK01C51Czb8eq +q0ETEtGx3HsR7im96wjWJFlVmjKq3fmxlz/8ahtK1TXdc4jjt9cvGljit2EQXbmIAAz9KXnYWVq8 +apjCSSDfJGwTr/yNAiMlFy/7oORvjepDihUk0CGFZFjDToDI/wCUZOMn7060vT7OXQVvpLadCwZY +wzBEkPL8JJPQBs5OcHp61E8Xs3+TwW867mQcy47gGgl+P+CtL4d0qyv9L1w6glwgLq6gEZHUY/lv +Wb3MmG5c4PKR9akYbaRinPczyRpuiO2QufTNRuqxNHdKvYsD/wBfagPFtEAegFFkbzcKq7A9aDDK +igx8qHb8zHCiiPQfhJrX+bcJQiWQyT2rGFyeu3Q/aropX0rzp4e8YS8JXqxPCJ7O5I89R+ZcfvCv +QOnXttqFlFeWkgkglUMjDuKzY1DvPegGoLRWqNFFbI7UK4g2O1CshgAcelGRFkUxt0YEH610jAo0 +LKD71qDzaI1g1Ke1JB5XZM/I4o1xGY2Db4FLcWRfgOML9OgS8kH0LH+4p75AmgDcwwa2wYYLplNi +BtiixzzRH4gWA7EVyM/h5miz8NPIRHMOUAADvnFENpYrO7X87wSnoU/tTC80bUoR5kEgvUG+xww+ +h/pUxJawr8cec02FxJE27HlHrQQOkTG2lliZCuGyA2xHtUxFKzfER9qYaxBLPci9tD5g5cSRj8w9 +x6ijW8+YsZHTegPLvuMDBp2JAIlbbcY6U2tpPMV8/LOKAceXjB2oJ3hCeN9Z3UMywvjbp0rt9bmS +OeeQZeQQrGN8AkFjt26ioLhed4uKIU5iOdXBH/GrLqDfEkaDc3BJ/wCKqv8ASirgbmxtLENZ6W7x +ae37cMQFYc+Qq8wOSAASfXvvVP8AEdndhLJzeZ+LfmBbJyc7VJPdW0ziW6hilkAzzFFBHyqJ8RLt +brTLec8gkacFiFwT8JoIK2lXlBYbVE8RMFaKT19/ejG5wAq0x1uVnhiB3IbagdwqHVc9MUSP9vdF +v3E+FaPptvdX7Q2NhDJPdzEJHGgySTWq2/hOukcMvd6teSC/EZZUhwY1OM4JxknttjrQZqYlDGRx +sOgNaX4M8Sw2Usmi6hOscUp5rcucAN3X60Xh3wf4k1e1ju7ua105GHMsUuWk/wCQH5flnNUnUrFo +NVms5MB7VzGeX+JTg4+1QemVwRkYxRGJzWd+H3HtrJDBo+sytDdKOSOdz8MnoCexrRRg4IIIPTFZ +aHRtutCiEkChWappJkLjJrkaEnJo8rCgjYxvW4MN8XrNYeNr3GR5ipJ09VFRWg3XnQlMnmTYg1dP +G+z/APGbO8HSa25D81J/oRWXQztYXvm78j7PVYqw6hFGSJOXp1OKbpzKOZExipGIxXduGT4sjam3 +lmJzG+du4NULW84YBZCPfaiXtosykxt13xXFCEEDOR39aNDKUk/KceuelEVq/gu4JcRrICD13okr +XBhE8x+IHDEdSOxNW55oHT4lXPqKjr+2je0kjHLlxjtRURZNiPYjfvXWbAIYCkbZuVeQnp1HvSU8 +yDIBG+2KIW055DxDp7W7AP5uCcA/Dj4v0zVnhdpZIWOxMZkP/Ni230IqrcMRo2tyTqCTDbyMBnbJ +HKP/AJVaLGRBcXDkhkiYRqADsFGB/KosSyxQOg50JPuKqvH7CJrG2RsjDyN+gFS9veTT3W4+HPrV +X48uObWljBwI4gMDsTRTbhvTbjXtfstHtZooprqURh5Gwq+5qa8ZeCLjgi6s7d9VttRSUkiSIcpV +h1BXP61VrW4ltikttIYZo2DI69Qw70e9utS1/VrSPULlp5ZZVXJwOpAJrSN7/wAO3B6Wul/9qL5R ++JuU5bcMu6Reo92/lWo2GmS6hqBv78AJGSLe3JyFH8Te/f2+dNOHEkt+FtOtRcRmNY1AePBGw2pT +U9RFoHSW9htI+txcu3KIl74z1JOwHr8qjUTkY/Fo0Vs7RW6bM46k98e9Y54+No6alYWtjbRw3MMZ +82RVwFQ9Ax9Sd9/ep3WvFzQ7G3FhollcXap8IkJ8tSPXfcn3xVP408QdP4k4Yk0M6GLbmkEvmGfn +JcHOTsPehqhpb2kjiTl53H72dqv/AADxs9hyaZquTaDaObmyY/Y+1Z000iAsRGF6DkJOPnSE1xME +ODzKR2rNiPTqyxTQrLDIskbDKspyCKFZ14IX7XGgXFm0vP5MpIBOcA9PpQrNaXlmwBtXRkkGkQwz +SqZyKSireMVh+I4TivAPitJ1JP8ApbY/ry1h+pW3OTgnHbevQPHsN3qOjwaDYcguNTnEILdAoBcn +/wBtYMRzKFIww2atxmmGl6nPps3lOeaPO4qz2t5aXyghwD71WLy18zJC71FwzSW8uCSKrK+XFpyn +nifJPXemp8xdgjZ+RqOtLucqG5+YEbUs2pyoCWJoHEqyn8sbkkdlNRV4uoKwPkuvtykVIwcQRK2G +yKPNrEEzpySFGXpk/Cw9DRVT1qeW1lWQoyCUZx03HWo1LtpG3JrdvBySz1fxAtbDUeG7PUomVjI8 +8SukAxs45ts5GPXrXp+z4b4chAaHQ9MjI6FbVB/SkMeG/CzTdS1jiRNOtdPu5TduiGZISyRKrcxZ +j6bV6l4a8FuDLSxWO7hvb+U7vJNM0ZLHr8KYx+taqkcUIVI41VewUYArsigSZGKuKoKeEHAMJDpo +rhvX8VKf5tTW78DfDW+na5udDleV8Zb8ZMOnyatLyCu/pQQgDqKDNbPwJ8M7W6juU4e8xozkLLcy +un1Utg/WlX8EfDsasurW+hm3u0bmUxXEiqp9lzj9K0gkAda7nK7VRVZeDdIeJwqzwzH8s6vzPGfU +Bsr+lecPGPh/ifh/iBbLW9Sl1GxkzJY3LfCGGdwR0DDv8/evWb7HOdqpvjLwsvGHh9qGnxIDf26G +4sX7iVRnl/5br9amFeQiOQ4yCPbrTeSLLl4zv1371DQX8yA+YCGHX2NBuJCshR0UgbdKiHdxd8jl +dwfSm348xnm5Oai/ibe/bflRj0P96ZXHNBLyMpA96GtG8G9cS34rWDIWO9Qxuv8AqG6n+n1oVV/D +S3a7490qNCwVZfNbHooJ/pQrnZjUejBjPSl12IG1Nhkt0pxGD1NSVcQ3F17HpeqcOalNJyRRaj5b +tnAAdGWsIlYJql7CSMpcOvX0Y1vvG3DZ4q4Yn0qO4W3nLLJDKwyEZTnt7ZFYxxfwJq3DV9LNNfR6 +j+xWeeRFK8nM3IOvXfG9bjNRbJzHAxnGMUwurJJVbb608t5gFywJJ9KcZXBBGRVREabM1rP5E26n +cGp0Q288PMNjTC/tkki5kwCB2613SblsCF2APT50AvNMj5TIuB3z3qNSMBvi7Hap65TbP9dqirhV +QFsYOelNMW7wn1ltC4uspjPy28kgSTmOwB6H23r2Fpl+rwhieY8ucj+9fP1794mJBAxWhcEeJHF1 +vYCCDU3Fra/xgNk9lyRnHtRXtA3S8iuCppvd38ceDg7HtWD6B40W7Wwe9tQZlHxRBsBv9p7VcND4 +80ni+ynbTGkt7222ntJtnXPQjsw9xV1V8XX7U8o5XO5H8v70dNcsOflZ+VvQmszvG1Yc7RJzxk5y +N+tIQX8sbjmkhBO551II+9T9GNfTUbaTHJMjfI04S4UjGay+zvp5CpJBP+kbCp2y1GRByFmPfc9K +foxb2lDE7/ShDIomUZ2OxHzquG8nZQyinNpdyNgEYYGmmPH3jLoFpo3iVrulEGBBcmaEqNuST4wP +pzY+lUu60i2EBImVpPbvWwf4z7Q23iBpeoRLgXenAE+pR2/owrCo7iUOAT9KrIvJJbyfDkYNSkFx +Hd24jlxzr+U0QeVdDHMFIG5NMJk8p/gl+1X0aj4Eaf5vFNzeFdrW2IB92IA/QGhVn8BdNntOF59U +uYyn46UeVnqyLtn5Ek/ahXLq/Wo0LlAbY0dAc9TSJOD70ojjY1iNU7iyM4yKqfF8IvNa1KyfcT8N +3DKD3aN1dftirVDKp9Kz/wAbLq60pdO1ywmWN+SazfftIvp8s1uJWNluQq/NsRnrTtJkbBUjGKj7 +OSOeE2rN8QGVNNj51jNljzRnvW2U7JKrKQoyR1phM/lSK64z1NKQOsq8/MObrmkJjzty8h9M1BOW +E63UIQMOalZdBkul+J8DtvVTE01pLzwyMuPQU4bi3UUQJkEjbJFMNSl1wzbwjnurxQvcLTC61OGO +FdO08IkS+nf5moDUtbvr5irynHtSNoxiIbO9WRFnVWEBfm3A9abadJxE16l3pN7cWk8eyyxyFDj0 +yO3tSNpqSD4ZcYPWp231e0t7blgYA43FBYLbxG474Zt+eW/ttULYDrOhJP1BGftVguOK+MNV4cte +I7LheG4sp8q01rO+Y5FPxKy9j36dMVlVzfSX1zsMhdzWm/4fePLPhniSTh7XJUj0bWCFLufggnH5 +XPoD+Un5HtRZVq4J8Q+GDpbza/FrMFzAc3Kxwq/kL/ER1K57gbbVpPDHFPAuuMo0jjbT5XbcQzv5 +T/8Apf8AtTTirwj0niC9N7a6oulXirmKSABiT/qGd1x2715q8SOAZ9D4gm0y4jWw1INlMHFtdL2a +Mn8pP8JqZFe1raxkKjlaGZT3RhinSWLKdosZr532mu8UcPXLQ2uralp8sZwViuHT+Rqaj8XPEeKE +xJxfqnKRjeXJ+53rWJrYv8cd9YC94ask5Wv44pXch90jJUAEe5B+1eaDKSMgkGlNV1HUNVvZL3Ur +ye7uZDl5ZnLMfqaluDODeIeLLvydHsHkjU4kuH+GKP5sf5DJ9qvgghI4YkM2/vWreEXhhfcQSxaz +r8Ulto6kMkbfC917DuF9+/b1GkcB+EPDvDix3mphdX1Jfi5pF/Yxn/Snf5tn5Cr9PP22A6Vi9Lhp +NHFEkcEEaxRRqFRFGAoHQAUKTnkHPmhXOqTcnNBGNChWI0VjbvgVnX+IiMtwbZzKQDHeA/dWoUK6 +c+s1g0N2/mrMuzKas0YS/tQ7LguvN8qFCt1kwV2tJ/KU5XNO2cqSQBkj0oUKBheFmzvudyah51Oe +tChViVyOFQfenCxKRvQoVUJSRgHrSTMQ2ATQoVYsOIZ3gBAJpve3Ek5BJ70KFIJXh/i/ibQZEl0r +Wbu2KkEKJCV/9J2q73Pi3xPxRZ/5JqENhNPe8tsLqSLLICcbD60KFLCM+4gDw63cWUkjTpaSNApb +qQpIrWPD/wAMeG+IuDLW/vWvYbqUNl4ZRjrtsQaFCs9XIs9XHhvwa4M04CS9hudVl5sg3EnKg/4r +jP1zWhW8dvY2kdpZW8NtbxjCRRIFVR7AUKFc9tawSSViDTSRsnehQqKbyjLbUKFCs0j/2Q== + +Cheerio! + +--Multipart_Sun_Oct_17_10:37:40_2010-1-- diff --git a/testdata/testdir4/multimime!2,FS b/testdata/testdir4/multimime!2,FS new file mode 100644 index 0000000..84f85aa --- /dev/null +++ b/testdata/testdir4/multimime!2,FS @@ -0,0 +1,27 @@ +Return-path: <> +Envelope-to: djcb@localhost +Delivery-date: Sun, 20 May 2012 09:59:51 +0300 +From: Steve Jobs +To: Bill Gates +Subject: multimime +User-agent: mu4e 0.9.8.4; emacs 23.3.1 +Date: Sat, 19 May 2012 20:57:56 +0100 +Message-ID: +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="=-=-=" + +--=-=-= +Content-Type: text/plain + +abc +--=-=-= +Content-Type: application/octet-stream +Content-Disposition: attachment; filename="test1.C" +Content-Transfer-Encoding: base64 + +aGVyZSBpcyBhIHNpbXBsZSB0ZXN0IGZpbGUuCg== +--=-=-= +Content-Type: text/plain + +def +--=-=-=-- diff --git a/testdata/testdir4/signed!2,S b/testdata/testdir4/signed!2,S new file mode 100644 index 0000000..7e1319a --- /dev/null +++ b/testdata/testdir4/signed!2,S @@ -0,0 +1,36 @@ +User-agent: mu4e 1.1.0; emacs 27.0.50 +From: Skipio +To: Hannibal +Subject: test 123 +Date: Sun, 24 Mar 2019 11:50:42 +0200 +Message-ID: <87zhpky51p.fsf@djcbsoftware.nl> +MIME-Version: 1.0 +Content-Type: multipart/signed; boundary="=-=-="; + micalg=pgp-sha256; protocol="application/pgp-signature" + +--=-=-= +Content-Type: text/plain + + +I am signed! + +--=-=-= +Content-Type: application/pgp-signature; name="signature.asc" + +-----BEGIN PGP SIGNATURE----- + +iQIzBAEBCAAdFiEEaYec7RdFk3UPFNqYEd3+qdzEoDYFAlyXUwAACgkQEd3+qdzE +oDbjdw//dAosaEyqSfyUMXjS++iJEeDIwKwO6AjEI0xCbJjHmxq93PA61ApE/BS3 +d/sKa1dsfN+plRS+Fh3NNGSA7evar9dXtMBUr6hwL0VTmm5NDwedaPeuW6mgyVcB +VNUn5x1e/QdnSClapnGd156sryfcM1pg/667fTHT6WC01Xe0sezpkV9l0j4pslYt +y6ud/Hejszax+NcwQY7vkCcVWfB9K4zbiapdoCjHi78S4YAcsbd//KmePOqn04Sa +Tg1XsmMzIh7L/3njkJdIOd9XctTwYEcN5geY1QKrHQ/3+gBeaEYvwsvrnqnVKqMY +WCg/aYibuXl+xNkPMcKHIj1dXA3m5MkL77RrxODiAYz0YkiQx1/DLZs8PV3IVoB4 +f0GGDqyiOwSmSDa4iuCottwO4yG1WM1i7r6pir22qAekIt43wSdwakOrT1IkS8q2 +o0VGiQtEPy27D+ufiw06t02Ryf20Q7i2YcueZxYeRBq41m11M41DJ4wH7LQcJsww +qG5iBOdwQFCTWpi1UrbbFjlxXXWvKMuIU+4k7nsamrEL4SDXmq1v13vtlcgJ6vnn +v7c9+MF7laqdfI+BYnlD1v/9LosPbFTm0hPdvK4yIOORp8Iwj/1PGzTOz6SCUxzA +kDu+Y+NN9/SM1ppStg1OikYPcfEXF8igWhuORwqcmpgHxVkIQ9I= +=wnkU +-----END PGP SIGNATURE----- +--=-=-=-- diff --git a/testdata/testdir4/signed-bad!2,S b/testdata/testdir4/signed-bad!2,S new file mode 100644 index 0000000..7a37ba9 --- /dev/null +++ b/testdata/testdir4/signed-bad!2,S @@ -0,0 +1,35 @@ +Return-path: <> +Envelope-to: skipio@localhost +Delivery-date: Fri, 11 May 2012 16:21:57 +0300 +Received: from localhost.roma.net([127.0.0.1] helo=borealis) + by borealis with esmtp (Exim 4.77) + id 1SSpnB-00038a-55 + for djcb@localhost; Fri, 11 May 2012 16:21:57 +0300 +From: Skipio +To: Hannibal +Subject: signed +User-agent: mu4e 0.9.8.5-dev1; emacs 24.1.50.8 +Date: Fri, 11 May 2012 16:20:45 +0300 +Message-ID: <878vgy97ma.fsf@roma.net> +MIME-Version: 1.0 +Content-Type: multipart/signed; boundary="=-=-="; micalg=pgp-sha1; + protocol="application/pgp-signature" + +--=-=-= +Content-Type: text/plain + + +I am signed! But it's not good because I added this later + +--=-=-= +Content-Type: application/pgp-signature + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v1.4.12 (GNU/Linux) + +iEYEARECAAYFAk+tEi0ACgkQ6WrHoQF92jxTzACeKd/XxY+P7bpymWL3JBRHaW9p +DpwAoKw7PDW4z/lNTkWjndVTjoO9jGhs +=blXz +-----END PGP SIGNATURE----- +--=-=-=-- + diff --git a/testdata/testdir4/signed-encrypted!2,S b/testdata/testdir4/signed-encrypted!2,S new file mode 100644 index 0000000..a3910e6 --- /dev/null +++ b/testdata/testdir4/signed-encrypted!2,S @@ -0,0 +1,54 @@ +Return-path: <> +Envelope-to: karjala@localhost +Delivery-date: Fri, 11 May 2012 16:37:57 +0300 +From: karjala@example.com +To: lapinkulta@example.com +Subject: signed + encrypted +User-agent: mu4e 0.9.8.5-dev1; emacs 24.1.50.8 +Date: Fri, 11 May 2012 16:36:08 +0300 +Message-ID: <874nrm96wn.fsf@example.com> +MIME-Version: 1.0 +Content-Type: multipart/encrypted; boundary="=-=-="; + protocol="application/pgp-encrypted" + +--=-=-= +Content-Type: application/pgp-encrypted + +Version: 1 + +--=-=-= +Content-Type: application/octet-stream + +-----BEGIN PGP MESSAGE----- +Version: GnuPG v1.4.12 (GNU/Linux) + +hQQOA1T38TPQrHD6EA/+K4kSpMa7zk+qihUkQnHSq28xYxisNQx6X5DVNjA/Qx16 +uZj/40ae+PoSMTVfklP+B2S/IomuTW6dwVqS7aQ3u4MTzi+YOi11k1lEMD7hR0Wb +L0i48o3/iCPuCTpnOsaLZvRL06g+oTi0BF2pgz/YdsgsBTGrTb3pkDGSlLIhvh/J +P8eE3OuzkXS6d8ymJKx2S2wQJrc1AFf1BgJfgc5T0iAvcV+zIMG+PIYcVd04zVpj +cORFEfvGgfxWkeX+Ks3tu/l5PA1EesnoqFdNFZm+RKBg3RFsOm8tBlJ46xJjfeHg +zLgifeSLy3tOX7CvWYs9torrx7s7UOI2gV8kzBqz+a7diyCMezceeQ9l0nIRybwW +C9Egp8Bpfb02iXTOGdE/vRiNItQH14GKmXf4nCSwdtQUm3yzaqY9yL3xBxAlW53e +YOFfPMESt+E7IlPn0c7llWGrcdrhJbUEoGOIPezES7kdeNPzi8G1lLtvT04/SSZJ +QxPH5FNzSFaYFAQSdI7TR69P7L7vtLL8ndkjY49HfLFXochQQzsqrzVxzRCruHxA +zbZSRptNf9SuXEaX9buO1vlFHheGvrCKzEWa6O7JD/DiyrE/zqy4jdlh9abMCouQ +GWGSbn8jk6SMTQQ2Yv/VOyFqifHZp0UJD59tyIdenpxoYu5M0lwHLNVDlRjLEwUQ +AIDz1tbLoM7lxs2FOKGr8QqbKIeMfL+NUmbvVIDc4mJrOlRnHh+cZYm4Z49iTl1v +bYNMYgR5nY7W6rqh0ae7ZOW0h2NzpkAwTzuf1YrSjNavd9KBwOCFtAoZhRwfwFVx +ju+ByHFNnf7g/R6DekHS0pSiatM0cPDJT05atEZb+13CRHHznonmLHi+VahXjrpg +cIUA8Lhjdfm6Fsabo7gNZnTTRxNBqUXKK2vJF/XLbNrH5K2BH2dCCmUNtm3yFWiM +DOzaw3665Y3S6MvZdyKpatbNrVoJdBpRgPxJ1YCSEituFUqHJBStay+aRb5fVkQR +w3+9hWw+Ob0+2EumKbgfQ7iMwTZBCZP4VOxkoqdHvs9aWm4N7wHtXsyCew3icbJx +lyUWsDx/FI+HlQRfOqeAMxmp8kKybmHNw8oGiw+uPPUHSD1NFYVm2DtwhYll3Fvs +YY7r5s3yP1ZnwxMqWI3OsExVUXs8MS4UTAgO+cggO7YidPcANbBDihBFP8mTXtni +Oo5n5v+/eRoLfHmnsGcaK8EkKsfFHpbqn4gxXGcBuHaTTJ/ZhbW6bi1WWZA9ExaJ +IeTDtp5Bks1pJvTjCDacvgwl3rEBM6yaeIvB7575Y/GPMTOZhawhfOxV1smMmTKI +JOWYb3+PuN2cvWetkjFgH8re4sRXq22DKBZHJEWYU8sH0sACAePnIr+pkrOtGeJB +t1zBqZUnrupH6ptk9n/AjbQ+XSMTEKu55gSjYLAYx1EHApx52QLkdh+ej5xCIVeY +6wS1Iipkoc6/r6F7CKctupXurNY2AlD4uQIOfD6kQgkqK4PY3hsRHQA+Zqj6oRfr +kxysFJZvhgt26IeBVapFs10WuYt9iHfpbPUBQUIZCLyPAh08UdVW64Uc2DvUPy+I +C+3RrmTHQPP/YNKgDQaZ3ySVEDkqjaDPmXr5K0Ibaib2dtPCLcA= +=pv03 +-----END PGP MESSAGE----- +--=-=-=-- + diff --git a/testdata/testdir4/special!2,Sabc b/testdata/testdir4/special!2,Sabc new file mode 100644 index 0000000..7f1de8e --- /dev/null +++ b/testdata/testdir4/special!2,Sabc @@ -0,0 +1,10 @@ +Date: Thu, 1 Jun 2012 14:57:25 -0200 +From: "Rocky Balboa" +To: "Ivan Drago" +Subject: currying and tail optimization +Message-id: <3BE9E653ef345@emss35m06.us.lmco.com> +MIME-version: 1.0 +Content-type: text/plain; charset=us-ascii +Content-transfer-encoding: 7BIT + +Test 123. I'm a special message with special flags. diff --git a/thirdparty/CLI11.hpp b/thirdparty/CLI11.hpp new file mode 100644 index 0000000..41027f0 --- /dev/null +++ b/thirdparty/CLI11.hpp @@ -0,0 +1,10966 @@ +// CLI11: Version 2.4.1 +// Originally designed by Henry Schreiner +// https://github.com/CLIUtils/CLI11 +// +// This is a standalone header file generated by MakeSingleHeader.py in CLI11/scripts +// from: v2.4.1 +// +// CLI11 2.4.1 Copyright (c) 2017-2024 University of Cincinnati, developed by Henry +// Schreiner under NSF AWARD 1414736. All rights reserved. +// +// Redistribution and use in source and binary forms of CLI11, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this +// list of conditions and the following disclaimer. +// 2. Redistributions in binary form must reproduce the above copyright notice, +// this list of conditions and the following disclaimer in the documentation +// and/or other materials provided with the distribution. +// 3. Neither the name of the copyright holder nor the names of its contributors +// may be used to endorse or promote products derived from this software without +// specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#pragma once + +// Standard combined includes: +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +#define CLI11_VERSION_MAJOR 2 +#define CLI11_VERSION_MINOR 4 +#define CLI11_VERSION_PATCH 1 +#define CLI11_VERSION "2.4.1" + + + + +// The following version macro is very similar to the one in pybind11 +#if !(defined(_MSC_VER) && __cplusplus == 199711L) && !defined(__INTEL_COMPILER) +#if __cplusplus >= 201402L +#define CLI11_CPP14 +#if __cplusplus >= 201703L +#define CLI11_CPP17 +#if __cplusplus > 201703L +#define CLI11_CPP20 +#endif +#endif +#endif +#elif defined(_MSC_VER) && __cplusplus == 199711L +// MSVC sets _MSVC_LANG rather than __cplusplus (supposedly until the standard is fully implemented) +// Unless you use the /Zc:__cplusplus flag on Visual Studio 2017 15.7 Preview 3 or newer +#if _MSVC_LANG >= 201402L +#define CLI11_CPP14 +#if _MSVC_LANG > 201402L && _MSC_VER >= 1910 +#define CLI11_CPP17 +#if _MSVC_LANG > 201703L && _MSC_VER >= 1910 +#define CLI11_CPP20 +#endif +#endif +#endif +#endif + +#if defined(CLI11_CPP14) +#define CLI11_DEPRECATED(reason) [[deprecated(reason)]] +#elif defined(_MSC_VER) +#define CLI11_DEPRECATED(reason) __declspec(deprecated(reason)) +#else +#define CLI11_DEPRECATED(reason) __attribute__((deprecated(reason))) +#endif + +// GCC < 10 doesn't ignore this in unevaluated contexts +#if !defined(CLI11_CPP17) || \ + (defined(__GNUC__) && !defined(__llvm__) && !defined(__INTEL_COMPILER) && __GNUC__ < 10 && __GNUC__ > 4) +#define CLI11_NODISCARD +#else +#define CLI11_NODISCARD [[nodiscard]] +#endif + +/** detection of rtti */ +#ifndef CLI11_USE_STATIC_RTTI +#if(defined(_HAS_STATIC_RTTI) && _HAS_STATIC_RTTI) +#define CLI11_USE_STATIC_RTTI 1 +#elif defined(__cpp_rtti) +#if(defined(_CPPRTTI) && _CPPRTTI == 0) +#define CLI11_USE_STATIC_RTTI 1 +#else +#define CLI11_USE_STATIC_RTTI 0 +#endif +#elif(defined(__GCC_RTTI) && __GXX_RTTI) +#define CLI11_USE_STATIC_RTTI 0 +#else +#define CLI11_USE_STATIC_RTTI 1 +#endif +#endif + +/** availability */ +#if defined CLI11_CPP17 && defined __has_include && !defined CLI11_HAS_FILESYSTEM +#if __has_include() +// Filesystem cannot be used if targeting macOS < 10.15 +#if defined __MAC_OS_X_VERSION_MIN_REQUIRED && __MAC_OS_X_VERSION_MIN_REQUIRED < 101500 +#define CLI11_HAS_FILESYSTEM 0 +#elif defined(__wasi__) +// As of wasi-sdk-14, filesystem is not implemented +#define CLI11_HAS_FILESYSTEM 0 +#else +#include +#if defined __cpp_lib_filesystem && __cpp_lib_filesystem >= 201703 +#if defined _GLIBCXX_RELEASE && _GLIBCXX_RELEASE >= 9 +#define CLI11_HAS_FILESYSTEM 1 +#elif defined(__GLIBCXX__) +// if we are using gcc and Version <9 default to no filesystem +#define CLI11_HAS_FILESYSTEM 0 +#else +#define CLI11_HAS_FILESYSTEM 1 +#endif +#else +#define CLI11_HAS_FILESYSTEM 0 +#endif +#endif +#endif +#endif + +/** availability */ +#if defined(__GNUC__) && !defined(__llvm__) && !defined(__INTEL_COMPILER) && __GNUC__ < 5 +#define CLI11_HAS_CODECVT 0 +#else +#define CLI11_HAS_CODECVT 1 +#include +#endif + +/** disable deprecations */ +#if defined(__GNUC__) // GCC or clang +#define CLI11_DIAGNOSTIC_PUSH _Pragma("GCC diagnostic push") +#define CLI11_DIAGNOSTIC_POP _Pragma("GCC diagnostic pop") + +#define CLI11_DIAGNOSTIC_IGNORE_DEPRECATED _Pragma("GCC diagnostic ignored \"-Wdeprecated-declarations\"") + +#elif defined(_MSC_VER) +#define CLI11_DIAGNOSTIC_PUSH __pragma(warning(push)) +#define CLI11_DIAGNOSTIC_POP __pragma(warning(pop)) + +#define CLI11_DIAGNOSTIC_IGNORE_DEPRECATED __pragma(warning(disable : 4996)) + +#else +#define CLI11_DIAGNOSTIC_PUSH +#define CLI11_DIAGNOSTIC_POP + +#define CLI11_DIAGNOSTIC_IGNORE_DEPRECATED + +#endif + +/** Inline macro **/ +#ifdef CLI11_COMPILE +#define CLI11_INLINE +#else +#define CLI11_INLINE inline +#endif + + + +#if defined CLI11_HAS_FILESYSTEM && CLI11_HAS_FILESYSTEM > 0 +#include // NOLINT(build/include) +#else +#include +#include +#endif + + + + +#ifdef CLI11_CPP17 +#include +#endif // CLI11_CPP17 + +#if defined CLI11_HAS_FILESYSTEM && CLI11_HAS_FILESYSTEM > 0 +#include +#include // NOLINT(build/include) +#endif // CLI11_HAS_FILESYSTEM + + + +#if defined(_WIN32) +#if !(defined(_AMD64_) || defined(_X86_) || defined(_ARM_)) +#if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || \ + defined(_M_AMD64) +#define _AMD64_ +#elif defined(i386) || defined(__i386) || defined(__i386__) || defined(__i386__) || defined(_M_IX86) +#define _X86_ +#elif defined(__arm__) || defined(_M_ARM) || defined(_M_ARMT) +#define _ARM_ +#elif defined(__aarch64__) || defined(_M_ARM64) +#define _ARM64_ +#elif defined(_M_ARM64EC) +#define _ARM64EC_ +#endif +#endif + +// first +#ifndef NOMINMAX +// if NOMINMAX is already defined we don't want to mess with that either way +#define NOMINMAX +#include +#undef NOMINMAX +#else +#include +#endif + +// second +#include +// third +#include +#include +#endif + + +namespace CLI { + + +/// Convert a wide string to a narrow string. +CLI11_INLINE std::string narrow(const std::wstring &str); +CLI11_INLINE std::string narrow(const wchar_t *str); +CLI11_INLINE std::string narrow(const wchar_t *str, std::size_t size); + +/// Convert a narrow string to a wide string. +CLI11_INLINE std::wstring widen(const std::string &str); +CLI11_INLINE std::wstring widen(const char *str); +CLI11_INLINE std::wstring widen(const char *str, std::size_t size); + +#ifdef CLI11_CPP17 +CLI11_INLINE std::string narrow(std::wstring_view str); +CLI11_INLINE std::wstring widen(std::string_view str); +#endif // CLI11_CPP17 + +#if defined CLI11_HAS_FILESYSTEM && CLI11_HAS_FILESYSTEM > 0 +/// Convert a char-string to a native path correctly. +CLI11_INLINE std::filesystem::path to_path(std::string_view str); +#endif // CLI11_HAS_FILESYSTEM + + + + +namespace detail { + +#if !CLI11_HAS_CODECVT +/// Attempt to set one of the acceptable unicode locales for conversion +CLI11_INLINE void set_unicode_locale() { + static const std::array unicode_locales{{"C.UTF-8", "en_US.UTF-8", ".UTF-8"}}; + + for(const auto &locale_name : unicode_locales) { + if(std::setlocale(LC_ALL, locale_name) != nullptr) { + return; + } + } + throw std::runtime_error("CLI::narrow: could not set locale to C.UTF-8"); +} + +template struct scope_guard_t { + F closure; + + explicit scope_guard_t(F closure_) : closure(closure_) {} + ~scope_guard_t() { closure(); } +}; + +template CLI11_NODISCARD CLI11_INLINE scope_guard_t scope_guard(F &&closure) { + return scope_guard_t{std::forward(closure)}; +} + +#endif // !CLI11_HAS_CODECVT + +CLI11_DIAGNOSTIC_PUSH +CLI11_DIAGNOSTIC_IGNORE_DEPRECATED + +CLI11_INLINE std::string narrow_impl(const wchar_t *str, std::size_t str_size) { +#if CLI11_HAS_CODECVT +#ifdef _WIN32 + return std::wstring_convert>().to_bytes(str, str + str_size); + +#else + return std::wstring_convert>().to_bytes(str, str + str_size); + +#endif // _WIN32 +#else // CLI11_HAS_CODECVT + (void)str_size; + std::mbstate_t state = std::mbstate_t(); + const wchar_t *it = str; + + std::string old_locale = std::setlocale(LC_ALL, nullptr); + auto sg = scope_guard([&] { std::setlocale(LC_ALL, old_locale.c_str()); }); + set_unicode_locale(); + + std::size_t new_size = std::wcsrtombs(nullptr, &it, 0, &state); + if(new_size == static_cast(-1)) { + throw std::runtime_error("CLI::narrow: conversion error in std::wcsrtombs at offset " + + std::to_string(it - str)); + } + std::string result(new_size, '\0'); + std::wcsrtombs(const_cast(result.data()), &str, new_size, &state); + + return result; + +#endif // CLI11_HAS_CODECVT +} + +CLI11_INLINE std::wstring widen_impl(const char *str, std::size_t str_size) { +#if CLI11_HAS_CODECVT +#ifdef _WIN32 + return std::wstring_convert>().from_bytes(str, str + str_size); + +#else + return std::wstring_convert>().from_bytes(str, str + str_size); + +#endif // _WIN32 +#else // CLI11_HAS_CODECVT + (void)str_size; + std::mbstate_t state = std::mbstate_t(); + const char *it = str; + + std::string old_locale = std::setlocale(LC_ALL, nullptr); + auto sg = scope_guard([&] { std::setlocale(LC_ALL, old_locale.c_str()); }); + set_unicode_locale(); + + std::size_t new_size = std::mbsrtowcs(nullptr, &it, 0, &state); + if(new_size == static_cast(-1)) { + throw std::runtime_error("CLI::widen: conversion error in std::mbsrtowcs at offset " + + std::to_string(it - str)); + } + std::wstring result(new_size, L'\0'); + std::mbsrtowcs(const_cast(result.data()), &str, new_size, &state); + + return result; + +#endif // CLI11_HAS_CODECVT +} + +CLI11_DIAGNOSTIC_POP + +} // namespace detail + +CLI11_INLINE std::string narrow(const wchar_t *str, std::size_t str_size) { return detail::narrow_impl(str, str_size); } +CLI11_INLINE std::string narrow(const std::wstring &str) { return detail::narrow_impl(str.data(), str.size()); } +// Flawfinder: ignore +CLI11_INLINE std::string narrow(const wchar_t *str) { return detail::narrow_impl(str, std::wcslen(str)); } + +CLI11_INLINE std::wstring widen(const char *str, std::size_t str_size) { return detail::widen_impl(str, str_size); } +CLI11_INLINE std::wstring widen(const std::string &str) { return detail::widen_impl(str.data(), str.size()); } +// Flawfinder: ignore +CLI11_INLINE std::wstring widen(const char *str) { return detail::widen_impl(str, std::strlen(str)); } + +#ifdef CLI11_CPP17 +CLI11_INLINE std::string narrow(std::wstring_view str) { return detail::narrow_impl(str.data(), str.size()); } +CLI11_INLINE std::wstring widen(std::string_view str) { return detail::widen_impl(str.data(), str.size()); } +#endif // CLI11_CPP17 + +#if defined CLI11_HAS_FILESYSTEM && CLI11_HAS_FILESYSTEM > 0 +CLI11_INLINE std::filesystem::path to_path(std::string_view str) { + return std::filesystem::path{ +#ifdef _WIN32 + widen(str) +#else + str +#endif // _WIN32 + }; +} +#endif // CLI11_HAS_FILESYSTEM + + + + +namespace detail { +#ifdef _WIN32 +/// Decode and return UTF-8 argv from GetCommandLineW. +CLI11_INLINE std::vector compute_win32_argv(); +#endif +} // namespace detail + + + +namespace detail { + +#ifdef _WIN32 +CLI11_INLINE std::vector compute_win32_argv() { + std::vector result; + int argc = 0; + + auto deleter = [](wchar_t **ptr) { LocalFree(ptr); }; + // NOLINTBEGIN(*-avoid-c-arrays) + auto wargv = std::unique_ptr(CommandLineToArgvW(GetCommandLineW(), &argc), deleter); + // NOLINTEND(*-avoid-c-arrays) + + if(wargv == nullptr) { + throw std::runtime_error("CommandLineToArgvW failed with code " + std::to_string(GetLastError())); + } + + result.reserve(static_cast(argc)); + for(size_t i = 0; i < static_cast(argc); ++i) { + result.push_back(narrow(wargv[i])); + } + + return result; +} +#endif + +} // namespace detail + + + + +/// Include the items in this namespace to get free conversion of enums to/from streams. +/// (This is available inside CLI as well, so CLI11 will use this without a using statement). +namespace enums { + +/// output streaming for enumerations +template ::value>::type> +std::ostream &operator<<(std::ostream &in, const T &item) { + // make sure this is out of the detail namespace otherwise it won't be found when needed + return in << static_cast::type>(item); +} + +} // namespace enums + +/// Export to CLI namespace +using enums::operator<<; + +namespace detail { +/// a constant defining an expected max vector size defined to be a big number that could be multiplied by 4 and not +/// produce overflow for some expected uses +constexpr int expected_max_vector_size{1 << 29}; +// Based on http://stackoverflow.com/questions/236129/split-a-string-in-c +/// Split a string by a delim +CLI11_INLINE std::vector split(const std::string &s, char delim); + +/// Simple function to join a string +template std::string join(const T &v, std::string delim = ",") { + std::ostringstream s; + auto beg = std::begin(v); + auto end = std::end(v); + if(beg != end) + s << *beg++; + while(beg != end) { + s << delim << *beg++; + } + return s.str(); +} + +/// Simple function to join a string from processed elements +template ::value>::type> +std::string join(const T &v, Callable func, std::string delim = ",") { + std::ostringstream s; + auto beg = std::begin(v); + auto end = std::end(v); + auto loc = s.tellp(); + while(beg != end) { + auto nloc = s.tellp(); + if(nloc > loc) { + s << delim; + loc = nloc; + } + s << func(*beg++); + } + return s.str(); +} + +/// Join a string in reverse order +template std::string rjoin(const T &v, std::string delim = ",") { + std::ostringstream s; + for(std::size_t start = 0; start < v.size(); start++) { + if(start > 0) + s << delim; + s << v[v.size() - start - 1]; + } + return s.str(); +} + +// Based roughly on http://stackoverflow.com/questions/25829143/c-trim-whitespace-from-a-string + +/// Trim whitespace from left of string +CLI11_INLINE std::string <rim(std::string &str); + +/// Trim anything from left of string +CLI11_INLINE std::string <rim(std::string &str, const std::string &filter); + +/// Trim whitespace from right of string +CLI11_INLINE std::string &rtrim(std::string &str); + +/// Trim anything from right of string +CLI11_INLINE std::string &rtrim(std::string &str, const std::string &filter); + +/// Trim whitespace from string +inline std::string &trim(std::string &str) { return ltrim(rtrim(str)); } + +/// Trim anything from string +inline std::string &trim(std::string &str, const std::string filter) { return ltrim(rtrim(str, filter), filter); } + +/// Make a copy of the string and then trim it +inline std::string trim_copy(const std::string &str) { + std::string s = str; + return trim(s); +} + +/// remove quotes at the front and back of a string either '"' or '\'' +CLI11_INLINE std::string &remove_quotes(std::string &str); + +/// remove quotes from all elements of a string vector and process escaped components +CLI11_INLINE void remove_quotes(std::vector &args); + +/// Add a leader to the beginning of all new lines (nothing is added +/// at the start of the first line). `"; "` would be for ini files +/// +/// Can't use Regex, or this would be a subs. +CLI11_INLINE std::string fix_newlines(const std::string &leader, std::string input); + +/// Make a copy of the string and then trim it, any filter string can be used (any char in string is filtered) +inline std::string trim_copy(const std::string &str, const std::string &filter) { + std::string s = str; + return trim(s, filter); +} +/// Print a two part "help" string +CLI11_INLINE std::ostream & +format_help(std::ostream &out, std::string name, const std::string &description, std::size_t wid); + +/// Print subcommand aliases +CLI11_INLINE std::ostream &format_aliases(std::ostream &out, const std::vector &aliases, std::size_t wid); + +/// Verify the first character of an option +/// - is a trigger character, ! has special meaning and new lines would just be annoying to deal with +template bool valid_first_char(T c) { + return ((c != '-') && (static_cast(c) > 33)); // space and '!' not allowed +} + +/// Verify following characters of an option +template bool valid_later_char(T c) { + // = and : are value separators, { has special meaning for option defaults, + // and control codes other than tab would just be annoying to deal with in many places allowing space here has too + // much potential for inadvertent entry errors and bugs + return ((c != '=') && (c != ':') && (c != '{') && ((static_cast(c) > 32) || c == '\t')); +} + +/// Verify an option/subcommand name +CLI11_INLINE bool valid_name_string(const std::string &str); + +/// Verify an app name +inline bool valid_alias_name_string(const std::string &str) { + static const std::string badChars(std::string("\n") + '\0'); + return (str.find_first_of(badChars) == std::string::npos); +} + +/// check if a string is a container segment separator (empty or "%%") +inline bool is_separator(const std::string &str) { + static const std::string sep("%%"); + return (str.empty() || str == sep); +} + +/// Verify that str consists of letters only +inline bool isalpha(const std::string &str) { + return std::all_of(str.begin(), str.end(), [](char c) { return std::isalpha(c, std::locale()); }); +} + +/// Return a lower case version of a string +inline std::string to_lower(std::string str) { + std::transform(std::begin(str), std::end(str), std::begin(str), [](const std::string::value_type &x) { + return std::tolower(x, std::locale()); + }); + return str; +} + +/// remove underscores from a string +inline std::string remove_underscore(std::string str) { + str.erase(std::remove(std::begin(str), std::end(str), '_'), std::end(str)); + return str; +} + +/// Find and replace a substring with another substring +CLI11_INLINE std::string find_and_replace(std::string str, std::string from, std::string to); + +/// check if the flag definitions has possible false flags +inline bool has_default_flag_values(const std::string &flags) { + return (flags.find_first_of("{!") != std::string::npos); +} + +CLI11_INLINE void remove_default_flag_values(std::string &flags); + +/// Check if a string is a member of a list of strings and optionally ignore case or ignore underscores +CLI11_INLINE std::ptrdiff_t find_member(std::string name, + const std::vector names, + bool ignore_case = false, + bool ignore_underscore = false); + +/// Find a trigger string and call a modify callable function that takes the current string and starting position of the +/// trigger and returns the position in the string to search for the next trigger string +template inline std::string find_and_modify(std::string str, std::string trigger, Callable modify) { + std::size_t start_pos = 0; + while((start_pos = str.find(trigger, start_pos)) != std::string::npos) { + start_pos = modify(str, start_pos); + } + return str; +} + +/// close a sequence of characters indicated by a closure character. Brackets allows sub sequences +/// recognized bracket sequences include "'`[(<{ other closure characters are assumed to be literal strings +CLI11_INLINE std::size_t close_sequence(const std::string &str, std::size_t start, char closure_char); + +/// Split a string '"one two" "three"' into 'one two', 'three' +/// Quote characters can be ` ' or " or bracket characters [{(< with matching to the matching bracket +CLI11_INLINE std::vector split_up(std::string str, char delimiter = '\0'); + +/// get the value of an environmental variable or empty string if empty +CLI11_INLINE std::string get_environment_value(const std::string &env_name); + +/// This function detects an equal or colon followed by an escaped quote after an argument +/// then modifies the string to replace the equality with a space. This is needed +/// to allow the split up function to work properly and is intended to be used with the find_and_modify function +/// the return value is the offset+1 which is required by the find_and_modify function. +CLI11_INLINE std::size_t escape_detect(std::string &str, std::size_t offset); + +/// @brief detect if a string has escapable characters +/// @param str the string to do the detection on +/// @return true if the string has escapable characters +CLI11_INLINE bool has_escapable_character(const std::string &str); + +/// @brief escape all escapable characters +/// @param str the string to escape +/// @return a string with the escapble characters escaped with '\' +CLI11_INLINE std::string add_escaped_characters(const std::string &str); + +/// @brief replace the escaped characters with their equivalent +CLI11_INLINE std::string remove_escaped_characters(const std::string &str); + +/// generate a string with all non printable characters escaped to hex codes +CLI11_INLINE std::string binary_escape_string(const std::string &string_to_escape); + +CLI11_INLINE bool is_binary_escaped_string(const std::string &escaped_string); + +/// extract an escaped binary_string +CLI11_INLINE std::string extract_binary_string(const std::string &escaped_string); + +/// process a quoted string, remove the quotes and if appropriate handle escaped characters +CLI11_INLINE bool process_quoted_string(std::string &str, char string_char = '\"', char literal_char = '\''); + +} // namespace detail + + + + +namespace detail { +CLI11_INLINE std::vector split(const std::string &s, char delim) { + std::vector elems; + // Check to see if empty string, give consistent result + if(s.empty()) { + elems.emplace_back(); + } else { + std::stringstream ss; + ss.str(s); + std::string item; + while(std::getline(ss, item, delim)) { + elems.push_back(item); + } + } + return elems; +} + +CLI11_INLINE std::string <rim(std::string &str) { + auto it = std::find_if(str.begin(), str.end(), [](char ch) { return !std::isspace(ch, std::locale()); }); + str.erase(str.begin(), it); + return str; +} + +CLI11_INLINE std::string <rim(std::string &str, const std::string &filter) { + auto it = std::find_if(str.begin(), str.end(), [&filter](char ch) { return filter.find(ch) == std::string::npos; }); + str.erase(str.begin(), it); + return str; +} + +CLI11_INLINE std::string &rtrim(std::string &str) { + auto it = std::find_if(str.rbegin(), str.rend(), [](char ch) { return !std::isspace(ch, std::locale()); }); + str.erase(it.base(), str.end()); + return str; +} + +CLI11_INLINE std::string &rtrim(std::string &str, const std::string &filter) { + auto it = + std::find_if(str.rbegin(), str.rend(), [&filter](char ch) { return filter.find(ch) == std::string::npos; }); + str.erase(it.base(), str.end()); + return str; +} + +CLI11_INLINE std::string &remove_quotes(std::string &str) { + if(str.length() > 1 && (str.front() == '"' || str.front() == '\'' || str.front() == '`')) { + if(str.front() == str.back()) { + str.pop_back(); + str.erase(str.begin(), str.begin() + 1); + } + } + return str; +} + +CLI11_INLINE std::string &remove_outer(std::string &str, char key) { + if(str.length() > 1 && (str.front() == key)) { + if(str.front() == str.back()) { + str.pop_back(); + str.erase(str.begin(), str.begin() + 1); + } + } + return str; +} + +CLI11_INLINE std::string fix_newlines(const std::string &leader, std::string input) { + std::string::size_type n = 0; + while(n != std::string::npos && n < input.size()) { + n = input.find('\n', n); + if(n != std::string::npos) { + input = input.substr(0, n + 1) + leader + input.substr(n + 1); + n += leader.size(); + } + } + return input; +} + +CLI11_INLINE std::ostream & +format_help(std::ostream &out, std::string name, const std::string &description, std::size_t wid) { + name = " " + name; + out << std::setw(static_cast(wid)) << std::left << name; + if(!description.empty()) { + if(name.length() >= wid) + out << "\n" << std::setw(static_cast(wid)) << ""; + for(const char c : description) { + out.put(c); + if(c == '\n') { + out << std::setw(static_cast(wid)) << ""; + } + } + } + out << "\n"; + return out; +} + +CLI11_INLINE std::ostream &format_aliases(std::ostream &out, const std::vector &aliases, std::size_t wid) { + if(!aliases.empty()) { + out << std::setw(static_cast(wid)) << " aliases: "; + bool front = true; + for(const auto &alias : aliases) { + if(!front) { + out << ", "; + } else { + front = false; + } + out << detail::fix_newlines(" ", alias); + } + out << "\n"; + } + return out; +} + +CLI11_INLINE bool valid_name_string(const std::string &str) { + if(str.empty() || !valid_first_char(str[0])) { + return false; + } + auto e = str.end(); + for(auto c = str.begin() + 1; c != e; ++c) + if(!valid_later_char(*c)) + return false; + return true; +} + +CLI11_INLINE std::string find_and_replace(std::string str, std::string from, std::string to) { + + std::size_t start_pos = 0; + + while((start_pos = str.find(from, start_pos)) != std::string::npos) { + str.replace(start_pos, from.length(), to); + start_pos += to.length(); + } + + return str; +} + +CLI11_INLINE void remove_default_flag_values(std::string &flags) { + auto loc = flags.find_first_of('{', 2); + while(loc != std::string::npos) { + auto finish = flags.find_first_of("},", loc + 1); + if((finish != std::string::npos) && (flags[finish] == '}')) { + flags.erase(flags.begin() + static_cast(loc), + flags.begin() + static_cast(finish) + 1); + } + loc = flags.find_first_of('{', loc + 1); + } + flags.erase(std::remove(flags.begin(), flags.end(), '!'), flags.end()); +} + +CLI11_INLINE std::ptrdiff_t +find_member(std::string name, const std::vector names, bool ignore_case, bool ignore_underscore) { + auto it = std::end(names); + if(ignore_case) { + if(ignore_underscore) { + name = detail::to_lower(detail::remove_underscore(name)); + it = std::find_if(std::begin(names), std::end(names), [&name](std::string local_name) { + return detail::to_lower(detail::remove_underscore(local_name)) == name; + }); + } else { + name = detail::to_lower(name); + it = std::find_if(std::begin(names), std::end(names), [&name](std::string local_name) { + return detail::to_lower(local_name) == name; + }); + } + + } else if(ignore_underscore) { + name = detail::remove_underscore(name); + it = std::find_if(std::begin(names), std::end(names), [&name](std::string local_name) { + return detail::remove_underscore(local_name) == name; + }); + } else { + it = std::find(std::begin(names), std::end(names), name); + } + + return (it != std::end(names)) ? (it - std::begin(names)) : (-1); +} + +static const std::string escapedChars("\b\t\n\f\r\"\\"); +static const std::string escapedCharsCode("btnfr\"\\"); +static const std::string bracketChars{"\"'`[(<{"}; +static const std::string matchBracketChars("\"'`])>}"); + +CLI11_INLINE bool has_escapable_character(const std::string &str) { + return (str.find_first_of(escapedChars) != std::string::npos); +} + +CLI11_INLINE std::string add_escaped_characters(const std::string &str) { + std::string out; + out.reserve(str.size() + 4); + for(char s : str) { + auto sloc = escapedChars.find_first_of(s); + if(sloc != std::string::npos) { + out.push_back('\\'); + out.push_back(escapedCharsCode[sloc]); + } else { + out.push_back(s); + } + } + return out; +} + +CLI11_INLINE std::uint32_t hexConvert(char hc) { + int hcode{0}; + if(hc >= '0' && hc <= '9') { + hcode = (hc - '0'); + } else if(hc >= 'A' && hc <= 'F') { + hcode = (hc - 'A' + 10); + } else if(hc >= 'a' && hc <= 'f') { + hcode = (hc - 'a' + 10); + } else { + hcode = -1; + } + return static_cast(hcode); +} + +CLI11_INLINE char make_char(std::uint32_t code) { return static_cast(static_cast(code)); } + +CLI11_INLINE void append_codepoint(std::string &str, std::uint32_t code) { + if(code < 0x80) { // ascii code equivalent + str.push_back(static_cast(code)); + } else if(code < 0x800) { // \u0080 to \u07FF + // 110yyyyx 10xxxxxx; 0x3f == 0b0011'1111 + str.push_back(make_char(0xC0 | code >> 6)); + str.push_back(make_char(0x80 | (code & 0x3F))); + } else if(code < 0x10000) { // U+0800...U+FFFF + if(0xD800 <= code && code <= 0xDFFF) { + throw std::invalid_argument("[0xD800, 0xDFFF] are not valid UTF-8."); + } + // 1110yyyy 10yxxxxx 10xxxxxx + str.push_back(make_char(0xE0 | code >> 12)); + str.push_back(make_char(0x80 | (code >> 6 & 0x3F))); + str.push_back(make_char(0x80 | (code & 0x3F))); + } else if(code < 0x110000) { // U+010000 ... U+10FFFF + // 11110yyy 10yyxxxx 10xxxxxx 10xxxxxx + str.push_back(make_char(0xF0 | code >> 18)); + str.push_back(make_char(0x80 | (code >> 12 & 0x3F))); + str.push_back(make_char(0x80 | (code >> 6 & 0x3F))); + str.push_back(make_char(0x80 | (code & 0x3F))); + } +} + +CLI11_INLINE std::string remove_escaped_characters(const std::string &str) { + + std::string out; + out.reserve(str.size()); + for(auto loc = str.begin(); loc < str.end(); ++loc) { + if(*loc == '\\') { + if(str.end() - loc < 2) { + throw std::invalid_argument("invalid escape sequence " + str); + } + auto ecloc = escapedCharsCode.find_first_of(*(loc + 1)); + if(ecloc != std::string::npos) { + out.push_back(escapedChars[ecloc]); + ++loc; + } else if(*(loc + 1) == 'u') { + // must have 4 hex characters + if(str.end() - loc < 6) { + throw std::invalid_argument("unicode sequence must have 4 hex codes " + str); + } + std::uint32_t code{0}; + std::uint32_t mplier{16 * 16 * 16}; + for(int ii = 2; ii < 6; ++ii) { + std::uint32_t res = hexConvert(*(loc + ii)); + if(res > 0x0F) { + throw std::invalid_argument("unicode sequence must have 4 hex codes " + str); + } + code += res * mplier; + mplier = mplier / 16; + } + append_codepoint(out, code); + loc += 5; + } else if(*(loc + 1) == 'U') { + // must have 8 hex characters + if(str.end() - loc < 10) { + throw std::invalid_argument("unicode sequence must have 8 hex codes " + str); + } + std::uint32_t code{0}; + std::uint32_t mplier{16 * 16 * 16 * 16 * 16 * 16 * 16}; + for(int ii = 2; ii < 10; ++ii) { + std::uint32_t res = hexConvert(*(loc + ii)); + if(res > 0x0F) { + throw std::invalid_argument("unicode sequence must have 8 hex codes " + str); + } + code += res * mplier; + mplier = mplier / 16; + } + append_codepoint(out, code); + loc += 9; + } else if(*(loc + 1) == '0') { + out.push_back('\0'); + ++loc; + } else { + throw std::invalid_argument(std::string("unrecognized escape sequence \\") + *(loc + 1) + " in " + str); + } + } else { + out.push_back(*loc); + } + } + return out; +} + +CLI11_INLINE std::size_t close_string_quote(const std::string &str, std::size_t start, char closure_char) { + std::size_t loc{0}; + for(loc = start + 1; loc < str.size(); ++loc) { + if(str[loc] == closure_char) { + break; + } + if(str[loc] == '\\') { + // skip the next character for escaped sequences + ++loc; + } + } + return loc; +} + +CLI11_INLINE std::size_t close_literal_quote(const std::string &str, std::size_t start, char closure_char) { + auto loc = str.find_first_of(closure_char, start + 1); + return (loc != std::string::npos ? loc : str.size()); +} + +CLI11_INLINE std::size_t close_sequence(const std::string &str, std::size_t start, char closure_char) { + + auto bracket_loc = matchBracketChars.find(closure_char); + switch(bracket_loc) { + case 0: + return close_string_quote(str, start, closure_char); + case 1: + case 2: + case std::string::npos: + return close_literal_quote(str, start, closure_char); + default: + break; + } + + std::string closures(1, closure_char); + auto loc = start + 1; + + while(loc < str.size()) { + if(str[loc] == closures.back()) { + closures.pop_back(); + if(closures.empty()) { + return loc; + } + } + bracket_loc = bracketChars.find(str[loc]); + if(bracket_loc != std::string::npos) { + switch(bracket_loc) { + case 0: + loc = close_string_quote(str, loc, str[loc]); + break; + case 1: + case 2: + loc = close_literal_quote(str, loc, str[loc]); + break; + default: + closures.push_back(matchBracketChars[bracket_loc]); + break; + } + } + ++loc; + } + if(loc > str.size()) { + loc = str.size(); + } + return loc; +} + +CLI11_INLINE std::vector split_up(std::string str, char delimiter) { + + auto find_ws = [delimiter](char ch) { + return (delimiter == '\0') ? std::isspace(ch, std::locale()) : (ch == delimiter); + }; + trim(str); + + std::vector output; + while(!str.empty()) { + if(bracketChars.find_first_of(str[0]) != std::string::npos) { + auto bracketLoc = bracketChars.find_first_of(str[0]); + auto end = close_sequence(str, 0, matchBracketChars[bracketLoc]); + if(end >= str.size()) { + output.push_back(std::move(str)); + str.clear(); + } else { + output.push_back(str.substr(0, end + 1)); + if(end + 2 < str.size()) { + str = str.substr(end + 2); + } else { + str.clear(); + } + } + + } else { + auto it = std::find_if(std::begin(str), std::end(str), find_ws); + if(it != std::end(str)) { + std::string value = std::string(str.begin(), it); + output.push_back(value); + str = std::string(it + 1, str.end()); + } else { + output.push_back(str); + str.clear(); + } + } + trim(str); + } + return output; +} + +CLI11_INLINE std::size_t escape_detect(std::string &str, std::size_t offset) { + auto next = str[offset + 1]; + if((next == '\"') || (next == '\'') || (next == '`')) { + auto astart = str.find_last_of("-/ \"\'`", offset - 1); + if(astart != std::string::npos) { + if(str[astart] == ((str[offset] == '=') ? '-' : '/')) + str[offset] = ' '; // interpret this as a space so the split_up works properly + } + } + return offset + 1; +} + +CLI11_INLINE std::string binary_escape_string(const std::string &string_to_escape) { + // s is our escaped output string + std::string escaped_string{}; + // loop through all characters + for(char c : string_to_escape) { + // check if a given character is printable + // the cast is necessary to avoid undefined behaviour + if(isprint(static_cast(c)) == 0) { + std::stringstream stream; + // if the character is not printable + // we'll convert it to a hex string using a stringstream + // note that since char is signed we have to cast it to unsigned first + stream << std::hex << static_cast(static_cast(c)); + std::string code = stream.str(); + escaped_string += std::string("\\x") + (code.size() < 2 ? "0" : "") + code; + + } else { + escaped_string.push_back(c); + } + } + if(escaped_string != string_to_escape) { + auto sqLoc = escaped_string.find('\''); + while(sqLoc != std::string::npos) { + escaped_string.replace(sqLoc, sqLoc + 1, "\\x27"); + sqLoc = escaped_string.find('\''); + } + escaped_string.insert(0, "'B\"("); + escaped_string.push_back(')'); + escaped_string.push_back('"'); + escaped_string.push_back('\''); + } + return escaped_string; +} + +CLI11_INLINE bool is_binary_escaped_string(const std::string &escaped_string) { + size_t ssize = escaped_string.size(); + if(escaped_string.compare(0, 3, "B\"(") == 0 && escaped_string.compare(ssize - 2, 2, ")\"") == 0) { + return true; + } + return (escaped_string.compare(0, 4, "'B\"(") == 0 && escaped_string.compare(ssize - 3, 3, ")\"'") == 0); +} + +CLI11_INLINE std::string extract_binary_string(const std::string &escaped_string) { + std::size_t start{0}; + std::size_t tail{0}; + size_t ssize = escaped_string.size(); + if(escaped_string.compare(0, 3, "B\"(") == 0 && escaped_string.compare(ssize - 2, 2, ")\"") == 0) { + start = 3; + tail = 2; + } else if(escaped_string.compare(0, 4, "'B\"(") == 0 && escaped_string.compare(ssize - 3, 3, ")\"'") == 0) { + start = 4; + tail = 3; + } + + if(start == 0) { + return escaped_string; + } + std::string outstring; + + outstring.reserve(ssize - start - tail); + std::size_t loc = start; + while(loc < ssize - tail) { + // ssize-2 to skip )" at the end + if(escaped_string[loc] == '\\' && (escaped_string[loc + 1] == 'x' || escaped_string[loc + 1] == 'X')) { + auto c1 = escaped_string[loc + 2]; + auto c2 = escaped_string[loc + 3]; + + std::uint32_t res1 = hexConvert(c1); + std::uint32_t res2 = hexConvert(c2); + if(res1 <= 0x0F && res2 <= 0x0F) { + loc += 4; + outstring.push_back(static_cast(res1 * 16 + res2)); + continue; + } + } + outstring.push_back(escaped_string[loc]); + ++loc; + } + return outstring; +} + +CLI11_INLINE void remove_quotes(std::vector &args) { + for(auto &arg : args) { + if(arg.front() == '\"' && arg.back() == '\"') { + remove_quotes(arg); + // only remove escaped for string arguments not literal strings + arg = remove_escaped_characters(arg); + } else { + remove_quotes(arg); + } + } +} + +CLI11_INLINE bool process_quoted_string(std::string &str, char string_char, char literal_char) { + if(str.size() <= 1) { + return false; + } + if(detail::is_binary_escaped_string(str)) { + str = detail::extract_binary_string(str); + return true; + } + if(str.front() == string_char && str.back() == string_char) { + detail::remove_outer(str, string_char); + if(str.find_first_of('\\') != std::string::npos) { + str = detail::remove_escaped_characters(str); + } + return true; + } + if((str.front() == literal_char || str.front() == '`') && str.back() == str.front()) { + detail::remove_outer(str, str.front()); + return true; + } + return false; +} + +std::string get_environment_value(const std::string &env_name) { + char *buffer = nullptr; + std::string ename_string; + +#ifdef _MSC_VER + // Windows version + std::size_t sz = 0; + if(_dupenv_s(&buffer, &sz, env_name.c_str()) == 0 && buffer != nullptr) { + ename_string = std::string(buffer); + free(buffer); + } +#else + // This also works on Windows, but gives a warning + buffer = std::getenv(env_name.c_str()); + if(buffer != nullptr) { + ename_string = std::string(buffer); + } +#endif + return ename_string; +} + +} // namespace detail + + + +// Use one of these on all error classes. +// These are temporary and are undef'd at the end of this file. +#define CLI11_ERROR_DEF(parent, name) \ + protected: \ + name(std::string ename, std::string msg, int exit_code) : parent(std::move(ename), std::move(msg), exit_code) {} \ + name(std::string ename, std::string msg, ExitCodes exit_code) \ + : parent(std::move(ename), std::move(msg), exit_code) {} \ + \ + public: \ + name(std::string msg, ExitCodes exit_code) : parent(#name, std::move(msg), exit_code) {} \ + name(std::string msg, int exit_code) : parent(#name, std::move(msg), exit_code) {} + +// This is added after the one above if a class is used directly and builds its own message +#define CLI11_ERROR_SIMPLE(name) \ + explicit name(std::string msg) : name(#name, msg, ExitCodes::name) {} + +/// These codes are part of every error in CLI. They can be obtained from e using e.exit_code or as a quick shortcut, +/// int values from e.get_error_code(). +enum class ExitCodes { + Success = 0, + IncorrectConstruction = 100, + BadNameString, + OptionAlreadyAdded, + FileError, + ConversionError, + ValidationError, + RequiredError, + RequiresError, + ExcludesError, + ExtrasError, + ConfigError, + InvalidError, + HorribleError, + OptionNotFound, + ArgumentMismatch, + BaseClass = 127 +}; + +// Error definitions + +/// @defgroup error_group Errors +/// @brief Errors thrown by CLI11 +/// +/// These are the errors that can be thrown. Some of them, like CLI::Success, are not really errors. +/// @{ + +/// All errors derive from this one +class Error : public std::runtime_error { + int actual_exit_code; + std::string error_name{"Error"}; + + public: + CLI11_NODISCARD int get_exit_code() const { return actual_exit_code; } + + CLI11_NODISCARD std::string get_name() const { return error_name; } + + Error(std::string name, std::string msg, int exit_code = static_cast(ExitCodes::BaseClass)) + : runtime_error(msg), actual_exit_code(exit_code), error_name(std::move(name)) {} + + Error(std::string name, std::string msg, ExitCodes exit_code) : Error(name, msg, static_cast(exit_code)) {} +}; + +// Note: Using Error::Error constructors does not work on GCC 4.7 + +/// Construction errors (not in parsing) +class ConstructionError : public Error { + CLI11_ERROR_DEF(Error, ConstructionError) +}; + +/// Thrown when an option is set to conflicting values (non-vector and multi args, for example) +class IncorrectConstruction : public ConstructionError { + CLI11_ERROR_DEF(ConstructionError, IncorrectConstruction) + CLI11_ERROR_SIMPLE(IncorrectConstruction) + static IncorrectConstruction PositionalFlag(std::string name) { + return IncorrectConstruction(name + ": Flags cannot be positional"); + } + static IncorrectConstruction Set0Opt(std::string name) { + return IncorrectConstruction(name + ": Cannot set 0 expected, use a flag instead"); + } + static IncorrectConstruction SetFlag(std::string name) { + return IncorrectConstruction(name + ": Cannot set an expected number for flags"); + } + static IncorrectConstruction ChangeNotVector(std::string name) { + return IncorrectConstruction(name + ": You can only change the expected arguments for vectors"); + } + static IncorrectConstruction AfterMultiOpt(std::string name) { + return IncorrectConstruction( + name + ": You can't change expected arguments after you've changed the multi option policy!"); + } + static IncorrectConstruction MissingOption(std::string name) { + return IncorrectConstruction("Option " + name + " is not defined"); + } + static IncorrectConstruction MultiOptionPolicy(std::string name) { + return IncorrectConstruction(name + ": multi_option_policy only works for flags and exact value options"); + } +}; + +/// Thrown on construction of a bad name +class BadNameString : public ConstructionError { + CLI11_ERROR_DEF(ConstructionError, BadNameString) + CLI11_ERROR_SIMPLE(BadNameString) + static BadNameString OneCharName(std::string name) { return BadNameString("Invalid one char name: " + name); } + static BadNameString MissingDash(std::string name) { + return BadNameString("Long names strings require 2 dashes " + name); + } + static BadNameString BadLongName(std::string name) { return BadNameString("Bad long name: " + name); } + static BadNameString BadPositionalName(std::string name) { + return BadNameString("Invalid positional Name: " + name); + } + static BadNameString DashesOnly(std::string name) { + return BadNameString("Must have a name, not just dashes: " + name); + } + static BadNameString MultiPositionalNames(std::string name) { + return BadNameString("Only one positional name allowed, remove: " + name); + } +}; + +/// Thrown when an option already exists +class OptionAlreadyAdded : public ConstructionError { + CLI11_ERROR_DEF(ConstructionError, OptionAlreadyAdded) + explicit OptionAlreadyAdded(std::string name) + : OptionAlreadyAdded(name + " is already added", ExitCodes::OptionAlreadyAdded) {} + static OptionAlreadyAdded Requires(std::string name, std::string other) { + return {name + " requires " + other, ExitCodes::OptionAlreadyAdded}; + } + static OptionAlreadyAdded Excludes(std::string name, std::string other) { + return {name + " excludes " + other, ExitCodes::OptionAlreadyAdded}; + } +}; + +// Parsing errors + +/// Anything that can error in Parse +class ParseError : public Error { + CLI11_ERROR_DEF(Error, ParseError) +}; + +// Not really "errors" + +/// This is a successful completion on parsing, supposed to exit +class Success : public ParseError { + CLI11_ERROR_DEF(ParseError, Success) + Success() : Success("Successfully completed, should be caught and quit", ExitCodes::Success) {} +}; + +/// -h or --help on command line +class CallForHelp : public Success { + CLI11_ERROR_DEF(Success, CallForHelp) + CallForHelp() : CallForHelp("This should be caught in your main function, see examples", ExitCodes::Success) {} +}; + +/// Usually something like --help-all on command line +class CallForAllHelp : public Success { + CLI11_ERROR_DEF(Success, CallForAllHelp) + CallForAllHelp() + : CallForAllHelp("This should be caught in your main function, see examples", ExitCodes::Success) {} +}; + +/// -v or --version on command line +class CallForVersion : public Success { + CLI11_ERROR_DEF(Success, CallForVersion) + CallForVersion() + : CallForVersion("This should be caught in your main function, see examples", ExitCodes::Success) {} +}; + +/// Does not output a diagnostic in CLI11_PARSE, but allows main() to return with a specific error code. +class RuntimeError : public ParseError { + CLI11_ERROR_DEF(ParseError, RuntimeError) + explicit RuntimeError(int exit_code = 1) : RuntimeError("Runtime error", exit_code) {} +}; + +/// Thrown when parsing an INI file and it is missing +class FileError : public ParseError { + CLI11_ERROR_DEF(ParseError, FileError) + CLI11_ERROR_SIMPLE(FileError) + static FileError Missing(std::string name) { return FileError(name + " was not readable (missing?)"); } +}; + +/// Thrown when conversion call back fails, such as when an int fails to coerce to a string +class ConversionError : public ParseError { + CLI11_ERROR_DEF(ParseError, ConversionError) + CLI11_ERROR_SIMPLE(ConversionError) + ConversionError(std::string member, std::string name) + : ConversionError("The value " + member + " is not an allowed value for " + name) {} + ConversionError(std::string name, std::vector results) + : ConversionError("Could not convert: " + name + " = " + detail::join(results)) {} + static ConversionError TooManyInputsFlag(std::string name) { + return ConversionError(name + ": too many inputs for a flag"); + } + static ConversionError TrueFalse(std::string name) { + return ConversionError(name + ": Should be true/false or a number"); + } +}; + +/// Thrown when validation of results fails +class ValidationError : public ParseError { + CLI11_ERROR_DEF(ParseError, ValidationError) + CLI11_ERROR_SIMPLE(ValidationError) + explicit ValidationError(std::string name, std::string msg) : ValidationError(name + ": " + msg) {} +}; + +/// Thrown when a required option is missing +class RequiredError : public ParseError { + CLI11_ERROR_DEF(ParseError, RequiredError) + explicit RequiredError(std::string name) : RequiredError(name + " is required", ExitCodes::RequiredError) {} + static RequiredError Subcommand(std::size_t min_subcom) { + if(min_subcom == 1) { + return RequiredError("A subcommand"); + } + return {"Requires at least " + std::to_string(min_subcom) + " subcommands", ExitCodes::RequiredError}; + } + static RequiredError + Option(std::size_t min_option, std::size_t max_option, std::size_t used, const std::string &option_list) { + if((min_option == 1) && (max_option == 1) && (used == 0)) + return RequiredError("Exactly 1 option from [" + option_list + "]"); + if((min_option == 1) && (max_option == 1) && (used > 1)) { + return {"Exactly 1 option from [" + option_list + "] is required and " + std::to_string(used) + + " were given", + ExitCodes::RequiredError}; + } + if((min_option == 1) && (used == 0)) + return RequiredError("At least 1 option from [" + option_list + "]"); + if(used < min_option) { + return {"Requires at least " + std::to_string(min_option) + " options used and only " + + std::to_string(used) + "were given from [" + option_list + "]", + ExitCodes::RequiredError}; + } + if(max_option == 1) + return {"Requires at most 1 options be given from [" + option_list + "]", ExitCodes::RequiredError}; + + return {"Requires at most " + std::to_string(max_option) + " options be used and " + std::to_string(used) + + "were given from [" + option_list + "]", + ExitCodes::RequiredError}; + } +}; + +/// Thrown when the wrong number of arguments has been received +class ArgumentMismatch : public ParseError { + CLI11_ERROR_DEF(ParseError, ArgumentMismatch) + CLI11_ERROR_SIMPLE(ArgumentMismatch) + ArgumentMismatch(std::string name, int expected, std::size_t received) + : ArgumentMismatch(expected > 0 ? ("Expected exactly " + std::to_string(expected) + " arguments to " + name + + ", got " + std::to_string(received)) + : ("Expected at least " + std::to_string(-expected) + " arguments to " + name + + ", got " + std::to_string(received)), + ExitCodes::ArgumentMismatch) {} + + static ArgumentMismatch AtLeast(std::string name, int num, std::size_t received) { + return ArgumentMismatch(name + ": At least " + std::to_string(num) + " required but received " + + std::to_string(received)); + } + static ArgumentMismatch AtMost(std::string name, int num, std::size_t received) { + return ArgumentMismatch(name + ": At Most " + std::to_string(num) + " required but received " + + std::to_string(received)); + } + static ArgumentMismatch TypedAtLeast(std::string name, int num, std::string type) { + return ArgumentMismatch(name + ": " + std::to_string(num) + " required " + type + " missing"); + } + static ArgumentMismatch FlagOverride(std::string name) { + return ArgumentMismatch(name + " was given a disallowed flag override"); + } + static ArgumentMismatch PartialType(std::string name, int num, std::string type) { + return ArgumentMismatch(name + ": " + type + " only partially specified: " + std::to_string(num) + + " required for each element"); + } +}; + +/// Thrown when a requires option is missing +class RequiresError : public ParseError { + CLI11_ERROR_DEF(ParseError, RequiresError) + RequiresError(std::string curname, std::string subname) + : RequiresError(curname + " requires " + subname, ExitCodes::RequiresError) {} +}; + +/// Thrown when an excludes option is present +class ExcludesError : public ParseError { + CLI11_ERROR_DEF(ParseError, ExcludesError) + ExcludesError(std::string curname, std::string subname) + : ExcludesError(curname + " excludes " + subname, ExitCodes::ExcludesError) {} +}; + +/// Thrown when too many positionals or options are found +class ExtrasError : public ParseError { + CLI11_ERROR_DEF(ParseError, ExtrasError) + explicit ExtrasError(std::vector args) + : ExtrasError((args.size() > 1 ? "The following arguments were not expected: " + : "The following argument was not expected: ") + + detail::rjoin(args, " "), + ExitCodes::ExtrasError) {} + ExtrasError(const std::string &name, std::vector args) + : ExtrasError(name, + (args.size() > 1 ? "The following arguments were not expected: " + : "The following argument was not expected: ") + + detail::rjoin(args, " "), + ExitCodes::ExtrasError) {} +}; + +/// Thrown when extra values are found in an INI file +class ConfigError : public ParseError { + CLI11_ERROR_DEF(ParseError, ConfigError) + CLI11_ERROR_SIMPLE(ConfigError) + static ConfigError Extras(std::string item) { return ConfigError("INI was not able to parse " + item); } + static ConfigError NotConfigurable(std::string item) { + return ConfigError(item + ": This option is not allowed in a configuration file"); + } +}; + +/// Thrown when validation fails before parsing +class InvalidError : public ParseError { + CLI11_ERROR_DEF(ParseError, InvalidError) + explicit InvalidError(std::string name) + : InvalidError(name + ": Too many positional arguments with unlimited expected args", ExitCodes::InvalidError) { + } +}; + +/// This is just a safety check to verify selection and parsing match - you should not ever see it +/// Strings are directly added to this error, but again, it should never be seen. +class HorribleError : public ParseError { + CLI11_ERROR_DEF(ParseError, HorribleError) + CLI11_ERROR_SIMPLE(HorribleError) +}; + +// After parsing + +/// Thrown when counting a non-existent option +class OptionNotFound : public Error { + CLI11_ERROR_DEF(Error, OptionNotFound) + explicit OptionNotFound(std::string name) : OptionNotFound(name + " not found", ExitCodes::OptionNotFound) {} +}; + +#undef CLI11_ERROR_DEF +#undef CLI11_ERROR_SIMPLE + +/// @} + + + + +// Type tools + +// Utilities for type enabling +namespace detail { +// Based generally on https://rmf.io/cxx11/almost-static-if +/// Simple empty scoped class +enum class enabler {}; + +/// An instance to use in EnableIf +constexpr enabler dummy = {}; +} // namespace detail + +/// A copy of enable_if_t from C++14, compatible with C++11. +/// +/// We could check to see if C++14 is being used, but it does not hurt to redefine this +/// (even Google does this: https://github.com/google/skia/blob/main/include/private/SkTLogic.h) +/// It is not in the std namespace anyway, so no harm done. +template using enable_if_t = typename std::enable_if::type; + +/// A copy of std::void_t from C++17 (helper for C++11 and C++14) +template struct make_void { + using type = void; +}; + +/// A copy of std::void_t from C++17 - same reasoning as enable_if_t, it does not hurt to redefine +template using void_t = typename make_void::type; + +/// A copy of std::conditional_t from C++14 - same reasoning as enable_if_t, it does not hurt to redefine +template using conditional_t = typename std::conditional::type; + +/// Check to see if something is bool (fail check by default) +template struct is_bool : std::false_type {}; + +/// Check to see if something is bool (true if actually a bool) +template <> struct is_bool : std::true_type {}; + +/// Check to see if something is a shared pointer +template struct is_shared_ptr : std::false_type {}; + +/// Check to see if something is a shared pointer (True if really a shared pointer) +template struct is_shared_ptr> : std::true_type {}; + +/// Check to see if something is a shared pointer (True if really a shared pointer) +template struct is_shared_ptr> : std::true_type {}; + +/// Check to see if something is copyable pointer +template struct is_copyable_ptr { + static bool const value = is_shared_ptr::value || std::is_pointer::value; +}; + +/// This can be specialized to override the type deduction for IsMember. +template struct IsMemberType { + using type = T; +}; + +/// The main custom type needed here is const char * should be a string. +template <> struct IsMemberType { + using type = std::string; +}; + +namespace detail { + +// These are utilities for IsMember and other transforming objects + +/// Handy helper to access the element_type generically. This is not part of is_copyable_ptr because it requires that +/// pointer_traits be valid. + +/// not a pointer +template struct element_type { + using type = T; +}; + +template struct element_type::value>::type> { + using type = typename std::pointer_traits::element_type; +}; + +/// Combination of the element type and value type - remove pointer (including smart pointers) and get the value_type of +/// the container +template struct element_value_type { + using type = typename element_type::type::value_type; +}; + +/// Adaptor for set-like structure: This just wraps a normal container in a few utilities that do almost nothing. +template struct pair_adaptor : std::false_type { + using value_type = typename T::value_type; + using first_type = typename std::remove_const::type; + using second_type = typename std::remove_const::type; + + /// Get the first value (really just the underlying value) + template static auto first(Q &&pair_value) -> decltype(std::forward(pair_value)) { + return std::forward(pair_value); + } + /// Get the second value (really just the underlying value) + template static auto second(Q &&pair_value) -> decltype(std::forward(pair_value)) { + return std::forward(pair_value); + } +}; + +/// Adaptor for map-like structure (true version, must have key_type and mapped_type). +/// This wraps a mapped container in a few utilities access it in a general way. +template +struct pair_adaptor< + T, + conditional_t, void>> + : std::true_type { + using value_type = typename T::value_type; + using first_type = typename std::remove_const::type; + using second_type = typename std::remove_const::type; + + /// Get the first value (really just the underlying value) + template static auto first(Q &&pair_value) -> decltype(std::get<0>(std::forward(pair_value))) { + return std::get<0>(std::forward(pair_value)); + } + /// Get the second value (really just the underlying value) + template static auto second(Q &&pair_value) -> decltype(std::get<1>(std::forward(pair_value))) { + return std::get<1>(std::forward(pair_value)); + } +}; + +// Warning is suppressed due to "bug" in gcc<5.0 and gcc 7.0 with c++17 enabled that generates a Wnarrowing warning +// in the unevaluated context even if the function that was using this wasn't used. The standard says narrowing in +// brace initialization shouldn't be allowed but for backwards compatibility gcc allows it in some contexts. It is a +// little fuzzy what happens in template constructs and I think that was something GCC took a little while to work out. +// But regardless some versions of gcc generate a warning when they shouldn't from the following code so that should be +// suppressed +#ifdef __GNUC__ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wnarrowing" +#endif +// check for constructibility from a specific type and copy assignable used in the parse detection +template class is_direct_constructible { + template + static auto test(int, std::true_type) -> decltype( +// NVCC warns about narrowing conversions here +#ifdef __CUDACC__ +#ifdef __NVCC_DIAG_PRAGMA_SUPPORT__ +#pragma nv_diag_suppress 2361 +#else +#pragma diag_suppress 2361 +#endif +#endif + TT{std::declval()} +#ifdef __CUDACC__ +#ifdef __NVCC_DIAG_PRAGMA_SUPPORT__ +#pragma nv_diag_default 2361 +#else +#pragma diag_default 2361 +#endif +#endif + , + std::is_move_assignable()); + + template static auto test(int, std::false_type) -> std::false_type; + + template static auto test(...) -> std::false_type; + + public: + static constexpr bool value = decltype(test(0, typename std::is_constructible::type()))::value; +}; +#ifdef __GNUC__ +#pragma GCC diagnostic pop +#endif + +// Check for output streamability +// Based on https://stackoverflow.com/questions/22758291/how-can-i-detect-if-a-type-can-be-streamed-to-an-stdostream + +template class is_ostreamable { + template + static auto test(int) -> decltype(std::declval() << std::declval(), std::true_type()); + + template static auto test(...) -> std::false_type; + + public: + static constexpr bool value = decltype(test(0))::value; +}; + +/// Check for input streamability +template class is_istreamable { + template + static auto test(int) -> decltype(std::declval() >> std::declval(), std::true_type()); + + template static auto test(...) -> std::false_type; + + public: + static constexpr bool value = decltype(test(0))::value; +}; + +/// Check for complex +template class is_complex { + template + static auto test(int) -> decltype(std::declval().real(), std::declval().imag(), std::true_type()); + + template static auto test(...) -> std::false_type; + + public: + static constexpr bool value = decltype(test(0))::value; +}; + +/// Templated operation to get a value from a stream +template ::value, detail::enabler> = detail::dummy> +bool from_stream(const std::string &istring, T &obj) { + std::istringstream is; + is.str(istring); + is >> obj; + return !is.fail() && !is.rdbuf()->in_avail(); +} + +template ::value, detail::enabler> = detail::dummy> +bool from_stream(const std::string & /*istring*/, T & /*obj*/) { + return false; +} + +// check to see if an object is a mutable container (fail by default) +template struct is_mutable_container : std::false_type {}; + +/// type trait to test if a type is a mutable container meaning it has a value_type, it has an iterator, a clear, and +/// end methods and an insert function. And for our purposes we exclude std::string and types that can be constructed +/// from a std::string +template +struct is_mutable_container< + T, + conditional_t().end()), + decltype(std::declval().clear()), + decltype(std::declval().insert(std::declval().end())>(), + std::declval()))>, + void>> : public conditional_t::value || + std::is_constructible::value, + std::false_type, + std::true_type> {}; + +// check to see if an object is a mutable container (fail by default) +template struct is_readable_container : std::false_type {}; + +/// type trait to test if a type is a container meaning it has a value_type, it has an iterator, a clear, and an end +/// methods and an insert function. And for our purposes we exclude std::string and types that can be constructed from +/// a std::string +template +struct is_readable_container< + T, + conditional_t().end()), decltype(std::declval().begin())>, void>> + : public std::true_type {}; + +// check to see if an object is a wrapper (fail by default) +template struct is_wrapper : std::false_type {}; + +// check if an object is a wrapper (it has a value_type defined) +template +struct is_wrapper, void>> : public std::true_type {}; + +// Check for tuple like types, as in classes with a tuple_size type trait +template class is_tuple_like { + template + // static auto test(int) + // -> decltype(std::conditional<(std::tuple_size::value > 0), std::true_type, std::false_type>::type()); + static auto test(int) -> decltype(std::tuple_size::type>::value, std::true_type{}); + template static auto test(...) -> std::false_type; + + public: + static constexpr bool value = decltype(test(0))::value; +}; + +/// Convert an object to a string (directly forward if this can become a string) +template ::value, detail::enabler> = detail::dummy> +auto to_string(T &&value) -> decltype(std::forward(value)) { + return std::forward(value); +} + +/// Construct a string from the object +template ::value && !std::is_convertible::value, + detail::enabler> = detail::dummy> +std::string to_string(const T &value) { + return std::string(value); // NOLINT(google-readability-casting) +} + +/// Convert an object to a string (streaming must be supported for that type) +template ::value && !std::is_constructible::value && + is_ostreamable::value, + detail::enabler> = detail::dummy> +std::string to_string(T &&value) { + std::stringstream stream; + stream << value; + return stream.str(); +} + +/// If conversion is not supported, return an empty string (streaming is not supported for that type) +template ::value && !is_ostreamable::value && + !is_readable_container::type>::value, + detail::enabler> = detail::dummy> +std::string to_string(T &&) { + return {}; +} + +/// convert a readable container to a string +template ::value && !is_ostreamable::value && + is_readable_container::value, + detail::enabler> = detail::dummy> +std::string to_string(T &&variable) { + auto cval = variable.begin(); + auto end = variable.end(); + if(cval == end) { + return {"{}"}; + } + std::vector defaults; + while(cval != end) { + defaults.emplace_back(CLI::detail::to_string(*cval)); + ++cval; + } + return {"[" + detail::join(defaults) + "]"}; +} + +/// special template overload +template ::value, detail::enabler> = detail::dummy> +auto checked_to_string(T &&value) -> decltype(to_string(std::forward(value))) { + return to_string(std::forward(value)); +} + +/// special template overload +template ::value, detail::enabler> = detail::dummy> +std::string checked_to_string(T &&) { + return std::string{}; +} +/// get a string as a convertible value for arithmetic types +template ::value, detail::enabler> = detail::dummy> +std::string value_string(const T &value) { + return std::to_string(value); +} +/// get a string as a convertible value for enumerations +template ::value, detail::enabler> = detail::dummy> +std::string value_string(const T &value) { + return std::to_string(static_cast::type>(value)); +} +/// for other types just use the regular to_string function +template ::value && !std::is_arithmetic::value, detail::enabler> = detail::dummy> +auto value_string(const T &value) -> decltype(to_string(value)) { + return to_string(value); +} + +/// template to get the underlying value type if it exists or use a default +template struct wrapped_type { + using type = def; +}; + +/// Type size for regular object types that do not look like a tuple +template struct wrapped_type::value>::type> { + using type = typename T::value_type; +}; + +/// This will only trigger for actual void type +template struct type_count_base { + static const int value{0}; +}; + +/// Type size for regular object types that do not look like a tuple +template +struct type_count_base::value && !is_mutable_container::value && + !std::is_void::value>::type> { + static constexpr int value{1}; +}; + +/// the base tuple size +template +struct type_count_base::value && !is_mutable_container::value>::type> { + static constexpr int value{std::tuple_size::value}; +}; + +/// Type count base for containers is the type_count_base of the individual element +template struct type_count_base::value>::type> { + static constexpr int value{type_count_base::value}; +}; + +/// Set of overloads to get the type size of an object + +/// forward declare the subtype_count structure +template struct subtype_count; + +/// forward declare the subtype_count_min structure +template struct subtype_count_min; + +/// This will only trigger for actual void type +template struct type_count { + static const int value{0}; +}; + +/// Type size for regular object types that do not look like a tuple +template +struct type_count::value && !is_tuple_like::value && !is_complex::value && + !std::is_void::value>::type> { + static constexpr int value{1}; +}; + +/// Type size for complex since it sometimes looks like a wrapper +template struct type_count::value>::type> { + static constexpr int value{2}; +}; + +/// Type size of types that are wrappers,except complex and tuples(which can also be wrappers sometimes) +template struct type_count::value>::type> { + static constexpr int value{subtype_count::value}; +}; + +/// Type size of types that are wrappers,except containers complex and tuples(which can also be wrappers sometimes) +template +struct type_count::value && !is_complex::value && !is_tuple_like::value && + !is_mutable_container::value>::type> { + static constexpr int value{type_count::value}; +}; + +/// 0 if the index > tuple size +template +constexpr typename std::enable_if::value, int>::type tuple_type_size() { + return 0; +} + +/// Recursively generate the tuple type name +template + constexpr typename std::enable_if < I::value, int>::type tuple_type_size() { + return subtype_count::type>::value + tuple_type_size(); +} + +/// Get the type size of the sum of type sizes for all the individual tuple types +template struct type_count::value>::type> { + static constexpr int value{tuple_type_size()}; +}; + +/// definition of subtype count +template struct subtype_count { + static constexpr int value{is_mutable_container::value ? expected_max_vector_size : type_count::value}; +}; + +/// This will only trigger for actual void type +template struct type_count_min { + static const int value{0}; +}; + +/// Type size for regular object types that do not look like a tuple +template +struct type_count_min< + T, + typename std::enable_if::value && !is_tuple_like::value && !is_wrapper::value && + !is_complex::value && !std::is_void::value>::type> { + static constexpr int value{type_count::value}; +}; + +/// Type size for complex since it sometimes looks like a wrapper +template struct type_count_min::value>::type> { + static constexpr int value{1}; +}; + +/// Type size min of types that are wrappers,except complex and tuples(which can also be wrappers sometimes) +template +struct type_count_min< + T, + typename std::enable_if::value && !is_complex::value && !is_tuple_like::value>::type> { + static constexpr int value{subtype_count_min::value}; +}; + +/// 0 if the index > tuple size +template +constexpr typename std::enable_if::value, int>::type tuple_type_size_min() { + return 0; +} + +/// Recursively generate the tuple type name +template + constexpr typename std::enable_if < I::value, int>::type tuple_type_size_min() { + return subtype_count_min::type>::value + tuple_type_size_min(); +} + +/// Get the type size of the sum of type sizes for all the individual tuple types +template struct type_count_min::value>::type> { + static constexpr int value{tuple_type_size_min()}; +}; + +/// definition of subtype count +template struct subtype_count_min { + static constexpr int value{is_mutable_container::value + ? ((type_count::value < expected_max_vector_size) ? type_count::value : 0) + : type_count_min::value}; +}; + +/// This will only trigger for actual void type +template struct expected_count { + static const int value{0}; +}; + +/// For most types the number of expected items is 1 +template +struct expected_count::value && !is_wrapper::value && + !std::is_void::value>::type> { + static constexpr int value{1}; +}; +/// number of expected items in a vector +template struct expected_count::value>::type> { + static constexpr int value{expected_max_vector_size}; +}; + +/// number of expected items in a vector +template +struct expected_count::value && is_wrapper::value>::type> { + static constexpr int value{expected_count::value}; +}; + +// Enumeration of the different supported categorizations of objects +enum class object_category : int { + char_value = 1, + integral_value = 2, + unsigned_integral = 4, + enumeration = 6, + boolean_value = 8, + floating_point = 10, + number_constructible = 12, + double_constructible = 14, + integer_constructible = 16, + // string like types + string_assignable = 23, + string_constructible = 24, + wstring_assignable = 25, + wstring_constructible = 26, + other = 45, + // special wrapper or container types + wrapper_value = 50, + complex_number = 60, + tuple_value = 70, + container_value = 80, + +}; + +/// Set of overloads to classify an object according to type + +/// some type that is not otherwise recognized +template struct classify_object { + static constexpr object_category value{object_category::other}; +}; + +/// Signed integers +template +struct classify_object< + T, + typename std::enable_if::value && !std::is_same::value && std::is_signed::value && + !is_bool::value && !std::is_enum::value>::type> { + static constexpr object_category value{object_category::integral_value}; +}; + +/// Unsigned integers +template +struct classify_object::value && std::is_unsigned::value && + !std::is_same::value && !is_bool::value>::type> { + static constexpr object_category value{object_category::unsigned_integral}; +}; + +/// single character values +template +struct classify_object::value && !std::is_enum::value>::type> { + static constexpr object_category value{object_category::char_value}; +}; + +/// Boolean values +template struct classify_object::value>::type> { + static constexpr object_category value{object_category::boolean_value}; +}; + +/// Floats +template struct classify_object::value>::type> { + static constexpr object_category value{object_category::floating_point}; +}; +#if defined _MSC_VER +// in MSVC wstring should take precedence if available this isn't as useful on other compilers due to the broader use of +// utf-8 encoding +#define WIDE_STRING_CHECK \ + !std::is_assignable::value && !std::is_constructible::value +#define STRING_CHECK true +#else +#define WIDE_STRING_CHECK true +#define STRING_CHECK !std::is_assignable::value && !std::is_constructible::value +#endif + +/// String and similar direct assignment +template +struct classify_object< + T, + typename std::enable_if::value && !std::is_integral::value && WIDE_STRING_CHECK && + std::is_assignable::value>::type> { + static constexpr object_category value{object_category::string_assignable}; +}; + +/// String and similar constructible and copy assignment +template +struct classify_object< + T, + typename std::enable_if::value && !std::is_integral::value && + !std::is_assignable::value && (type_count::value == 1) && + WIDE_STRING_CHECK && std::is_constructible::value>::type> { + static constexpr object_category value{object_category::string_constructible}; +}; + +/// Wide strings +template +struct classify_object::value && !std::is_integral::value && + STRING_CHECK && std::is_assignable::value>::type> { + static constexpr object_category value{object_category::wstring_assignable}; +}; + +template +struct classify_object< + T, + typename std::enable_if::value && !std::is_integral::value && + !std::is_assignable::value && (type_count::value == 1) && + STRING_CHECK && std::is_constructible::value>::type> { + static constexpr object_category value{object_category::wstring_constructible}; +}; + +/// Enumerations +template struct classify_object::value>::type> { + static constexpr object_category value{object_category::enumeration}; +}; + +template struct classify_object::value>::type> { + static constexpr object_category value{object_category::complex_number}; +}; + +/// Handy helper to contain a bunch of checks that rule out many common types (integers, string like, floating point, +/// vectors, and enumerations +template struct uncommon_type { + using type = typename std::conditional< + !std::is_floating_point::value && !std::is_integral::value && + !std::is_assignable::value && !std::is_constructible::value && + !std::is_assignable::value && !std::is_constructible::value && + !is_complex::value && !is_mutable_container::value && !std::is_enum::value, + std::true_type, + std::false_type>::type; + static constexpr bool value = type::value; +}; + +/// wrapper type +template +struct classify_object::value && is_wrapper::value && + !is_tuple_like::value && uncommon_type::value)>::type> { + static constexpr object_category value{object_category::wrapper_value}; +}; + +/// Assignable from double or int +template +struct classify_object::value && type_count::value == 1 && + !is_wrapper::value && is_direct_constructible::value && + is_direct_constructible::value>::type> { + static constexpr object_category value{object_category::number_constructible}; +}; + +/// Assignable from int +template +struct classify_object::value && type_count::value == 1 && + !is_wrapper::value && !is_direct_constructible::value && + is_direct_constructible::value>::type> { + static constexpr object_category value{object_category::integer_constructible}; +}; + +/// Assignable from double +template +struct classify_object::value && type_count::value == 1 && + !is_wrapper::value && is_direct_constructible::value && + !is_direct_constructible::value>::type> { + static constexpr object_category value{object_category::double_constructible}; +}; + +/// Tuple type +template +struct classify_object< + T, + typename std::enable_if::value && + ((type_count::value >= 2 && !is_wrapper::value) || + (uncommon_type::value && !is_direct_constructible::value && + !is_direct_constructible::value) || + (uncommon_type::value && type_count::value >= 2))>::type> { + static constexpr object_category value{object_category::tuple_value}; + // the condition on this class requires it be like a tuple, but on some compilers (like Xcode) tuples can be + // constructed from just the first element so tuples of can be constructed from a string, which + // could lead to issues so there are two variants of the condition, the first isolates things with a type size >=2 + // mainly to get tuples on Xcode with the exception of wrappers, the second is the main one and just separating out + // those cases that are caught by other object classifications +}; + +/// container type +template struct classify_object::value>::type> { + static constexpr object_category value{object_category::container_value}; +}; + +// Type name print + +/// Was going to be based on +/// http://stackoverflow.com/questions/1055452/c-get-name-of-type-in-template +/// But this is cleaner and works better in this case + +template ::value == object_category::char_value, detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "CHAR"; +} + +template ::value == object_category::integral_value || + classify_object::value == object_category::integer_constructible, + detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "INT"; +} + +template ::value == object_category::unsigned_integral, detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "UINT"; +} + +template ::value == object_category::floating_point || + classify_object::value == object_category::number_constructible || + classify_object::value == object_category::double_constructible, + detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "FLOAT"; +} + +/// Print name for enumeration types +template ::value == object_category::enumeration, detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "ENUM"; +} + +/// Print name for enumeration types +template ::value == object_category::boolean_value, detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "BOOLEAN"; +} + +/// Print name for enumeration types +template ::value == object_category::complex_number, detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "COMPLEX"; +} + +/// Print for all other types +template ::value >= object_category::string_assignable && + classify_object::value <= object_category::other, + detail::enabler> = detail::dummy> +constexpr const char *type_name() { + return "TEXT"; +} +/// typename for tuple value +template ::value == object_category::tuple_value && type_count_base::value >= 2, + detail::enabler> = detail::dummy> +std::string type_name(); // forward declaration + +/// Generate type name for a wrapper or container value +template ::value == object_category::container_value || + classify_object::value == object_category::wrapper_value, + detail::enabler> = detail::dummy> +std::string type_name(); // forward declaration + +/// Print name for single element tuple types +template ::value == object_category::tuple_value && type_count_base::value == 1, + detail::enabler> = detail::dummy> +inline std::string type_name() { + return type_name::type>::type>(); +} + +/// Empty string if the index > tuple size +template +inline typename std::enable_if::value, std::string>::type tuple_name() { + return std::string{}; +} + +/// Recursively generate the tuple type name +template +inline typename std::enable_if<(I < type_count_base::value), std::string>::type tuple_name() { + auto str = std::string{type_name::type>::type>()} + ',' + + tuple_name(); + if(str.back() == ',') + str.pop_back(); + return str; +} + +/// Print type name for tuples with 2 or more elements +template ::value == object_category::tuple_value && type_count_base::value >= 2, + detail::enabler>> +inline std::string type_name() { + auto tname = std::string(1, '[') + tuple_name(); + tname.push_back(']'); + return tname; +} + +/// get the type name for a type that has a value_type member +template ::value == object_category::container_value || + classify_object::value == object_category::wrapper_value, + detail::enabler>> +inline std::string type_name() { + return type_name(); +} + +// Lexical cast + +/// Convert to an unsigned integral +template ::value, detail::enabler> = detail::dummy> +bool integral_conversion(const std::string &input, T &output) noexcept { + if(input.empty() || input.front() == '-') { + return false; + } + char *val{nullptr}; + errno = 0; + std::uint64_t output_ll = std::strtoull(input.c_str(), &val, 0); + if(errno == ERANGE) { + return false; + } + output = static_cast(output_ll); + if(val == (input.c_str() + input.size()) && static_cast(output) == output_ll) { + return true; + } + val = nullptr; + std::int64_t output_sll = std::strtoll(input.c_str(), &val, 0); + if(val == (input.c_str() + input.size())) { + output = (output_sll < 0) ? static_cast(0) : static_cast(output_sll); + return (static_cast(output) == output_sll); + } + // remove separators + if(input.find_first_of("_'") != std::string::npos) { + std::string nstring = input; + nstring.erase(std::remove(nstring.begin(), nstring.end(), '_'), nstring.end()); + nstring.erase(std::remove(nstring.begin(), nstring.end(), '\''), nstring.end()); + return integral_conversion(nstring, output); + } + if(input.compare(0, 2, "0o") == 0) { + val = nullptr; + errno = 0; + output_ll = std::strtoull(input.c_str() + 2, &val, 8); + if(errno == ERANGE) { + return false; + } + output = static_cast(output_ll); + return (val == (input.c_str() + input.size()) && static_cast(output) == output_ll); + } + if(input.compare(0, 2, "0b") == 0) { + val = nullptr; + errno = 0; + output_ll = std::strtoull(input.c_str() + 2, &val, 2); + if(errno == ERANGE) { + return false; + } + output = static_cast(output_ll); + return (val == (input.c_str() + input.size()) && static_cast(output) == output_ll); + } + return false; +} + +/// Convert to a signed integral +template ::value, detail::enabler> = detail::dummy> +bool integral_conversion(const std::string &input, T &output) noexcept { + if(input.empty()) { + return false; + } + char *val = nullptr; + errno = 0; + std::int64_t output_ll = std::strtoll(input.c_str(), &val, 0); + if(errno == ERANGE) { + return false; + } + output = static_cast(output_ll); + if(val == (input.c_str() + input.size()) && static_cast(output) == output_ll) { + return true; + } + if(input == "true") { + // this is to deal with a few oddities with flags and wrapper int types + output = static_cast(1); + return true; + } + // remove separators + if(input.find_first_of("_'") != std::string::npos) { + std::string nstring = input; + nstring.erase(std::remove(nstring.begin(), nstring.end(), '_'), nstring.end()); + nstring.erase(std::remove(nstring.begin(), nstring.end(), '\''), nstring.end()); + return integral_conversion(nstring, output); + } + if(input.compare(0, 2, "0o") == 0) { + val = nullptr; + errno = 0; + output_ll = std::strtoll(input.c_str() + 2, &val, 8); + if(errno == ERANGE) { + return false; + } + output = static_cast(output_ll); + return (val == (input.c_str() + input.size()) && static_cast(output) == output_ll); + } + if(input.compare(0, 2, "0b") == 0) { + val = nullptr; + errno = 0; + output_ll = std::strtoll(input.c_str() + 2, &val, 2); + if(errno == ERANGE) { + return false; + } + output = static_cast(output_ll); + return (val == (input.c_str() + input.size()) && static_cast(output) == output_ll); + } + return false; +} + +/// Convert a flag into an integer value typically binary flags sets errno to nonzero if conversion failed +inline std::int64_t to_flag_value(std::string val) noexcept { + static const std::string trueString("true"); + static const std::string falseString("false"); + if(val == trueString) { + return 1; + } + if(val == falseString) { + return -1; + } + val = detail::to_lower(val); + std::int64_t ret = 0; + if(val.size() == 1) { + if(val[0] >= '1' && val[0] <= '9') { + return (static_cast(val[0]) - '0'); + } + switch(val[0]) { + case '0': + case 'f': + case 'n': + case '-': + ret = -1; + break; + case 't': + case 'y': + case '+': + ret = 1; + break; + default: + errno = EINVAL; + return -1; + } + return ret; + } + if(val == trueString || val == "on" || val == "yes" || val == "enable") { + ret = 1; + } else if(val == falseString || val == "off" || val == "no" || val == "disable") { + ret = -1; + } else { + char *loc_ptr{nullptr}; + ret = std::strtoll(val.c_str(), &loc_ptr, 0); + if(loc_ptr != (val.c_str() + val.size()) && errno == 0) { + errno = EINVAL; + } + } + return ret; +} + +/// Integer conversion +template ::value == object_category::integral_value || + classify_object::value == object_category::unsigned_integral, + detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + return integral_conversion(input, output); +} + +/// char values +template ::value == object_category::char_value, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + if(input.size() == 1) { + output = static_cast(input[0]); + return true; + } + return integral_conversion(input, output); +} + +/// Boolean values +template ::value == object_category::boolean_value, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + errno = 0; + auto out = to_flag_value(input); + if(errno == 0) { + output = (out > 0); + } else if(errno == ERANGE) { + output = (input[0] != '-'); + } else { + return false; + } + return true; +} + +/// Floats +template ::value == object_category::floating_point, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + if(input.empty()) { + return false; + } + char *val = nullptr; + auto output_ld = std::strtold(input.c_str(), &val); + output = static_cast(output_ld); + if(val == (input.c_str() + input.size())) { + return true; + } + // remove separators + if(input.find_first_of("_'") != std::string::npos) { + std::string nstring = input; + nstring.erase(std::remove(nstring.begin(), nstring.end(), '_'), nstring.end()); + nstring.erase(std::remove(nstring.begin(), nstring.end(), '\''), nstring.end()); + return lexical_cast(nstring, output); + } + return false; +} + +/// complex +template ::value == object_category::complex_number, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + using XC = typename wrapped_type::type; + XC x{0.0}, y{0.0}; + auto str1 = input; + bool worked = false; + auto nloc = str1.find_last_of("+-"); + if(nloc != std::string::npos && nloc > 0) { + worked = lexical_cast(str1.substr(0, nloc), x); + str1 = str1.substr(nloc); + if(str1.back() == 'i' || str1.back() == 'j') + str1.pop_back(); + worked = worked && lexical_cast(str1, y); + } else { + if(str1.back() == 'i' || str1.back() == 'j') { + str1.pop_back(); + worked = lexical_cast(str1, y); + x = XC{0}; + } else { + worked = lexical_cast(str1, x); + y = XC{0}; + } + } + if(worked) { + output = T{x, y}; + return worked; + } + return from_stream(input, output); +} + +/// String and similar direct assignment +template ::value == object_category::string_assignable, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + output = input; + return true; +} + +/// String and similar constructible and copy assignment +template < + typename T, + enable_if_t::value == object_category::string_constructible, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + output = T(input); + return true; +} + +/// Wide strings +template < + typename T, + enable_if_t::value == object_category::wstring_assignable, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + output = widen(input); + return true; +} + +template < + typename T, + enable_if_t::value == object_category::wstring_constructible, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + output = T{widen(input)}; + return true; +} + +/// Enumerations +template ::value == object_category::enumeration, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + typename std::underlying_type::type val; + if(!integral_conversion(input, val)) { + return false; + } + output = static_cast(val); + return true; +} + +/// wrapper types +template ::value == object_category::wrapper_value && + std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + typename T::value_type val; + if(lexical_cast(input, val)) { + output = val; + return true; + } + return from_stream(input, output); +} + +template ::value == object_category::wrapper_value && + !std::is_assignable::value && std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + typename T::value_type val; + if(lexical_cast(input, val)) { + output = T{val}; + return true; + } + return from_stream(input, output); +} + +/// Assignable from double or int +template < + typename T, + enable_if_t::value == object_category::number_constructible, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + int val = 0; + if(integral_conversion(input, val)) { + output = T(val); + return true; + } + + double dval = 0.0; + if(lexical_cast(input, dval)) { + output = T{dval}; + return true; + } + + return from_stream(input, output); +} + +/// Assignable from int +template < + typename T, + enable_if_t::value == object_category::integer_constructible, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + int val = 0; + if(integral_conversion(input, val)) { + output = T(val); + return true; + } + return from_stream(input, output); +} + +/// Assignable from double +template < + typename T, + enable_if_t::value == object_category::double_constructible, detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + double val = 0.0; + if(lexical_cast(input, val)) { + output = T{val}; + return true; + } + return from_stream(input, output); +} + +/// Non-string convertible from an int +template ::value == object_category::other && std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + int val = 0; + if(integral_conversion(input, val)) { +#ifdef _MSC_VER +#pragma warning(push) +#pragma warning(disable : 4800) +#endif + // with Atomic this could produce a warning due to the conversion but if atomic gets here it is an old style + // so will most likely still work + output = val; +#ifdef _MSC_VER +#pragma warning(pop) +#endif + return true; + } + // LCOV_EXCL_START + // This version of cast is only used for odd cases in an older compilers the fail over + // from_stream is tested elsewhere an not relevant for coverage here + return from_stream(input, output); + // LCOV_EXCL_STOP +} + +/// Non-string parsable by a stream +template ::value == object_category::other && !std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_cast(const std::string &input, T &output) { + static_assert(is_istreamable::value, + "option object type must have a lexical cast overload or streaming input operator(>>) defined, if it " + "is convertible from another type use the add_option(...) with XC being the known type"); + return from_stream(input, output); +} + +/// Assign a value through lexical cast operations +/// Strings can be empty so we need to do a little different +template ::value && + (classify_object::value == object_category::string_assignable || + classify_object::value == object_category::string_constructible || + classify_object::value == object_category::wstring_assignable || + classify_object::value == object_category::wstring_constructible), + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, AssignTo &output) { + return lexical_cast(input, output); +} + +/// Assign a value through lexical cast operations +template ::value && std::is_assignable::value && + classify_object::value != object_category::string_assignable && + classify_object::value != object_category::string_constructible && + classify_object::value != object_category::wstring_assignable && + classify_object::value != object_category::wstring_constructible, + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, AssignTo &output) { + if(input.empty()) { + output = AssignTo{}; + return true; + } + + return lexical_cast(input, output); +} + +/// Assign a value through lexical cast operations +template ::value && !std::is_assignable::value && + classify_object::value == object_category::wrapper_value, + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, AssignTo &output) { + if(input.empty()) { + typename AssignTo::value_type emptyVal{}; + output = emptyVal; + return true; + } + return lexical_cast(input, output); +} + +/// Assign a value through lexical cast operations for int compatible values +/// mainly for atomic operations on some compilers +template ::value && !std::is_assignable::value && + classify_object::value != object_category::wrapper_value && + std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, AssignTo &output) { + if(input.empty()) { + output = 0; + return true; + } + int val{0}; + if(lexical_cast(input, val)) { +#if defined(__clang__) +/* on some older clang compilers */ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wsign-conversion" +#endif + output = val; +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + return true; + } + return false; +} + +/// Assign a value converted from a string in lexical cast to the output value directly +template ::value && std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, AssignTo &output) { + ConvertTo val{}; + bool parse_result = (!input.empty()) ? lexical_cast(input, val) : true; + if(parse_result) { + output = val; + } + return parse_result; +} + +/// Assign a value from a lexical cast through constructing a value and move assigning it +template < + typename AssignTo, + typename ConvertTo, + enable_if_t::value && !std::is_assignable::value && + std::is_move_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_assign(const std::string &input, AssignTo &output) { + ConvertTo val{}; + bool parse_result = input.empty() ? true : lexical_cast(input, val); + if(parse_result) { + output = AssignTo(val); // use () form of constructor to allow some implicit conversions + } + return parse_result; +} + +/// primary lexical conversion operation, 1 string to 1 type of some kind +template ::value <= object_category::other && + classify_object::value <= object_category::wrapper_value, + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + return lexical_assign(strings[0], output); +} + +/// Lexical conversion if there is only one element but the conversion type is for two, then call a two element +/// constructor +template ::value <= 2) && expected_count::value == 1 && + is_tuple_like::value && type_count_base::value == 2, + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + // the remove const is to handle pair types coming from a container + using FirstType = typename std::remove_const::type>::type; + using SecondType = typename std::tuple_element<1, ConvertTo>::type; + FirstType v1; + SecondType v2; + bool retval = lexical_assign(strings[0], v1); + retval = retval && lexical_assign((strings.size() > 1) ? strings[1] : std::string{}, v2); + if(retval) { + output = AssignTo{v1, v2}; + } + return retval; +} + +/// Lexical conversion of a container types of single elements +template ::value && is_mutable_container::value && + type_count::value == 1, + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + output.erase(output.begin(), output.end()); + if(strings.empty()) { + return true; + } + if(strings.size() == 1 && strings[0] == "{}") { + return true; + } + bool skip_remaining = false; + if(strings.size() == 2 && strings[0] == "{}" && is_separator(strings[1])) { + skip_remaining = true; + } + for(const auto &elem : strings) { + typename AssignTo::value_type out; + bool retval = lexical_assign(elem, out); + if(!retval) { + return false; + } + output.insert(output.end(), std::move(out)); + if(skip_remaining) { + break; + } + } + return (!output.empty()); +} + +/// Lexical conversion for complex types +template ::value, detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + + if(strings.size() >= 2 && !strings[1].empty()) { + using XC2 = typename wrapped_type::type; + XC2 x{0.0}, y{0.0}; + auto str1 = strings[1]; + if(str1.back() == 'i' || str1.back() == 'j') { + str1.pop_back(); + } + auto worked = lexical_cast(strings[0], x) && lexical_cast(str1, y); + if(worked) { + output = ConvertTo{x, y}; + } + return worked; + } + return lexical_assign(strings[0], output); +} + +/// Conversion to a vector type using a particular single type as the conversion type +template ::value && (expected_count::value == 1) && + (type_count::value == 1), + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + bool retval = true; + output.clear(); + output.reserve(strings.size()); + for(const auto &elem : strings) { + + output.emplace_back(); + retval = retval && lexical_assign(elem, output.back()); + } + return (!output.empty()) && retval; +} + +// forward declaration + +/// Lexical conversion of a container types with conversion type of two elements +template ::value && is_mutable_container::value && + type_count_base::value == 2, + detail::enabler> = detail::dummy> +bool lexical_conversion(std::vector strings, AssignTo &output); + +/// Lexical conversion of a vector types with type_size >2 forward declaration +template ::value && is_mutable_container::value && + type_count_base::value != 2 && + ((type_count::value > 2) || + (type_count::value > type_count_base::value)), + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output); + +/// Conversion for tuples +template ::value && is_tuple_like::value && + (type_count_base::value != type_count::value || + type_count::value > 2), + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output); // forward declaration + +/// Conversion for operations where the assigned type is some class but the conversion is a mutable container or large +/// tuple +template ::value && !is_mutable_container::value && + classify_object::value != object_category::wrapper_value && + (is_mutable_container::value || type_count::value > 2), + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + + if(strings.size() > 1 || (!strings.empty() && !(strings.front().empty()))) { + ConvertTo val; + auto retval = lexical_conversion(strings, val); + output = AssignTo{val}; + return retval; + } + output = AssignTo{}; + return true; +} + +/// function template for converting tuples if the static Index is greater than the tuple size +template +inline typename std::enable_if<(I >= type_count_base::value), bool>::type +tuple_conversion(const std::vector &, AssignTo &) { + return true; +} + +/// Conversion of a tuple element where the type size ==1 and not a mutable container +template +inline typename std::enable_if::value && type_count::value == 1, bool>::type +tuple_type_conversion(std::vector &strings, AssignTo &output) { + auto retval = lexical_assign(strings[0], output); + strings.erase(strings.begin()); + return retval; +} + +/// Conversion of a tuple element where the type size !=1 but the size is fixed and not a mutable container +template +inline typename std::enable_if::value && (type_count::value > 1) && + type_count::value == type_count_min::value, + bool>::type +tuple_type_conversion(std::vector &strings, AssignTo &output) { + auto retval = lexical_conversion(strings, output); + strings.erase(strings.begin(), strings.begin() + type_count::value); + return retval; +} + +/// Conversion of a tuple element where the type is a mutable container or a type with different min and max type sizes +template +inline typename std::enable_if::value || + type_count::value != type_count_min::value, + bool>::type +tuple_type_conversion(std::vector &strings, AssignTo &output) { + + std::size_t index{subtype_count_min::value}; + const std::size_t mx_count{subtype_count::value}; + const std::size_t mx{(std::min)(mx_count, strings.size() - 1)}; + + while(index < mx) { + if(is_separator(strings[index])) { + break; + } + ++index; + } + bool retval = lexical_conversion( + std::vector(strings.begin(), strings.begin() + static_cast(index)), output); + if(strings.size() > index) { + strings.erase(strings.begin(), strings.begin() + static_cast(index) + 1); + } else { + strings.clear(); + } + return retval; +} + +/// Tuple conversion operation +template +inline typename std::enable_if<(I < type_count_base::value), bool>::type +tuple_conversion(std::vector strings, AssignTo &output) { + bool retval = true; + using ConvertToElement = typename std:: + conditional::value, typename std::tuple_element::type, ConvertTo>::type; + if(!strings.empty()) { + retval = retval && tuple_type_conversion::type, ConvertToElement>( + strings, std::get(output)); + } + retval = retval && tuple_conversion(std::move(strings), output); + return retval; +} + +/// Lexical conversion of a container types with tuple elements of size 2 +template ::value && is_mutable_container::value && + type_count_base::value == 2, + detail::enabler>> +bool lexical_conversion(std::vector strings, AssignTo &output) { + output.clear(); + while(!strings.empty()) { + + typename std::remove_const::type>::type v1; + typename std::tuple_element<1, typename ConvertTo::value_type>::type v2; + bool retval = tuple_type_conversion(strings, v1); + if(!strings.empty()) { + retval = retval && tuple_type_conversion(strings, v2); + } + if(retval) { + output.insert(output.end(), typename AssignTo::value_type{v1, v2}); + } else { + return false; + } + } + return (!output.empty()); +} + +/// lexical conversion of tuples with type count>2 or tuples of types of some element with a type size>=2 +template ::value && is_tuple_like::value && + (type_count_base::value != type_count::value || + type_count::value > 2), + detail::enabler>> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + static_assert( + !is_tuple_like::value || type_count_base::value == type_count_base::value, + "if the conversion type is defined as a tuple it must be the same size as the type you are converting to"); + return tuple_conversion(strings, output); +} + +/// Lexical conversion of a vector types for everything but tuples of two elements and types of size 1 +template ::value && is_mutable_container::value && + type_count_base::value != 2 && + ((type_count::value > 2) || + (type_count::value > type_count_base::value)), + detail::enabler>> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + bool retval = true; + output.clear(); + std::vector temp; + std::size_t ii{0}; + std::size_t icount{0}; + std::size_t xcm{type_count::value}; + auto ii_max = strings.size(); + while(ii < ii_max) { + temp.push_back(strings[ii]); + ++ii; + ++icount; + if(icount == xcm || is_separator(temp.back()) || ii == ii_max) { + if(static_cast(xcm) > type_count_min::value && is_separator(temp.back())) { + temp.pop_back(); + } + typename AssignTo::value_type temp_out; + retval = retval && + lexical_conversion(temp, temp_out); + temp.clear(); + if(!retval) { + return false; + } + output.insert(output.end(), std::move(temp_out)); + icount = 0; + } + } + return retval; +} + +/// conversion for wrapper types +template ::value == object_category::wrapper_value && + std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + if(strings.empty() || strings.front().empty()) { + output = ConvertTo{}; + return true; + } + typename ConvertTo::value_type val; + if(lexical_conversion(strings, val)) { + output = ConvertTo{val}; + return true; + } + return false; +} + +/// conversion for wrapper types +template ::value == object_category::wrapper_value && + !std::is_assignable::value, + detail::enabler> = detail::dummy> +bool lexical_conversion(const std::vector &strings, AssignTo &output) { + using ConvertType = typename ConvertTo::value_type; + if(strings.empty() || strings.front().empty()) { + output = ConvertType{}; + return true; + } + ConvertType val; + if(lexical_conversion(strings, val)) { + output = val; + return true; + } + return false; +} + +/// Sum a vector of strings +inline std::string sum_string_vector(const std::vector &values) { + double val{0.0}; + bool fail{false}; + std::string output; + for(const auto &arg : values) { + double tv{0.0}; + auto comp = lexical_cast(arg, tv); + if(!comp) { + errno = 0; + auto fv = detail::to_flag_value(arg); + fail = (errno != 0); + if(fail) { + break; + } + tv = static_cast(fv); + } + val += tv; + } + if(fail) { + for(const auto &arg : values) { + output.append(arg); + } + } else { + std::ostringstream out; + out.precision(16); + out << val; + output = out.str(); + } + return output; +} + +} // namespace detail + + + +namespace detail { + +// Returns false if not a short option. Otherwise, sets opt name and rest and returns true +CLI11_INLINE bool split_short(const std::string ¤t, std::string &name, std::string &rest); + +// Returns false if not a long option. Otherwise, sets opt name and other side of = and returns true +CLI11_INLINE bool split_long(const std::string ¤t, std::string &name, std::string &value); + +// Returns false if not a windows style option. Otherwise, sets opt name and value and returns true +CLI11_INLINE bool split_windows_style(const std::string ¤t, std::string &name, std::string &value); + +// Splits a string into multiple long and short names +CLI11_INLINE std::vector split_names(std::string current); + +/// extract default flag values either {def} or starting with a ! +CLI11_INLINE std::vector> get_default_flag_values(const std::string &str); + +/// Get a vector of short names, one of long names, and a single name +CLI11_INLINE std::tuple, std::vector, std::string> +get_names(const std::vector &input); + +} // namespace detail + + + +namespace detail { + +CLI11_INLINE bool split_short(const std::string ¤t, std::string &name, std::string &rest) { + if(current.size() > 1 && current[0] == '-' && valid_first_char(current[1])) { + name = current.substr(1, 1); + rest = current.substr(2); + return true; + } + return false; +} + +CLI11_INLINE bool split_long(const std::string ¤t, std::string &name, std::string &value) { + if(current.size() > 2 && current.compare(0, 2, "--") == 0 && valid_first_char(current[2])) { + auto loc = current.find_first_of('='); + if(loc != std::string::npos) { + name = current.substr(2, loc - 2); + value = current.substr(loc + 1); + } else { + name = current.substr(2); + value = ""; + } + return true; + } + return false; +} + +CLI11_INLINE bool split_windows_style(const std::string ¤t, std::string &name, std::string &value) { + if(current.size() > 1 && current[0] == '/' && valid_first_char(current[1])) { + auto loc = current.find_first_of(':'); + if(loc != std::string::npos) { + name = current.substr(1, loc - 1); + value = current.substr(loc + 1); + } else { + name = current.substr(1); + value = ""; + } + return true; + } + return false; +} + +CLI11_INLINE std::vector split_names(std::string current) { + std::vector output; + std::size_t val = 0; + while((val = current.find(',')) != std::string::npos) { + output.push_back(trim_copy(current.substr(0, val))); + current = current.substr(val + 1); + } + output.push_back(trim_copy(current)); + return output; +} + +CLI11_INLINE std::vector> get_default_flag_values(const std::string &str) { + std::vector flags = split_names(str); + flags.erase(std::remove_if(flags.begin(), + flags.end(), + [](const std::string &name) { + return ((name.empty()) || (!(((name.find_first_of('{') != std::string::npos) && + (name.back() == '}')) || + (name[0] == '!')))); + }), + flags.end()); + std::vector> output; + output.reserve(flags.size()); + for(auto &flag : flags) { + auto def_start = flag.find_first_of('{'); + std::string defval = "false"; + if((def_start != std::string::npos) && (flag.back() == '}')) { + defval = flag.substr(def_start + 1); + defval.pop_back(); + flag.erase(def_start, std::string::npos); // NOLINT(readability-suspicious-call-argument) + } + flag.erase(0, flag.find_first_not_of("-!")); + output.emplace_back(flag, defval); + } + return output; +} + +CLI11_INLINE std::tuple, std::vector, std::string> +get_names(const std::vector &input) { + + std::vector short_names; + std::vector long_names; + std::string pos_name; + for(std::string name : input) { + if(name.length() == 0) { + continue; + } + if(name.length() > 1 && name[0] == '-' && name[1] != '-') { + if(name.length() == 2 && valid_first_char(name[1])) + short_names.emplace_back(1, name[1]); + else if(name.length() > 2) + throw BadNameString::MissingDash(name); + else + throw BadNameString::OneCharName(name); + } else if(name.length() > 2 && name.substr(0, 2) == "--") { + name = name.substr(2); + if(valid_name_string(name)) + long_names.push_back(name); + else + throw BadNameString::BadLongName(name); + } else if(name == "-" || name == "--") { + throw BadNameString::DashesOnly(name); + } else { + if(!pos_name.empty()) + throw BadNameString::MultiPositionalNames(name); + if(valid_name_string(name)) { + pos_name = name; + } else { + throw BadNameString::BadPositionalName(name); + } + } + } + return std::make_tuple(short_names, long_names, pos_name); +} + +} // namespace detail + + + +class App; + +/// Holds values to load into Options +struct ConfigItem { + /// This is the list of parents + std::vector parents{}; + + /// This is the name + std::string name{}; + /// Listing of inputs + std::vector inputs{}; + + /// The list of parents and name joined by "." + CLI11_NODISCARD std::string fullname() const { + std::vector tmp = parents; + tmp.emplace_back(name); + return detail::join(tmp, "."); + } +}; + +/// This class provides a converter for configuration files. +class Config { + protected: + std::vector items{}; + + public: + /// Convert an app into a configuration + virtual std::string to_config(const App *, bool, bool, std::string) const = 0; + + /// Convert a configuration into an app + virtual std::vector from_config(std::istream &) const = 0; + + /// Get a flag value + CLI11_NODISCARD virtual std::string to_flag(const ConfigItem &item) const { + if(item.inputs.size() == 1) { + return item.inputs.at(0); + } + if(item.inputs.empty()) { + return "{}"; + } + throw ConversionError::TooManyInputsFlag(item.fullname()); // LCOV_EXCL_LINE + } + + /// Parse a config file, throw an error (ParseError:ConfigParseError or FileError) on failure + CLI11_NODISCARD std::vector from_file(const std::string &name) const { + std::ifstream input{name}; + if(!input.good()) + throw FileError::Missing(name); + + return from_config(input); + } + + /// Virtual destructor + virtual ~Config() = default; +}; + +/// This converter works with INI/TOML files; to write INI files use ConfigINI +class ConfigBase : public Config { + protected: + /// the character used for comments + char commentChar = '#'; + /// the character used to start an array '\0' is a default to not use + char arrayStart = '['; + /// the character used to end an array '\0' is a default to not use + char arrayEnd = ']'; + /// the character used to separate elements in an array + char arraySeparator = ','; + /// the character used separate the name from the value + char valueDelimiter = '='; + /// the character to use around strings + char stringQuote = '"'; + /// the character to use around single characters and literal strings + char literalQuote = '\''; + /// the maximum number of layers to allow + uint8_t maximumLayers{255}; + /// the separator used to separator parent layers + char parentSeparatorChar{'.'}; + /// Specify the configuration index to use for arrayed sections + int16_t configIndex{-1}; + /// Specify the configuration section that should be used + std::string configSection{}; + + public: + std::string + to_config(const App * /*app*/, bool default_also, bool write_description, std::string prefix) const override; + + std::vector from_config(std::istream &input) const override; + /// Specify the configuration for comment characters + ConfigBase *comment(char cchar) { + commentChar = cchar; + return this; + } + /// Specify the start and end characters for an array + ConfigBase *arrayBounds(char aStart, char aEnd) { + arrayStart = aStart; + arrayEnd = aEnd; + return this; + } + /// Specify the delimiter character for an array + ConfigBase *arrayDelimiter(char aSep) { + arraySeparator = aSep; + return this; + } + /// Specify the delimiter between a name and value + ConfigBase *valueSeparator(char vSep) { + valueDelimiter = vSep; + return this; + } + /// Specify the quote characters used around strings and literal strings + ConfigBase *quoteCharacter(char qString, char literalChar) { + stringQuote = qString; + literalQuote = literalChar; + return this; + } + /// Specify the maximum number of parents + ConfigBase *maxLayers(uint8_t layers) { + maximumLayers = layers; + return this; + } + /// Specify the separator to use for parent layers + ConfigBase *parentSeparator(char sep) { + parentSeparatorChar = sep; + return this; + } + /// get a reference to the configuration section + std::string §ionRef() { return configSection; } + /// get the section + CLI11_NODISCARD const std::string §ion() const { return configSection; } + /// specify a particular section of the configuration file to use + ConfigBase *section(const std::string §ionName) { + configSection = sectionName; + return this; + } + + /// get a reference to the configuration index + int16_t &indexRef() { return configIndex; } + /// get the section index + CLI11_NODISCARD int16_t index() const { return configIndex; } + /// specify a particular index in the section to use (-1) for all sections to use + ConfigBase *index(int16_t sectionIndex) { + configIndex = sectionIndex; + return this; + } +}; + +/// the default Config is the TOML file format +using ConfigTOML = ConfigBase; + +/// ConfigINI generates a "standard" INI compliant output +class ConfigINI : public ConfigTOML { + + public: + ConfigINI() { + commentChar = ';'; + arrayStart = '\0'; + arrayEnd = '\0'; + arraySeparator = ' '; + valueDelimiter = '='; + } +}; + + + +class Option; + +/// @defgroup validator_group Validators + +/// @brief Some validators that are provided +/// +/// These are simple `std::string(const std::string&)` validators that are useful. They return +/// a string if the validation fails. A custom struct is provided, as well, with the same user +/// semantics, but with the ability to provide a new type name. +/// @{ + +/// +class Validator { + protected: + /// This is the description function, if empty the description_ will be used + std::function desc_function_{[]() { return std::string{}; }}; + + /// This is the base function that is to be called. + /// Returns a string error message if validation fails. + std::function func_{[](std::string &) { return std::string{}; }}; + /// The name for search purposes of the Validator + std::string name_{}; + /// A Validator will only apply to an indexed value (-1 is all elements) + int application_index_ = -1; + /// Enable for Validator to allow it to be disabled if need be + bool active_{true}; + /// specify that a validator should not modify the input + bool non_modifying_{false}; + + Validator(std::string validator_desc, std::function func) + : desc_function_([validator_desc]() { return validator_desc; }), func_(std::move(func)) {} + + public: + Validator() = default; + /// Construct a Validator with just the description string + explicit Validator(std::string validator_desc) : desc_function_([validator_desc]() { return validator_desc; }) {} + /// Construct Validator from basic information + Validator(std::function op, std::string validator_desc, std::string validator_name = "") + : desc_function_([validator_desc]() { return validator_desc; }), func_(std::move(op)), + name_(std::move(validator_name)) {} + /// Set the Validator operation function + Validator &operation(std::function op) { + func_ = std::move(op); + return *this; + } + /// This is the required operator for a Validator - provided to help + /// users (CLI11 uses the member `func` directly) + std::string operator()(std::string &str) const; + + /// This is the required operator for a Validator - provided to help + /// users (CLI11 uses the member `func` directly) + std::string operator()(const std::string &str) const { + std::string value = str; + return (active_) ? func_(value) : std::string{}; + } + + /// Specify the type string + Validator &description(std::string validator_desc) { + desc_function_ = [validator_desc]() { return validator_desc; }; + return *this; + } + /// Specify the type string + CLI11_NODISCARD Validator description(std::string validator_desc) const; + + /// Generate type description information for the Validator + CLI11_NODISCARD std::string get_description() const { + if(active_) { + return desc_function_(); + } + return std::string{}; + } + /// Specify the type string + Validator &name(std::string validator_name) { + name_ = std::move(validator_name); + return *this; + } + /// Specify the type string + CLI11_NODISCARD Validator name(std::string validator_name) const { + Validator newval(*this); + newval.name_ = std::move(validator_name); + return newval; + } + /// Get the name of the Validator + CLI11_NODISCARD const std::string &get_name() const { return name_; } + /// Specify whether the Validator is active or not + Validator &active(bool active_val = true) { + active_ = active_val; + return *this; + } + /// Specify whether the Validator is active or not + CLI11_NODISCARD Validator active(bool active_val = true) const { + Validator newval(*this); + newval.active_ = active_val; + return newval; + } + + /// Specify whether the Validator can be modifying or not + Validator &non_modifying(bool no_modify = true) { + non_modifying_ = no_modify; + return *this; + } + /// Specify the application index of a validator + Validator &application_index(int app_index) { + application_index_ = app_index; + return *this; + } + /// Specify the application index of a validator + CLI11_NODISCARD Validator application_index(int app_index) const { + Validator newval(*this); + newval.application_index_ = app_index; + return newval; + } + /// Get the current value of the application index + CLI11_NODISCARD int get_application_index() const { return application_index_; } + /// Get a boolean if the validator is active + CLI11_NODISCARD bool get_active() const { return active_; } + + /// Get a boolean if the validator is allowed to modify the input returns true if it can modify the input + CLI11_NODISCARD bool get_modifying() const { return !non_modifying_; } + + /// Combining validators is a new validator. Type comes from left validator if function, otherwise only set if the + /// same. + Validator operator&(const Validator &other) const; + + /// Combining validators is a new validator. Type comes from left validator if function, otherwise only set if the + /// same. + Validator operator|(const Validator &other) const; + + /// Create a validator that fails when a given validator succeeds + Validator operator!() const; + + private: + void _merge_description(const Validator &val1, const Validator &val2, const std::string &merger); +}; + +/// Class wrapping some of the accessors of Validator +class CustomValidator : public Validator { + public: +}; +// The implementation of the built in validators is using the Validator class; +// the user is only expected to use the const (static) versions (since there's no setup). +// Therefore, this is in detail. +namespace detail { + +/// CLI enumeration of different file types +enum class path_type { nonexistent, file, directory }; + +/// get the type of the path from a file name +CLI11_INLINE path_type check_path(const char *file) noexcept; + +/// Check for an existing file (returns error message if check fails) +class ExistingFileValidator : public Validator { + public: + ExistingFileValidator(); +}; + +/// Check for an existing directory (returns error message if check fails) +class ExistingDirectoryValidator : public Validator { + public: + ExistingDirectoryValidator(); +}; + +/// Check for an existing path +class ExistingPathValidator : public Validator { + public: + ExistingPathValidator(); +}; + +/// Check for an non-existing path +class NonexistentPathValidator : public Validator { + public: + NonexistentPathValidator(); +}; + +/// Validate the given string is a legal ipv4 address +class IPV4Validator : public Validator { + public: + IPV4Validator(); +}; + +class EscapedStringTransformer : public Validator { + public: + EscapedStringTransformer(); +}; + +} // namespace detail + +// Static is not needed here, because global const implies static. + +/// Check for existing file (returns error message if check fails) +const detail::ExistingFileValidator ExistingFile; + +/// Check for an existing directory (returns error message if check fails) +const detail::ExistingDirectoryValidator ExistingDirectory; + +/// Check for an existing path +const detail::ExistingPathValidator ExistingPath; + +/// Check for an non-existing path +const detail::NonexistentPathValidator NonexistentPath; + +/// Check for an IP4 address +const detail::IPV4Validator ValidIPV4; + +/// convert escaped characters into their associated values +const detail::EscapedStringTransformer EscapedString; + +/// Validate the input as a particular type +template class TypeValidator : public Validator { + public: + explicit TypeValidator(const std::string &validator_name) + : Validator(validator_name, [](std::string &input_string) { + using CLI::detail::lexical_cast; + auto val = DesiredType(); + if(!lexical_cast(input_string, val)) { + return std::string("Failed parsing ") + input_string + " as a " + detail::type_name(); + } + return std::string(); + }) {} + TypeValidator() : TypeValidator(detail::type_name()) {} +}; + +/// Check for a number +const TypeValidator Number("NUMBER"); + +/// Modify a path if the file is a particular default location, can be used as Check or transform +/// with the error return optionally disabled +class FileOnDefaultPath : public Validator { + public: + explicit FileOnDefaultPath(std::string default_path, bool enableErrorReturn = true); +}; + +/// Produce a range (factory). Min and max are inclusive. +class Range : public Validator { + public: + /// This produces a range with min and max inclusive. + /// + /// Note that the constructor is templated, but the struct is not, so C++17 is not + /// needed to provide nice syntax for Range(a,b). + template + Range(T min_val, T max_val, const std::string &validator_name = std::string{}) : Validator(validator_name) { + if(validator_name.empty()) { + std::stringstream out; + out << detail::type_name() << " in [" << min_val << " - " << max_val << "]"; + description(out.str()); + } + + func_ = [min_val, max_val](std::string &input) { + using CLI::detail::lexical_cast; + T val; + bool converted = lexical_cast(input, val); + if((!converted) || (val < min_val || val > max_val)) { + std::stringstream out; + out << "Value " << input << " not in range ["; + out << min_val << " - " << max_val << "]"; + return out.str(); + } + return std::string{}; + }; + } + + /// Range of one value is 0 to value + template + explicit Range(T max_val, const std::string &validator_name = std::string{}) + : Range(static_cast(0), max_val, validator_name) {} +}; + +/// Check for a non negative number +const Range NonNegativeNumber((std::numeric_limits::max)(), "NONNEGATIVE"); + +/// Check for a positive valued number (val>0.0), ::min here is the smallest positive number +const Range PositiveNumber((std::numeric_limits::min)(), (std::numeric_limits::max)(), "POSITIVE"); + +/// Produce a bounded range (factory). Min and max are inclusive. +class Bound : public Validator { + public: + /// This bounds a value with min and max inclusive. + /// + /// Note that the constructor is templated, but the struct is not, so C++17 is not + /// needed to provide nice syntax for Range(a,b). + template Bound(T min_val, T max_val) { + std::stringstream out; + out << detail::type_name() << " bounded to [" << min_val << " - " << max_val << "]"; + description(out.str()); + + func_ = [min_val, max_val](std::string &input) { + using CLI::detail::lexical_cast; + T val; + bool converted = lexical_cast(input, val); + if(!converted) { + return std::string("Value ") + input + " could not be converted"; + } + if(val < min_val) + input = detail::to_string(min_val); + else if(val > max_val) + input = detail::to_string(max_val); + + return std::string{}; + }; + } + + /// Range of one value is 0 to value + template explicit Bound(T max_val) : Bound(static_cast(0), max_val) {} +}; + +namespace detail { +template ::type>::value, detail::enabler> = detail::dummy> +auto smart_deref(T value) -> decltype(*value) { + return *value; +} + +template < + typename T, + enable_if_t::type>::value, detail::enabler> = detail::dummy> +typename std::remove_reference::type &smart_deref(T &value) { + return value; +} +/// Generate a string representation of a set +template std::string generate_set(const T &set) { + using element_t = typename detail::element_type::type; + using iteration_type_t = typename detail::pair_adaptor::value_type; // the type of the object pair + std::string out(1, '{'); + out.append(detail::join( + detail::smart_deref(set), + [](const iteration_type_t &v) { return detail::pair_adaptor::first(v); }, + ",")); + out.push_back('}'); + return out; +} + +/// Generate a string representation of a map +template std::string generate_map(const T &map, bool key_only = false) { + using element_t = typename detail::element_type::type; + using iteration_type_t = typename detail::pair_adaptor::value_type; // the type of the object pair + std::string out(1, '{'); + out.append(detail::join( + detail::smart_deref(map), + [key_only](const iteration_type_t &v) { + std::string res{detail::to_string(detail::pair_adaptor::first(v))}; + + if(!key_only) { + res.append("->"); + res += detail::to_string(detail::pair_adaptor::second(v)); + } + return res; + }, + ",")); + out.push_back('}'); + return out; +} + +template struct has_find { + template + static auto test(int) -> decltype(std::declval().find(std::declval()), std::true_type()); + template static auto test(...) -> decltype(std::false_type()); + + static const auto value = decltype(test(0))::value; + using type = std::integral_constant; +}; + +/// A search function +template ::value, detail::enabler> = detail::dummy> +auto search(const T &set, const V &val) -> std::pair { + using element_t = typename detail::element_type::type; + auto &setref = detail::smart_deref(set); + auto it = std::find_if(std::begin(setref), std::end(setref), [&val](decltype(*std::begin(setref)) v) { + return (detail::pair_adaptor::first(v) == val); + }); + return {(it != std::end(setref)), it}; +} + +/// A search function that uses the built in find function +template ::value, detail::enabler> = detail::dummy> +auto search(const T &set, const V &val) -> std::pair { + auto &setref = detail::smart_deref(set); + auto it = setref.find(val); + return {(it != std::end(setref)), it}; +} + +/// A search function with a filter function +template +auto search(const T &set, const V &val, const std::function &filter_function) + -> std::pair { + using element_t = typename detail::element_type::type; + // do the potentially faster first search + auto res = search(set, val); + if((res.first) || (!(filter_function))) { + return res; + } + // if we haven't found it do the longer linear search with all the element translations + auto &setref = detail::smart_deref(set); + auto it = std::find_if(std::begin(setref), std::end(setref), [&](decltype(*std::begin(setref)) v) { + V a{detail::pair_adaptor::first(v)}; + a = filter_function(a); + return (a == val); + }); + return {(it != std::end(setref)), it}; +} + +// the following suggestion was made by Nikita Ofitserov(@himikof) +// done in templates to prevent compiler warnings on negation of unsigned numbers + +/// Do a check for overflow on signed numbers +template +inline typename std::enable_if::value, T>::type overflowCheck(const T &a, const T &b) { + if((a > 0) == (b > 0)) { + return ((std::numeric_limits::max)() / (std::abs)(a) < (std::abs)(b)); + } + return ((std::numeric_limits::min)() / (std::abs)(a) > -(std::abs)(b)); +} +/// Do a check for overflow on unsigned numbers +template +inline typename std::enable_if::value, T>::type overflowCheck(const T &a, const T &b) { + return ((std::numeric_limits::max)() / a < b); +} + +/// Performs a *= b; if it doesn't cause integer overflow. Returns false otherwise. +template typename std::enable_if::value, bool>::type checked_multiply(T &a, T b) { + if(a == 0 || b == 0 || a == 1 || b == 1) { + a *= b; + return true; + } + if(a == (std::numeric_limits::min)() || b == (std::numeric_limits::min)()) { + return false; + } + if(overflowCheck(a, b)) { + return false; + } + a *= b; + return true; +} + +/// Performs a *= b; if it doesn't equal infinity. Returns false otherwise. +template +typename std::enable_if::value, bool>::type checked_multiply(T &a, T b) { + T c = a * b; + if(std::isinf(c) && !std::isinf(a) && !std::isinf(b)) { + return false; + } + a = c; + return true; +} + +} // namespace detail +/// Verify items are in a set +class IsMember : public Validator { + public: + using filter_fn_t = std::function; + + /// This allows in-place construction using an initializer list + template + IsMember(std::initializer_list values, Args &&...args) + : IsMember(std::vector(values), std::forward(args)...) {} + + /// This checks to see if an item is in a set (empty function) + template explicit IsMember(T &&set) : IsMember(std::forward(set), nullptr) {} + + /// This checks to see if an item is in a set: pointer or copy version. You can pass in a function that will filter + /// both sides of the comparison before computing the comparison. + template explicit IsMember(T set, F filter_function) { + + // Get the type of the contained item - requires a container have ::value_type + // if the type does not have first_type and second_type, these are both value_type + using element_t = typename detail::element_type::type; // Removes (smart) pointers if needed + using item_t = typename detail::pair_adaptor::first_type; // Is value_type if not a map + + using local_item_t = typename IsMemberType::type; // This will convert bad types to good ones + // (const char * to std::string) + + // Make a local copy of the filter function, using a std::function if not one already + std::function filter_fn = filter_function; + + // This is the type name for help, it will take the current version of the set contents + desc_function_ = [set]() { return detail::generate_set(detail::smart_deref(set)); }; + + // This is the function that validates + // It stores a copy of the set pointer-like, so shared_ptr will stay alive + func_ = [set, filter_fn](std::string &input) { + using CLI::detail::lexical_cast; + local_item_t b; + if(!lexical_cast(input, b)) { + throw ValidationError(input); // name is added later + } + if(filter_fn) { + b = filter_fn(b); + } + auto res = detail::search(set, b, filter_fn); + if(res.first) { + // Make sure the version in the input string is identical to the one in the set + if(filter_fn) { + input = detail::value_string(detail::pair_adaptor::first(*(res.second))); + } + + // Return empty error string (success) + return std::string{}; + } + + // If you reach this point, the result was not found + return input + " not in " + detail::generate_set(detail::smart_deref(set)); + }; + } + + /// You can pass in as many filter functions as you like, they nest (string only currently) + template + IsMember(T &&set, filter_fn_t filter_fn_1, filter_fn_t filter_fn_2, Args &&...other) + : IsMember( + std::forward(set), + [filter_fn_1, filter_fn_2](std::string a) { return filter_fn_2(filter_fn_1(a)); }, + other...) {} +}; + +/// definition of the default transformation object +template using TransformPairs = std::vector>; + +/// Translate named items to other or a value set +class Transformer : public Validator { + public: + using filter_fn_t = std::function; + + /// This allows in-place construction + template + Transformer(std::initializer_list> values, Args &&...args) + : Transformer(TransformPairs(values), std::forward(args)...) {} + + /// direct map of std::string to std::string + template explicit Transformer(T &&mapping) : Transformer(std::forward(mapping), nullptr) {} + + /// This checks to see if an item is in a set: pointer or copy version. You can pass in a function that will filter + /// both sides of the comparison before computing the comparison. + template explicit Transformer(T mapping, F filter_function) { + + static_assert(detail::pair_adaptor::type>::value, + "mapping must produce value pairs"); + // Get the type of the contained item - requires a container have ::value_type + // if the type does not have first_type and second_type, these are both value_type + using element_t = typename detail::element_type::type; // Removes (smart) pointers if needed + using item_t = typename detail::pair_adaptor::first_type; // Is value_type if not a map + using local_item_t = typename IsMemberType::type; // Will convert bad types to good ones + // (const char * to std::string) + + // Make a local copy of the filter function, using a std::function if not one already + std::function filter_fn = filter_function; + + // This is the type name for help, it will take the current version of the set contents + desc_function_ = [mapping]() { return detail::generate_map(detail::smart_deref(mapping)); }; + + func_ = [mapping, filter_fn](std::string &input) { + using CLI::detail::lexical_cast; + local_item_t b; + if(!lexical_cast(input, b)) { + return std::string(); + // there is no possible way we can match anything in the mapping if we can't convert so just return + } + if(filter_fn) { + b = filter_fn(b); + } + auto res = detail::search(mapping, b, filter_fn); + if(res.first) { + input = detail::value_string(detail::pair_adaptor::second(*res.second)); + } + return std::string{}; + }; + } + + /// You can pass in as many filter functions as you like, they nest + template + Transformer(T &&mapping, filter_fn_t filter_fn_1, filter_fn_t filter_fn_2, Args &&...other) + : Transformer( + std::forward(mapping), + [filter_fn_1, filter_fn_2](std::string a) { return filter_fn_2(filter_fn_1(a)); }, + other...) {} +}; + +/// translate named items to other or a value set +class CheckedTransformer : public Validator { + public: + using filter_fn_t = std::function; + + /// This allows in-place construction + template + CheckedTransformer(std::initializer_list> values, Args &&...args) + : CheckedTransformer(TransformPairs(values), std::forward(args)...) {} + + /// direct map of std::string to std::string + template explicit CheckedTransformer(T mapping) : CheckedTransformer(std::move(mapping), nullptr) {} + + /// This checks to see if an item is in a set: pointer or copy version. You can pass in a function that will filter + /// both sides of the comparison before computing the comparison. + template explicit CheckedTransformer(T mapping, F filter_function) { + + static_assert(detail::pair_adaptor::type>::value, + "mapping must produce value pairs"); + // Get the type of the contained item - requires a container have ::value_type + // if the type does not have first_type and second_type, these are both value_type + using element_t = typename detail::element_type::type; // Removes (smart) pointers if needed + using item_t = typename detail::pair_adaptor::first_type; // Is value_type if not a map + using local_item_t = typename IsMemberType::type; // Will convert bad types to good ones + // (const char * to std::string) + using iteration_type_t = typename detail::pair_adaptor::value_type; // the type of the object pair + + // Make a local copy of the filter function, using a std::function if not one already + std::function filter_fn = filter_function; + + auto tfunc = [mapping]() { + std::string out("value in "); + out += detail::generate_map(detail::smart_deref(mapping)) + " OR {"; + out += detail::join( + detail::smart_deref(mapping), + [](const iteration_type_t &v) { return detail::to_string(detail::pair_adaptor::second(v)); }, + ","); + out.push_back('}'); + return out; + }; + + desc_function_ = tfunc; + + func_ = [mapping, tfunc, filter_fn](std::string &input) { + using CLI::detail::lexical_cast; + local_item_t b; + bool converted = lexical_cast(input, b); + if(converted) { + if(filter_fn) { + b = filter_fn(b); + } + auto res = detail::search(mapping, b, filter_fn); + if(res.first) { + input = detail::value_string(detail::pair_adaptor::second(*res.second)); + return std::string{}; + } + } + for(const auto &v : detail::smart_deref(mapping)) { + auto output_string = detail::value_string(detail::pair_adaptor::second(v)); + if(output_string == input) { + return std::string(); + } + } + + return "Check " + input + " " + tfunc() + " FAILED"; + }; + } + + /// You can pass in as many filter functions as you like, they nest + template + CheckedTransformer(T &&mapping, filter_fn_t filter_fn_1, filter_fn_t filter_fn_2, Args &&...other) + : CheckedTransformer( + std::forward(mapping), + [filter_fn_1, filter_fn_2](std::string a) { return filter_fn_2(filter_fn_1(a)); }, + other...) {} +}; + +/// Helper function to allow ignore_case to be passed to IsMember or Transform +inline std::string ignore_case(std::string item) { return detail::to_lower(item); } + +/// Helper function to allow ignore_underscore to be passed to IsMember or Transform +inline std::string ignore_underscore(std::string item) { return detail::remove_underscore(item); } + +/// Helper function to allow checks to ignore spaces to be passed to IsMember or Transform +inline std::string ignore_space(std::string item) { + item.erase(std::remove(std::begin(item), std::end(item), ' '), std::end(item)); + item.erase(std::remove(std::begin(item), std::end(item), '\t'), std::end(item)); + return item; +} + +/// Multiply a number by a factor using given mapping. +/// Can be used to write transforms for SIZE or DURATION inputs. +/// +/// Example: +/// With mapping = `{"b"->1, "kb"->1024, "mb"->1024*1024}` +/// one can recognize inputs like "100", "12kb", "100 MB", +/// that will be automatically transformed to 100, 14448, 104857600. +/// +/// Output number type matches the type in the provided mapping. +/// Therefore, if it is required to interpret real inputs like "0.42 s", +/// the mapping should be of a type or . +class AsNumberWithUnit : public Validator { + public: + /// Adjust AsNumberWithUnit behavior. + /// CASE_SENSITIVE/CASE_INSENSITIVE controls how units are matched. + /// UNIT_OPTIONAL/UNIT_REQUIRED throws ValidationError + /// if UNIT_REQUIRED is set and unit literal is not found. + enum Options { + CASE_SENSITIVE = 0, + CASE_INSENSITIVE = 1, + UNIT_OPTIONAL = 0, + UNIT_REQUIRED = 2, + DEFAULT = CASE_INSENSITIVE | UNIT_OPTIONAL + }; + + template + explicit AsNumberWithUnit(std::map mapping, + Options opts = DEFAULT, + const std::string &unit_name = "UNIT") { + description(generate_description(unit_name, opts)); + validate_mapping(mapping, opts); + + // transform function + func_ = [mapping, opts](std::string &input) -> std::string { + Number num{}; + + detail::rtrim(input); + if(input.empty()) { + throw ValidationError("Input is empty"); + } + + // Find split position between number and prefix + auto unit_begin = input.end(); + while(unit_begin > input.begin() && std::isalpha(*(unit_begin - 1), std::locale())) { + --unit_begin; + } + + std::string unit{unit_begin, input.end()}; + input.resize(static_cast(std::distance(input.begin(), unit_begin))); + detail::trim(input); + + if(opts & UNIT_REQUIRED && unit.empty()) { + throw ValidationError("Missing mandatory unit"); + } + if(opts & CASE_INSENSITIVE) { + unit = detail::to_lower(unit); + } + if(unit.empty()) { + using CLI::detail::lexical_cast; + if(!lexical_cast(input, num)) { + throw ValidationError(std::string("Value ") + input + " could not be converted to " + + detail::type_name()); + } + // No need to modify input if no unit passed + return {}; + } + + // find corresponding factor + auto it = mapping.find(unit); + if(it == mapping.end()) { + throw ValidationError(unit + + " unit not recognized. " + "Allowed values: " + + detail::generate_map(mapping, true)); + } + + if(!input.empty()) { + using CLI::detail::lexical_cast; + bool converted = lexical_cast(input, num); + if(!converted) { + throw ValidationError(std::string("Value ") + input + " could not be converted to " + + detail::type_name()); + } + // perform safe multiplication + bool ok = detail::checked_multiply(num, it->second); + if(!ok) { + throw ValidationError(detail::to_string(num) + " multiplied by " + unit + + " factor would cause number overflow. Use smaller value."); + } + } else { + num = static_cast(it->second); + } + + input = detail::to_string(num); + + return {}; + }; + } + + private: + /// Check that mapping contains valid units. + /// Update mapping for CASE_INSENSITIVE mode. + template static void validate_mapping(std::map &mapping, Options opts) { + for(auto &kv : mapping) { + if(kv.first.empty()) { + throw ValidationError("Unit must not be empty."); + } + if(!detail::isalpha(kv.first)) { + throw ValidationError("Unit must contain only letters."); + } + } + + // make all units lowercase if CASE_INSENSITIVE + if(opts & CASE_INSENSITIVE) { + std::map lower_mapping; + for(auto &kv : mapping) { + auto s = detail::to_lower(kv.first); + if(lower_mapping.count(s)) { + throw ValidationError(std::string("Several matching lowercase unit representations are found: ") + + s); + } + lower_mapping[detail::to_lower(kv.first)] = kv.second; + } + mapping = std::move(lower_mapping); + } + } + + /// Generate description like this: NUMBER [UNIT] + template static std::string generate_description(const std::string &name, Options opts) { + std::stringstream out; + out << detail::type_name() << ' '; + if(opts & UNIT_REQUIRED) { + out << name; + } else { + out << '[' << name << ']'; + } + return out.str(); + } +}; + +inline AsNumberWithUnit::Options operator|(const AsNumberWithUnit::Options &a, const AsNumberWithUnit::Options &b) { + return static_cast(static_cast(a) | static_cast(b)); +} + +/// Converts a human-readable size string (with unit literal) to uin64_t size. +/// Example: +/// "100" => 100 +/// "1 b" => 100 +/// "10Kb" => 10240 // you can configure this to be interpreted as kilobyte (*1000) or kibibyte (*1024) +/// "10 KB" => 10240 +/// "10 kb" => 10240 +/// "10 kib" => 10240 // *i, *ib are always interpreted as *bibyte (*1024) +/// "10kb" => 10240 +/// "2 MB" => 2097152 +/// "2 EiB" => 2^61 // Units up to exibyte are supported +class AsSizeValue : public AsNumberWithUnit { + public: + using result_t = std::uint64_t; + + /// If kb_is_1000 is true, + /// interpret 'kb', 'k' as 1000 and 'kib', 'ki' as 1024 + /// (same applies to higher order units as well). + /// Otherwise, interpret all literals as factors of 1024. + /// The first option is formally correct, but + /// the second interpretation is more wide-spread + /// (see https://en.wikipedia.org/wiki/Binary_prefix). + explicit AsSizeValue(bool kb_is_1000); + + private: + /// Get mapping + static std::map init_mapping(bool kb_is_1000); + + /// Cache calculated mapping + static std::map get_mapping(bool kb_is_1000); +}; + +namespace detail { +/// Split a string into a program name and command line arguments +/// the string is assumed to contain a file name followed by other arguments +/// the return value contains is a pair with the first argument containing the program name and the second +/// everything else. +CLI11_INLINE std::pair split_program_name(std::string commandline); + +} // namespace detail +/// @} + + + + +CLI11_INLINE std::string Validator::operator()(std::string &str) const { + std::string retstring; + if(active_) { + if(non_modifying_) { + std::string value = str; + retstring = func_(value); + } else { + retstring = func_(str); + } + } + return retstring; +} + +CLI11_NODISCARD CLI11_INLINE Validator Validator::description(std::string validator_desc) const { + Validator newval(*this); + newval.desc_function_ = [validator_desc]() { return validator_desc; }; + return newval; +} + +CLI11_INLINE Validator Validator::operator&(const Validator &other) const { + Validator newval; + + newval._merge_description(*this, other, " AND "); + + // Give references (will make a copy in lambda function) + const std::function &f1 = func_; + const std::function &f2 = other.func_; + + newval.func_ = [f1, f2](std::string &input) { + std::string s1 = f1(input); + std::string s2 = f2(input); + if(!s1.empty() && !s2.empty()) + return std::string("(") + s1 + ") AND (" + s2 + ")"; + return s1 + s2; + }; + + newval.active_ = active_ && other.active_; + newval.application_index_ = application_index_; + return newval; +} + +CLI11_INLINE Validator Validator::operator|(const Validator &other) const { + Validator newval; + + newval._merge_description(*this, other, " OR "); + + // Give references (will make a copy in lambda function) + const std::function &f1 = func_; + const std::function &f2 = other.func_; + + newval.func_ = [f1, f2](std::string &input) { + std::string s1 = f1(input); + std::string s2 = f2(input); + if(s1.empty() || s2.empty()) + return std::string(); + + return std::string("(") + s1 + ") OR (" + s2 + ")"; + }; + newval.active_ = active_ && other.active_; + newval.application_index_ = application_index_; + return newval; +} + +CLI11_INLINE Validator Validator::operator!() const { + Validator newval; + const std::function &dfunc1 = desc_function_; + newval.desc_function_ = [dfunc1]() { + auto str = dfunc1(); + return (!str.empty()) ? std::string("NOT ") + str : std::string{}; + }; + // Give references (will make a copy in lambda function) + const std::function &f1 = func_; + + newval.func_ = [f1, dfunc1](std::string &test) -> std::string { + std::string s1 = f1(test); + if(s1.empty()) { + return std::string("check ") + dfunc1() + " succeeded improperly"; + } + return std::string{}; + }; + newval.active_ = active_; + newval.application_index_ = application_index_; + return newval; +} + +CLI11_INLINE void +Validator::_merge_description(const Validator &val1, const Validator &val2, const std::string &merger) { + + const std::function &dfunc1 = val1.desc_function_; + const std::function &dfunc2 = val2.desc_function_; + + desc_function_ = [=]() { + std::string f1 = dfunc1(); + std::string f2 = dfunc2(); + if((f1.empty()) || (f2.empty())) { + return f1 + f2; + } + return std::string(1, '(') + f1 + ')' + merger + '(' + f2 + ')'; + }; +} + +namespace detail { + +#if defined CLI11_HAS_FILESYSTEM && CLI11_HAS_FILESYSTEM > 0 +CLI11_INLINE path_type check_path(const char *file) noexcept { + std::error_code ec; + auto stat = std::filesystem::status(to_path(file), ec); + if(ec) { + return path_type::nonexistent; + } + switch(stat.type()) { + case std::filesystem::file_type::none: // LCOV_EXCL_LINE + case std::filesystem::file_type::not_found: + return path_type::nonexistent; // LCOV_EXCL_LINE + case std::filesystem::file_type::directory: + return path_type::directory; + case std::filesystem::file_type::symlink: + case std::filesystem::file_type::block: + case std::filesystem::file_type::character: + case std::filesystem::file_type::fifo: + case std::filesystem::file_type::socket: + case std::filesystem::file_type::regular: + case std::filesystem::file_type::unknown: + default: + return path_type::file; + } +} +#else +CLI11_INLINE path_type check_path(const char *file) noexcept { +#if defined(_MSC_VER) + struct __stat64 buffer; + if(_stat64(file, &buffer) == 0) { + return ((buffer.st_mode & S_IFDIR) != 0) ? path_type::directory : path_type::file; + } +#else + struct stat buffer; + if(stat(file, &buffer) == 0) { + return ((buffer.st_mode & S_IFDIR) != 0) ? path_type::directory : path_type::file; + } +#endif + return path_type::nonexistent; +} +#endif + +CLI11_INLINE ExistingFileValidator::ExistingFileValidator() : Validator("FILE") { + func_ = [](std::string &filename) { + auto path_result = check_path(filename.c_str()); + if(path_result == path_type::nonexistent) { + return "File does not exist: " + filename; + } + if(path_result == path_type::directory) { + return "File is actually a directory: " + filename; + } + return std::string(); + }; +} + +CLI11_INLINE ExistingDirectoryValidator::ExistingDirectoryValidator() : Validator("DIR") { + func_ = [](std::string &filename) { + auto path_result = check_path(filename.c_str()); + if(path_result == path_type::nonexistent) { + return "Directory does not exist: " + filename; + } + if(path_result == path_type::file) { + return "Directory is actually a file: " + filename; + } + return std::string(); + }; +} + +CLI11_INLINE ExistingPathValidator::ExistingPathValidator() : Validator("PATH(existing)") { + func_ = [](std::string &filename) { + auto path_result = check_path(filename.c_str()); + if(path_result == path_type::nonexistent) { + return "Path does not exist: " + filename; + } + return std::string(); + }; +} + +CLI11_INLINE NonexistentPathValidator::NonexistentPathValidator() : Validator("PATH(non-existing)") { + func_ = [](std::string &filename) { + auto path_result = check_path(filename.c_str()); + if(path_result != path_type::nonexistent) { + return "Path already exists: " + filename; + } + return std::string(); + }; +} + +CLI11_INLINE IPV4Validator::IPV4Validator() : Validator("IPV4") { + func_ = [](std::string &ip_addr) { + auto result = CLI::detail::split(ip_addr, '.'); + if(result.size() != 4) { + return std::string("Invalid IPV4 address must have four parts (") + ip_addr + ')'; + } + int num = 0; + for(const auto &var : result) { + using CLI::detail::lexical_cast; + bool retval = lexical_cast(var, num); + if(!retval) { + return std::string("Failed parsing number (") + var + ')'; + } + if(num < 0 || num > 255) { + return std::string("Each IP number must be between 0 and 255 ") + var; + } + } + return std::string{}; + }; +} + +CLI11_INLINE EscapedStringTransformer::EscapedStringTransformer() { + func_ = [](std::string &str) { + try { + if(str.size() > 1 && (str.front() == '\"' || str.front() == '\'' || str.front() == '`') && + str.front() == str.back()) { + process_quoted_string(str); + } else if(str.find_first_of('\\') != std::string::npos) { + if(detail::is_binary_escaped_string(str)) { + str = detail::extract_binary_string(str); + } else { + str = remove_escaped_characters(str); + } + } + return std::string{}; + } catch(const std::invalid_argument &ia) { + return std::string(ia.what()); + } + }; +} +} // namespace detail + +CLI11_INLINE FileOnDefaultPath::FileOnDefaultPath(std::string default_path, bool enableErrorReturn) + : Validator("FILE") { + func_ = [default_path, enableErrorReturn](std::string &filename) { + auto path_result = detail::check_path(filename.c_str()); + if(path_result == detail::path_type::nonexistent) { + std::string test_file_path = default_path; + if(default_path.back() != '/' && default_path.back() != '\\') { + // Add folder separator + test_file_path += '/'; + } + test_file_path.append(filename); + path_result = detail::check_path(test_file_path.c_str()); + if(path_result == detail::path_type::file) { + filename = test_file_path; + } else { + if(enableErrorReturn) { + return "File does not exist: " + filename; + } + } + } + return std::string{}; + }; +} + +CLI11_INLINE AsSizeValue::AsSizeValue(bool kb_is_1000) : AsNumberWithUnit(get_mapping(kb_is_1000)) { + if(kb_is_1000) { + description("SIZE [b, kb(=1000b), kib(=1024b), ...]"); + } else { + description("SIZE [b, kb(=1024b), ...]"); + } +} + +CLI11_INLINE std::map AsSizeValue::init_mapping(bool kb_is_1000) { + std::map m; + result_t k_factor = kb_is_1000 ? 1000 : 1024; + result_t ki_factor = 1024; + result_t k = 1; + result_t ki = 1; + m["b"] = 1; + for(std::string p : {"k", "m", "g", "t", "p", "e"}) { + k *= k_factor; + ki *= ki_factor; + m[p] = k; + m[p + "b"] = k; + m[p + "i"] = ki; + m[p + "ib"] = ki; + } + return m; +} + +CLI11_INLINE std::map AsSizeValue::get_mapping(bool kb_is_1000) { + if(kb_is_1000) { + static auto m = init_mapping(true); + return m; + } + static auto m = init_mapping(false); + return m; +} + +namespace detail { + +CLI11_INLINE std::pair split_program_name(std::string commandline) { + // try to determine the programName + std::pair vals; + trim(commandline); + auto esp = commandline.find_first_of(' ', 1); + while(detail::check_path(commandline.substr(0, esp).c_str()) != path_type::file) { + esp = commandline.find_first_of(' ', esp + 1); + if(esp == std::string::npos) { + // if we have reached the end and haven't found a valid file just assume the first argument is the + // program name + if(commandline[0] == '"' || commandline[0] == '\'' || commandline[0] == '`') { + bool embeddedQuote = false; + auto keyChar = commandline[0]; + auto end = commandline.find_first_of(keyChar, 1); + while((end != std::string::npos) && (commandline[end - 1] == '\\')) { // deal with escaped quotes + end = commandline.find_first_of(keyChar, end + 1); + embeddedQuote = true; + } + if(end != std::string::npos) { + vals.first = commandline.substr(1, end - 1); + esp = end + 1; + if(embeddedQuote) { + vals.first = find_and_replace(vals.first, std::string("\\") + keyChar, std::string(1, keyChar)); + } + } else { + esp = commandline.find_first_of(' ', 1); + } + } else { + esp = commandline.find_first_of(' ', 1); + } + + break; + } + } + if(vals.first.empty()) { + vals.first = commandline.substr(0, esp); + rtrim(vals.first); + } + + // strip the program name + vals.second = (esp < commandline.length() - 1) ? commandline.substr(esp + 1) : std::string{}; + ltrim(vals.second); + return vals; +} + +} // namespace detail +/// @} + + + + +class Option; +class App; + +/// This enum signifies the type of help requested +/// +/// This is passed in by App; all user classes must accept this as +/// the second argument. + +enum class AppFormatMode { + Normal, ///< The normal, detailed help + All, ///< A fully expanded help + Sub, ///< Used when printed as part of expanded subcommand +}; + +/// This is the minimum requirements to run a formatter. +/// +/// A user can subclass this is if they do not care at all +/// about the structure in CLI::Formatter. +class FormatterBase { + protected: + /// @name Options + ///@{ + + /// The width of the first column + std::size_t column_width_{30}; + + /// @brief The required help printout labels (user changeable) + /// Values are Needs, Excludes, etc. + std::map labels_{}; + + ///@} + /// @name Basic + ///@{ + + public: + FormatterBase() = default; + FormatterBase(const FormatterBase &) = default; + FormatterBase(FormatterBase &&) = default; + FormatterBase &operator=(const FormatterBase &) = default; + FormatterBase &operator=(FormatterBase &&) = default; + + /// Adding a destructor in this form to work around bug in GCC 4.7 + virtual ~FormatterBase() noexcept {} // NOLINT(modernize-use-equals-default) + + /// This is the key method that puts together help + virtual std::string make_help(const App *, std::string, AppFormatMode) const = 0; + + ///@} + /// @name Setters + ///@{ + + /// Set the "REQUIRED" label + void label(std::string key, std::string val) { labels_[key] = val; } + + /// Set the column width + void column_width(std::size_t val) { column_width_ = val; } + + ///@} + /// @name Getters + ///@{ + + /// Get the current value of a name (REQUIRED, etc.) + CLI11_NODISCARD std::string get_label(std::string key) const { + if(labels_.find(key) == labels_.end()) + return key; + return labels_.at(key); + } + + /// Get the current column width + CLI11_NODISCARD std::size_t get_column_width() const { return column_width_; } + + ///@} +}; + +/// This is a specialty override for lambda functions +class FormatterLambda final : public FormatterBase { + using funct_t = std::function; + + /// The lambda to hold and run + funct_t lambda_; + + public: + /// Create a FormatterLambda with a lambda function + explicit FormatterLambda(funct_t funct) : lambda_(std::move(funct)) {} + + /// Adding a destructor (mostly to make GCC 4.7 happy) + ~FormatterLambda() noexcept override {} // NOLINT(modernize-use-equals-default) + + /// This will simply call the lambda function + std::string make_help(const App *app, std::string name, AppFormatMode mode) const override { + return lambda_(app, name, mode); + } +}; + +/// This is the default Formatter for CLI11. It pretty prints help output, and is broken into quite a few +/// overridable methods, to be highly customizable with minimal effort. +class Formatter : public FormatterBase { + public: + Formatter() = default; + Formatter(const Formatter &) = default; + Formatter(Formatter &&) = default; + Formatter &operator=(const Formatter &) = default; + Formatter &operator=(Formatter &&) = default; + + /// @name Overridables + ///@{ + + /// This prints out a group of options with title + /// + CLI11_NODISCARD virtual std::string + make_group(std::string group, bool is_positional, std::vector opts) const; + + /// This prints out just the positionals "group" + virtual std::string make_positionals(const App *app) const; + + /// This prints out all the groups of options + std::string make_groups(const App *app, AppFormatMode mode) const; + + /// This prints out all the subcommands + virtual std::string make_subcommands(const App *app, AppFormatMode mode) const; + + /// This prints out a subcommand + virtual std::string make_subcommand(const App *sub) const; + + /// This prints out a subcommand in help-all + virtual std::string make_expanded(const App *sub) const; + + /// This prints out all the groups of options + virtual std::string make_footer(const App *app) const; + + /// This displays the description line + virtual std::string make_description(const App *app) const; + + /// This displays the usage line + virtual std::string make_usage(const App *app, std::string name) const; + + /// This puts everything together + std::string make_help(const App * /*app*/, std::string, AppFormatMode) const override; + + ///@} + /// @name Options + ///@{ + + /// This prints out an option help line, either positional or optional form + virtual std::string make_option(const Option *opt, bool is_positional) const { + std::stringstream out; + detail::format_help( + out, make_option_name(opt, is_positional) + make_option_opts(opt), make_option_desc(opt), column_width_); + return out.str(); + } + + /// @brief This is the name part of an option, Default: left column + virtual std::string make_option_name(const Option *, bool) const; + + /// @brief This is the options part of the name, Default: combined into left column + virtual std::string make_option_opts(const Option *) const; + + /// @brief This is the description. Default: Right column, on new line if left column too large + virtual std::string make_option_desc(const Option *) const; + + /// @brief This is used to print the name on the USAGE line + virtual std::string make_option_usage(const Option *opt) const; + + ///@} +}; + + + + +using results_t = std::vector; +/// callback function definition +using callback_t = std::function; + +class Option; +class App; + +using Option_p = std::unique_ptr