From 25f1dbdf54947bd6bbce653bc64f6c45b2473792 Mon Sep 17 00:00:00 2001 From: Guilhem Moulin Date: Fri, 5 Jul 2019 01:03:36 +0200 Subject: Refactor documentation. In particular, move manpages to the 'doc' directory, and generate HTML documentation with `make html`. --- .gitignore | 4 +- Changelog | 3 + INSTALL | 31 ---- Makefile | 34 +++- README | 6 +- doc/build.md | 99 ++++++++++ doc/development.md | 4 +- doc/index.md | 20 ++ doc/interimap.1.md | 526 +++++++++++++++++++++++++++++++++++++++++++++++++++++ doc/pullimap.1.md | 375 ++++++++++++++++++++++++++++++++++++++ doc/template.html | 76 ++++++++ interimap.md | 526 ----------------------------------------------------- pullimap.md | 375 -------------------------------------- 13 files changed, 1133 insertions(+), 946 deletions(-) delete mode 100644 INSTALL create mode 100644 doc/build.md create mode 100644 doc/index.md create mode 100644 doc/interimap.1.md create mode 100644 doc/pullimap.1.md create mode 100644 doc/template.html delete mode 100644 interimap.md delete mode 100644 pullimap.md diff --git a/.gitignore b/.gitignore index d0e940e..9dae7e6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ *~ -/*.1 +/doc/*.1 +/doc/*.html +!/doc/template.html /.pc/ diff --git a/Changelog b/Changelog index 49f49d9..4cc66ba 100644 --- a/Changelog +++ b/Changelog @@ -19,6 +19,9 @@ interimap (0.5) upstream; synchronization, for instance with the newly provided systemd template unit file). * Add a test-suite. (Requires dovecot-imapd, pkill(1) and xxd(1).) + * Completely refactor the documentation. In particular, move manpages + to a new 'doc' directory, and generate HTML documentation with `make + html`. + interimap: write which --target to use in --delete command suggestions. + interimap: avoid caching hierarchy delimiters forever in the diff --git a/INSTALL b/INSTALL deleted file mode 100644 index 69afb26..0000000 --- a/INSTALL +++ /dev/null @@ -1,31 +0,0 @@ -InterIMAP depends on Perl >=5.20 and the following Perl modules: - - - Compress::Raw::Zlib (core module) - - Config::Tiny - - DBI - - DBD::SQLite - - Errno (core module) - - Getopt::Long (core module) - - MIME::Base64 (core module) if authentication is required - - List::Util (core module) - - Net::SSLeay >=1.73 - - POSIX (core module) - - Socket (core module) - - Time::HiRes (core module) if 'logfile' is set - -On Debian GNU/Linux systems, these modules can be installed with the -following command: - - apt-get install libconfig-tiny-perl libdbi-perl libdbd-sqlite3-perl libnet-ssleay-perl - -However Debian GNU/Linux users can also use gbp(1) from git-buildpackage -to build their own package: - - $ git checkout debian - $ AUTO_DEBSIGN=no gbp buildpackage - -Alternatively, for the development version: - - $ git checkout debian - $ git merge master - $ AUTO_DEBSIGN=no gbp buildpackage --git-force-create --git-upstream-tree=BRANCH diff --git a/Makefile b/Makefile index b0e4d59..fda0fe0 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,11 @@ -all: pullimap.1 interimap.1 +all: manual + +MANUALS = $(patsubst %.md,%,$(wildcard ./doc/*.[1-9].md)) +manual: $(MANUALS) # upper case the headers and remove the links -%.1: %.md - @pandoc -f markdown -t json "$<" | \ +$(MANUALS): %: %.md + @pandoc -f markdown -t json -- "$<" | \ jq " \ def fixheaders: \ if .t == \"Header\" then \ @@ -29,12 +32,29 @@ all: pullimap.1 interimap.1 }" | \ pandoc -s -f json -t man+smart -o "$@" -install: - test: @for t in tests/*; do if [ -f "$$t/run" ]; then ./tests/run "$$t" || exit 1; fi; done +HTML_ROOTDIR ?= ./doc +CSS ?= /usr/share/javascript/bootstrap/css/bootstrap.min.css +HTML_TEMPLATE ?= ./doc/template.html + +HTML_FILES = $(addprefix $(HTML_ROOTDIR)/,$(patsubst ./doc/%.md,%.html,$(wildcard ./doc/*.md))) +html: $(HTML_FILES) + +## CSS="https://guilhem.org/static/css/bootstrap.min.css" HTML_ROOTDIR="$XDG_RUNTIME_DIR/Downloads" make html +$(HTML_ROOTDIR)/%.html: ./doc/%.md $(HTML_TEMPLATE) + mtime="$$(git --no-pager log -1 --pretty="format:%ct" -- "$<" 2>/dev/null)"; \ + [ -n "$$mtime" ] || mtime="$$(date +%s -r "$<")"; \ + pandoc -sp -f markdown -t html+smart --css=$(CSS) --template=$(HTML_TEMPLATE) \ + --variable=date:"$$(LC_TIME=C date +"Last modified on %a, %d %b %Y at %T %z" -d @"$$mtime")" \ + --output="$@" -- "$<" + +doc: manual html + +install: + clean: - rm -f pullimap.1 interimap.1 + rm -f $(MANUALS) $(HTML_FILES) -.PHONY: all install clean test +.PHONY: all manual html doc test install clean diff --git a/README b/README index a930512..fbc4ed7 100644 --- a/README +++ b/README @@ -1,9 +1,7 @@ InterIMAP is a fast bidirectional synchronization program for QRESYNC-capable IMAP4rev1 servers. PullIMAP retrieves messages a remote IMAP mailbox and -deliver them to an SMTP session. Consult the manuals for more information. - - https://guilhem.org/man/interimap.1.html - https://guilhem.org/man/pullimap.1.html +deliver them to an SMTP session. Visit https://guilhem.org/interimap +for more information. _______________________________________________________________________ diff --git a/doc/build.md b/doc/build.md new file mode 100644 index 0000000..38d1bfb --- /dev/null +++ b/doc/build.md @@ -0,0 +1,99 @@ +% Build instructions +% Guilhem Moulin + +On Debian 9 (codename *Stretch*) and later, installing [`interimap`(1)] +is a single command away: + + $ apt-get install interimap + +This document is for those who are running other systems, and/or who +wish to install from [source](https://git.guilhem.org/interimap). + + +Dependencies +============ + +[`interimap`(1)](interimap.1.html) depends on Perl ≥5.20 and the +following Perl modules: + + * [`Compress::Raw::Zlib`](https://perldoc.perl.org/Compress/Raw/Zlib.html) (*core module*) + * [`Config::Tiny`](https://metacpan.org/pod/Config::Tiny) + * [`DBI`](https://metacpan.org/pod/DBI) + * [`DBD::SQLite`](https://metacpan.org/pod/DBD::SQLite) + * [`Errno`](https://perldoc.perl.org/Errno.html) (*core module*) + * [`Getopt::Long`](https://perldoc.perl.org/Getopt/Long.html) (*core module*) + * [`MIME::Base64`](https://perldoc.perl.org/MIME/Base64.html) (*core module*) — if authentication is required + * [`List::Util`](https://perldoc.perl.org/List/Util.html) (*core module*) + * [`Net::SSLeay`](https://metacpan.org/pod/Net::SSLeay) ≥1.73 + * [`POSIX`](https://perldoc.perl.org/POSIX.html) (*core module*) + * [`Socket`](https://perldoc.perl.org/Socket.html) (*core module*) + * [`Time::HiRes`](https://perldoc.perl.org/Time/HiRes.html) (*core module*) — if `logfile` is set + +On Debian GNU/Linux systems, the dependencies can be installed with the +following command: + + $ apt install libconfig-tiny-perl \ + libdbi-perl \ + libdbd-sqlite3-perl \ + libnet-ssleay-perl + +Additional packages are required in order to run the test suite: + + $ apt install dovecot-imapd procps sqlite3 xxd + + $ make test + + +Generate documentation +====================== + +Yet another set of packages is needed to generate the documentation: + + $ apt install jq pandoc + +Run `` `make manual` `` (or just `` `make` ``) in order to generate the +manpages. You'll find them at `doc/*.[1-9]`. Use for instance `` `man +-l doc/interimap.1` `` in order to read your copy of the [`interimap`(1)] +manpage. + +The HTML documentation can be built with `` `make html` ``. HTML files +are generated alongside their Markdown source by default, but you can +choose another target directory using the `HTML_ROOTDIR` environment +variable (the value of which defaults to `./doc`). Moreover the +[`libjs-bootstrap`](https://tracker.debian.org/libjs-bootstrap) is +needed by default for the local CSS file; this can be controlled with +the `CSS` environment variable (the value of which defaults to +`/usr/share/javascript/bootstrap/css/bootstrap.min.css`). + +For instance, use + + $ CSS="https://guilhem.org/static/css/bootstrap.min.css" \ + HTML_ROOTDIR="$XDG_RUNTIME_DIR/interimap" \ + make html + +to generate the HTML documentation under directory `$XDG_RUNTIME_DIR/interimap` +(which needs to exist) using a remote CSS file. + +The `doc` target generates all documentation, manpages as well as HTML +pages. + + +Build custom Debian package +=========================== + +Debian GNU/Linux users can also use [`gbp`(1)] from +[`git-buildpackage`](https://tracker.debian.org/pkg/git-buildpackage) in +order to build their own package: + + $ git checkout debian + $ gbp buildpackage + +Alternatively, for the development version: + + $ git checkout debian + $ git merge master + $ gbp buildpackage --git-force-create --git-upstream-tree=BRANCH + + +[`interimap`(1)]: interimap.1.html +[`gbp`(1)]: https://manpages.debian.org/git-buildpackage/gbp.1.en.html diff --git a/doc/development.md b/doc/development.md index 2cb1367..406207a 100644 --- a/doc/development.md +++ b/doc/development.md @@ -202,7 +202,7 @@ recursively remove the directory `$BASEDIR`. [IMAP4rev1]: https://tools.ietf.org/html/rfc3501 -[`interimap`(1)]: https://guilhem.org/man/interimap.1.html -[`pullimap`(1)]: https://guilhem.org/man/pullimap.1.html +[`interimap`(1)]: interimap.1.html +[`pullimap`(1)]: pullimap.1.html [`doveadm`(1)]: https://wiki.dovecot.org/Tools/Doveadm [`%`-variable]: https://wiki.dovecot.org/Variables diff --git a/doc/index.md b/doc/index.md new file mode 100644 index 0000000..12de956 --- /dev/null +++ b/doc/index.md @@ -0,0 +1,20 @@ +% [`interimap`(1)] and [`pullimap`(1)] documentation +% Guilhem Moulin + +Manuals (HTML versions) +----------------------- + + * [`interimap`(1)] — Fast bidirectional synchronization for + QRESYNC-capable IMAP servers + * [`pullimap`(1)] — Pull mails from an IMAP mailbox and deliver them + to an SMTP session + +Resources for developers +------------------------ + + * [Source-code repository](https://git.guilhem.org/interimap) + * [Build instructions](build.html) + * [Test environment setup](development.html) + +[`interimap`(1)]: interimap.1.html +[`pullimap`(1)]: pullimap.1.html diff --git a/doc/interimap.1.md b/doc/interimap.1.md new file mode 100644 index 0000000..387850a --- /dev/null +++ b/doc/interimap.1.md @@ -0,0 +1,526 @@ +% interimap(1) +% [Guilhem Moulin](mailto:guilhem@fripost.org) +% July 2015 + +Name +==== + +InterIMAP - Fast bidirectional synchronization for QRESYNC-capable IMAP servers + +Synopsis +======== + +`interimap` [*OPTION* ...] [*COMMAND*] [*MAILBOX* ...] + +Description +=========== + +`interimap` performs stateful synchronization between two IMAP4rev1 +servers. +Such synchronization is made possible by the [`QRESYNC` IMAP +extension][RFC 7162]; for convenience reasons servers must also support +the [`LIST-EXTENDED`][RFC 5258], [`LIST-STATUS`][RFC 5819] (or +[`NOTIFY`][RFC 5465]) and [`UIDPLUS`][RFC 4315] IMAP extensions. +See also the **[supported extensions](#supported-extensions)** section +below. + +Stateful synchronization is only possible for mailboxes supporting +persistent message Unique Identifiers (UID) and persistent storage of +mod-sequences (MODSEQ); any non-compliant mailbox will cause `interimap` +to abort. +Furthermore, because UIDs are allocated not by the client but by the +server, `interimap` needs to keep track of associations between local +and remote UIDs for each mailbox. +The synchronization state of a mailbox consists of its `UIDNEXT` and +`HIGHESTMODSEQ` values on each server; it is then assumed that each +message with UID smaller than `UIDNEXT` have been replicated to the +other server, and that the metadata (such as flags) of each message with +MODSEQ at most `HIGHESTMODSEQ` have been synchronized. +Conceptually, the synchronization algorithm is derived from [RFC 4549] +with the [RFC 7162] (sec. 6) amendments, and works as follows: + + 1. `SELECT` (on both servers) a mailbox the current `UIDNEXT` or `HIGHESTMODSEQ` + values of which differ from the values found in the database (for + either server). Use the `QRESYNC` `SELECT` parameter from [RFC + 7162] to list changes (vanished messages and flag updates) since + `HIGHESTMODSEQ` to messages with UID smaller than `UIDNEXT`. + + 2. Propagate these changes onto the other server: get the corresponding + UIDs from the database, then: + a. issue a `UID STORE` command, followed by `UID EXPUNGE`, to + remove messages that have not already been deleted on both + servers; and + b. issue some `UID STORE` commands to propagate flag updates (send + a single command for each flag list in order the reduce the + number of round trips). + + (Conflicts may occur if the metadata of a message has been updated + on both servers with different flag lists; in that case, `interimap` + issues a warning and updates the message on each server with the + union of both flag lists.) + Repeat this step if the server sent some updates in the meantime. + Otherwise, update the `HIGHESTMODSEQ` value in the database. + + 3. Process new messages (if the current `UIDNEXT` value of the mailbox + differs from the one found in the database) by issuing a `UID FETCH` + command; process each received message on-the-fly by issuing an + `APPEND` command with the message's `RFC822` body, `FLAGS` and + `INTERNALDATE`. + Repeat this step if the server received new messages in the + meantime. Otherwise, update the `UIDNEXT` value in the database. + Go back to step 2 if the server sent some metadata (such as flag) + updates in the meantime. + + 4. Go back to step 1 to proceed with the next unsynchronized mailbox. + +Commands +======== + +By default, `interimap` synchronizes each mailbox listed by the `LIST "" +"*"` IMAP command; +the *list-mailbox*, *list-select-opts* and *ignore-mailbox* options from +the [configuration file](#configuration-file) can be used to shrink that +list and save bandwidth. +However if some extra argument are provided on the command line, +`interimap` ignores these options and synchronizes the given +*MAILBOX*es instead. Note that each *MAILBOX* is taken “as is”; in +particular, it must be [UTF-7 encoded][RFC 2152], unquoted, and the list +wildcards ‘\*’ and ‘%’ are passed verbatim to the IMAP server. + +If the synchronization was interrupted during a previous run while some +messages were being replicated (but before the `UIDNEXT` or +`HIGHESTMODSEQ` values have been updated), `interimap` performs a “full +synchronization” on theses messages: downloading the whole UID and flag +lists on each servers allows `interimap` to detect messages that have +been removed or for which their flags have changed in the meantime. +Finally, after propagating the offline changes for these messages, +`interimap` resumes the synchronization for the rest of the mailbox. + +Specifying one of the commands below makes `interimap` perform an action +other than the default [`QRESYNC`][RFC 7162]-based synchronization. + +`--repair` [*MAILBOX* ...] + +: List the database anomalies and try to repair them. (Consider only + the given *MAILBOX*es if non-optional arguments are provided.) + This is done by performing a so-called “full synchronization”, + namely: + 1/ download all UIDs along with their flag list both from the + local and remote servers; + 2/ ensure that each entry in the database corresponds to an + existing UID; and + 3/ ensure that both flag lists match. + Any message found on a server but not in the database is replicated + on the other server (which in the worst case, might yield a message + duplicate). + Flag conflicts are solved by updating each message to the union of + both lists. + +`--delete` *MAILBOX* [*MAILBOX* ...] + +: Delete the given *MAILBOX*es on each target (by default each server + plus the database, unless `--target` specifies otherwise) where it + exists. + Note that per the [IMAP4rev1 standard][RFC 3501] deletion is not + recursive. Thus *MAILBOX*'s children are not deleted. + +`--rename` *SOURCE* *DEST* + +: Rename the mailbox *SOURCE* to *DEST* on each target (by default + each server plus the database, unless `--target` specifies + otherwise) where it exists. + `interimap` aborts if *DEST* already exists on either target. + Note that per the [IMAP4rev1 standard][RFC 3501] renaming is + recursive. Thus *SOURCE*'s children are moved to become *DEST*'s + children instead. + +Options +======= + +`--config=`*FILE* + +: Specify an alternate [configuration file](#configuration-file). + Relative paths start from *$XDG_CONFIG_HOME/interimap*, or *~/.config/interimap* + if the `XDG_CONFIG_HOME` environment variable is unset. + +`--target={local,remote,database}` + +: Limit the scope of a `--delete` or `--rename` command to the given + target. Can be repeated to act on multiple targets. By default all + three targets are considered. + +`--watch`[`=`*seconds*] + +: Don't exit after a successful synchronization. Instead, keep + synchronizing forever. Sleep for the given number of *seconds* (by + default 1 minute if `--notify` is unset, and 15 minutes if + `--notify` is set) between two synchronizations. Setting this + options enables `SO_KEEPALIVE` on the socket for *type*s other than + `tunnel`. + +`--notify` + +: Whether to use the [IMAP `NOTIFY` extension][RFC 5465] to instruct + the server to automatically send updates to the client. (Both local + and remote servers must support [RFC 5465] for this to work.) + This greatly reduces IMAP traffic since `interimap` can rely on + server notifications instead of manually polling for updates. + If the connection remains idle for 15 minutes (configurable with + `--watch`), then `interimap` sends a `NOOP` command to avoid being + logged out for inactivity. + +`-q`, `--quiet` + +: Try to be quiet. + +`--debug` + +: Turn on debug mode. Debug messages are written to the given *logfile*. + Note that this include all IMAP traffic (except literals). + Depending on the chosen authentication mechanism, this might include + authentication credentials. + +`-h`, `--help` + +: Output a brief help and exit. + +`--version` + +: Show the version number and exit. + +Configuration file +================== + +Unless told otherwise by the `--config=FILE` command-line option, +`interimap` reads its configuration from *$XDG_CONFIG_HOME/interimap/config* +(or *~/.config/interimap/config* if the `XDG_CONFIG_HOME` environment +variable is unset) as an [INI file]. +The syntax of the configuration file is a series of `OPTION=VALUE` +lines organized under some `[SECTION]`; lines starting with a ‘#’ or +‘;’ character are ignored as comments. +The `[local]` and `[remote]` sections define the two IMAP servers to +synchronize. +Valid options are: + +*database* + +: SQLite version 3 database file to use to keep track of associations + between local and remote UIDs, as well as the `UIDVALIDITY`, + `UIDNEXT` and `HIGHESTMODSEQ` of each known mailbox on both servers. + Relative paths start from *$XDG_DATA_HOME/interimap*, or + *~/.local/share/interimap* if the `XDG_DATA_HOME` environment + variable is unset. This option is only available in the default + section. + (Default: `HOST.db`, where *HOST* is taken from the `[remote]` or + `[local]` sections, in that order.) + +*list-reference* + +: An optional “reference name” to use for the initial `LIST` command, + indicating the context in which the *MAILBOX*es are interpreted. + For instance, by specifying `list-reference=perso/` in the `[local]` + section, *MAILBOX* names are interpreted relative to `perso/` on the + local server; in other words the remote mailbox hierarchy is mapped + to the `perso/` sub-hierarchy on the local server. This is useful + for synchronizing multiple remote servers against different + namespaces belonging to the same local IMAP server (using a + different `interimap` instance for each local namespace ↔ remote + synchronization). + + (Note that if the reference name is not a level of mailbox hierarchy + and/or does not end with the hierarchy delimiter, by [RFC 3501] its + interpretation by the IMAP server is implementation-dependent.) + +*list-mailbox* + +: A space separated list of mailbox patterns to use when issuing the + initial `LIST` command (overridden by the *MAILBOX*es given as + command-line arguments). + Names containing special characters such as spaces or brackets need + to be enclosed in double quotes. Within double quotes C-style + backslash escape sequences can be used (‘\\t’ for an horizontal tab, + ‘\\n’ for a new line, ‘\\\\’ for a backslash, etc.), as well as + hexadecimal escape sequences ‘\\xHH’. + Furthermore, non-ASCII names must be [UTF-7 encoded][RFC 2152]. + Two wildcards are available, and passed verbatim to the IMAP server: + a ‘\*’ character matches zero or more characters, while a ‘%’ + character matches zero or more characters up to the hierarchy + delimiter. + This option is only available in the default section. + (The default pattern, `*`, matches all visible mailboxes on the + server.) + +*list-select-opts* + +: An optional space separated list of selectors for the initial `LIST` + command. (Requires a server supporting the [`LIST-EXTENDED` IMAP + extension][RFC 5258].) Useful values are `SUBSCRIBED` (to list only + subscribed mailboxes), `REMOTE` (to also list remote mailboxes on a + server supporting mailbox referrals), and `RECURSIVEMATCH` (to + list parent mailboxes with children matching one of the above + *list-mailbox* patterns). This option is only available in the + default section. + +*ignore-mailbox* + +: An optional Perl Compatible Regular Expressions ([PCRE]) covering + mailboxes to exclude: any ([UTF-7 encoded][RFC 2152] and unquoted) + mailbox listed in the initial `LIST` responses is ignored if it + matches the given expression. + Note that the *MAILBOX*es given as command-line arguments bypass the + check and are always considered for synchronization. This option is + only available in the default section. + +*logfile* + +: A file name to use to log debug and informational messages. (By + default these messages are written to the error output.) This + option is only available in the default section. + +*type* + +: One of `imap`, `imaps` or `tunnel`. + `type=imap` and `type=imaps` are respectively used for IMAP and IMAP + over SSL/TLS connections over an INET socket. + `type=tunnel` causes `interimap` to create an unnamed pair of + connected sockets for interprocess communication with a *command* + instead of opening a network socket. + Note that specifying `type=tunnel` in the `[remote]` section makes + the default *database* to be `localhost.db`. + (Default: `imaps`.) + +*host* + +: Server hostname, for `type=imap` and `type=imaps`. + (Default: `localhost`.) + +*port* + +: Server port. + (Default: `143` for `type=imap`, `993` for `type=imaps`.) + +*proxy* + +: An optional SOCKS proxy to use for TCP connections to the IMAP + server (`type=imap` and `type=imaps` only), formatted as + `PROTOCOL://[USER:PASSWORD@]PROXYHOST[:PROXYPORT]`. + If `PROXYPORT` is omitted, it is assumed at port 1080. + Only [SOCKSv5][RFC 1928] is supported (with optional + [username/password authentication][RFC 1929]), in two flavors: + `socks5://` to resolve *hostname* locally, and `socks5h://` to let + the proxy resolve *hostname*. + +*command* + +: Command to use for `type=tunnel`. Must speak the [IMAP4rev1 + protocol][RFC 3501] on its standard output, and understand it on its + standard input. The value is passed to `` `/bin/sh -c` `` if it + contains shell metacharacters; otherwise it is split into words and + the resulting list is passed to `execvp`(3). + +*STARTTLS* + +: Whether to use the [`STARTTLS`][RFC 2595] directive to upgrade to a + secure connection. Setting this to `YES` for a server not + advertising the `STARTTLS` capability causes `interimap` to + immediately abort the connection. + (Ignored for *type*s other than `imap`. Default: `YES`.) + +*auth* + +: Space-separated list of preferred authentication mechanisms. + `interimap` uses the first mechanism in that list that is also + advertised (prefixed with `AUTH=`) in the server's capability list. + Supported authentication mechanisms are `PLAIN` and `LOGIN`. + (Default: `PLAIN LOGIN`.) + +*username*, *password* + +: Username and password to authenticate with. Can be required for non + pre-authenticated connections, depending on the chosen + authentication mechanism. + +*compress* + +: Whether to use the [`IMAP COMPRESS` extension][RFC 4978] for servers + advertising it. + (Default: `NO` for the `[local]` section, `YES` for the `[remote]` + section.) + +*null-stderr* + +: Whether to redirect *command*'s standard error to `/dev/null` for + `type=tunnel`. (Default: `NO`.) + +*SSL_protocols* + +: A space-separated list of SSL protocols to enable or disable (if + prefixed with an exclamation mark `!`. Known protocols are `SSLv2`, + `SSLv3`, `TLSv1`, `TLSv1.1`, `TLSv1.2`, and `TLSv1.3`. Enabling a + protocol is a short-hand for disabling all other protocols. + (Default: `!SSLv2 !SSLv3 !TLSv1 !TLSv1.1`, i.e., only enable TLSv1.2 + and above.) + +*SSL_cipher_list* + +: The cipher list to send to the server. Although the server + determines which cipher suite is used, it should take the first + supported cipher in the list sent by the client. See + [`ciphers`(1ssl)] for more information. + +*SSL_fingerprint* + +: Fingerprint of the server certificate's Subject Public Key Info, in + the form `[ALGO$]DIGEST_HEX` where `ALGO` is the used algorithm (by + default `sha256`). + Attempting to connect to a server with a non-matching certificate + SPKI fingerprint causes `interimap` to abort the connection during + the SSL/TLS handshake. + The following command can be used to compute the SHA-256 digest of a + certificate's Subject Public Key Info: + + openssl x509 -in /path/to/server/certificate.pem -pubkey \ + | openssl pkey -pubin -outform DER \ + | openssl dgst -sha256 + +*SSL_verify* + +: Whether to verify the server certificate chain. + Note that using *SSL_fingerprint* to specify the fingerprint of the + server certificate is an orthogonal authentication measure as it + ignores the CA chain. + (Default: `YES`.) + +*SSL_CApath* + +: Directory to use for server certificate verification if + `SSL_verify=YES`. + This directory must be in “hash format”, see [`verify`(1ssl)] for + more information. + +*SSL_CAfile* + +: File containing trusted certificates to use during server + certificate authentication if `SSL_verify=YES`. + +Supported extensions +==================== + +`interimap` takes advantage of servers supporting the following +extensions to the [IMAP4rev1 protocol][RFC 3501] (those marked as +“recommended” give the most significant performance gain): + + * `LITERAL+` ([RFC 2088], recommended); + * `MULTIAPPEND` ([RFC 3502], recommended); + * `COMPRESS=DEFLATE` ([RFC 4978], recommended); + * `NOTIFY` ([RFC 5465], recommended); + * `SASL-IR` ([RFC 4959]); and + * `UNSELECT` ([RFC 3691]). + +Known bugs and limitations +========================== + + * Using `interimap` on two identical servers with a non-existent or + empty *database* will duplicate each message due to the absence of + local ↔ remote UID association. Hence one needs to manually empty + the mail store on one end when migrating to `interimap` from another + synchronisation solution. + + * `interimap` is single threaded and doesn't use IMAP command + pipelining. Synchronization could be boosted up by sending + independent commands (such as the initial `LIST` and `STATUS` + commands) to both servers in parallel, and for a given server, by + sending independent commands (such as flag updates) in a pipeline. + + * Because the [IMAP protocol][RFC 3501] doesn't have a specific + response code for when a message is moved to another mailbox (either + using the `MOVE` command from [RFC 6851], or via `COPY` + `STORE` + + `EXPUNGE`), moving a message causes `interimap` to believe that it + was deleted while another one (which is replicated again) was added + to the other mailbox in the meantime. + + * `PLAIN` and `LOGIN` are the only authentication mechanisms currently + supported. + + * `interimap` will probably not work with non [RFC][RFC 3501]-compliant + servers. In particular, no work-around is currently implemented + beside the tunables in the [configuration file](#configuration-file). + Moreover, few IMAP servers have been tested so far. + +Standards +========= + + * M. Leech, M. Ganis, Y. Lee, R. Kuris, D. Koblas and L. Jones, + _SOCKS Protocol Version 5_, + [RFC 1928], March 1996. + * M. Leech, _Username/Password Authentication for SOCKS V5_, + [RFC 1929], March 1996. + * J. Myers, _IMAP4 non-synchronizing literals_, + [RFC 2088], January 1997. + * D. Goldsmith and M. Davis, + _A Mail-Safe Transformation Format of Unicode_, + [RFC 2152], May 1997. + * C. Newman, _Using TLS with IMAP, POP3 and ACAP_, + [RFC 2595], June 1999. + * M. Crispin, _Internet Message Access Protocol - Version 4rev1_, + [RFC 3501], March 2003. + * M. Crispin, + _Internet Message Access Protocol (IMAP) - `MULTIAPPEND` Extension_, + [RFC 3502], March 2003. + * A. Melnikov, + _Internet Message Access Protocol (IMAP) `UNSELECT` command_, + [RFC 3691], February 2004. + * M. Crispin, + _Internet Message Access Protocol (IMAP) - `UIDPLUS` extension_, + [RFC 4315], December 2005. + * A. Melnikov, + _Synchronization Operations for Disconnected IMAP4 Clients_, + [RFC 4549], June 2006. + * A. Gulbrandsen, _The IMAP `COMPRESS` Extension_, + [RFC 4978], August 2007. + * R. Siemborski and A. Gulbrandsen, _IMAP Extension for Simple + Authentication and Security Layer (SASL) Initial Client Response_, + [RFC 4959], September 2007. + * A. Gulbrandsen and A. Melnikov, + _The IMAP `ENABLE` Extension_, + [RFC 5161], March 2008. + * B. Leiba and A. Melnikov, + _Internet Message Access Protocol version 4 - `LIST` Command Extensions_, + [RFC 5258], June 2008. + * A. Gulbrandsen, C. King and A. Melnikov, + _The IMAP `NOTIFY` Extension_, + [RFC 5465], February 2009. + * A. Melnikov and T. Sirainen, + _IMAP4 Extension for Returning `STATUS` Information in Extended LIST_, + [RFC 5819], March 2010. + * A. Gulbrandsen and N. Freed, + _Internet Message Access Protocol (IMAP) - `MOVE` Extension_, + [RFC 6851], January 2013. + * A. Melnikov and D. Cridland, + _IMAP Extensions: Quick Flag Changes Resynchronization (`CONDSTORE`) + and Quick Mailbox Resynchronization (`QRESYNC`)_, + [RFC 7162], May 2014. + +[RFC 7162]: https://tools.ietf.org/html/rfc7162 +[RFC 5258]: https://tools.ietf.org/html/rfc5258 +[RFC 5819]: https://tools.ietf.org/html/rfc5819 +[RFC 4315]: https://tools.ietf.org/html/rfc4315 +[RFC 4549]: https://tools.ietf.org/html/rfc4549 +[RFC 2152]: https://tools.ietf.org/html/rfc2152 +[RFC 3501]: https://tools.ietf.org/html/rfc3501 +[RFC 1928]: https://tools.ietf.org/html/rfc1928 +[RFC 1929]: https://tools.ietf.org/html/rfc1929 +[RFC 2595]: https://tools.ietf.org/html/rfc2595 +[RFC 4978]: https://tools.ietf.org/html/rfc4978 +[RFC 2088]: https://tools.ietf.org/html/rfc2088 +[RFC 3502]: https://tools.ietf.org/html/rfc3502 +[RFC 4959]: https://tools.ietf.org/html/rfc4959 +[RFC 3691]: https://tools.ietf.org/html/rfc3691 +[RFC 6851]: https://tools.ietf.org/html/rfc6851 +[RFC 5161]: https://tools.ietf.org/html/rfc5161 +[RFC 5465]: https://tools.ietf.org/html/rfc5465 + +[INI file]: https://en.wikipedia.org/wiki/INI_file +[PCRE]: https://en.wikipedia.org/wiki/Perl_Compatible_Regular_Expressions +[`ciphers`(1ssl)]: https://www.openssl.org/docs/manmaster/apps/ciphers.html +[`verify`(1ssl)]: https://www.openssl.org/docs/manmaster/apps/verify.html diff --git a/doc/pullimap.1.md b/doc/pullimap.1.md new file mode 100644 index 0000000..1b2e509 --- /dev/null +++ b/doc/pullimap.1.md @@ -0,0 +1,375 @@ +% pullimap(1) +% [Guilhem Moulin](mailto:guilhem@fripost.org) +% March 2016 + +Name +==== + +PullIMAP - Pull mails from an IMAP mailbox and deliver them to an SMTP session + +Synopsis +======== + +`pullimap` [**\-\-config=***FILE*] [**\-\-idle**[**=***SECONDS*]] +[**\-\-no-delivery**] [**\-\-quiet**] *SECTION* + +Description +=========== + +`pullimap` retrieves messages from an IMAP mailbox and deliver them to +an SMTP or LMTP transmission channel. It can also remove old messages +after a configurable retention period. + +A *statefile* is used to keep track of the mailbox's `UIDVALIDITY` and +`UIDNEXT` values. While `pullimap` is running, the *statefile* is also +used to keep track of UIDs being delivered, which avoids duplicate +deliveries in case the process is interrupted. +See the **[control flow](#control-flow)** section below for details. + +Options +======= + +`--config=`*FILE* + +: Specify an alternate [configuration file](#configuration-file). + Relative paths start from *$XDG_CONFIG_HOME/pullimap*, or *~/.config/pullimap* + if the `XDG_CONFIG_HOME` environment variable is unset. + +`--idle`[`=`*seconds*] + +: Don't exit after a successful poll. Instead, keep the connection open + and issue `IDLE` commands (require an IMAP server supporting [RFC + 2177]) to watch for updates in the mailbox. This also enables + `SO_KEEPALIVE` on the socket. + Each `IDLE` command is terminated after at most *seconds* (29 + minutes by default) to avoid being logged out for inactivity. + +`--no-delivery` + +: Update the *statefile*, but skip SMTP/LMTP delivery. This is mostly + useful for initializing the *statefile* when migrating to `pullimap` + from another similar program such as [`fetchmail`(1)] or + [`getmail`(1)]. + +`-q`, `--quiet` + +: Try to be quiet. + +`--debug` + +: Turn on debug mode. Debug messages are written to the error output. + Note that this include all IMAP traffic (except literals). + Depending on the chosen authentication mechanism, this might include + authentication credentials. + +`-h`, `--help` + +: Output a brief help and exit. + +`--version` + +: Show the version number and exit. + +Configuration file +================== + +Unless told otherwise by the `--config=FILE` command-line option, +`pullimap` reads its configuration from *$XDG_CONFIG_HOME/pullimap/config* +(or *~/.config/pullimap/config* if the `XDG_CONFIG_HOME` environment variable +is unset) as an [INI file]. +The syntax of the configuration file is a series of `OPTION=VALUE` +lines organized under some `[SECTION]`; lines starting with a ‘#’ or +‘;’ character are ignored as comments. +Valid options are: + +*statefile* + +: State file to use to keep track of the *mailbox*'s `UIDVALIDITY` and + `UIDNEXT` values. Relative paths start from + *$XDG_DATA_HOME/pullimap*, or *~/.local/share/pullimap* if the + `XDG_DATA_HOME` environment variable is unset. + (Default: the parent section name of the option.) + +*mailbox* + +: The IMAP mailbox ([UTF-7 encoded][RFC 2152] and unquoted) to pull + messages from. Support for persistent message Unique Identifiers + (UID) is required. (Default: `INBOX`.) + +*deliver-method* + +: `PROTOCOL:[ADDRESS]:PORT` where to deliver messages. Both + [SMTP][RFC 5321] and [LMTP][RFC 2033] servers are supported, and + [SMTP pipelining][RFC 2920] is used when possible. + (Default: `smtp:[127.0.0.1]:25`.) + +*deliver-ehlo* + +: Hostname to use in `EHLO` or `LHLO` commands. + (Default: `localhost.localdomain`.) + +*deliver-rcpt* + +: Message recipient. Note that the local part needs to quoted if it + contains special characters; see [RFC 5321] for details. + (Default: the username associated with the effective uid of the + `pullimap` process.) + +*purge-after* + +: Retention period (in days), after which messages are removed from + the IMAP server. (The value is at best 24h accurate due to the IMAP + `SEARCH` criterion ignoring time and timezone.) + If *purge-after* is set to `0` then messages are deleted immediately + after delivery. Otherwise `pullimap` issues an IMAP `SEARCH` (or + extended `SEARCH` on servers advertizing the [`ESEARCH`][RFC 4731] + capability) command to list old messages; if `--idle` is set then + the `SEARCH` command is issued again every 12 hours. + +*type* + +: One of `imap`, `imaps` or `tunnel`. + `type=imap` and `type=imaps` are respectively used for IMAP and IMAP + over SSL/TLS connections over an INET socket. + `type=tunnel` causes `pullimap` to create an unnamed pair of + connected sockets for interprocess communication with a *command* + instead of opening a network socket. + (Default: `imaps`.) + +*host* + +: Server hostname, for `type=imap` and `type=imaps`. + (Default: `localhost`.) + +*port* + +: Server port. + (Default: `143` for `type=imap`, `993` for `type=imaps`.) + +*proxy* + +: An optional SOCKS proxy to use for TCP connections to the IMAP + server (`type=imap` and `type=imaps` only), formatted as + `PROTOCOL://[USER:PASSWORD@]PROXYHOST[:PROXYPORT]`. + If `PROXYPORT` is omitted, it is assumed at port 1080. + Only [SOCKSv5][RFC 1928] is supported (with optional + [username/password authentication][RFC 1929]), in two flavors: + `socks5://` to resolve *hostname* locally, and `socks5h://` to let + the proxy resolve *hostname*. + +*command* + +: Command to use for `type=tunnel`. Must speak the [IMAP4rev1 + protocol][RFC 3501] on its standard output, and understand it on its + standard input. The value is passed to `` `/bin/sh -c` `` if it + contains shell metacharacters; otherwise it is split into words and + the resulting list is passed to `execvp`(3). + +*STARTTLS* + +: Whether to use the [`STARTTLS`][RFC 2595] directive to upgrade to a + secure connection. Setting this to `YES` for a server not + advertising the `STARTTLS` capability causes `pullimap` to + immediately abort the connection. + (Ignored for *type*s other than `imap`. Default: `YES`.) + +*auth* + +: Space-separated list of preferred authentication mechanisms. + `pullimap` uses the first mechanism in that list that is also + advertised (prefixed with `AUTH=`) in the server's capability list. + Supported authentication mechanisms are `PLAIN` and `LOGIN`. + (Default: `PLAIN LOGIN`.) + +*username*, *password* + +: Username and password to authenticate with. Can be required for non + pre-authenticated connections, depending on the chosen + authentication mechanism. + +*compress* + +: Whether to use the [`IMAP COMPRESS` extension][RFC 4978] for servers + advertising it. (Default: `YES`.) + +*null-stderr* + +: Whether to redirect *command*'s standard error to `/dev/null` for + `type=tunnel`. (Default: `NO`.) + +*SSL_protocols* + +: A space-separated list of SSL protocols to enable or disable (if + prefixed with an exclamation mark `!`. Known protocols are `SSLv2`, + `SSLv3`, `TLSv1`, `TLSv1.1`, `TLSv1.2`, and `TLSv1.3`. Enabling a + protocol is a short-hand for disabling all other protocols. + (Default: `!SSLv2 !SSLv3 !TLSv1 !TLSv1.1`, i.e., only enable TLSv1.2 + and above.) + +*SSL_cipher_list* + +: The cipher list to send to the server. Although the server + determines which cipher suite is used, it should take the first + supported cipher in the list sent by the client. See + [`ciphers`(1ssl)] for more information. + +*SSL_fingerprint* + +: Fingerprint of the server certificate's Subject Public Key Info, in + the form `[ALGO$]DIGEST_HEX` where `ALGO` is the used algorithm (by + default `sha256`). + Attempting to connect to a server with a non-matching certificate + SPKI fingerprint causes `pullimap` to abort the connection during + the SSL/TLS handshake. + The following command can be used to compute the SHA-256 digest of a + certificate's Subject Public Key Info: + + openssl x509 -in /path/to/server/certificate.pem -pubkey \ + | openssl pkey -pubin -outform DER \ + | openssl dgst -sha256 + +*SSL_verify* + +: Whether to verify the server certificate chain. + Note that using *SSL_fingerprint* to specify the fingerprint of the + server certificate is an orthogonal authentication measure as it + ignores the CA chain. + (Default: `YES`.) + +*SSL_CApath* + +: Directory to use for server certificate verification if + `SSL_verify=YES`. + This directory must be in “hash format”, see [`verify`(1ssl)] for + more information. + +*SSL_CAfile* + +: File containing trusted certificates to use during server + certificate authentication if `SSL_verify=YES`. + +Control flow +============ + +`pullimap` opens the *statefile* corresponding to a given configuration +*SECTION* with `O_DSYNC` to ensure that written data is flushed to the +underlying hardware by the time [`write`(2)] returns. Moreover an +exclusive lock is placed on the file descriptor immediately after +opening to prevent multiple `pullimap` processes from accessing the +*statefile* concurrently. + +Each *statefile* consists of a series of 32-bits big-endian integers. +Usually there are only two integers: the first is the *mailbox*'s +`UIDVALIDITY` value, and the second is the *mailbox*'s last seen +`UIDNEXT` value (`pullimap` then assumes that all messages with UID +smaller than this `UIDNEXT` value have already been retrieved and +delivered). +The [IMAP4rev1 specification][RFC 3501] does not guaranty that untagged +`FETCH` responses are sent ordered by UID in response to a `UID FETCH` +command. Thus it would be unsafe for `pullimap` to update the `UIDNEXT` +value in its *statefile* while the `UID FETCH` command is progress. +Instead, for each untagged `FETCH` response received while the `UID +FETCH` command is in progress, `pullimap` delivers the message `RFC822` +body to the SMTP or LMTP server (specified with *deliver-method*) then +appends the message UID to the *statefile*. +When the `UID FETCH` command eventually terminates, `pullimap` updates +the `UIDNEXT` value in the *statefile* and truncate the file down to 8 +bytes. Keeping track of message UIDs as they are received avoids +duplicate in the event of a crash or connection loss while the `UID +FETCH` command is in progress. + +In more details, `pullimap` works as follows: + + 1. Issue a `UID FETCH` command to retrieve message `ENVELOPE` and + `RFC822` (and `UID`) with UID bigger or equal than the `UIDNEXT` + value found in the *statefile*. + While the `UID FETCH` command is in progress, perform the following + for each untagged `FETCH` response sent by the server: + + i. if no SMTP/LMTP transmission channel was opened, open one to the + server specified with *deliver-method* and send an `EHLO` (or + `LHO`) command with the domain specified by *deliver-ehlo* (the + channel is kept open and shared for all messages retrieved while + the `UID FETCH` IMAP command is in progress); + + i. perform a mail transaction (using [SMTP pipelining][RFC 2920] if + possible) to deliver the retrieved message `RFC822` body to the + SMTP or LMTP session; and + + i. append the message UID to the *statefile*. + + 2. If an SMTP/LMTP transmission channel was opened, send a `QUIT` command + to terminate it gracefully. + + 3. Issue a `UID STORE` command to mark all retrieved messages (and + stalled UIDs found in the *statefile* after the eigth byte) as + `\Seen`. + + 4. Update the *statefile* with the new UIDNEXT value (bytes 5-8). + + 5. Truncate the *statefile* down to 8 bytes (so that it contains only + two 32-bits integers, respectively the *mailbox*'s current + `UIDVALIDITY` and `UIDNEXT` values). + + 6. If `--idle` was set, issue an `IDLE` command; stop idling and go + back to step 1 when a new message is received (or when the `IDLE` + timeout expires). + +Standards +========= + + * M. Leech, M. Ganis, Y. Lee, R. Kuris, D. Koblas and L. Jones, + _SOCKS Protocol Version 5_, + [RFC 1928], March 1996. + * M. Leech, _Username/Password Authentication for SOCKS V5_, + [RFC 1929], March 1996. + * J. Myers, _Local Mail Transfer Protocol_, + [RFC 2033], October 1996. + * J. Myers, _IMAP4 non-synchronizing literals_, + [RFC 2088], January 1997. + * D. Goldsmith and M. Davis, + _A Mail-Safe Transformation Format of Unicode_, + [RFC 2152], May 1997. + * B. Leiba, _IMAP4 `IDLE` command_, + [RFC 2177], June 1997. + * C. Newman, _Using TLS with IMAP, POP3 and ACAP_, + [RFC 2595], June 1999. + * N. Freed, _SMTP Service Extension for Command Pipelining_, + [RFC 2920], September 2000. + * M. Crispin, _Internet Message Access Protocol - Version 4rev1_, + [RFC 3501], March 2003. + * M. Crispin, + _Internet Message Access Protocol (IMAP) - `UIDPLUS` extension_, + [RFC 4315], December 2005. + * A. Gulbrandsen, _The IMAP `COMPRESS` Extension_, + [RFC 4978], August 2007. + * A. Melnikov and D. Cridland, _IMAP4 Extension to SEARCH Command for + Controlling What Kind of Information Is Returned_, + [RFC 4731], November 2006. + * R. Siemborski and A. Gulbrandsen, _IMAP Extension for Simple + Authentication and Security Layer (SASL) Initial Client Response_, + [RFC 4959], September 2007. + * J. Klensin, _Simple Mail Transfer Protocol_, + [RFC 5321], October 2008. + +[RFC 4315]: https://tools.ietf.org/html/rfc4315 +[RFC 2177]: https://tools.ietf.org/html/rfc2177 +[RFC 2595]: https://tools.ietf.org/html/rfc2595 +[RFC 4959]: https://tools.ietf.org/html/rfc4959 +[RFC 2152]: https://tools.ietf.org/html/rfc2152 +[RFC 2088]: https://tools.ietf.org/html/rfc2088 +[RFC 5321]: https://tools.ietf.org/html/rfc5321 +[RFC 2033]: https://tools.ietf.org/html/rfc2033 +[RFC 2920]: https://tools.ietf.org/html/rfc2920 +[RFC 3501]: https://tools.ietf.org/html/rfc3501 +[RFC 4978]: https://tools.ietf.org/html/rfc4978 +[RFC 1928]: https://tools.ietf.org/html/rfc1928 +[RFC 1929]: https://tools.ietf.org/html/rfc1929 +[RFC 4731]: https://tools.ietf.org/html/rfc4731 + +[INI file]: https://en.wikipedia.org/wiki/INI_file +[`fetchmail`(1)]: http://www.fetchmail.info/ +[`getmail`(1)]: http://pyropus.ca/software/getmail/ +[`write`(2)]: http://man7.org/linux/man-pages/man2/write.2.html +[`ciphers`(1ssl)]: https://www.openssl.org/docs/manmaster/apps/ciphers.html +[`verify`(1ssl)]: https://www.openssl.org/docs/manmaster/apps/verify.html diff --git a/doc/template.html b/doc/template.html new file mode 100644 index 0000000..e17f0e3 --- /dev/null +++ b/doc/template.html @@ -0,0 +1,76 @@ + + + + + + +$for(author-meta)$ + +$endfor$ +$if(date-meta)$ + +$endif$ +$if(keywords)$ + +$endif$ + $if(title-prefix)$$title-prefix$ – $endif$$pagetitle$ + +$if(highlighting-css)$ + +$endif$ +$for(css)$ + +$endfor$ + +$if(math)$ + $math$ +$endif$ +$for(header-includes)$ + $header-includes$ +$endfor$ + + + +$for(include-before)$ +$include-before$ +$endfor$ +
+
+$if(title)$ +

$title$

+$endif$ + +$body$ +
+ + +
+ + diff --git a/interimap.md b/interimap.md deleted file mode 100644 index 387850a..0000000 --- a/interimap.md +++ /dev/null @@ -1,526 +0,0 @@ -% interimap(1) -% [Guilhem Moulin](mailto:guilhem@fripost.org) -% July 2015 - -Name -==== - -InterIMAP - Fast bidirectional synchronization for QRESYNC-capable IMAP servers - -Synopsis -======== - -`interimap` [*OPTION* ...] [*COMMAND*] [*MAILBOX* ...] - -Description -=========== - -`interimap` performs stateful synchronization between two IMAP4rev1 -servers. -Such synchronization is made possible by the [`QRESYNC` IMAP -extension][RFC 7162]; for convenience reasons servers must also support -the [`LIST-EXTENDED`][RFC 5258], [`LIST-STATUS`][RFC 5819] (or -[`NOTIFY`][RFC 5465]) and [`UIDPLUS`][RFC 4315] IMAP extensions. -See also the **[supported extensions](#supported-extensions)** section -below. - -Stateful synchronization is only possible for mailboxes supporting -persistent message Unique Identifiers (UID) and persistent storage of -mod-sequences (MODSEQ); any non-compliant mailbox will cause `interimap` -to abort. -Furthermore, because UIDs are allocated not by the client but by the -server, `interimap` needs to keep track of associations between local -and remote UIDs for each mailbox. -The synchronization state of a mailbox consists of its `UIDNEXT` and -`HIGHESTMODSEQ` values on each server; it is then assumed that each -message with UID smaller than `UIDNEXT` have been replicated to the -other server, and that the metadata (such as flags) of each message with -MODSEQ at most `HIGHESTMODSEQ` have been synchronized. -Conceptually, the synchronization algorithm is derived from [RFC 4549] -with the [RFC 7162] (sec. 6) amendments, and works as follows: - - 1. `SELECT` (on both servers) a mailbox the current `UIDNEXT` or `HIGHESTMODSEQ` - values of which differ from the values found in the database (for - either server). Use the `QRESYNC` `SELECT` parameter from [RFC - 7162] to list changes (vanished messages and flag updates) since - `HIGHESTMODSEQ` to messages with UID smaller than `UIDNEXT`. - - 2. Propagate these changes onto the other server: get the corresponding - UIDs from the database, then: - a. issue a `UID STORE` command, followed by `UID EXPUNGE`, to - remove messages that have not already been deleted on both - servers; and - b. issue some `UID STORE` commands to propagate flag updates (send - a single command for each flag list in order the reduce the - number of round trips). - - (Conflicts may occur if the metadata of a message has been updated - on both servers with different flag lists; in that case, `interimap` - issues a warning and updates the message on each server with the - union of both flag lists.) - Repeat this step if the server sent some updates in the meantime. - Otherwise, update the `HIGHESTMODSEQ` value in the database. - - 3. Process new messages (if the current `UIDNEXT` value of the mailbox - differs from the one found in the database) by issuing a `UID FETCH` - command; process each received message on-the-fly by issuing an - `APPEND` command with the message's `RFC822` body, `FLAGS` and - `INTERNALDATE`. - Repeat this step if the server received new messages in the - meantime. Otherwise, update the `UIDNEXT` value in the database. - Go back to step 2 if the server sent some metadata (such as flag) - updates in the meantime. - - 4. Go back to step 1 to proceed with the next unsynchronized mailbox. - -Commands -======== - -By default, `interimap` synchronizes each mailbox listed by the `LIST "" -"*"` IMAP command; -the *list-mailbox*, *list-select-opts* and *ignore-mailbox* options from -the [configuration file](#configuration-file) can be used to shrink that -list and save bandwidth. -However if some extra argument are provided on the command line, -`interimap` ignores these options and synchronizes the given -*MAILBOX*es instead. Note that each *MAILBOX* is taken “as is”; in -particular, it must be [UTF-7 encoded][RFC 2152], unquoted, and the list -wildcards ‘\*’ and ‘%’ are passed verbatim to the IMAP server. - -If the synchronization was interrupted during a previous run while some -messages were being replicated (but before the `UIDNEXT` or -`HIGHESTMODSEQ` values have been updated), `interimap` performs a “full -synchronization” on theses messages: downloading the whole UID and flag -lists on each servers allows `interimap` to detect messages that have -been removed or for which their flags have changed in the meantime. -Finally, after propagating the offline changes for these messages, -`interimap` resumes the synchronization for the rest of the mailbox. - -Specifying one of the commands below makes `interimap` perform an action -other than the default [`QRESYNC`][RFC 7162]-based synchronization. - -`--repair` [*MAILBOX* ...] - -: List the database anomalies and try to repair them. (Consider only - the given *MAILBOX*es if non-optional arguments are provided.) - This is done by performing a so-called “full synchronization”, - namely: - 1/ download all UIDs along with their flag list both from the - local and remote servers; - 2/ ensure that each entry in the database corresponds to an - existing UID; and - 3/ ensure that both flag lists match. - Any message found on a server but not in the database is replicated - on the other server (which in the worst case, might yield a message - duplicate). - Flag conflicts are solved by updating each message to the union of - both lists. - -`--delete` *MAILBOX* [*MAILBOX* ...] - -: Delete the given *MAILBOX*es on each target (by default each server - plus the database, unless `--target` specifies otherwise) where it - exists. - Note that per the [IMAP4rev1 standard][RFC 3501] deletion is not - recursive. Thus *MAILBOX*'s children are not deleted. - -`--rename` *SOURCE* *DEST* - -: Rename the mailbox *SOURCE* to *DEST* on each target (by default - each server plus the database, unless `--target` specifies - otherwise) where it exists. - `interimap` aborts if *DEST* already exists on either target. - Note that per the [IMAP4rev1 standard][RFC 3501] renaming is - recursive. Thus *SOURCE*'s children are moved to become *DEST*'s - children instead. - -Options -======= - -`--config=`*FILE* - -: Specify an alternate [configuration file](#configuration-file). - Relative paths start from *$XDG_CONFIG_HOME/interimap*, or *~/.config/interimap* - if the `XDG_CONFIG_HOME` environment variable is unset. - -`--target={local,remote,database}` - -: Limit the scope of a `--delete` or `--rename` command to the given - target. Can be repeated to act on multiple targets. By default all - three targets are considered. - -`--watch`[`=`*seconds*] - -: Don't exit after a successful synchronization. Instead, keep - synchronizing forever. Sleep for the given number of *seconds* (by - default 1 minute if `--notify` is unset, and 15 minutes if - `--notify` is set) between two synchronizations. Setting this - options enables `SO_KEEPALIVE` on the socket for *type*s other than - `tunnel`. - -`--notify` - -: Whether to use the [IMAP `NOTIFY` extension][RFC 5465] to instruct - the server to automatically send updates to the client. (Both local - and remote servers must support [RFC 5465] for this to work.) - This greatly reduces IMAP traffic since `interimap` can rely on - server notifications instead of manually polling for updates. - If the connection remains idle for 15 minutes (configurable with - `--watch`), then `interimap` sends a `NOOP` command to avoid being - logged out for inactivity. - -`-q`, `--quiet` - -: Try to be quiet. - -`--debug` - -: Turn on debug mode. Debug messages are written to the given *logfile*. - Note that this include all IMAP traffic (except literals). - Depending on the chosen authentication mechanism, this might include - authentication credentials. - -`-h`, `--help` - -: Output a brief help and exit. - -`--version` - -: Show the version number and exit. - -Configuration file -================== - -Unless told otherwise by the `--config=FILE` command-line option, -`interimap` reads its configuration from *$XDG_CONFIG_HOME/interimap/config* -(or *~/.config/interimap/config* if the `XDG_CONFIG_HOME` environment -variable is unset) as an [INI file]. -The syntax of the configuration file is a series of `OPTION=VALUE` -lines organized under some `[SECTION]`; lines starting with a ‘#’ or -‘;’ character are ignored as comments. -The `[local]` and `[remote]` sections define the two IMAP servers to -synchronize. -Valid options are: - -*database* - -: SQLite version 3 database file to use to keep track of associations - between local and remote UIDs, as well as the `UIDVALIDITY`, - `UIDNEXT` and `HIGHESTMODSEQ` of each known mailbox on both servers. - Relative paths start from *$XDG_DATA_HOME/interimap*, or - *~/.local/share/interimap* if the `XDG_DATA_HOME` environment - variable is unset. This option is only available in the default - section. - (Default: `HOST.db`, where *HOST* is taken from the `[remote]` or - `[local]` sections, in that order.) - -*list-reference* - -: An optional “reference name” to use for the initial `LIST` command, - indicating the context in which the *MAILBOX*es are interpreted. - For instance, by specifying `list-reference=perso/` in the `[local]` - section, *MAILBOX* names are interpreted relative to `perso/` on the - local server; in other words the remote mailbox hierarchy is mapped - to the `perso/` sub-hierarchy on the local server. This is useful - for synchronizing multiple remote servers against different - namespaces belonging to the same local IMAP server (using a - different `interimap` instance for each local namespace ↔ remote - synchronization). - - (Note that if the reference name is not a level of mailbox hierarchy - and/or does not end with the hierarchy delimiter, by [RFC 3501] its - interpretation by the IMAP server is implementation-dependent.) - -*list-mailbox* - -: A space separated list of mailbox patterns to use when issuing the - initial `LIST` command (overridden by the *MAILBOX*es given as - command-line arguments). - Names containing special characters such as spaces or brackets need - to be enclosed in double quotes. Within double quotes C-style - backslash escape sequences can be used (‘\\t’ for an horizontal tab, - ‘\\n’ for a new line, ‘\\\\’ for a backslash, etc.), as well as - hexadecimal escape sequences ‘\\xHH’. - Furthermore, non-ASCII names must be [UTF-7 encoded][RFC 2152]. - Two wildcards are available, and passed verbatim to the IMAP server: - a ‘\*’ character matches zero or more characters, while a ‘%’ - character matches zero or more characters up to the hierarchy - delimiter. - This option is only available in the default section. - (The default pattern, `*`, matches all visible mailboxes on the - server.) - -*list-select-opts* - -: An optional space separated list of selectors for the initial `LIST` - command. (Requires a server supporting the [`LIST-EXTENDED` IMAP - extension][RFC 5258].) Useful values are `SUBSCRIBED` (to list only - subscribed mailboxes), `REMOTE` (to also list remote mailboxes on a - server supporting mailbox referrals), and `RECURSIVEMATCH` (to - list parent mailboxes with children matching one of the above - *list-mailbox* patterns). This option is only available in the - default section. - -*ignore-mailbox* - -: An optional Perl Compatible Regular Expressions ([PCRE]) covering - mailboxes to exclude: any ([UTF-7 encoded][RFC 2152] and unquoted) - mailbox listed in the initial `LIST` responses is ignored if it - matches the given expression. - Note that the *MAILBOX*es given as command-line arguments bypass the - check and are always considered for synchronization. This option is - only available in the default section. - -*logfile* - -: A file name to use to log debug and informational messages. (By - default these messages are written to the error output.) This - option is only available in the default section. - -*type* - -: One of `imap`, `imaps` or `tunnel`. - `type=imap` and `type=imaps` are respectively used for IMAP and IMAP - over SSL/TLS connections over an INET socket. - `type=tunnel` causes `interimap` to create an unnamed pair of - connected sockets for interprocess communication with a *command* - instead of opening a network socket. - Note that specifying `type=tunnel` in the `[remote]` section makes - the default *database* to be `localhost.db`. - (Default: `imaps`.) - -*host* - -: Server hostname, for `type=imap` and `type=imaps`. - (Default: `localhost`.) - -*port* - -: Server port. - (Default: `143` for `type=imap`, `993` for `type=imaps`.) - -*proxy* - -: An optional SOCKS proxy to use for TCP connections to the IMAP - server (`type=imap` and `type=imaps` only), formatted as - `PROTOCOL://[USER:PASSWORD@]PROXYHOST[:PROXYPORT]`. - If `PROXYPORT` is omitted, it is assumed at port 1080. - Only [SOCKSv5][RFC 1928] is supported (with optional - [username/password authentication][RFC 1929]), in two flavors: - `socks5://` to resolve *hostname* locally, and `socks5h://` to let - the proxy resolve *hostname*. - -*command* - -: Command to use for `type=tunnel`. Must speak the [IMAP4rev1 - protocol][RFC 3501] on its standard output, and understand it on its - standard input. The value is passed to `` `/bin/sh -c` `` if it - contains shell metacharacters; otherwise it is split into words and - the resulting list is passed to `execvp`(3). - -*STARTTLS* - -: Whether to use the [`STARTTLS`][RFC 2595] directive to upgrade to a - secure connection. Setting this to `YES` for a server not - advertising the `STARTTLS` capability causes `interimap` to - immediately abort the connection. - (Ignored for *type*s other than `imap`. Default: `YES`.) - -*auth* - -: Space-separated list of preferred authentication mechanisms. - `interimap` uses the first mechanism in that list that is also - advertised (prefixed with `AUTH=`) in the server's capability list. - Supported authentication mechanisms are `PLAIN` and `LOGIN`. - (Default: `PLAIN LOGIN`.) - -*username*, *password* - -: Username and password to authenticate with. Can be required for non - pre-authenticated connections, depending on the chosen - authentication mechanism. - -*compress* - -: Whether to use the [`IMAP COMPRESS` extension][RFC 4978] for servers - advertising it. - (Default: `NO` for the `[local]` section, `YES` for the `[remote]` - section.) - -*null-stderr* - -: Whether to redirect *command*'s standard error to `/dev/null` for - `type=tunnel`. (Default: `NO`.) - -*SSL_protocols* - -: A space-separated list of SSL protocols to enable or disable (if - prefixed with an exclamation mark `!`. Known protocols are `SSLv2`, - `SSLv3`, `TLSv1`, `TLSv1.1`, `TLSv1.2`, and `TLSv1.3`. Enabling a - protocol is a short-hand for disabling all other protocols. - (Default: `!SSLv2 !SSLv3 !TLSv1 !TLSv1.1`, i.e., only enable TLSv1.2 - and above.) - -*SSL_cipher_list* - -: The cipher list to send to the server. Although the server - determines which cipher suite is used, it should take the first - supported cipher in the list sent by the client. See - [`ciphers`(1ssl)] for more information. - -*SSL_fingerprint* - -: Fingerprint of the server certificate's Subject Public Key Info, in - the form `[ALGO$]DIGEST_HEX` where `ALGO` is the used algorithm (by - default `sha256`). - Attempting to connect to a server with a non-matching certificate - SPKI fingerprint causes `interimap` to abort the connection during - the SSL/TLS handshake. - The following command can be used to compute the SHA-256 digest of a - certificate's Subject Public Key Info: - - openssl x509 -in /path/to/server/certificate.pem -pubkey \ - | openssl pkey -pubin -outform DER \ - | openssl dgst -sha256 - -*SSL_verify* - -: Whether to verify the server certificate chain. - Note that using *SSL_fingerprint* to specify the fingerprint of the - server certificate is an orthogonal authentication measure as it - ignores the CA chain. - (Default: `YES`.) - -*SSL_CApath* - -: Directory to use for server certificate verification if - `SSL_verify=YES`. - This directory must be in “hash format”, see [`verify`(1ssl)] for - more information. - -*SSL_CAfile* - -: File containing trusted certificates to use during server - certificate authentication if `SSL_verify=YES`. - -Supported extensions -==================== - -`interimap` takes advantage of servers supporting the following -extensions to the [IMAP4rev1 protocol][RFC 3501] (those marked as -“recommended” give the most significant performance gain): - - * `LITERAL+` ([RFC 2088], recommended); - * `MULTIAPPEND` ([RFC 3502], recommended); - * `COMPRESS=DEFLATE` ([RFC 4978], recommended); - * `NOTIFY` ([RFC 5465], recommended); - * `SASL-IR` ([RFC 4959]); and - * `UNSELECT` ([RFC 3691]). - -Known bugs and limitations -========================== - - * Using `interimap` on two identical servers with a non-existent or - empty *database* will duplicate each message due to the absence of - local ↔ remote UID association. Hence one needs to manually empty - the mail store on one end when migrating to `interimap` from another - synchronisation solution. - - * `interimap` is single threaded and doesn't use IMAP command - pipelining. Synchronization could be boosted up by sending - independent commands (such as the initial `LIST` and `STATUS` - commands) to both servers in parallel, and for a given server, by - sending independent commands (such as flag updates) in a pipeline. - - * Because the [IMAP protocol][RFC 3501] doesn't have a specific - response code for when a message is moved to another mailbox (either - using the `MOVE` command from [RFC 6851], or via `COPY` + `STORE` + - `EXPUNGE`), moving a message causes `interimap` to believe that it - was deleted while another one (which is replicated again) was added - to the other mailbox in the meantime. - - * `PLAIN` and `LOGIN` are the only authentication mechanisms currently - supported. - - * `interimap` will probably not work with non [RFC][RFC 3501]-compliant - servers. In particular, no work-around is currently implemented - beside the tunables in the [configuration file](#configuration-file). - Moreover, few IMAP servers have been tested so far. - -Standards -========= - - * M. Leech, M. Ganis, Y. Lee, R. Kuris, D. Koblas and L. Jones, - _SOCKS Protocol Version 5_, - [RFC 1928], March 1996. - * M. Leech, _Username/Password Authentication for SOCKS V5_, - [RFC 1929], March 1996. - * J. Myers, _IMAP4 non-synchronizing literals_, - [RFC 2088], January 1997. - * D. Goldsmith and M. Davis, - _A Mail-Safe Transformation Format of Unicode_, - [RFC 2152], May 1997. - * C. Newman, _Using TLS with IMAP, POP3 and ACAP_, - [RFC 2595], June 1999. - * M. Crispin, _Internet Message Access Protocol - Version 4rev1_, - [RFC 3501], March 2003. - * M. Crispin, - _Internet Message Access Protocol (IMAP) - `MULTIAPPEND` Extension_, - [RFC 3502], March 2003. - * A. Melnikov, - _Internet Message Access Protocol (IMAP) `UNSELECT` command_, - [RFC 3691], February 2004. - * M. Crispin, - _Internet Message Access Protocol (IMAP) - `UIDPLUS` extension_, - [RFC 4315], December 2005. - * A. Melnikov, - _Synchronization Operations for Disconnected IMAP4 Clients_, - [RFC 4549], June 2006. - * A. Gulbrandsen, _The IMAP `COMPRESS` Extension_, - [RFC 4978], August 2007. - * R. Siemborski and A. Gulbrandsen, _IMAP Extension for Simple - Authentication and Security Layer (SASL) Initial Client Response_, - [RFC 4959], September 2007. - * A. Gulbrandsen and A. Melnikov, - _The IMAP `ENABLE` Extension_, - [RFC 5161], March 2008. - * B. Leiba and A. Melnikov, - _Internet Message Access Protocol version 4 - `LIST` Command Extensions_, - [RFC 5258], June 2008. - * A. Gulbrandsen, C. King and A. Melnikov, - _The IMAP `NOTIFY` Extension_, - [RFC 5465], February 2009. - * A. Melnikov and T. Sirainen, - _IMAP4 Extension for Returning `STATUS` Information in Extended LIST_, - [RFC 5819], March 2010. - * A. Gulbrandsen and N. Freed, - _Internet Message Access Protocol (IMAP) - `MOVE` Extension_, - [RFC 6851], January 2013. - * A. Melnikov and D. Cridland, - _IMAP Extensions: Quick Flag Changes Resynchronization (`CONDSTORE`) - and Quick Mailbox Resynchronization (`QRESYNC`)_, - [RFC 7162], May 2014. - -[RFC 7162]: https://tools.ietf.org/html/rfc7162 -[RFC 5258]: https://tools.ietf.org/html/rfc5258 -[RFC 5819]: https://tools.ietf.org/html/rfc5819 -[RFC 4315]: https://tools.ietf.org/html/rfc4315 -[RFC 4549]: https://tools.ietf.org/html/rfc4549 -[RFC 2152]: https://tools.ietf.org/html/rfc2152 -[RFC 3501]: https://tools.ietf.org/html/rfc3501 -[RFC 1928]: https://tools.ietf.org/html/rfc1928 -[RFC 1929]: https://tools.ietf.org/html/rfc1929 -[RFC 2595]: https://tools.ietf.org/html/rfc2595 -[RFC 4978]: https://tools.ietf.org/html/rfc4978 -[RFC 2088]: https://tools.ietf.org/html/rfc2088 -[RFC 3502]: https://tools.ietf.org/html/rfc3502 -[RFC 4959]: https://tools.ietf.org/html/rfc4959 -[RFC 3691]: https://tools.ietf.org/html/rfc3691 -[RFC 6851]: https://tools.ietf.org/html/rfc6851 -[RFC 5161]: https://tools.ietf.org/html/rfc5161 -[RFC 5465]: https://tools.ietf.org/html/rfc5465 - -[INI file]: https://en.wikipedia.org/wiki/INI_file -[PCRE]: https://en.wikipedia.org/wiki/Perl_Compatible_Regular_Expressions -[`ciphers`(1ssl)]: https://www.openssl.org/docs/manmaster/apps/ciphers.html -[`verify`(1ssl)]: https://www.openssl.org/docs/manmaster/apps/verify.html diff --git a/pullimap.md b/pullimap.md deleted file mode 100644 index 1b2e509..0000000 --- a/pullimap.md +++ /dev/null @@ -1,375 +0,0 @@ -% pullimap(1) -% [Guilhem Moulin](mailto:guilhem@fripost.org) -% March 2016 - -Name -==== - -PullIMAP - Pull mails from an IMAP mailbox and deliver them to an SMTP session - -Synopsis -======== - -`pullimap` [**\-\-config=***FILE*] [**\-\-idle**[**=***SECONDS*]] -[**\-\-no-delivery**] [**\-\-quiet**] *SECTION* - -Description -=========== - -`pullimap` retrieves messages from an IMAP mailbox and deliver them to -an SMTP or LMTP transmission channel. It can also remove old messages -after a configurable retention period. - -A *statefile* is used to keep track of the mailbox's `UIDVALIDITY` and -`UIDNEXT` values. While `pullimap` is running, the *statefile* is also -used to keep track of UIDs being delivered, which avoids duplicate -deliveries in case the process is interrupted. -See the **[control flow](#control-flow)** section below for details. - -Options -======= - -`--config=`*FILE* - -: Specify an alternate [configuration file](#configuration-file). - Relative paths start from *$XDG_CONFIG_HOME/pullimap*, or *~/.config/pullimap* - if the `XDG_CONFIG_HOME` environment variable is unset. - -`--idle`[`=`*seconds*] - -: Don't exit after a successful poll. Instead, keep the connection open - and issue `IDLE` commands (require an IMAP server supporting [RFC - 2177]) to watch for updates in the mailbox. This also enables - `SO_KEEPALIVE` on the socket. - Each `IDLE` command is terminated after at most *seconds* (29 - minutes by default) to avoid being logged out for inactivity. - -`--no-delivery` - -: Update the *statefile*, but skip SMTP/LMTP delivery. This is mostly - useful for initializing the *statefile* when migrating to `pullimap` - from another similar program such as [`fetchmail`(1)] or - [`getmail`(1)]. - -`-q`, `--quiet` - -: Try to be quiet. - -`--debug` - -: Turn on debug mode. Debug messages are written to the error output. - Note that this include all IMAP traffic (except literals). - Depending on the chosen authentication mechanism, this might include - authentication credentials. - -`-h`, `--help` - -: Output a brief help and exit. - -`--version` - -: Show the version number and exit. - -Configuration file -================== - -Unless told otherwise by the `--config=FILE` command-line option, -`pullimap` reads its configuration from *$XDG_CONFIG_HOME/pullimap/config* -(or *~/.config/pullimap/config* if the `XDG_CONFIG_HOME` environment variable -is unset) as an [INI file]. -The syntax of the configuration file is a series of `OPTION=VALUE` -lines organized under some `[SECTION]`; lines starting with a ‘#’ or -‘;’ character are ignored as comments. -Valid options are: - -*statefile* - -: State file to use to keep track of the *mailbox*'s `UIDVALIDITY` and - `UIDNEXT` values. Relative paths start from - *$XDG_DATA_HOME/pullimap*, or *~/.local/share/pullimap* if the - `XDG_DATA_HOME` environment variable is unset. - (Default: the parent section name of the option.) - -*mailbox* - -: The IMAP mailbox ([UTF-7 encoded][RFC 2152] and unquoted) to pull - messages from. Support for persistent message Unique Identifiers - (UID) is required. (Default: `INBOX`.) - -*deliver-method* - -: `PROTOCOL:[ADDRESS]:PORT` where to deliver messages. Both - [SMTP][RFC 5321] and [LMTP][RFC 2033] servers are supported, and - [SMTP pipelining][RFC 2920] is used when possible. - (Default: `smtp:[127.0.0.1]:25`.) - -*deliver-ehlo* - -: Hostname to use in `EHLO` or `LHLO` commands. - (Default: `localhost.localdomain`.) - -*deliver-rcpt* - -: Message recipient. Note that the local part needs to quoted if it - contains special characters; see [RFC 5321] for details. - (Default: the username associated with the effective uid of the - `pullimap` process.) - -*purge-after* - -: Retention period (in days), after which messages are removed from - the IMAP server. (The value is at best 24h accurate due to the IMAP - `SEARCH` criterion ignoring time and timezone.) - If *purge-after* is set to `0` then messages are deleted immediately - after delivery. Otherwise `pullimap` issues an IMAP `SEARCH` (or - extended `SEARCH` on servers advertizing the [`ESEARCH`][RFC 4731] - capability) command to list old messages; if `--idle` is set then - the `SEARCH` command is issued again every 12 hours. - -*type* - -: One of `imap`, `imaps` or `tunnel`. - `type=imap` and `type=imaps` are respectively used for IMAP and IMAP - over SSL/TLS connections over an INET socket. - `type=tunnel` causes `pullimap` to create an unnamed pair of - connected sockets for interprocess communication with a *command* - instead of opening a network socket. - (Default: `imaps`.) - -*host* - -: Server hostname, for `type=imap` and `type=imaps`. - (Default: `localhost`.) - -*port* - -: Server port. - (Default: `143` for `type=imap`, `993` for `type=imaps`.) - -*proxy* - -: An optional SOCKS proxy to use for TCP connections to the IMAP - server (`type=imap` and `type=imaps` only), formatted as - `PROTOCOL://[USER:PASSWORD@]PROXYHOST[:PROXYPORT]`. - If `PROXYPORT` is omitted, it is assumed at port 1080. - Only [SOCKSv5][RFC 1928] is supported (with optional - [username/password authentication][RFC 1929]), in two flavors: - `socks5://` to resolve *hostname* locally, and `socks5h://` to let - the proxy resolve *hostname*. - -*command* - -: Command to use for `type=tunnel`. Must speak the [IMAP4rev1 - protocol][RFC 3501] on its standard output, and understand it on its - standard input. The value is passed to `` `/bin/sh -c` `` if it - contains shell metacharacters; otherwise it is split into words and - the resulting list is passed to `execvp`(3). - -*STARTTLS* - -: Whether to use the [`STARTTLS`][RFC 2595] directive to upgrade to a - secure connection. Setting this to `YES` for a server not - advertising the `STARTTLS` capability causes `pullimap` to - immediately abort the connection. - (Ignored for *type*s other than `imap`. Default: `YES`.) - -*auth* - -: Space-separated list of preferred authentication mechanisms. - `pullimap` uses the first mechanism in that list that is also - advertised (prefixed with `AUTH=`) in the server's capability list. - Supported authentication mechanisms are `PLAIN` and `LOGIN`. - (Default: `PLAIN LOGIN`.) - -*username*, *password* - -: Username and password to authenticate with. Can be required for non - pre-authenticated connections, depending on the chosen - authentication mechanism. - -*compress* - -: Whether to use the [`IMAP COMPRESS` extension][RFC 4978] for servers - advertising it. (Default: `YES`.) - -*null-stderr* - -: Whether to redirect *command*'s standard error to `/dev/null` for - `type=tunnel`. (Default: `NO`.) - -*SSL_protocols* - -: A space-separated list of SSL protocols to enable or disable (if - prefixed with an exclamation mark `!`. Known protocols are `SSLv2`, - `SSLv3`, `TLSv1`, `TLSv1.1`, `TLSv1.2`, and `TLSv1.3`. Enabling a - protocol is a short-hand for disabling all other protocols. - (Default: `!SSLv2 !SSLv3 !TLSv1 !TLSv1.1`, i.e., only enable TLSv1.2 - and above.) - -*SSL_cipher_list* - -: The cipher list to send to the server. Although the server - determines which cipher suite is used, it should take the first - supported cipher in the list sent by the client. See - [`ciphers`(1ssl)] for more information. - -*SSL_fingerprint* - -: Fingerprint of the server certificate's Subject Public Key Info, in - the form `[ALGO$]DIGEST_HEX` where `ALGO` is the used algorithm (by - default `sha256`). - Attempting to connect to a server with a non-matching certificate - SPKI fingerprint causes `pullimap` to abort the connection during - the SSL/TLS handshake. - The following command can be used to compute the SHA-256 digest of a - certificate's Subject Public Key Info: - - openssl x509 -in /path/to/server/certificate.pem -pubkey \ - | openssl pkey -pubin -outform DER \ - | openssl dgst -sha256 - -*SSL_verify* - -: Whether to verify the server certificate chain. - Note that using *SSL_fingerprint* to specify the fingerprint of the - server certificate is an orthogonal authentication measure as it - ignores the CA chain. - (Default: `YES`.) - -*SSL_CApath* - -: Directory to use for server certificate verification if - `SSL_verify=YES`. - This directory must be in “hash format”, see [`verify`(1ssl)] for - more information. - -*SSL_CAfile* - -: File containing trusted certificates to use during server - certificate authentication if `SSL_verify=YES`. - -Control flow -============ - -`pullimap` opens the *statefile* corresponding to a given configuration -*SECTION* with `O_DSYNC` to ensure that written data is flushed to the -underlying hardware by the time [`write`(2)] returns. Moreover an -exclusive lock is placed on the file descriptor immediately after -opening to prevent multiple `pullimap` processes from accessing the -*statefile* concurrently. - -Each *statefile* consists of a series of 32-bits big-endian integers. -Usually there are only two integers: the first is the *mailbox*'s -`UIDVALIDITY` value, and the second is the *mailbox*'s last seen -`UIDNEXT` value (`pullimap` then assumes that all messages with UID -smaller than this `UIDNEXT` value have already been retrieved and -delivered). -The [IMAP4rev1 specification][RFC 3501] does not guaranty that untagged -`FETCH` responses are sent ordered by UID in response to a `UID FETCH` -command. Thus it would be unsafe for `pullimap` to update the `UIDNEXT` -value in its *statefile* while the `UID FETCH` command is progress. -Instead, for each untagged `FETCH` response received while the `UID -FETCH` command is in progress, `pullimap` delivers the message `RFC822` -body to the SMTP or LMTP server (specified with *deliver-method*) then -appends the message UID to the *statefile*. -When the `UID FETCH` command eventually terminates, `pullimap` updates -the `UIDNEXT` value in the *statefile* and truncate the file down to 8 -bytes. Keeping track of message UIDs as they are received avoids -duplicate in the event of a crash or connection loss while the `UID -FETCH` command is in progress. - -In more details, `pullimap` works as follows: - - 1. Issue a `UID FETCH` command to retrieve message `ENVELOPE` and - `RFC822` (and `UID`) with UID bigger or equal than the `UIDNEXT` - value found in the *statefile*. - While the `UID FETCH` command is in progress, perform the following - for each untagged `FETCH` response sent by the server: - - i. if no SMTP/LMTP transmission channel was opened, open one to the - server specified with *deliver-method* and send an `EHLO` (or - `LHO`) command with the domain specified by *deliver-ehlo* (the - channel is kept open and shared for all messages retrieved while - the `UID FETCH` IMAP command is in progress); - - i. perform a mail transaction (using [SMTP pipelining][RFC 2920] if - possible) to deliver the retrieved message `RFC822` body to the - SMTP or LMTP session; and - - i. append the message UID to the *statefile*. - - 2. If an SMTP/LMTP transmission channel was opened, send a `QUIT` command - to terminate it gracefully. - - 3. Issue a `UID STORE` command to mark all retrieved messages (and - stalled UIDs found in the *statefile* after the eigth byte) as - `\Seen`. - - 4. Update the *statefile* with the new UIDNEXT value (bytes 5-8). - - 5. Truncate the *statefile* down to 8 bytes (so that it contains only - two 32-bits integers, respectively the *mailbox*'s current - `UIDVALIDITY` and `UIDNEXT` values). - - 6. If `--idle` was set, issue an `IDLE` command; stop idling and go - back to step 1 when a new message is received (or when the `IDLE` - timeout expires). - -Standards -========= - - * M. Leech, M. Ganis, Y. Lee, R. Kuris, D. Koblas and L. Jones, - _SOCKS Protocol Version 5_, - [RFC 1928], March 1996. - * M. Leech, _Username/Password Authentication for SOCKS V5_, - [RFC 1929], March 1996. - * J. Myers, _Local Mail Transfer Protocol_, - [RFC 2033], October 1996. - * J. Myers, _IMAP4 non-synchronizing literals_, - [RFC 2088], January 1997. - * D. Goldsmith and M. Davis, - _A Mail-Safe Transformation Format of Unicode_, - [RFC 2152], May 1997. - * B. Leiba, _IMAP4 `IDLE` command_, - [RFC 2177], June 1997. - * C. Newman, _Using TLS with IMAP, POP3 and ACAP_, - [RFC 2595], June 1999. - * N. Freed, _SMTP Service Extension for Command Pipelining_, - [RFC 2920], September 2000. - * M. Crispin, _Internet Message Access Protocol - Version 4rev1_, - [RFC 3501], March 2003. - * M. Crispin, - _Internet Message Access Protocol (IMAP) - `UIDPLUS` extension_, - [RFC 4315], December 2005. - * A. Gulbrandsen, _The IMAP `COMPRESS` Extension_, - [RFC 4978], August 2007. - * A. Melnikov and D. Cridland, _IMAP4 Extension to SEARCH Command for - Controlling What Kind of Information Is Returned_, - [RFC 4731], November 2006. - * R. Siemborski and A. Gulbrandsen, _IMAP Extension for Simple - Authentication and Security Layer (SASL) Initial Client Response_, - [RFC 4959], September 2007. - * J. Klensin, _Simple Mail Transfer Protocol_, - [RFC 5321], October 2008. - -[RFC 4315]: https://tools.ietf.org/html/rfc4315 -[RFC 2177]: https://tools.ietf.org/html/rfc2177 -[RFC 2595]: https://tools.ietf.org/html/rfc2595 -[RFC 4959]: https://tools.ietf.org/html/rfc4959 -[RFC 2152]: https://tools.ietf.org/html/rfc2152 -[RFC 2088]: https://tools.ietf.org/html/rfc2088 -[RFC 5321]: https://tools.ietf.org/html/rfc5321 -[RFC 2033]: https://tools.ietf.org/html/rfc2033 -[RFC 2920]: https://tools.ietf.org/html/rfc2920 -[RFC 3501]: https://tools.ietf.org/html/rfc3501 -[RFC 4978]: https://tools.ietf.org/html/rfc4978 -[RFC 1928]: https://tools.ietf.org/html/rfc1928 -[RFC 1929]: https://tools.ietf.org/html/rfc1929 -[RFC 4731]: https://tools.ietf.org/html/rfc4731 - -[INI file]: https://en.wikipedia.org/wiki/INI_file -[`fetchmail`(1)]: http://www.fetchmail.info/ -[`getmail`(1)]: http://pyropus.ca/software/getmail/ -[`write`(2)]: http://man7.org/linux/man-pages/man2/write.2.html -[`ciphers`(1ssl)]: https://www.openssl.org/docs/manmaster/apps/ciphers.html -[`verify`(1ssl)]: https://www.openssl.org/docs/manmaster/apps/verify.html -- cgit v1.2.3