Merge branch 'master' into debian

21 files changed, 533 insertions, 110 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..9dae7e6
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,5 @@
+*~
+/doc/*.1
+/doc/*.html
+!/doc/template.html
+/.pc/
diff --git a/Changelog b/Changelog
index a13801a..4cc66ba 100644
--- a/Changelog
+++ b/Changelog
@@ -10,7 +10,7 @@ interimap (0.5) upstream;
    namespace.
  * libinterimap: in tunnel mode, use a socketpair rather than two pipes
    for IPC between the interimap and the IMAP server.  Also, use
-   SOCK_CLOEXEC to save a fcntl() call when setting the close-on-exec
+   SOCK_CLOEXEC to save an fcntl() call when setting the close-on-exec
    flag on the socket.
  * interimap: new option 'list-reference' to specify a reference name.
    This is useful for synchronizing multiple remote servers against
@@ -18,7 +18,10 @@ interimap (0.5) upstream;
    different InterIMAP instance for each local namespace <-> remote
    synchronization, for instance with the newly provided systemd
    template unit file).
- * Add a small test-suite (requires dovecot-imapd).
+ * Add a test-suite.  (Requires dovecot-imapd, pkill(1) and xxd(1).)
+ * Completely refactor the documentation.  In particular, move manpages
+   to a new 'doc' directory, and generate HTML documentation  with `make
+   html`.
  + interimap: write which --target to use in --delete command
    suggestions.
  + interimap: avoid caching hierarchy delimiters forever in the
@@ -29,12 +32,17 @@ interimap (0.5) upstream;
    BLOB.
  + interimap: use the 'user_version' SQLite PRAGMA for database schema
    version.
+ + interimap, pullimap: in the manpage, clarify that the tunnel command
+   is run following Perl's `exec` semantics: it is passed to `/bin/sh -c`
+   when it contains shell metacharacters; and split into words and passed
+   to execvp(3) otherwise.
  - libinterimap: bugfix: hierarchy delimiters in LIST responses were
    returned as an escaped quoted special, like "\\", not as a single
    character (backslash in this case).
  - libinterimap: the parser choked on responses with non-quoted/literal
    astring containing ']' characters.  And LIST responses with
-   non-quoted/literal list-mailbox names '%', '*' or ']' characters.
+   non-quoted/literal list-mailbox names containing '%', '*' or ']'
+   characters.
  - libinterimap: quote() the empty string as "" instead of a 0-length
    literal.  (This saves 3 bytes + one round-trip on servers not
    supporting non-synchronizing literals, and 4 bytes otherwise.)
@@ -86,7 +94,7 @@ interimap (0.4) upstream;
 interimap (0.3) upstream;
 
   + New script 'pullimap', to pull mails from an IMAP mailbox and
-    deliver them to a SMTP session.
+    deliver them to an SMTP session.
   + Convert manpage format from groff to pandoc.
   + interimap: Add support for IMAP NOTIFY [RFC 5465].
   + 'fingerprint' now only pins the cert's SPKI, not the cert itself
diff --git a/INSTALL b/INSTALL
deleted file mode 100644
index 69afb26..0000000
--- a/INSTALL
+++ /dev/null
@@ -1,31 +0,0 @@
-InterIMAP depends on Perl >=5.20 and the following Perl modules:
-
-  - Compress::Raw::Zlib (core module)
-  - Config::Tiny
-  - DBI
-  - DBD::SQLite
-  - Errno (core module)
-  - Getopt::Long (core module)
-  - MIME::Base64 (core module) if authentication is required
-  - List::Util (core module)
-  - Net::SSLeay >=1.73
-  - POSIX (core module)
-  - Socket (core module)
-  - Time::HiRes (core module) if 'logfile' is set
-
-On Debian GNU/Linux systems, these modules can be installed with the
-following command:
-
-  apt-get install libconfig-tiny-perl libdbi-perl libdbd-sqlite3-perl libnet-ssleay-perl
-
-However Debian GNU/Linux users can also use gbp(1) from git-buildpackage
-to build their own package:
-
-  $ git checkout debian
-  $ AUTO_DEBSIGN=no gbp buildpackage
-
-Alternatively, for the development version:
-
-  $ git checkout debian
-  $ git merge master
-  $ AUTO_DEBSIGN=no gbp buildpackage --git-force-create --git-upstream-tree=BRANCH
diff --git a/Makefile b/Makefile
index ec35011..fda0fe0 100644
--- a/Makefile
+++ b/Makefile
@@ -1,8 +1,11 @@
-all: pullimap.1 interimap.1
+all: manual
+
+MANUALS = $(patsubst %.md,%,$(wildcard ./doc/*.[1-9].md))
+manual: $(MANUALS)
 
 # upper case the headers and remove the links
-%.1: %.md
-	@pandoc -f markdown -t json "$<" | \
+$(MANUALS): %: %.md
+	@pandoc -f markdown -t json -- "$<" | \
 	jq "                                                                        \
 	    def fixheaders:                                                         \
 	        if .t == \"Header\" then                                            \
@@ -29,12 +32,29 @@ all: pullimap.1 interimap.1
 	    }" | \
 	pandoc -s -f json -t man+smart -o "$@"
 
-install:
-
 test:
-	@for t in tests/*; do [ -d "$$t" ] || continue; ./tests/run "$$t" || exit 1; done
+	@for t in tests/*; do if [ -f "$$t/run" ]; then ./tests/run "$$t" || exit 1; fi; done
+
+HTML_ROOTDIR ?= ./doc
+CSS ?= /usr/share/javascript/bootstrap/css/bootstrap.min.css
+HTML_TEMPLATE ?= ./doc/template.html
+
+HTML_FILES = $(addprefix $(HTML_ROOTDIR)/,$(patsubst ./doc/%.md,%.html,$(wildcard ./doc/*.md)))
+html: $(HTML_FILES)
+
+## CSS="https://guilhem.org/static/css/bootstrap.min.css" HTML_ROOTDIR="$XDG_RUNTIME_DIR/Downloads" make html
+$(HTML_ROOTDIR)/%.html: ./doc/%.md $(HTML_TEMPLATE)
+	mtime="$$(git --no-pager log -1 --pretty="format:%ct" -- "$<" 2>/dev/null)"; \
+	[ -n "$$mtime" ] || mtime="$$(date +%s -r "$<")"; \
+	pandoc -sp -f markdown -t html+smart --css=$(CSS) --template=$(HTML_TEMPLATE) \
+		--variable=date:"$$(LC_TIME=C date +"Last modified on %a, %d %b %Y at %T %z" -d @"$$mtime")" \
+		--output="$@" -- "$<"
+
+doc: manual html
+
+install:
 
 clean:
-	rm -f pullimap.1 interimap.1
+	rm -f $(MANUALS) $(HTML_FILES)
 
-.PHONY: all install clean test
+.PHONY: all manual html doc test install clean
diff --git a/README b/README
index 06f328f..fbc4ed7 100644
--- a/README
+++ b/README
@@ -1,9 +1,7 @@
 InterIMAP is a fast bidirectional synchronization program for QRESYNC-capable
 IMAP4rev1 servers.  PullIMAP retrieves messages a remote IMAP mailbox and
-deliver them to an SMTP session.  Consult the manuals for more information.
-
-    https://guilhem.org/man/interimap.1.html
-    https://guilhem.org/man/pullimap.1.html
+deliver them to an SMTP session.  Visit https://guilhem.org/interimap
+for more information.
 
 _______________________________________________________________________
 
@@ -39,7 +37,7 @@ IMAP traffic is mostly text (beside message bodies perhaps) hence
 compresses pretty well: enabling compression can save a great amount of
 network resources.
 
-However establishing a SSL/TLS connection (type=imaps, or type=imap and
+However establishing an SSL/TLS connection (type=imaps, or type=imap and
 STARTTLS=YES) yields a small overhead due to the SSL/TLS handshake.
 
 On the other hand if SSH access is allowed on the remote server, one can
diff --git a/doc/build.md b/doc/build.md
new file mode 100644
index 0000000..38d1bfb
--- /dev/null
+++ b/doc/build.md
@@ -0,0 +1,99 @@
+% Build instructions
+% Guilhem Moulin <guilhem@fripost.org>
+
+On Debian 9 (codename *Stretch*) and later, installing [`interimap`(1)]
+is a single command away:
+
+    $ apt-get install interimap
+
+This document is for those who are running other systems, and/or who
+wish to install from [source](https://git.guilhem.org/interimap).
+
+
+Dependencies
+============
+
+[`interimap`(1)](interimap.1.html) depends on Perl ≥5.20 and the
+following Perl modules:
+
+  * [`Compress::Raw::Zlib`](https://perldoc.perl.org/Compress/Raw/Zlib.html) (*core module*)
+  * [`Config::Tiny`](https://metacpan.org/pod/Config::Tiny)
+  * [`DBI`](https://metacpan.org/pod/DBI)
+  * [`DBD::SQLite`](https://metacpan.org/pod/DBD::SQLite)
+  * [`Errno`](https://perldoc.perl.org/Errno.html) (*core module*)
+  * [`Getopt::Long`](https://perldoc.perl.org/Getopt/Long.html) (*core module*)
+  * [`MIME::Base64`](https://perldoc.perl.org/MIME/Base64.html) (*core module*) — if authentication is required
+  * [`List::Util`](https://perldoc.perl.org/List/Util.html) (*core module*)
+  * [`Net::SSLeay`](https://metacpan.org/pod/Net::SSLeay) ≥1.73
+  * [`POSIX`](https://perldoc.perl.org/POSIX.html) (*core module*)
+  * [`Socket`](https://perldoc.perl.org/Socket.html) (*core module*)
+  * [`Time::HiRes`](https://perldoc.perl.org/Time/HiRes.html) (*core module*) — if `logfile` is set
+
+On Debian GNU/Linux systems, the dependencies can be installed with the
+following command:
+
+    $ apt install libconfig-tiny-perl \
+		libdbi-perl \
+		libdbd-sqlite3-perl \
+		libnet-ssleay-perl
+
+Additional packages are required in order to run the test suite:
+
+    $ apt install dovecot-imapd procps sqlite3 xxd
+<!-- -->
+    $ make test
+
+
+Generate documentation
+======================
+
+Yet another set of packages is needed to generate the documentation:
+
+    $ apt install jq pandoc
+
+Run `` `make manual` `` (or just `` `make` ``) in order to generate the
+manpages.  You'll find them at `doc/*.[1-9]`.  Use for instance `` `man
+-l doc/interimap.1` `` in order to read your copy of the [`interimap`(1)]
+manpage.
+
+The HTML documentation can be built with `` `make html` ``.  HTML files
+are generated alongside their Markdown source by default, but you can
+choose another target directory using the `HTML_ROOTDIR` environment
+variable (the value of which defaults to `./doc`).  Moreover the
+[`libjs-bootstrap`](https://tracker.debian.org/libjs-bootstrap) is
+needed by default for the local CSS file; this can be controlled with
+the `CSS` environment variable (the value of which defaults to
+`/usr/share/javascript/bootstrap/css/bootstrap.min.css`).
+
+For instance, use
+
+    $ CSS="https://guilhem.org/static/css/bootstrap.min.css" \
+		HTML_ROOTDIR="$XDG_RUNTIME_DIR/interimap" \
+		make html
+
+to generate the HTML documentation under directory `$XDG_RUNTIME_DIR/interimap`
+(which needs to exist) using a remote CSS file.
+
+The `doc` target generates all documentation, manpages as well as HTML
+pages.
+
+
+Build custom Debian package
+===========================
+
+Debian GNU/Linux users can also use [`gbp`(1)] from
+[`git-buildpackage`](https://tracker.debian.org/pkg/git-buildpackage) in
+order to build their own package:
+
+    $ git checkout debian
+    $ gbp buildpackage
+
+Alternatively, for the development version:
+
+    $ git checkout debian
+    $ git merge master
+    $ gbp buildpackage --git-force-create --git-upstream-tree=BRANCH
+
+
+[`interimap`(1)]: interimap.1.html
+[`gbp`(1)]: https://manpages.debian.org/git-buildpackage/gbp.1.en.html
diff --git a/doc/development.md b/doc/development.md
new file mode 100644
index 0000000..406207a
--- /dev/null
+++ b/doc/development.md
@@ -0,0 +1,208 @@
+% Test environment setup for [`interimap`(1)] and [`pullimap`(1)]
+% Guilhem Moulin <guilhem@fripost.org>
+
+Introduction
+============
+
+This document describes how to create dummy mail storage for
+[`interimap`(1)] and/or [`pullimap`(1)] development, using
+[Dovecot](https://dovecot.org) as [IMAP4rev1] server.  Start by creating
+a new temporary directory:
+
+    $ BASEDIR="$(mktemp --tmpdir --directory)"
+
+(The leading `$ ` in this document are command-line prompt strings, and
+are not part of the command themselves.)
+
+
+Dovecot configuration
+=====================
+
+Create a file `$BASEDIR/dovecot.conf`, which will be used as
+configuration for the various Dovecot commands (the system configuration
+will be skipped).
+
+    $ cat >"$BASEDIR/dovecot.conf" <<-EOF
+		log_path      = "$BASEDIR/dovecot.log"
+		ssl           = no
+		mail_home     = "$BASEDIR/%u"
+		mail_location = maildir:~/mail
+	EOF
+
+Here are some details on the above:
+
+`log_path`
+
+  : Dovecot [logs to syslog](https://wiki.dovecot.org/Logging) by default.
+    It's annoying to clutter syslog with test entries, so instead we make it
+    log to a file under `$BASEDIR`.
+
+`ssl`
+
+  : Not required, but turned off here so dumping the configuration with
+    `` `doveconf -c "$BASEDIR/dovecot.conf" -n` `` doesn't spew a warning.
+
+`mail_home`
+
+  : Dovecot needs the name of the user to (pre-)authenticate.  It is shown
+    in the greeting line, and also used in [`%`-variable] expansion.
+    Several [`doveadm`(1)] sub-commands have a `-u` (or `-d`) option which
+    can be used to determine the username.  When this option is not set,
+    the username is taken from the `USER` environment variable.  If that
+    environment variable is unset as well, then the return string of
+    [`getlogin`(3)](https://pubs.opengroup.org/onlinepubs/9699919799/functions/getlogin.html)
+    is used.
+
+    Similarly, the user's home directory is used in (`~`- and)
+    [`%`-variable] expansion.  It's taken from the `HOME` environment
+    variable when the `mail_home` setting is left unset in the Dovecot
+    configuration (and not overridden by the [user database](https://wiki.dovecot.org/UserDatabase)).
+
+    `mail_home` can therefore be left unset if the `HOME` environment
+    variable is consistently set to `$BASEDIR/$USER`.  However it's
+    safer to explicitly set it in the configuration file: otherwise a
+    command run in a non-curated environment might mess up with your own
+    mail storage…
+
+`mail_location`
+
+  : The user's mail storage resides — in [Maildir](https://wiki.dovecot.org/MailLocation/Maildir)
+    format — in a directory `mail` under their home directory.  This is
+    enough if you're fine with the default IMAP hierarchy delimiter
+    (which depends on the mail format) is used, and if you need a single
+    [IMAP namespace](https://tools.ietf.org/html/rfc2342).  For more
+    complex setups you'll need one or more
+    [`namespace {…}` block](https://wiki.dovecot.org/Namespaces).
+
+
+Mail storage access
+===================
+
+Feel free to point a mail client at the dummy mail storage.  To start a
+pre-authenticated [IMAP4rev1] in the test environment for username
+`testuser`, list mailboxes, and exit, run:
+
+    $ env -i PATH="/usr/bin:/bin" USER="testuser" \
+		doveadm -c "$BASEDIR/dovecot.conf" exec imap
+    * PREAUTH [CAPABILITY IMAP4rev1 …] Logged in as testuser
+    a LIST "" "*"
+    * LIST (\HasNoChildren) "." INBOX
+    a OK List completed (0.002 + 0.000 + 0.001 secs).
+    q LOGOUT
+    q OK Logout completed (0.001 + 0.000 secs).
+
+For mailbox (create, delete, rename) and message (add, flag update)
+manipulation you can use your mail client, the relevant [IMAP4rev1]
+commands, or simply the [`doveadm`(1)] tools.  Here is an example using
+the latter to create a mailbox `foo`, add a sample message to it, and
+finally mark it as `\Seen`.
+
+    $ env -i PATH="/usr/bin:/bin" USER="testuser" \
+		doveadm -c "$BASEDIR/dovecot.conf" mailbox create "foo"
+<!-- -->
+    $ env -i PATH="/usr/bin:/bin" USER="testuser" HOME="$BASEDIR/testuser" \
+		doveadm -c "$BASEDIR/dovecot.conf" exec dovecot-lda -e -m "foo" <<-EOF
+			From: <sender@example.net>
+			To: <recipient@example.net>
+			Subject: Hello world!
+			Date: $(date -R)
+			Message-ID: <$(</proc/sys/kernel/random/uuid)@example.net>
+
+			Hello world!
+		EOF
+<!-- -->
+    $ env -i PATH="/usr/bin:/bin" USER="testuser" \
+		doveadm -c "$BASEDIR/dovecot.conf" flags add "\\Seen" mailbox "foo" "*"
+
+Normally [`dovecot-lda`(1)](https://wiki.dovecot.org/LDA) tries to do a
+userdb lookup in order to determine the user's home directory.  Since we
+didn't configure a user database we need to explicitly set the `HOME`
+environment variable.
+
+
+InterIMAP configuration and test
+================================
+
+In this example the peers to synchronize are sharing the same Dovecot
+configuration file `$BASEDIR/dovecot.conf`.  Of course, it's also
+possible to use a different configuration on each “server”, for instance
+in order to specify different hierarchy delimiters, namespaces, or mail
+storage format.
+
+Create an [`interimap`(1)] configuration file to synchronize the `local`
+and `remote` accounts.
+
+    $ cat >"$BASEDIR/interimap.conf" <<-EOF
+		database = $BASEDIR/interimap.db
+
+		[local]
+		type = tunnel
+		command = env -i PATH="$PATH" USER="local" doveadm -c "$BASEDIR/dovecot.conf" exec imap
+
+		[remote]
+		type = tunnel
+		command = env -i PATH="$PATH" USER="remote" doveadm -c "$BASEDIR/dovecot.conf" exec imap
+	EOF
+
+Run [`interimap`(1)] without `--watch` in order to create the database.
+
+    $ env -i PATH="$PATH" perl -I./lib -T ./interimap --config="$BASEDIR/interimap.conf"
+    Creating new schema in database file …/interimap.db
+    database: Created mailbox INBOX
+    […]
+
+You can now run [`interimap`(1)] with `--watch` set, here to one second
+to observe synchronisation steps early.
+
+    $ env -i PATH="$PATH" perl -I./lib -T ./interimap --config="$BASEDIR/interimap.conf" \
+		--watch=1 --debug
+
+Use instructions from the [previous section][Mail storage access]
+(substituting `testuser` with `local` or `remote`) in order to simulate
+activity on either end to synchronize.  If you run these commands in
+another shell, then make sure to re-set the `BASEDIR` environment
+variable!
+
+
+PullIMAP configuration and test
+===============================
+
+Create a [`pullimap`(1)] configuration file with as section `[foo]`.
+
+    $ cat >"$BASEDIR/pullimap.conf" <<-EOF
+		[foo]
+		type = tunnel
+		command = env -i PATH="$PATH" USER="testuser" doveadm -c "$BASEDIR/dovecot.conf" exec imap
+		statefile = $BASEDIR/pullimap.foo
+	EOF
+
+Run [`pullimap`(1)] without `--idle` in order to create the state file.
+
+    $ env -i PATH="$PATH" perl -I./lib -T ./pullimap --config="$BASEDIR/pullimap.conf" \
+		--no-delivery foo
+
+You can now run [`pullimap`(1)] with `--idle` set.
+
+    $ env -i PATH="$PATH" perl -I./lib -T ./pullimap --config="$BASEDIR/pullimap.conf" \
+		--no-delivery --idle --debug foo
+
+Use instructions from the [previous section][Mail storage access]
+in order to simulate activity on the “remote” server (in the relevant
+mailbox — `INBOX` by default).  If you run these commands in another
+shell, then make sure to re-set the `BASEDIR` environment variable!
+
+
+Cleanup
+=======
+
+To remove temporary directories and the message they contain, simply
+recursively remove the directory `$BASEDIR`.
+
+    $ rm -rf -- "$BASEDIR"
+
+
+[IMAP4rev1]: https://tools.ietf.org/html/rfc3501
+[`interimap`(1)]: interimap.1.html
+[`pullimap`(1)]: pullimap.1.html
+[`doveadm`(1)]: https://wiki.dovecot.org/Tools/Doveadm
+[`%`-variable]: https://wiki.dovecot.org/Variables
diff --git a/doc/index.md b/doc/index.md
new file mode 100644
index 0000000..12de956
--- /dev/null
+++ b/doc/index.md
@@ -0,0 +1,20 @@
+% [`interimap`(1)] and [`pullimap`(1)] documentation
+% Guilhem Moulin <guilhem@fripost.org>
+
+Manuals (HTML versions)
+-----------------------
+
+  * [`interimap`(1)] — Fast bidirectional synchronization for
+    QRESYNC-capable IMAP servers
+  * [`pullimap`(1)] — Pull mails from an IMAP mailbox and deliver them
+    to an SMTP session
+
+Resources for developers
+------------------------
+
+  * [Source-code repository](https://git.guilhem.org/interimap)
+  * [Build instructions](build.html)
+  * [Test environment setup](development.html)
+
+[`interimap`(1)]: interimap.1.html
+[`pullimap`(1)]: pullimap.1.html
diff --git a/interimap.md b/doc/interimap.1.md
index 50c1832..387850a 100644
--- a/interimap.md
+++ b/doc/interimap.1.md
@@ -1,4 +1,4 @@
-% intermap(1)
+% interimap(1)
 % [Guilhem Moulin](mailto:guilhem@fripost.org)
 % July 2015
 
@@ -47,7 +47,7 @@ with the [RFC 7162] (sec. 6) amendments, and works as follows:
 
  2. Propagate these changes onto the other server: get the corresponding
     UIDs from the database, then:
-     a. issue an `UID STORE` command, followed by `UID EXPUNGE`, to
+     a. issue a `UID STORE` command, followed by `UID EXPUNGE`, to
         remove messages that have not already been deleted on both
         servers; and
      b. issue some `UID STORE` commands to propagate flag updates (send
@@ -62,9 +62,9 @@ with the [RFC 7162] (sec. 6) amendments, and works as follows:
     Otherwise, update the `HIGHESTMODSEQ` value in the database.
 
  3. Process new messages (if the current `UIDNEXT` value of the mailbox
-    differs from the one found in the database) by issuing an `UID
-    FETCH` command; process each received message on-the-fly by issuing
-    an `APPEND` command with the message's `RFC822` body, `FLAGS` and
+    differs from the one found in the database) by issuing a `UID FETCH`
+    command; process each received message on-the-fly by issuing an
+    `APPEND` command with the message's `RFC822` body, `FLAGS` and
     `INTERNALDATE`.
     Repeat this step if the server received new messages in the
     meantime.  Otherwise, update the `UIDNEXT` value in the database.
@@ -111,8 +111,8 @@ other than the default [`QRESYNC`][RFC 7162]-based synchronization.
          existing UID; and
       3/ ensure that both flag lists match.
     Any message found on a server but not in the database is replicated
-    on the other server (which in the worst case, might lead to a
-    message duplicate).
+    on the other server (which in the worst case, might yield a message
+    duplicate).
     Flag conflicts are solved by updating each message to the union of
     both lists.
 
@@ -224,7 +224,7 @@ Valid options are:
     to the `perso/` sub-hierarchy on the local server.  This is useful
     for synchronizing multiple remote servers against different
     namespaces belonging to the same local IMAP server (using a
-    different InterIMAP instance for each local namespace ↔ remote
+    different `interimap` instance for each local namespace ↔ remote
     synchronization).
 
     (Note that if the reference name is not a level of mailbox hierarchy
@@ -281,10 +281,10 @@ Valid options are:
 
 :   One of `imap`, `imaps` or `tunnel`.
     `type=imap` and `type=imaps` are respectively used for IMAP and IMAP
-    over SSL/TLS connections over a INET socket.
+    over SSL/TLS connections over an INET socket.
     `type=tunnel` causes `interimap` to create an unnamed pair of
     connected sockets for interprocess communication with a *command*
-    instead of a opening a network socket.
+    instead of opening a network socket.
     Note that specifying `type=tunnel` in the `[remote]` section makes
     the default *database* to be `localhost.db`.
     (Default: `imaps`.)
@@ -314,7 +314,9 @@ Valid options are:
 
 :   Command to use for `type=tunnel`.  Must speak the [IMAP4rev1
     protocol][RFC 3501] on its standard output, and understand it on its
-    standard input.
+    standard input.  The value is passed to `` `/bin/sh -c` `` if it
+    contains shell metacharacters; otherwise it is split into words and
+    the resulting list is passed to `execvp`(3).
 
 *STARTTLS*
 
@@ -348,7 +350,7 @@ Valid options are:
 *null-stderr*
 
 :   Whether to redirect *command*'s standard error to `/dev/null` for
-    type `type=tunnel`.  (Default: `NO`.)
+    `type=tunnel`.  (Default: `NO`.)
 
 *SSL_protocols*
 
@@ -374,9 +376,8 @@ Valid options are:
     Attempting to connect to a server with a non-matching certificate
     SPKI fingerprint causes `interimap` to abort the connection during
     the SSL/TLS handshake.
-
-    You can use the following command to compute the SHA-256 digest of
-    certificate's Subject Public Key Info.
+    The following command can be used to compute the SHA-256 digest of a
+    certificate's Subject Public Key Info:
 
         openssl x509 -in /path/to/server/certificate.pem -pubkey \
         | openssl pkey -pubin -outform DER \
diff --git a/pullimap.md b/doc/pullimap.1.md
index a367dd1..1b2e509 100644
--- a/pullimap.md
+++ b/doc/pullimap.1.md
@@ -5,7 +5,7 @@
 Name
 ====
 
-PullIMAP - Pull mails from an IMAP mailbox and deliver them to a SMTP session
+PullIMAP - Pull mails from an IMAP mailbox and deliver them to an SMTP session
 
 Synopsis
 ========
@@ -16,8 +16,8 @@ Synopsis
 Description
 ===========
 
-`pullimap` retrieves messages from an IMAP mailbox and deliver them to a
-SMTP or LMTP transmission channel.  It can also remove old messages
+`pullimap` retrieves messages from an IMAP mailbox and deliver them to
+an SMTP or LMTP transmission channel.  It can also remove old messages
 after a configurable retention period.
 
 A *statefile* is used to keep track of the mailbox's `UIDVALIDITY` and
@@ -130,9 +130,10 @@ Valid options are:
 
 :   One of `imap`, `imaps` or `tunnel`.
     `type=imap` and `type=imaps` are respectively used for IMAP and IMAP
-    over SSL/TLS connections over a INET socket.
-    `type=tunnel` causes `pullimap` to open a pipe to a *command*
-    instead of a raw socket.
+    over SSL/TLS connections over an INET socket.
+    `type=tunnel` causes `pullimap` to create an unnamed pair of
+    connected sockets for interprocess communication with a *command*
+    instead of opening a network socket.
     (Default: `imaps`.)
 
 *host*
@@ -160,7 +161,9 @@ Valid options are:
 
 :   Command to use for `type=tunnel`.  Must speak the [IMAP4rev1
     protocol][RFC 3501] on its standard output, and understand it on its
-    standard input.
+    standard input.  The value is passed to `` `/bin/sh -c` `` if it
+    contains shell metacharacters; otherwise it is split into words and
+    the resulting list is passed to `execvp`(3).
 
 *STARTTLS*
 
@@ -192,7 +195,7 @@ Valid options are:
 *null-stderr*
 
 :   Whether to redirect *command*'s standard error to `/dev/null` for
-    type `type=tunnel`.  (Default: `NO`.)
+    `type=tunnel`.  (Default: `NO`.)
 
 *SSL_protocols*
 
@@ -218,9 +221,8 @@ Valid options are:
     Attempting to connect to a server with a non-matching certificate
     SPKI fingerprint causes `pullimap` to abort the connection during
     the SSL/TLS handshake.
-
-    You can use the following command to compute the SHA-256 digest of
-    certificate's Subject Public Key Info.
+    The following command can be used to compute the SHA-256 digest of a
+    certificate's Subject Public Key Info:
 
         openssl x509 -in /path/to/server/certificate.pem -pubkey \
         | openssl pkey -pubin -outform DER \
@@ -263,7 +265,7 @@ Usually there are only two integers: the first is the *mailbox*'s
 smaller than this `UIDNEXT` value have already been retrieved and
 delivered).
 The [IMAP4rev1 specification][RFC 3501] does not guaranty that untagged
-`FETCH` responses are sent ordered by UID in response to an `UID FETCH`
+`FETCH` responses are sent ordered by UID in response to a `UID FETCH`
 command.  Thus it would be unsafe for `pullimap` to update the `UIDNEXT`
 value in its *statefile* while the `UID FETCH` command is progress.
 Instead, for each untagged `FETCH` response received while the `UID
@@ -278,7 +280,7 @@ FETCH` command is in progress.
 
 In more details, `pullimap` works as follows:
 
- 1. Issue an `UID FETCH` command to retrieve message `ENVELOPE` and
+ 1. Issue a `UID FETCH` command to retrieve message `ENVELOPE` and
     `RFC822` (and `UID`) with UID bigger or equal than the `UIDNEXT`
     value found in the *statefile*.
     While the `UID FETCH` command is in progress, perform the following
@@ -296,10 +298,10 @@ In more details, `pullimap` works as follows:
 
      i. append the message UID to the *statefile*.
 
- 2. If a SMTP/LMTP transmission channel was opened, send a `QUIT` command
+ 2. If an SMTP/LMTP transmission channel was opened, send a `QUIT` command
     to terminate it gracefully.
 
- 3. Issue an `UID STORE` command to mark all retrieved messages (and
+ 3. Issue a `UID STORE` command to mark all retrieved messages (and
     stalled UIDs found in the *statefile* after the eigth byte) as
     `\Seen`.
 
diff --git a/doc/template.html b/doc/template.html
new file mode 100644
index 0000000..e17f0e3
--- /dev/null
+++ b/doc/template.html
@@ -0,0 +1,76 @@
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" lang="$lang$" xml:lang="$lang$"$if(dir)$ dir="$dir$"$endif$>
+<head>
+  <meta charset="utf-8" />
+  <meta name="generator" content="pandoc" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
+$for(author-meta)$
+  <meta name="author" content="$author-meta$" />
+$endfor$
+$if(date-meta)$
+  <meta name="dcterms.date" content="$date-meta$" />
+$endif$
+$if(keywords)$
+  <meta name="keywords" content="$for(keywords)$$keywords$$sep$, $endfor$" />
+$endif$
+  <title>$if(title-prefix)$$title-prefix$ – $endif$$pagetitle$</title>
+  <style type="text/css">
+      code{white-space: pre-wrap;}
+      span.smallcaps{font-variant: small-caps;}
+      span.underline{text-decoration: underline;}
+      div.column{display: inline-block; vertical-align: top; width: 50%;}
+$if(quotes)$
+      q { quotes: "“" "”" "‘" "’"; }
+$endif$
+  </style>
+$if(highlighting-css)$
+  <style type="text/css">
+$highlighting-css$
+  </style>
+$endif$
+$for(css)$
+  <link rel="stylesheet" href="$css$" />
+$endfor$
+  <style type="text/css">
+    @media(max-width: 1440px) { .container{ max-width: 1080px; } }
+    @media(max-width: 1280px) { .container{ max-width: 960px; } }
+    @media(max-width: 1024px) { .container{ max-width: 768px; } }
+  </style>
+$if(math)$
+  $math$
+$endif$
+$for(header-includes)$
+  $header-includes$
+$endfor$
+</head>
+
+<body>
+$for(include-before)$
+$include-before$
+$endfor$
+<div class="container text-justify">
+  <div class="content">
+$if(title)$
+    <div class="page-header"><h1>$title$</h1></div>
+$endif$
+
+$body$
+  </div>
+
+  <footer>
+    <hr/>
+    <div class="row">
+      <div class="col-md-8 text-muted">
+$if(author)$
+        <a href="https://git.guilhem.org/interimap/plain/COPYING">&copy;</a>
+        $for(author)$$author$$sep$, $endfor$
+$endif$
+      </div>
+      <div class="col-md-4 text-muted text-right small">
+        $if(date)$$date$$endif$
+      </div>
+    </div>
+  </footer>
+</div>
+</body>
+</html>
diff --git a/lib/Net/IMAP/InterIMAP.pm b/lib/Net/IMAP/InterIMAP.pm
index 1dd54b7..19895c4 100644
--- a/lib/Net/IMAP/InterIMAP.pm
+++ b/lib/Net/IMAP/InterIMAP.pm
@@ -26,7 +26,8 @@ use Errno qw/EEXIST EINTR/;
 use Net::SSLeay 1.73 ();
 use List::Util qw/all first/;
 use POSIX ':signal_h';
-use Socket qw/SOCK_STREAM SOCK_RAW IPPROTO_TCP AF_UNIX AF_INET AF_INET6 PF_UNSPEC SOCK_CLOEXEC :addrinfo/;
+use Socket qw/SOCK_STREAM SOCK_RAW SOCK_CLOEXEC IPPROTO_TCP SHUT_RDWR
+              AF_UNIX AF_INET AF_INET6 PF_UNSPEC :addrinfo/;
 
 use Exporter 'import';
 BEGIN {
@@ -506,7 +507,7 @@ sub stats($) {
 }
 
 
-# Log out when the Net::IMAP::InterIMAP object is destroyed.
+# Destroy a Net::IMAP::InterIMAP object.
 sub DESTROY($) {
     my $self = shift;
     $self->{_STATE} = 'LOGOUT';
@@ -514,8 +515,12 @@ sub DESTROY($) {
     Net::SSLeay::free($self->{_SSL}) if defined $self->{_SSL};
     Net::SSLeay::CTX_free($self->{_SSL_CTX}) if defined $self->{_SSL_CTX};
 
-    shutdown($self->{S}, 2) if $self->{type} ne 'tunnel' and defined $self->{S};
-    $self->{S}->close() if defined $self->{S} and $self->{S}->opened();
+    if (defined (my $s = $self->{S})) {
+        # for type=tunnel we assume the child won't linger around once
+        # we close its standard input and output.
+        shutdown($s, SHUT_RDWR);
+        $s->close() if $s->opened();
+    }
     $self->stats() unless $self->{quiet};
 }
 
@@ -606,7 +611,7 @@ sub incapable($@) {
 
 
 # $self->search($criterion)
-#   Issue an UID SEARCH command with the given $criterion.  For the "normal"
+#   Issue a UID SEARCH command with the given $criterion.  For the "normal"
 #   UID SEARCH command from RFC 3501, return the list of matching UIDs;
 #   for the extended UID SEARCH command from RFC 4731 (ensuring ESEARCH
 #   capability is the caller's responsibility), return an optional "UID"
@@ -917,7 +922,7 @@ sub append($$@) {
 
 
 # $self->fetch($set, $flags, [$callback])
-#   Issue an UID FETCH command with the given UID $set, $flags, and
+#   Issue a UID FETCH command with the given UID $set, $flags, and
 #   optional $callback.
 sub fetch($$$;&) {
     my ($self, $set, $flags, $callback) = @_;
diff --git a/pullimap b/pullimap
index 84587fe..e1c96e8 100755
--- a/pullimap
+++ b/pullimap
@@ -1,7 +1,7 @@
 #!/usr/bin/perl -T
 
 #----------------------------------------------------------------------
-# Pull mails from an IMAP mailbox and deliver them to a SMTP session
+# Pull mails from an IMAP mailbox and deliver them to an SMTP session
 # Copyright © 2016-2018 Guilhem Moulin <guilhem@fripost.org>
 #
 # This program is free software: you can redistribute it and/or modify
diff --git a/pullimap@.service b/pullimap@.service
index 53694da..a9ce09a 100644
--- a/pullimap@.service
+++ b/pullimap@.service
@@ -1,5 +1,5 @@
 [Unit]
-Description=Pull mails from an IMAP mailbox and deliver them to a SMTP session (instance %i)
+Description=Pull mails from an IMAP mailbox and deliver them to an SMTP session (instance %i)
 Documentation=man:pullimap(1)
 Wants=network-online.target
 After=network-online.target
diff --git a/tests/00-db-exclusive/run b/tests/00-db-exclusive/run
index 1ae27b6..1528b3b 100644
--- a/tests/00-db-exclusive/run
+++ b/tests/00-db-exclusive/run
@@ -9,8 +9,8 @@ interimap
 interimap --watch=60 & pid=$!
 cleanup() {
     # kill interimap process and its children
-    pkill -P "$pid" -TERM
-    kill -TERM "$pid"
+    pkill -P "$pid" -TERM || true
+    kill -TERM "$pid" || true
     wait
 }
 trap cleanup EXIT INT TERM
diff --git a/tests/00-db-migration-0-to-1/run b/tests/00-db-migration-0-to-1/run
index e4eb770..757fe04 100644
--- a/tests/00-db-migration-0-to-1/run
+++ b/tests/00-db-migration-0-to-1/run
@@ -18,9 +18,16 @@ sqlite3 "$XDG_DATA_HOME/interimap/remote.db" >"$TMPDIR/dump.sql" <<-EOF
 	.dump
 EOF
 
+# re-import and dump the expected dump to work around SQLite format
+# differences across versions
+sqlite3 "$XDG_DATA_HOME/interimap/remote2.db" <"$TESTDIR/after.sql"
+sqlite3 "$XDG_DATA_HOME/interimap/remote2.db" >"$TMPDIR/dump-expected.sql" <<-EOF
+	.dump
+EOF
+
 # XXX need 'user_version' PRAGMA in the dump for future migrations
 # http://sqlite.1065341.n5.nabble.com/dump-command-and-user-version-td101228.html
 diff -u --label="a/dump.sql" --label="b/dump.sql" \
-    "$TESTDIR/after.sql" "$TMPDIR/dump.sql"
+    "$TMPDIR/dump-expected.sql" "$TMPDIR/dump.sql"
 
 # vim: set filetype=sh :
diff --git a/tests/05-repair/run b/tests/05-repair/run
index 747b974..15553e0 100644
--- a/tests/05-repair/run
+++ b/tests/05-repair/run
@@ -65,7 +65,7 @@ spoof UIDNEXT "local"
 spoof HIGHESTMODSEQ "local" "remote"
 
 # now repair
-interimap --repair "baz" "foo.bar" 
+interimap --repair "baz" "foo.bar"
 
 # 6 updates with \Answered (luid 4,8,11:13,16), 2 of which (luid 12,13) vanished from remote
 # 3 updates with \Seen (ruid 6,8,10), 1 of which (uid 10) vanished from remote
diff --git a/tests/06-largeint/run b/tests/06-largeint/run
index edcbd31..b08bcfa 100644
--- a/tests/06-largeint/run
+++ b/tests/06-largeint/run
@@ -7,9 +7,9 @@ doveadm -u "local"  mailbox update --uid-validity 2147483648 "bar"   # 2^31
 doveadm -u "local"  mailbox update --uid-validity 4294967295 "baz"   # 2^32-1
 
 doveadm -u "remote" mailbox update --uid-validity 4294967295 "INBOX" # 2^32-1
-doveadm -u "remote" mailbox update --uid-validity 2147483648 "foo"   # 2^31 
+doveadm -u "remote" mailbox update --uid-validity 2147483648 "foo"   # 2^31
 doveadm -u "remote" mailbox update --uid-validity 2147483647 "bar"   # 2^31-1
-doveadm -u "remote" mailbox update --uid-validity          1 "baz"   # 
+doveadm -u "remote" mailbox update --uid-validity          1 "baz"   #
 
 run() {
     local u m
diff --git a/tests/07-sync-live-multi/run b/tests/07-sync-live-multi/run
index bf0d2f5..15a27fd 100644
--- a/tests/07-sync-live-multi/run
+++ b/tests/07-sync-live-multi/run
@@ -15,12 +15,12 @@ interimap --config="config3" --watch=1 & pid3=$!
 
 abort() {
     # kill interimap process and its children
-    pkill -P "$pid" -TERM
-    kill -TERM "$pid"
-    pkill -P "$pid2" -TERM
-    kill -TERM "$pid2"
-    pkill -P "$pid3" -TERM
-    kill -TERM "$pid3"
+    pkill -P "$pid" -TERM || true
+    kill -TERM "$pid" || true
+    pkill -P "$pid2" -TERM || true
+    kill -TERM "$pid2" || true
+    pkill -P "$pid3" -TERM || true
+    kill -TERM "$pid3" || true
     wait
 }
 trap abort EXIT INT TERM
@@ -101,7 +101,7 @@ while [ $(date +%s) -le $timer ]; do
     done
 
     # sleep a little bit
-    sleep "0.$(shuf -n1 -i1-99)"
+    sleep "$(printf "0.%03d" "$(shuf -n1 -i1-999)")"
 done
 
 # wait a little longer so interimap has time to run loop() again and
diff --git a/tests/07-sync-live/run b/tests/07-sync-live/run
index 1950e0b..04d8247 100644
--- a/tests/07-sync-live/run
+++ b/tests/07-sync-live/run
@@ -6,8 +6,8 @@ interimap --watch=1 & pid=$!
 
 abort() {
     # kill interimap process and its children
-    pkill -P "$pid" -TERM
-    kill -TERM "$pid"
+    pkill -P "$pid" -TERM || true
+    kill -TERM "$pid" || true
     wait
 }
 trap abort EXIT INT TERM
@@ -63,7 +63,7 @@ while [ $(date +%s) -le $timer ]; do
     done
 
     # sleep a little bit
-    sleep "0.$(shuf -n1 -i1-99)"
+    sleep "$(printf "0.%03d" "$(shuf -n1 -i1-999)")"
 done
 
 # wait a little longer so interimap has time to run loop() again and
diff --git a/tests/run b/tests/run
index 31af03e..ee11757 100755
--- a/tests/run
+++ b/tests/run
@@ -36,7 +36,7 @@ if [ ! -d "$TESTDIR" ]; then
     exit 1
 fi
 
-ROOTDIR="$(mktemp --tmpdir=/dev/shm --directory "$NAME.XXXXXXXXXX")"
+ROOTDIR="$(mktemp --tmpdir="${TMPDIR:-/dev/shm}" --directory "$NAME.XXXXXXXXXX")"
 trap 'rm -rf -- "$ROOTDIR"' EXIT INT TERM
 
 STDOUT="$ROOTDIR/stdout"
@@ -73,6 +73,7 @@ prepare() {
         cat >"$home/.config/dovecot/config" <<-EOF
 			log_path = /dev/null
 			mail_home = $ROOTDIR/home/%u
+			mailbox_list_index = yes
 			ssl = no
 		EOF
         cat >>"$home/.config/dovecot/config" <"$src"
@@ -102,12 +103,12 @@ prepare() {
 			[local]
 			type = tunnel
 			command = exec ${HOME_local@Q}/.local/bin/doveadm exec imap
-			null-stderr = YES
+			null-stderr = NO
 
 			[remote]
 			type = tunnel
 			command = exec ${home@Q}/.local/bin/doveadm exec imap
-			null-stderr = YES
+			null-stderr = NO
 		EOF
     done
 }
@@ -129,14 +130,18 @@ doveadm() {
     shift 2
     "$home/.local/bin/doveadm" "$@"
 }
+sqlite3() {
+    command sqlite3 -init /dev/null "$@"
+}
 
 # Sample (random) message
 sample_message() {
+    local date="$(date +"%s.%N")"
     cat <<-EOF
 		From: <sender@example.net>
 		To: <recipient@example.net>
-		Date: $(date -R)
-		Message-ID: <$(< /proc/sys/kernel/random/uuid)@example.net>
+		Date: $(date -R -d@"$date")
+		Message-ID: <$date@example.net>
 
 	EOF
     local len="$(shuf -i1-4096 -n1)"
@@ -156,7 +161,7 @@ deliver() {
     doveadm "${argv[@]}" exec dovecot-lda -e "$@"
 }
 
-# Dump test results 
+# Dump test results
 dump_test_result() {
     local below=">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
     local above="<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<"
@@ -215,7 +220,7 @@ check_mailbox_status2() {
     read MESSAGES < <( sqlite3 "$XDG_DATA_HOME/interimap/$u.db" <<-EOF
 			.mode csv
 			.separator " " "\\n"
-            SELECT COUNT(*)
+			SELECT COUNT(*)
 			FROM mailboxes a JOIN mapping b ON a.idx = b.idx
 			WHERE mailbox = $blob
 		EOF
@@ -319,7 +324,7 @@ xcgrep() {
 declare -a ENVIRON=()
 environ_set "local"
 export TMPDIR TESTDIR STDOUT STDERR "${ENVIRON[@]}"
-export -f environ_set doveadm interimap sample_message deliver
+export -f environ_set doveadm interimap sqlite3 sample_message deliver
 export -f check_mailbox_status check_mailbox_status_values check_mailbox_status2
 export -f check_mailboxes_status check_mailbox_list xgrep xcgrep
 printf "%s..." "$TEST"