Merge branch 'master' into debian

7 files changed, 336 insertions, 52 deletions

diff --git a/Makefile b/Makefile
index fdfb717..b232602 100644
--- a/Makefile
+++ b/Makefile
@@ -5,32 +5,7 @@ manual: $(MANUALS)
 
 # upper case the headers and remove the links
 $(MANUALS): %: %.md
-	@pandoc -f markdown -t json -- "$<" | \
-	jq "                                                                        \
-	    def fixheaders:                                                         \
-	        if .t == \"Header\" then                                            \
-	            .c[2][] |= (if .t == \"Str\" then .c |= ascii_upcase else . end)\
-	        else                                                                \
-	            .                                                               \
-	        end;                                                                \
-	    def fixlinks:                                                           \
-	        if type == \"object\" then                                          \
-	            if .t == \"Link\" then                                          \
-	                if .c[2][0][0:7] == \"mailto:\" then . else .c[1][] end     \
-	            else                                                            \
-	                map_values(fixlinks)                                        \
-	            end                                                             \
-	        else if type == \"array\" then                                      \
-	                map(fixlinks)                                               \
-	            else                                                            \
-	                .                                                           \
-	            end                                                             \
-	        end;                                                                \
-	    { \"pandoc-api-version\"                                                \
-	    , meta                                                                  \
-	    , blocks: .blocks | map(fixheaders) | map(fixlinks)                     \
-	    }" | \
-	pandoc -s -f json -t man+smart -o "$@"
+	pandoc -f markdown -t json -- "$<" | ./pandoc2man.jq | pandoc -s -f json -t man+smart -o "$@"
 
 test:
 	@./tests/run-all
diff --git a/doc/development.md b/doc/development.md
index 49e8d74..b5624f0 100644
--- a/doc/development.md
+++ b/doc/development.md
@@ -11,9 +11,6 @@ a new temporary directory:
 
     $ BASEDIR="$(mktemp --tmpdir --directory)"
 
-(The leading `$ ` in this document are command-line prompt strings, and
-are not part of the command themselves.)
-
 
 Dovecot configuration
 =====================
@@ -26,7 +23,7 @@ will be skipped).
 		log_path      = "$BASEDIR/dovecot.log"
 		ssl           = no
 		mail_home     = "$BASEDIR/%u"
-		mail_location = maildir:~/mail
+		mail_location = maildir:~/Mail
 	EOF
 
 Here are some details on the above:
@@ -37,11 +34,6 @@ Here are some details on the above:
     It's annoying to clutter syslog with test entries, so instead we make it
     log to a file under `$BASEDIR`.
 
-`ssl`
-
-  : Not required, but turned off here so dumping the configuration with
-    `` `doveconf -c "$BASEDIR/dovecot.conf" -n` `` doesn't spew a warning.
-
 `mail_home`
 
   : Dovecot needs the name of the user to (pre-)authenticate.  It is shown
@@ -67,7 +59,7 @@ Here are some details on the above:
 `mail_location`
 
   : The user's mail storage resides — in [Maildir](https://wiki.dovecot.org/MailLocation/Maildir)
-    format — in a directory `mail` under their home directory.  This is
+    format — in a directory `Mail` under their home directory.  This is
     enough if you're fine with the default IMAP hierarchy delimiter
     (which depends on the mail format) is used, and if you need a single
     [IMAP namespace](https://tools.ietf.org/html/rfc2342).  For more
@@ -84,12 +76,13 @@ pre-authenticated [IMAP4rev1] in the test environment for username
 
     $ env -i PATH="/usr/bin:/bin" USER="testuser" \
         doveadm -c "$BASEDIR/dovecot.conf" exec imap
-    * PREAUTH [CAPABILITY IMAP4rev1 …] Logged in as testuser
-    a LIST "" "*"
-    * LIST (\HasNoChildren) "." INBOX
-    a OK List completed (0.002 + 0.000 + 0.001 secs).
-    q LOGOUT
-    q OK Logout completed (0.001 + 0.000 secs).
+    S: * PREAUTH [CAPABILITY IMAP4rev1 …] Logged in as testuser
+    C: a LIST "" "*"
+    S: * LIST (\HasNoChildren) "." INBOX
+    S: a OK List completed (0.002 + 0.000 + 0.001 secs).
+    C: q LOGOUT
+    S: * BYE Logging out
+    S: q OK Logout completed (0.001 + 0.000 secs).
 
 For mailbox (create, delete, rename) and message (add, flag update)
 manipulation you can use your mail client, the relevant [IMAP4rev1]
diff --git a/doc/getting-started.md b/doc/getting-started.md
new file mode 100644
index 0000000..371449d
--- /dev/null
+++ b/doc/getting-started.md
@@ -0,0 +1,286 @@
+% Getting started with InterIMAP
+% [Guilhem Moulin](mailto:guilhem@fripost.org);
+  [Gustav Eek](mailto:gustav.eek@fripost.org)
+
+This document describes the setup of InterIMAP for a rather usual
+user-case, where messages on a remote IMAP server `imap.example.net`
+need to be synchronized locally in a bidirectional fashion (changes on
+either server are replicated on the other one).
+
+
+Local IMAP server
+=================
+
+Background and rationale
+------------------------
+
+On a workstation, one's mail storage is typically found under `~/Maildir`
+(in [*Maildir* format][Maildir]) or in `/var/mail/$USER` (in [*mbox*
+format][mbox]).  Local mail clients usually access it directly.  They
+also often maintain their own cache in order to speed up message header
+listing and searches.
+
+While most bidirectional synchronisation software (such as [OfflineIMAP])
+are able to handle a mail storage in Maildir format, *InterIMAP is
+not*.  Instead, InterIMAP needs an [IMAP4rev1] server on *both* peers
+to synchronize.  This may sound like a severe limitation at first, but by
+seeing both local and remote mail storage though the same “IMAP lens”,
+InterIMAP is able to take advantage of the abstraction layer and
+perform significant optimizations, yielding much faster synchronization.
+(*TODO* link to benchmark.)
+*Note*: InterIMAP uses the [Quick Mailbox Resynchronization][RFC 7162]
+extension for stateful synchronization, hence won't work on IMAP servers
+that don't advertise support for that extension.
+
+Installing an [IMAP4rev1] server on a single-user workstation may sound
+overkill, but we argue that most systems, not only servers, come with a
+[Message Transfer Agent][MTA] preinstalled.  Just like one may use
+`/usr/sbin/sendmail` (or a compatible interface) in order to send mail
+out, we propose to use an `imap` binary to access them.
+
+In order to take full advantage of the abstraction layer and of
+InterIMAP's optimizations, one should *always* access the mail storage
+through the local [IMAP4rev1] server and *never directly*.  Otherwise
+the IMAP server will invalidate its cache each time it notices
+inconsistencies, potentially causing a severe performance hit.  (*Or
+worse*: very likely many [IMAP4rev1] servers are not able to gracefully
+reconcile cache inconsistencies.)  As far as the mail client is
+concerned, the cost of abstraction seems to be negligible.  (*TODO* link
+to benchmark.)  Furthermore, we think that approach is in line with the
+[Unix philosophy]: the mail client only takes care of the rendering
+part, leaving the rest to the IMAP server (searches, sorting/threading,
+as well as storage and caching logic).
+
+Installation
+------------
+
+While this document focuses on [Dovecot](https://dovecot.org), a popular
+[IMAP4rev1] server, any other [`QRESYNC`][RFC 7162]-capable server
+should work equally well.  Run the following command to install the
+Dovecot IMAP server on a Debian GNU/Linux system.
+
+    $ sudo apt install dovecot-imapd
+
+(The leading `$ ` in this document are command-line prompt strings,
+which are not part of the command themselves.)
+
+Configuration
+-------------
+
+Our [`interimap`(1)] instance will use the `imap` command from Dovecot's
+`libexec_dir` in order to access the local mail storage.  We assume that
+the mail client can access it in the same fashion.  In other words, that
+it can spawn a command and use its standard input (resp. output) for
+[IMAP4rev1] commands (resp. responses).  [Mutt] is an example of such a
+mail client, for which we propose a configuration snippet
+[below](#mutt-config).
+
+Since we don't need the Dovecot services nor master process in this
+example, we disable them and create a local configuration file under
+`$XDG_CONFIG_HOME/dovecot`.  If you need to keep the system-wise
+services (for instance because your [MTA] uses the [LMTP server] for
+mailbox delivery) then don't disable them, and modify Dovecot's system
+wide configuration instead.  Same thing if your mail client isn't able
+to spawn a command for IMAP communication, and instead insists on
+connecting to a network socket (in that case you'll even need to
+configure [user authentication](https://wiki.dovecot.org/Authentication)
+for the IMAP service, which is out of scope for the present document).
+
+Run the following command to terminate and disable the system-wide
+Dovecot processes.
+
+    $ sudo systemctl mask --now dovecot.socket dovecot.service
+
+
+Create a new directory `$XDG_CONFIG_HOME/dovecot` holding the local
+Dovecot configuration:
+
+    $ install -m0700 -vd ${XDG_CONFIG_HOME:-~/.config}/dovecot
+<!-- -->
+    $ cat >${XDG_CONFIG_HOME:-~/.config}/dovecot/dovecot.conf <<-EOF
+		ssl = no
+		mail_location = maildir:~/Mail
+		namespace {
+		    inbox     = yes
+		    list      = yes
+		    separator = /
+		}
+	EOF
+
+Some remarks on the above:
+
+  * SSL/TLS is explicitly turned off in order to avoid warnings when
+    running `` `doveconf -nc ${XDG_CONFIG_HOME:-~/.config}/dovecot/dovecot.conf` ``.
+  * Messages will be stored in Maildir format under `~/Mail`.  Ensure
+    the directory is either *empty* or *doesn't exist* before
+    continuing!  You may want to choose a different [format](https://wiki.dovecot.org/MailboxFormat)
+    here, or simply append `:LAYOUT=fs` to the `mail_location` value in
+    order to use a nicer (File System like) Maildir layout.
+  * The `separator` setting defines the IMAP hierarchy delimiter.  This
+    is orthogonal to the Maildir layout delimiter, and you can safely
+    change it later (even on an existing mail store).  Popular hierarchy
+    delimiters include `/` (slash) and `.` (period).
+
+Now test the configuration by starting a pre-authenticated [IMAP4rev1]
+session and issuing two commands, first `` `LIST "" "*"` `` to
+recursively list all mailboxes (along with their hierarchy delimiter),
+then `` `LOGOUT` `` to… log out and exit.  (The "`C: `" and "`S: `"
+prefixes respectively denote client commands and server responses.)
+
+    $ doveadm -c ${XDG_CONFIG_HOME:-~/.config}/dovecot/dovecot.conf exec imap
+    S: * PREAUTH [CAPABILITY IMAP4rev1 …] Logged in as myuser
+    C: a LIST "" "*"
+    S: * LIST (\HasNoChildren) "/" INBOX
+    S: a OK List completed (0.001 + 0.000 secs).
+    C: q LOGOUT
+    S: * BYE Logging out
+    S: q OK Logout completed (0.001 + 0.000 secs).
+
+Create a wrapper under `~/.local/bin` in order to avoid hard-coding the
+local Dovecot configuration path:
+
+    $ install -Dm 0755 /dev/stdin ~/.local/bin/dovecot-imap <<-EOF
+		#!/bin/sh
+		set -ue
+		export PATH="/usr/bin:/bin"
+		exec env -i PATH="\$PATH" HOME="\$HOME" USER="\$USER" \\
+		    doveadm -c "\${XDG_CONFIG_HOME:-\$HOME/.config}/dovecot/dovecot.conf" \\
+		        exec imap
+	EOF
+
+You can now start a pre-authenticated [IMAP4rev1] session like the one
+above by simply running `` `~/.local/bin/dovecot-imap` ``.
+
+
+InterIMAP
+========
+
+On Debian 10 (codename *Buster*) and later, installing the package is
+one command away.  Simply run the following:
+
+    $ sudo apt install interimap
+
+Create directories for the InterIMAP configuration and data files:
+
+    $ install -m0700 -vd ${XDG_CONFIG_HOME:-~/.config}/interimap ${XDG_DATA_HOME:-~/.local/share}/interimap
+
+Create the configuration file.  The included sample file
+`/usr/share/doc/interimap/interimap.sample` can be used as baseline, but
+for the sake of clarity we start from an empty file here.
+
+    $ install -m0600 /dev/null ${XDG_CONFIG_HOME:-~/.config}/interimap/config
+
+ 1. The file is in [INI format][INI file].  First, set general options
+    in the default section:
+
+        $ cat >${XDG_CONFIG_HOME:-~/.config}/interimap/config <<-EOF
+			# only consider subscribed mailboxes
+			list-select-opts = SUBSCRIBED
+			#list-mailbox = "*"
+
+			# ignore the mailbox named 'virtual' and its descendants
+			# WARN: for version 0.4 and earlier it should be ^virtual(?:/|$)
+			ignore-mailbox = ^virtual(?:\x00|$)
+
+		EOF
+
+ 2. Next, append a `[local]` section pointing to the wrapper defined
+    above:
+
+        $ cat >>${XDG_CONFIG_HOME:-~/.config}/interimap/config <<-EOF
+			[local]
+			type = tunnel
+			command = exec ~/.local/bin/dovecot-imap
+
+		EOF
+
+ 3. And finally append a `[remote]` section with your account
+    information at `imap.example.org` (adapt the values accordingly):
+
+        $ cat >>${XDG_CONFIG_HOME:-~/.config}/interimap/config <<-EOF
+			[remote]
+			type = imaps
+			host = imap.example.net
+			username = myname
+			password = xxxxxxxx
+		EOF
+
+At this point running `` `interimap` `` should create the database and
+copy the entire remote mail store locally.  (If `~/Mail` was not empty,
+it will also copy its content remotely, possibly *yielding duplicates*.)
+This might take a while depending on the volume of messages to
+synchronize.
+
+    $ interimap
+    Creating new schema in database file …/imap.example.net.db
+    database: Created mailbox INBOX
+    […]
+
+A user unit for systemd is provided.  Run the following command to
+enable and start the service:
+
+    $ systemctl --user enable --now interimap
+
+By default the connection to the IMAP servers remains open, and a status
+update is requested every minute.  Thanks to the [`QRESYNC`][RFC 7162]
+IMAP extension a status update scales linearly with the number of
+mailboxes (unlike [OfflineIMAP] *not* with the number of messages).  And
+thanks to the `COMPRESS` extension, the typical volume of data exchanged
+is rather small (*TODO* metrics).  You may even want to override the
+default settings and reduce the interval between status updates to 20s:
+
+    $ mkdir -p ${XDG_CONFIG_HOME:-~/.config}/systemd/user/interimap.service.d
+<!-- -->
+    $ cat >${XDG_CONFIG_HOME:-~/.config}/systemd/user/interimap.service.d/override.conf <<-EOF
+		[Service]
+		ExecStart=
+		ExecStart=/usr/bin/interimap --watch=20
+	EOF
+<!-- -->
+    $ systemctl --user daemon-reload
+<!-- -->
+    $ systemctl --user restart interimap
+
+
+Email client configuration
+==========================
+
+[Mutt]  {#mutt-config}
+------
+
+Add the following snippet to the configuration file:
+
+    $ cat >>~/.muttrc <<-EOF
+		set tunnel    = "exec ~/.local/bin/dovecot-imap"
+		set folder    = "imap://foo"
+		set spoolfile = "imap://foo"
+	EOF
+
+
+Further Reading and Resources
+=============================
+
+Other use-cases:
+
+:   *TODO*
+
+Benchmarks:
+
+:   *TODO*
+
+Manual
+
+:   [`interimap`(1)]
+
+
+[IMAP4rev1]: https://tools.ietf.org/html/rfc3501
+[INI file]: https://en.wikipedia.org/wiki/INI_file
+[`interimap`(1)]: interimap.1.html
+[LMTP server]: https://doc.dovecot.org/configuration_manual/protocols/lmtp_server/
+[Maildir]: https://en.wikipedia.org/wiki/Maildir
+[mbox]: https://en.wikipedia.org/wiki/Mbox
+[MTA]: https://en.wikipedia.org/wiki/Message_transfer_agent
+[Mutt]: http://mutt.org/
+[OfflineIMAP]: https://www.offlineimap.org/
+[RFC 7162]: https://tools.ietf.org/html/rfc7162
+[Unix philosophy]: https://en.wikipedia.org/wiki/Unix_philosophy
diff --git a/doc/index.md b/doc/index.md
index a403a3e..9225e12 100644
--- a/doc/index.md
+++ b/doc/index.md
@@ -1,13 +1,18 @@
-% [`interimap`(1)] and [`pullimap`(1)]
+% InterIMAP & PullIMAP
 % [Guilhem Moulin](mailto:guilhem@fripost.org)
 
+General documentation
+---------------------
+
+ * [Getting started with InterIMAP](getting-started.html)
+
 Manuals (HTML versions)
 -----------------------
 
-  * [`interimap`(1)] — Fast bidirectional synchronization for
-    QRESYNC-capable IMAP servers
-  * [`pullimap`(1)] — Pull mails from an IMAP mailbox and deliver them
-    to an SMTP session
+  * [`interimap`(1)](interimap.1.html) — Fast bidirectional
+    synchronization for QRESYNC-capable IMAP servers
+  * [`pullimap`(1)](pullimap.1.html) — Pull mails from an IMAP mailbox
+    and deliver them to an SMTP session
 
 Resources for developers
 ------------------------
@@ -15,6 +20,3 @@ Resources for developers
   * [Source-code repository](https://git.guilhem.org/interimap)
   * [Build instructions](build.html)
   * [Test environment setup](development.html)
-
-[`interimap`(1)]: interimap.1.html
-[`pullimap`(1)]: pullimap.1.html
diff --git a/doc/template.html b/doc/template.html
index 2cd7cc9..dbcc0e6 100644
--- a/doc/template.html
+++ b/doc/template.html
@@ -68,7 +68,7 @@ $endif$
 $endif$
 
 $body$
-</div>
+  </div>
 
   <footer>
     <hr/>
diff --git a/lib/Net/IMAP/InterIMAP.pm b/lib/Net/IMAP/InterIMAP.pm
index 02ae65f..ef1c20b 100644
--- a/lib/Net/IMAP/InterIMAP.pm
+++ b/lib/Net/IMAP/InterIMAP.pm
@@ -450,7 +450,7 @@ sub new($%) {
             $self->fail("Unsupported authentication mechanism: $mech");
         }
 
-        my $dbg; 
+        my $dbg;
         delete $self->{password}; # no need to remember passwords
         if (($self->{debug} // 0) == 1) {
             $dbg = $self->{debug}--;
diff --git a/pandoc2man.jq b/pandoc2man.jq
new file mode 100755
index 0000000..69802a5
--- /dev/null
+++ b/pandoc2man.jq
@@ -0,0 +1,28 @@
+#!/usr/bin/jq -f
+
+def fixheaders:
+    if .t == "Header" then
+        .c[2][] |= (if .t == "Str" then .c |= ascii_upcase else . end)
+    else
+        .
+    end;
+
+def fixlinks:
+    if type == "object" then
+        if .t == "Link" then
+            if .c[2][0][0:7] == "mailto:" then . else .c[1][] end
+        else
+            map_values(fixlinks)
+        end
+    else if type == "array" then
+            map(fixlinks)
+        else
+            .
+        end
+    end;
+
+{
+    "pandoc-api-version"
+  , meta
+  , blocks: .blocks | map(fixheaders | fixlinks)
+}