aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--.gitignore5
-rw-r--r--Changelog16
-rw-r--r--INSTALL31
-rw-r--r--Makefile36
-rw-r--r--README8
-rw-r--r--doc/build.md99
-rw-r--r--doc/development.md208
-rw-r--r--doc/index.md20
-rw-r--r--doc/interimap.1.md (renamed from interimap.md)31
-rw-r--r--doc/pullimap.1.md (renamed from pullimap.md)32
-rw-r--r--doc/template.html76
-rw-r--r--lib/Net/IMAP/InterIMAP.pm17
-rwxr-xr-xpullimap2
-rw-r--r--pullimap@.service2
-rw-r--r--tests/00-db-exclusive/run4
-rw-r--r--tests/00-db-migration-0-to-1/run9
-rw-r--r--tests/05-repair/run2
-rw-r--r--tests/06-largeint/run4
-rw-r--r--tests/07-sync-live-multi/run14
-rw-r--r--tests/07-sync-live/run6
-rwxr-xr-xtests/run21
21 files changed, 533 insertions, 110 deletions
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..9dae7e6
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,5 @@
+*~
+/doc/*.1
+/doc/*.html
+!/doc/template.html
+/.pc/
diff --git a/Changelog b/Changelog
index a13801a..4cc66ba 100644
--- a/Changelog
+++ b/Changelog
@@ -10,7 +10,7 @@ interimap (0.5) upstream;
namespace.
* libinterimap: in tunnel mode, use a socketpair rather than two pipes
for IPC between the interimap and the IMAP server. Also, use
- SOCK_CLOEXEC to save a fcntl() call when setting the close-on-exec
+ SOCK_CLOEXEC to save an fcntl() call when setting the close-on-exec
flag on the socket.
* interimap: new option 'list-reference' to specify a reference name.
This is useful for synchronizing multiple remote servers against
@@ -18,7 +18,10 @@ interimap (0.5) upstream;
different InterIMAP instance for each local namespace <-> remote
synchronization, for instance with the newly provided systemd
template unit file).
- * Add a small test-suite (requires dovecot-imapd).
+ * Add a test-suite. (Requires dovecot-imapd, pkill(1) and xxd(1).)
+ * Completely refactor the documentation. In particular, move manpages
+ to a new 'doc' directory, and generate HTML documentation with `make
+ html`.
+ interimap: write which --target to use in --delete command
suggestions.
+ interimap: avoid caching hierarchy delimiters forever in the
@@ -29,12 +32,17 @@ interimap (0.5) upstream;
BLOB.
+ interimap: use the 'user_version' SQLite PRAGMA for database schema
version.
+ + interimap, pullimap: in the manpage, clarify that the tunnel command
+ is run following Perl's `exec` semantics: it is passed to `/bin/sh -c`
+ when it contains shell metacharacters; and split into words and passed
+ to execvp(3) otherwise.
- libinterimap: bugfix: hierarchy delimiters in LIST responses were
returned as an escaped quoted special, like "\\", not as a single
character (backslash in this case).
- libinterimap: the parser choked on responses with non-quoted/literal
astring containing ']' characters. And LIST responses with
- non-quoted/literal list-mailbox names '%', '*' or ']' characters.
+ non-quoted/literal list-mailbox names containing '%', '*' or ']'
+ characters.
- libinterimap: quote() the empty string as "" instead of a 0-length
literal. (This saves 3 bytes + one round-trip on servers not
supporting non-synchronizing literals, and 4 bytes otherwise.)
@@ -86,7 +94,7 @@ interimap (0.4) upstream;
interimap (0.3) upstream;
+ New script 'pullimap', to pull mails from an IMAP mailbox and
- deliver them to a SMTP session.
+ deliver them to an SMTP session.
+ Convert manpage format from groff to pandoc.
+ interimap: Add support for IMAP NOTIFY [RFC 5465].
+ 'fingerprint' now only pins the cert's SPKI, not the cert itself
diff --git a/INSTALL b/INSTALL
deleted file mode 100644
index 69afb26..0000000
--- a/INSTALL
+++ /dev/null
@@ -1,31 +0,0 @@
-InterIMAP depends on Perl >=5.20 and the following Perl modules:
-
- - Compress::Raw::Zlib (core module)
- - Config::Tiny
- - DBI
- - DBD::SQLite
- - Errno (core module)
- - Getopt::Long (core module)
- - MIME::Base64 (core module) if authentication is required
- - List::Util (core module)
- - Net::SSLeay >=1.73
- - POSIX (core module)
- - Socket (core module)
- - Time::HiRes (core module) if 'logfile' is set
-
-On Debian GNU/Linux systems, these modules can be installed with the
-following command:
-
- apt-get install libconfig-tiny-perl libdbi-perl libdbd-sqlite3-perl libnet-ssleay-perl
-
-However Debian GNU/Linux users can also use gbp(1) from git-buildpackage
-to build their own package:
-
- $ git checkout debian
- $ AUTO_DEBSIGN=no gbp buildpackage
-
-Alternatively, for the development version:
-
- $ git checkout debian
- $ git merge master
- $ AUTO_DEBSIGN=no gbp buildpackage --git-force-create --git-upstream-tree=BRANCH
diff --git a/Makefile b/Makefile
index ec35011..fda0fe0 100644
--- a/Makefile
+++ b/Makefile
@@ -1,8 +1,11 @@
-all: pullimap.1 interimap.1
+all: manual
+
+MANUALS = $(patsubst %.md,%,$(wildcard ./doc/*.[1-9].md))
+manual: $(MANUALS)
# upper case the headers and remove the links
-%.1: %.md
- @pandoc -f markdown -t json "$<" | \
+$(MANUALS): %: %.md
+ @pandoc -f markdown -t json -- "$<" | \
jq " \
def fixheaders: \
if .t == \"Header\" then \
@@ -29,12 +32,29 @@ all: pullimap.1 interimap.1
}" | \
pandoc -s -f json -t man+smart -o "$@"
-install:
-
test:
- @for t in tests/*; do [ -d "$$t" ] || continue; ./tests/run "$$t" || exit 1; done
+ @for t in tests/*; do if [ -f "$$t/run" ]; then ./tests/run "$$t" || exit 1; fi; done
+
+HTML_ROOTDIR ?= ./doc
+CSS ?= /usr/share/javascript/bootstrap/css/bootstrap.min.css
+HTML_TEMPLATE ?= ./doc/template.html
+
+HTML_FILES = $(addprefix $(HTML_ROOTDIR)/,$(patsubst ./doc/%.md,%.html,$(wildcard ./doc/*.md)))
+html: $(HTML_FILES)
+
+## CSS="https://guilhem.org/static/css/bootstrap.min.css" HTML_ROOTDIR="$XDG_RUNTIME_DIR/Downloads" make html
+$(HTML_ROOTDIR)/%.html: ./doc/%.md $(HTML_TEMPLATE)
+ mtime="$$(git --no-pager log -1 --pretty="format:%ct" -- "$<" 2>/dev/null)"; \
+ [ -n "$$mtime" ] || mtime="$$(date +%s -r "$<")"; \
+ pandoc -sp -f markdown -t html+smart --css=$(CSS) --template=$(HTML_TEMPLATE) \
+ --variable=date:"$$(LC_TIME=C date +"Last modified on %a, %d %b %Y at %T %z" -d @"$$mtime")" \
+ --output="$@" -- "$<"
+
+doc: manual html
+
+install:
clean:
- rm -f pullimap.1 interimap.1
+ rm -f $(MANUALS) $(HTML_FILES)
-.PHONY: all install clean test
+.PHONY: all manual html doc test install clean
diff --git a/README b/README
index 06f328f..fbc4ed7 100644
--- a/README
+++ b/README
@@ -1,9 +1,7 @@
InterIMAP is a fast bidirectional synchronization program for QRESYNC-capable
IMAP4rev1 servers. PullIMAP retrieves messages a remote IMAP mailbox and
-deliver them to an SMTP session. Consult the manuals for more information.
-
- https://guilhem.org/man/interimap.1.html
- https://guilhem.org/man/pullimap.1.html
+deliver them to an SMTP session. Visit https://guilhem.org/interimap
+for more information.
_______________________________________________________________________
@@ -39,7 +37,7 @@ IMAP traffic is mostly text (beside message bodies perhaps) hence
compresses pretty well: enabling compression can save a great amount of
network resources.
-However establishing a SSL/TLS connection (type=imaps, or type=imap and
+However establishing an SSL/TLS connection (type=imaps, or type=imap and
STARTTLS=YES) yields a small overhead due to the SSL/TLS handshake.
On the other hand if SSH access is allowed on the remote server, one can
diff --git a/doc/build.md b/doc/build.md
new file mode 100644
index 0000000..38d1bfb
--- /dev/null
+++ b/doc/build.md
@@ -0,0 +1,99 @@
+% Build instructions
+% Guilhem Moulin <guilhem@fripost.org>
+
+On Debian 9 (codename *Stretch*) and later, installing [`interimap`(1)]
+is a single command away:
+
+ $ apt-get install interimap
+
+This document is for those who are running other systems, and/or who
+wish to install from [source](https://git.guilhem.org/interimap).
+
+
+Dependencies
+============
+
+[`interimap`(1)](interimap.1.html) depends on Perl ≥5.20 and the
+following Perl modules:
+
+ * [`Compress::Raw::Zlib`](https://perldoc.perl.org/Compress/Raw/Zlib.html) (*core module*)
+ * [`Config::Tiny`](https://metacpan.org/pod/Config::Tiny)
+ * [`DBI`](https://metacpan.org/pod/DBI)
+ * [`DBD::SQLite`](https://metacpan.org/pod/DBD::SQLite)
+ * [`Errno`](https://perldoc.perl.org/Errno.html) (*core module*)
+ * [`Getopt::Long`](https://perldoc.perl.org/Getopt/Long.html) (*core module*)
+ * [`MIME::Base64`](https://perldoc.perl.org/MIME/Base64.html) (*core module*) — if authentication is required
+ * [`List::Util`](https://perldoc.perl.org/List/Util.html) (*core module*)
+ * [`Net::SSLeay`](https://metacpan.org/pod/Net::SSLeay) ≥1.73
+ * [`POSIX`](https://perldoc.perl.org/POSIX.html) (*core module*)
+ * [`Socket`](https://perldoc.perl.org/Socket.html) (*core module*)
+ * [`Time::HiRes`](https://perldoc.perl.org/Time/HiRes.html) (*core module*) — if `logfile` is set
+
+On Debian GNU/Linux systems, the dependencies can be installed with the
+following command:
+
+ $ apt install libconfig-tiny-perl \
+ libdbi-perl \
+ libdbd-sqlite3-perl \
+ libnet-ssleay-perl
+
+Additional packages are required in order to run the test suite:
+
+ $ apt install dovecot-imapd procps sqlite3 xxd
+<!-- -->
+ $ make test
+
+
+Generate documentation
+======================
+
+Yet another set of packages is needed to generate the documentation:
+
+ $ apt install jq pandoc
+
+Run `` `make manual` `` (or just `` `make` ``) in order to generate the
+manpages. You'll find them at `doc/*.[1-9]`. Use for instance `` `man
+-l doc/interimap.1` `` in order to read your copy of the [`interimap`(1)]
+manpage.
+
+The HTML documentation can be built with `` `make html` ``. HTML files
+are generated alongside their Markdown source by default, but you can
+choose another target directory using the `HTML_ROOTDIR` environment
+variable (the value of which defaults to `./doc`). Moreover the
+[`libjs-bootstrap`](https://tracker.debian.org/libjs-bootstrap) is
+needed by default for the local CSS file; this can be controlled with
+the `CSS` environment variable (the value of which defaults to
+`/usr/share/javascript/bootstrap/css/bootstrap.min.css`).
+
+For instance, use
+
+ $ CSS="https://guilhem.org/static/css/bootstrap.min.css" \
+ HTML_ROOTDIR="$XDG_RUNTIME_DIR/interimap" \
+ make html
+
+to generate the HTML documentation under directory `$XDG_RUNTIME_DIR/interimap`
+(which needs to exist) using a remote CSS file.
+
+The `doc` target generates all documentation, manpages as well as HTML
+pages.
+
+
+Build custom Debian package
+===========================
+
+Debian GNU/Linux users can also use [`gbp`(1)] from
+[`git-buildpackage`](https://tracker.debian.org/pkg/git-buildpackage) in
+order to build their own package:
+
+ $ git checkout debian
+ $ gbp buildpackage
+
+Alternatively, for the development version:
+
+ $ git checkout debian
+ $ git merge master
+ $ gbp buildpackage --git-force-create --git-upstream-tree=BRANCH
+
+
+[`interimap`(1)]: interimap.1.html
+[`gbp`(1)]: https://manpages.debian.org/git-buildpackage/gbp.1.en.html
diff --git a/doc/development.md b/doc/development.md
new file mode 100644
index 0000000..406207a
--- /dev/null
+++ b/doc/development.md
@@ -0,0 +1,208 @@
+% Test environment setup for [`interimap`(1)] and [`pullimap`(1)]
+% Guilhem Moulin <guilhem@fripost.org>
+
+Introduction
+============
+
+This document describes how to create dummy mail storage for
+[`interimap`(1)] and/or [`pullimap`(1)] development, using
+[Dovecot](https://dovecot.org) as [IMAP4rev1] server. Start by creating
+a new temporary directory:
+
+ $ BASEDIR="$(mktemp --tmpdir --directory)"
+
+(The leading `$ ` in this document are command-line prompt strings, and
+are not part of the command themselves.)
+
+
+Dovecot configuration
+=====================
+
+Create a file `$BASEDIR/dovecot.conf`, which will be used as
+configuration for the various Dovecot commands (the system configuration
+will be skipped).
+
+ $ cat >"$BASEDIR/dovecot.conf" <<-EOF
+ log_path = "$BASEDIR/dovecot.log"
+ ssl = no
+ mail_home = "$BASEDIR/%u"
+ mail_location = maildir:~/mail
+ EOF
+
+Here are some details on the above:
+
+`log_path`
+
+ : Dovecot [logs to syslog](https://wiki.dovecot.org/Logging) by default.
+ It's annoying to clutter syslog with test entries, so instead we make it
+ log to a file under `$BASEDIR`.
+
+`ssl`
+
+ : Not required, but turned off here so dumping the configuration with
+ `` `doveconf -c "$BASEDIR/dovecot.conf" -n` `` doesn't spew a warning.
+
+`mail_home`
+
+ : Dovecot needs the name of the user to (pre-)authenticate. It is shown
+ in the greeting line, and also used in [`%`-variable] expansion.
+ Several [`doveadm`(1)] sub-commands have a `-u` (or `-d`) option which
+ can be used to determine the username. When this option is not set,
+ the username is taken from the `USER` environment variable. If that
+ environment variable is unset as well, then the return string of
+ [`getlogin`(3)](https://pubs.opengroup.org/onlinepubs/9699919799/functions/getlogin.html)
+ is used.
+
+ Similarly, the user's home directory is used in (`~`- and)
+ [`%`-variable] expansion. It's taken from the `HOME` environment
+ variable when the `mail_home` setting is left unset in the Dovecot
+ configuration (and not overridden by the [user database](https://wiki.dovecot.org/UserDatabase)).
+
+ `mail_home` can therefore be left unset if the `HOME` environment
+ variable is consistently set to `$BASEDIR/$USER`. However it's
+ safer to explicitly set it in the configuration file: otherwise a
+ command run in a non-curated environment might mess up with your own
+ mail storage…
+
+`mail_location`
+
+ : The user's mail storage resides — in [Maildir](https://wiki.dovecot.org/MailLocation/Maildir)
+ format — in a directory `mail` under their home directory. This is
+ enough if you're fine with the default IMAP hierarchy delimiter
+ (which depends on the mail format) is used, and if you need a single
+ [IMAP namespace](https://tools.ietf.org/html/rfc2342). For more
+ complex setups you'll need one or more
+ [`namespace {…}` block](https://wiki.dovecot.org/Namespaces).
+
+
+Mail storage access
+===================
+
+Feel free to point a mail client at the dummy mail storage. To start a
+pre-authenticated [IMAP4rev1] in the test environment for username
+`testuser`, list mailboxes, and exit, run:
+
+ $ env -i PATH="/usr/bin:/bin" USER="testuser" \
+ doveadm -c "$BASEDIR/dovecot.conf" exec imap
+ * PREAUTH [CAPABILITY IMAP4rev1 …] Logged in as testuser
+ a LIST "" "*"
+ * LIST (\HasNoChildren) "." INBOX
+ a OK List completed (0.002 + 0.000 + 0.001 secs).
+ q LOGOUT
+ q OK Logout completed (0.001 + 0.000 secs).
+
+For mailbox (create, delete, rename) and message (add, flag update)
+manipulation you can use your mail client, the relevant [IMAP4rev1]
+commands, or simply the [`doveadm`(1)] tools. Here is an example using
+the latter to create a mailbox `foo`, add a sample message to it, and
+finally mark it as `\Seen`.
+
+ $ env -i PATH="/usr/bin:/bin" USER="testuser" \
+ doveadm -c "$BASEDIR/dovecot.conf" mailbox create "foo"
+<!-- -->
+ $ env -i PATH="/usr/bin:/bin" USER="testuser" HOME="$BASEDIR/testuser" \
+ doveadm -c "$BASEDIR/dovecot.conf" exec dovecot-lda -e -m "foo" <<-EOF
+ From: <sender@example.net>
+ To: <recipient@example.net>
+ Subject: Hello world!
+ Date: $(date -R)
+ Message-ID: <$(</proc/sys/kernel/random/uuid)@example.net>
+
+ Hello world!
+ EOF
+<!-- -->
+ $ env -i PATH="/usr/bin:/bin" USER="testuser" \
+ doveadm -c "$BASEDIR/dovecot.conf" flags add "\\Seen" mailbox "foo" "*"
+
+Normally [`dovecot-lda`(1)](https://wiki.dovecot.org/LDA) tries to do a
+userdb lookup in order to determine the user's home directory. Since we
+didn't configure a user database we need to explicitly set the `HOME`
+environment variable.
+
+
+InterIMAP configuration and test
+================================
+
+In this example the peers to synchronize are sharing the same Dovecot
+configuration file `$BASEDIR/dovecot.conf`. Of course, it's also
+possible to use a different configuration on each “server”, for instance
+in order to specify different hierarchy delimiters, namespaces, or mail
+storage format.
+
+Create an [`interimap`(1)] configuration file to synchronize the `local`
+and `remote` accounts.
+
+ $ cat >"$BASEDIR/interimap.conf" <<-EOF
+ database = $BASEDIR/interimap.db
+
+ [local]
+ type = tunnel
+ command = env -i PATH="$PATH" USER="local" doveadm -c "$BASEDIR/dovecot.conf" exec imap
+
+ [remote]
+ type = tunnel
+ command = env -i PATH="$PATH" USER="remote" doveadm -c "$BASEDIR/dovecot.conf" exec imap
+ EOF
+
+Run [`interimap`(1)] without `--watch` in order to create the database.
+
+ $ env -i PATH="$PATH" perl -I./lib -T ./interimap --config="$BASEDIR/interimap.conf"
+ Creating new schema in database file …/interimap.db
+ database: Created mailbox INBOX
+ […]
+
+You can now run [`interimap`(1)] with `--watch` set, here to one second
+to observe synchronisation steps early.
+
+ $ env -i PATH="$PATH" perl -I./lib -T ./interimap --config="$BASEDIR/interimap.conf" \
+ --watch=1 --debug
+
+Use instructions from the [previous section][Mail storage access]
+(substituting `testuser` with `local` or `remote`) in order to simulate
+activity on either end to synchronize. If you run these commands in
+another shell, then make sure to re-set the `BASEDIR` environment
+variable!
+
+
+PullIMAP configuration and test
+===============================
+
+Create a [`pullimap`(1)] configuration file with as section `[foo]`.
+
+ $ cat >"$BASEDIR/pullimap.conf" <<-EOF
+ [foo]
+ type = tunnel
+ command = env -i PATH="$PATH" USER="testuser" doveadm -c "$BASEDIR/dovecot.conf" exec imap
+ statefile = $BASEDIR/pullimap.foo
+ EOF
+
+Run [`pullimap`(1)] without `--idle` in order to create the state file.
+
+ $ env -i PATH="$PATH" perl -I./lib -T ./pullimap --config="$BASEDIR/pullimap.conf" \
+ --no-delivery foo
+
+You can now run [`pullimap`(1)] with `--idle` set.
+
+ $ env -i PATH="$PATH" perl -I./lib -T ./pullimap --config="$BASEDIR/pullimap.conf" \
+ --no-delivery --idle --debug foo
+
+Use instructions from the [previous section][Mail storage access]
+in order to simulate activity on the “remote” server (in the relevant
+mailbox — `INBOX` by default). If you run these commands in another
+shell, then make sure to re-set the `BASEDIR` environment variable!
+
+
+Cleanup
+=======
+
+To remove temporary directories and the message they contain, simply
+recursively remove the directory `$BASEDIR`.
+
+ $ rm -rf -- "$BASEDIR"
+
+
+[IMAP4rev1]: https://tools.ietf.org/html/rfc3501
+[`interimap`(1)]: interimap.1.html
+[`pullimap`(1)]: pullimap.1.html
+[`doveadm`(1)]: https://wiki.dovecot.org/Tools/Doveadm
+[`%`-variable]: https://wiki.dovecot.org/Variables
diff --git a/doc/index.md b/doc/index.md
new file mode 100644
index 0000000..12de956
--- /dev/null
+++ b/doc/index.md
@@ -0,0 +1,20 @@
+% [`interimap`(1)] and [`pullimap`(1)] documentation
+% Guilhem Moulin <guilhem@fripost.org>
+
+Manuals (HTML versions)
+-----------------------
+
+ * [`interimap`(1)] — Fast bidirectional synchronization for
+ QRESYNC-capable IMAP servers
+ * [`pullimap`(1)] — Pull mails from an IMAP mailbox and deliver them
+ to an SMTP session
+
+Resources for developers
+------------------------
+
+ * [Source-code repository](https://git.guilhem.org/interimap)
+ * [Build instructions](build.html)
+ * [Test environment setup](development.html)
+
+[`interimap`(1)]: interimap.1.html
+[`pullimap`(1)]: pullimap.1.html
diff --git a/interimap.md b/doc/interimap.1.md
index 50c1832..387850a 100644
--- a/interimap.md
+++ b/doc/interimap.1.md
@@ -1,4 +1,4 @@
-% intermap(1)
+% interimap(1)
% [Guilhem Moulin](mailto:guilhem@fripost.org)
% July 2015
@@ -47,7 +47,7 @@ with the [RFC 7162] (sec. 6) amendments, and works as follows:
2. Propagate these changes onto the other server: get the corresponding
UIDs from the database, then:
- a. issue an `UID STORE` command, followed by `UID EXPUNGE`, to
+ a. issue a `UID STORE` command, followed by `UID EXPUNGE`, to
remove messages that have not already been deleted on both
servers; and
b. issue some `UID STORE` commands to propagate flag updates (send
@@ -62,9 +62,9 @@ with the [RFC 7162] (sec. 6) amendments, and works as follows:
Otherwise, update the `HIGHESTMODSEQ` value in the database.
3. Process new messages (if the current `UIDNEXT` value of the mailbox
- differs from the one found in the database) by issuing an `UID
- FETCH` command; process each received message on-the-fly by issuing
- an `APPEND` command with the message's `RFC822` body, `FLAGS` and
+ differs from the one found in the database) by issuing a `UID FETCH`
+ command; process each received message on-the-fly by issuing an
+ `APPEND` command with the message's `RFC822` body, `FLAGS` and
`INTERNALDATE`.
Repeat this step if the server received new messages in the
meantime. Otherwise, update the `UIDNEXT` value in the database.
@@ -111,8 +111,8 @@ other than the default [`QRESYNC`][RFC 7162]-based synchronization.
existing UID; and
3/ ensure that both flag lists match.
Any message found on a server but not in the database is replicated
- on the other server (which in the worst case, might lead to a
- message duplicate).
+ on the other server (which in the worst case, might yield a message
+ duplicate).
Flag conflicts are solved by updating each message to the union of
both lists.
@@ -224,7 +224,7 @@ Valid options are:
to the `perso/` sub-hierarchy on the local server. This is useful
for synchronizing multiple remote servers against different
namespaces belonging to the same local IMAP server (using a
- different InterIMAP instance for each local namespace ↔ remote
+ different `interimap` instance for each local namespace ↔ remote
synchronization).
(Note that if the reference name is not a level of mailbox hierarchy
@@ -281,10 +281,10 @@ Valid options are:
: One of `imap`, `imaps` or `tunnel`.
`type=imap` and `type=imaps` are respectively used for IMAP and IMAP
- over SSL/TLS connections over a INET socket.
+ over SSL/TLS connections over an INET socket.
`type=tunnel` causes `interimap` to create an unnamed pair of
connected sockets for interprocess communication with a *command*
- instead of a opening a network socket.
+ instead of opening a network socket.
Note that specifying `type=tunnel` in the `[remote]` section makes
the default *database* to be `localhost.db`.
(Default: `imaps`.)
@@ -314,7 +314,9 @@ Valid options are:
: Command to use for `type=tunnel`. Must speak the [IMAP4rev1
protocol][RFC 3501] on its standard output, and understand it on its
- standard input.
+ standard input. The value is passed to `` `/bin/sh -c` `` if it
+ contains shell metacharacters; otherwise it is split into words and
+ the resulting list is passed to `execvp`(3).
*STARTTLS*
@@ -348,7 +350,7 @@ Valid options are:
*null-stderr*
: Whether to redirect *command*'s standard error to `/dev/null` for
- type `type=tunnel`. (Default: `NO`.)
+ `type=tunnel`. (Default: `NO`.)
*SSL_protocols*
@@ -374,9 +376,8 @@ Valid options are:
Attempting to connect to a server with a non-matching certificate
SPKI fingerprint causes `interimap` to abort the connection during
the SSL/TLS handshake.
-
- You can use the following command to compute the SHA-256 digest of
- certificate's Subject Public Key Info.
+ The following command can be used to compute the SHA-256 digest of a
+ certificate's Subject Public Key Info:
openssl x509 -in /path/to/server/certificate.pem -pubkey \
| openssl pkey -pubin -outform DER \
diff --git a/pullimap.md b/doc/pullimap.1.md
index a367dd1..1b2e509 100644
--- a/pullimap.md
+++ b/doc/pullimap.1.md
@@ -5,7 +5,7 @@
Name
====
-PullIMAP - Pull mails from an IMAP mailbox and deliver them to a SMTP session
+PullIMAP - Pull mails from an IMAP mailbox and deliver them to an SMTP session
Synopsis
========
@@ -16,8 +16,8 @@ Synopsis
Description
===========
-`pullimap` retrieves messages from an IMAP mailbox and deliver them to a
-SMTP or LMTP transmission channel. It can also remove old messages
+`pullimap` retrieves messages from an IMAP mailbox and deliver them to
+an SMTP or LMTP transmission channel. It can also remove old messages
after a configurable retention period.
A *statefile* is used to keep track of the mailbox's `UIDVALIDITY` and
@@ -130,9 +130,10 @@ Valid options are:
: One of `imap`, `imaps` or `tunnel`.
`type=imap` and `type=imaps` are respectively used for IMAP and IMAP
- over SSL/TLS connections over a INET socket.
- `type=tunnel` causes `pullimap` to open a pipe to a *command*
- instead of a raw socket.
+ over SSL/TLS connections over an INET socket.
+ `type=tunnel` causes `pullimap` to create an unnamed pair of
+ connected sockets for interprocess communication with a *command*
+ instead of opening a network socket.
(Default: `imaps`.)
*host*
@@ -160,7 +161,9 @@ Valid options are:
: Command to use for `type=tunnel`. Must speak the [IMAP4rev1
protocol][RFC 3501] on its standard output, and understand it on its
- standard input.
+ standard input. The value is passed to `` `/bin/sh -c` `` if it
+ contains shell metacharacters; otherwise it is split into words and
+ the resulting list is passed to `execvp`(3).
*STARTTLS*
@@ -192,7 +195,7 @@ Valid options are:
*null-stderr*
: Whether to redirect *command*'s standard error to `/dev/null` for
- type `type=tunnel`. (Default: `NO`.)
+ `type=tunnel`. (Default: `NO`.)
*SSL_protocols*
@@ -218,9 +221,8 @@ Valid options are:
Attempting to connect to a server with a non-matching certificate
SPKI fingerprint causes `pullimap` to abort the connection during
the SSL/TLS handshake.
-
- You can use the following command to compute the SHA-256 digest of
- certificate's Subject Public Key Info.
+ The following command can be used to compute the SHA-256 digest of a
+ certificate's Subject Public Key Info:
openssl x509 -in /path/to/server/certificate.pem -pubkey \
| openssl pkey -pubin -outform DER \
@@ -263,7 +265,7 @@ Usually there are only two integers: the first is the *mailbox*'s
smaller than this `UIDNEXT` value have already been retrieved and
delivered).
The [IMAP4rev1 specification][RFC 3501] does not guaranty that untagged
-`FETCH` responses are sent ordered by UID in response to an `UID FETCH`
+`FETCH` responses are sent ordered by UID in response to a `UID FETCH`
command. Thus it would be unsafe for `pullimap` to update the `UIDNEXT`
value in its *statefile* while the `UID FETCH` command is progress.
Instead, for each untagged `FETCH` response received while the `UID
@@ -278,7 +280,7 @@ FETCH` command is in progress.
In more details, `pullimap` works as follows:
- 1. Issue an `UID FETCH` command to retrieve message `ENVELOPE` and
+ 1. Issue a `UID FETCH` command to retrieve message `ENVELOPE` and
`RFC822` (and `UID`) with UID bigger or equal than the `UIDNEXT`
value found in the *statefile*.
While the `UID FETCH` command is in progress, perform the following
@@ -296,10 +298,10 @@ In more details, `pullimap` works as follows:
i. append the message UID to the *statefile*.
- 2. If a SMTP/LMTP transmission channel was opened, send a `QUIT` command
+ 2. If an SMTP/LMTP transmission channel was opened, send a `QUIT` command
to terminate it gracefully.
- 3. Issue an `UID STORE` command to mark all retrieved messages (and
+ 3. Issue a `UID STORE` command to mark all retrieved messages (and
stalled UIDs found in the *statefile* after the eigth byte) as
`\Seen`.
diff --git a/doc/template.html b/doc/template.html
new file mode 100644
index 0000000..e17f0e3
--- /dev/null
+++ b/doc/template.html
@@ -0,0 +1,76 @@
+<!DOCTYPE html>
+<html xmlns="http://www.w3.org/1999/xhtml" lang="$lang$" xml:lang="$lang$"$if(dir)$ dir="$dir$"$endif$>
+<head>
+ <meta charset="utf-8" />
+ <meta name="generator" content="pandoc" />
+ <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" />
+$for(author-meta)$
+ <meta name="author" content="$author-meta$" />
+$endfor$
+$if(date-meta)$
+ <meta name="dcterms.date" content="$date-meta$" />
+$endif$
+$if(keywords)$
+ <meta name="keywords" content="$for(keywords)$$keywords$$sep$, $endfor$" />
+$endif$
+ <title>$if(title-prefix)$$title-prefix$ – $endif$$pagetitle$</title>
+ <style type="text/css">
+ code{white-space: pre-wrap;}
+ span.smallcaps{font-variant: small-caps;}
+ span.underline{text-decoration: underline;}
+ div.column{display: inline-block; vertical-align: top; width: 50%;}
+$if(quotes)$
+ q { quotes: "“" "”" "‘" "’"; }
+$endif$
+ </style>
+$if(highlighting-css)$
+ <style type="text/css">
+$highlighting-css$
+ </style>
+$endif$
+$for(css)$
+ <link rel="stylesheet" href="$css$" />
+$endfor$
+ <style type="text/css">
+ @media(max-width: 1440px) { .container{ max-width: 1080px; } }
+ @media(max-width: 1280px) { .container{ max-width: 960px; } }
+ @media(max-width: 1024px) { .container{ max-width: 768px; } }
+ </style>
+$if(math)$
+ $math$
+$endif$
+$for(header-includes)$
+ $header-includes$
+$endfor$
+</head>
+
+<body>
+$for(include-before)$
+$include-before$
+$endfor$
+<div class="container text-justify">
+ <div class="content">
+$if(title)$
+ <div class="page-header"><h1>$title$</h1></div>
+$endif$
+
+$body$
+ </div>
+
+ <footer>
+ <hr/>
+ <div class="row">
+ <div class="col-md-8 text-muted">
+$if(author)$
+ <a href="https://git.guilhem.org/interimap/plain/COPYING">&copy;</a>
+ $for(author)$$author$$sep$, $endfor$
+$endif$
+ </div>
+ <div class="col-md-4 text-muted text-right small">
+ $if(date)$$date$$endif$
+ </div>
+ </div>
+ </footer>
+</div>
+</body>
+</html>
diff --git a/lib/Net/IMAP/InterIMAP.pm b/lib/Net/IMAP/InterIMAP.pm
index 1dd54b7..19895c4 100644
--- a/lib/Net/IMAP/InterIMAP.pm
+++ b/lib/Net/IMAP/InterIMAP.pm
@@ -26,7 +26,8 @@ use Errno qw/EEXIST EINTR/;
use Net::SSLeay 1.73 ();
use List::Util qw/all first/;
use POSIX ':signal_h';
-use Socket qw/SOCK_STREAM SOCK_RAW IPPROTO_TCP AF_UNIX AF_INET AF_INET6 PF_UNSPEC SOCK_CLOEXEC :addrinfo/;
+use Socket qw/SOCK_STREAM SOCK_RAW SOCK_CLOEXEC IPPROTO_TCP SHUT_RDWR
+ AF_UNIX AF_INET AF_INET6 PF_UNSPEC :addrinfo/;
use Exporter 'import';
BEGIN {
@@ -506,7 +507,7 @@ sub stats($) {
}
-# Log out when the Net::IMAP::InterIMAP object is destroyed.
+# Destroy a Net::IMAP::InterIMAP object.
sub DESTROY($) {
my $self = shift;
$self->{_STATE} = 'LOGOUT';
@@ -514,8 +515,12 @@ sub DESTROY($) {
Net::SSLeay::free($self->{_SSL}) if defined $self->{_SSL};
Net::SSLeay::CTX_free($self->{_SSL_CTX}) if defined $self->{_SSL_CTX};
- shutdown($self->{S}, 2) if $self->{type} ne 'tunnel' and defined $self->{S};
- $self->{S}->close() if defined $self->{S} and $self->{S}->opened();
+ if (defined (my $s = $self->{S})) {
+ # for type=tunnel we assume the child won't linger around once
+ # we close its standard input and output.
+ shutdown($s, SHUT_RDWR);
+ $s->close() if $s->opened();
+ }
$self->stats() unless $self->{quiet};
}
@@ -606,7 +611,7 @@ sub incapable($@) {
# $self->search($criterion)
-# Issue an UID SEARCH command with the given $criterion. For the "normal"
+# Issue a UID SEARCH command with the given $criterion. For the "normal"
# UID SEARCH command from RFC 3501, return the list of matching UIDs;
# for the extended UID SEARCH command from RFC 4731 (ensuring ESEARCH
# capability is the caller's responsibility), return an optional "UID"
@@ -917,7 +922,7 @@ sub append($$@) {
# $self->fetch($set, $flags, [$callback])
-# Issue an UID FETCH command with the given UID $set, $flags, and
+# Issue a UID FETCH command with the given UID $set, $flags, and
# optional $callback.
sub fetch($$$;&) {
my ($self, $set, $flags, $callback) = @_;
diff --git a/pullimap b/pullimap
index 84587fe..e1c96e8 100755
--- a/pullimap
+++ b/pullimap
@@ -1,7 +1,7 @@
#!/usr/bin/perl -T
#----------------------------------------------------------------------
-# Pull mails from an IMAP mailbox and deliver them to a SMTP session
+# Pull mails from an IMAP mailbox and deliver them to an SMTP session
# Copyright © 2016-2018 Guilhem Moulin <guilhem@fripost.org>
#
# This program is free software: you can redistribute it and/or modify
diff --git a/pullimap@.service b/pullimap@.service
index 53694da..a9ce09a 100644
--- a/pullimap@.service
+++ b/pullimap@.service
@@ -1,5 +1,5 @@
[Unit]
-Description=Pull mails from an IMAP mailbox and deliver them to a SMTP session (instance %i)
+Description=Pull mails from an IMAP mailbox and deliver them to an SMTP session (instance %i)
Documentation=man:pullimap(1)
Wants=network-online.target
After=network-online.target
diff --git a/tests/00-db-exclusive/run b/tests/00-db-exclusive/run
index 1ae27b6..1528b3b 100644
--- a/tests/00-db-exclusive/run
+++ b/tests/00-db-exclusive/run
@@ -9,8 +9,8 @@ interimap
interimap --watch=60 & pid=$!
cleanup() {
# kill interimap process and its children
- pkill -P "$pid" -TERM
- kill -TERM "$pid"
+ pkill -P "$pid" -TERM || true
+ kill -TERM "$pid" || true
wait
}
trap cleanup EXIT INT TERM
diff --git a/tests/00-db-migration-0-to-1/run b/tests/00-db-migration-0-to-1/run
index e4eb770..757fe04 100644
--- a/tests/00-db-migration-0-to-1/run
+++ b/tests/00-db-migration-0-to-1/run
@@ -18,9 +18,16 @@ sqlite3 "$XDG_DATA_HOME/interimap/remote.db" >"$TMPDIR/dump.sql" <<-EOF
.dump
EOF
+# re-import and dump the expected dump to work around SQLite format
+# differences across versions
+sqlite3 "$XDG_DATA_HOME/interimap/remote2.db" <"$TESTDIR/after.sql"
+sqlite3 "$XDG_DATA_HOME/interimap/remote2.db" >"$TMPDIR/dump-expected.sql" <<-EOF
+ .dump
+EOF
+
# XXX need 'user_version' PRAGMA in the dump for future migrations
# http://sqlite.1065341.n5.nabble.com/dump-command-and-user-version-td101228.html
diff -u --label="a/dump.sql" --label="b/dump.sql" \
- "$TESTDIR/after.sql" "$TMPDIR/dump.sql"
+ "$TMPDIR/dump-expected.sql" "$TMPDIR/dump.sql"
# vim: set filetype=sh :
diff --git a/tests/05-repair/run b/tests/05-repair/run
index 747b974..15553e0 100644
--- a/tests/05-repair/run
+++ b/tests/05-repair/run
@@ -65,7 +65,7 @@ spoof UIDNEXT "local"
spoof HIGHESTMODSEQ "local" "remote"
# now repair
-interimap --repair "baz" "foo.bar"
+interimap --repair "baz" "foo.bar"
# 6 updates with \Answered (luid 4,8,11:13,16), 2 of which (luid 12,13) vanished from remote
# 3 updates with \Seen (ruid 6,8,10), 1 of which (uid 10) vanished from remote
diff --git a/tests/06-largeint/run b/tests/06-largeint/run
index edcbd31..b08bcfa 100644
--- a/tests/06-largeint/run
+++ b/tests/06-largeint/run
@@ -7,9 +7,9 @@ doveadm -u "local" mailbox update --uid-validity 2147483648 "bar" # 2^31
doveadm -u "local" mailbox update --uid-validity 4294967295 "baz" # 2^32-1
doveadm -u "remote" mailbox update --uid-validity 4294967295 "INBOX" # 2^32-1
-doveadm -u "remote" mailbox update --uid-validity 2147483648 "foo" # 2^31
+doveadm -u "remote" mailbox update --uid-validity 2147483648 "foo" # 2^31
doveadm -u "remote" mailbox update --uid-validity 2147483647 "bar" # 2^31-1
-doveadm -u "remote" mailbox update --uid-validity 1 "baz" #
+doveadm -u "remote" mailbox update --uid-validity 1 "baz" #
run() {
local u m
diff --git a/tests/07-sync-live-multi/run b/tests/07-sync-live-multi/run
index bf0d2f5..15a27fd 100644
--- a/tests/07-sync-live-multi/run
+++ b/tests/07-sync-live-multi/run
@@ -15,12 +15,12 @@ interimap --config="config3" --watch=1 & pid3=$!
abort() {
# kill interimap process and its children
- pkill -P "$pid" -TERM
- kill -TERM "$pid"
- pkill -P "$pid2" -TERM
- kill -TERM "$pid2"
- pkill -P "$pid3" -TERM
- kill -TERM "$pid3"
+ pkill -P "$pid" -TERM || true
+ kill -TERM "$pid" || true
+ pkill -P "$pid2" -TERM || true
+ kill -TERM "$pid2" || true
+ pkill -P "$pid3" -TERM || true
+ kill -TERM "$pid3" || true
wait
}
trap abort EXIT INT TERM
@@ -101,7 +101,7 @@ while [ $(date +%s) -le $timer ]; do
done
# sleep a little bit
- sleep "0.$(shuf -n1 -i1-99)"
+ sleep "$(printf "0.%03d" "$(shuf -n1 -i1-999)")"
done
# wait a little longer so interimap has time to run loop() again and
diff --git a/tests/07-sync-live/run b/tests/07-sync-live/run
index 1950e0b..04d8247 100644
--- a/tests/07-sync-live/run
+++ b/tests/07-sync-live/run
@@ -6,8 +6,8 @@ interimap --watch=1 & pid=$!
abort() {
# kill interimap process and its children
- pkill -P "$pid" -TERM
- kill -TERM "$pid"
+ pkill -P "$pid" -TERM || true
+ kill -TERM "$pid" || true
wait
}
trap abort EXIT INT TERM
@@ -63,7 +63,7 @@ while [ $(date +%s) -le $timer ]; do
done
# sleep a little bit
- sleep "0.$(shuf -n1 -i1-99)"
+ sleep "$(printf "0.%03d" "$(shuf -n1 -i1-999)")"
done
# wait a little longer so interimap has time to run loop() again and
diff --git a/tests/run b/tests/run
index 31af03e..ee11757 100755
--- a/tests/run
+++ b/tests/run
@@ -36,7 +36,7 @@ if [ ! -d "$TESTDIR" ]; then
exit 1
fi
-ROOTDIR="$(mktemp --tmpdir=/dev/shm --directory "$NAME.XXXXXXXXXX")"
+ROOTDIR="$(mktemp --tmpdir="${TMPDIR:-/dev/shm}" --directory "$NAME.XXXXXXXXXX")"
trap 'rm -rf -- "$ROOTDIR"' EXIT INT TERM
STDOUT="$ROOTDIR/stdout"
@@ -73,6 +73,7 @@ prepare() {
cat >"$home/.config/dovecot/config" <<-EOF
log_path = /dev/null
mail_home = $ROOTDIR/home/%u
+ mailbox_list_index = yes
ssl = no
EOF
cat >>"$home/.config/dovecot/config" <"$src"
@@ -102,12 +103,12 @@ prepare() {
[local]
type = tunnel
command = exec ${HOME_local@Q}/.local/bin/doveadm exec imap
- null-stderr = YES
+ null-stderr = NO
[remote]
type = tunnel
command = exec ${home@Q}/.local/bin/doveadm exec imap
- null-stderr = YES
+ null-stderr = NO
EOF
done
}
@@ -129,14 +130,18 @@ doveadm() {
shift 2
"$home/.local/bin/doveadm" "$@"
}
+sqlite3() {
+ command sqlite3 -init /dev/null "$@"
+}
# Sample (random) message
sample_message() {
+ local date="$(date +"%s.%N")"
cat <<-EOF
From: <sender@example.net>
To: <recipient@example.net>
- Date: $(date -R)
- Message-ID: <$(< /proc/sys/kernel/random/uuid)@example.net>
+ Date: $(date -R -d@"$date")
+ Message-ID: <$date@example.net>
EOF
local len="$(shuf -i1-4096 -n1)"
@@ -156,7 +161,7 @@ deliver() {
doveadm "${argv[@]}" exec dovecot-lda -e "$@"
}
-# Dump test results
+# Dump test results
dump_test_result() {
local below=">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"
local above="<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<"
@@ -215,7 +220,7 @@ check_mailbox_status2() {
read MESSAGES < <( sqlite3 "$XDG_DATA_HOME/interimap/$u.db" <<-EOF
.mode csv
.separator " " "\\n"
- SELECT COUNT(*)
+ SELECT COUNT(*)
FROM mailboxes a JOIN mapping b ON a.idx = b.idx
WHERE mailbox = $blob
EOF
@@ -319,7 +324,7 @@ xcgrep() {
declare -a ENVIRON=()
environ_set "local"
export TMPDIR TESTDIR STDOUT STDERR "${ENVIRON[@]}"
-export -f environ_set doveadm interimap sample_message deliver
+export -f environ_set doveadm interimap sqlite3 sample_message deliver
export -f check_mailbox_status check_mailbox_status_values check_mailbox_status2
export -f check_mailboxes_status check_mailbox_list xgrep xcgrep
printf "%s..." "$TEST"
generated by cgit v1.2.3 (git 2.25.1) at 2024-11-26 08:30:58 +0000