diff options
-rw-r--r-- | .gitignore | 5 | ||||
-rw-r--r-- | Changelog | 16 | ||||
-rw-r--r-- | INSTALL | 31 | ||||
-rw-r--r-- | Makefile | 36 | ||||
-rw-r--r-- | README | 8 | ||||
-rw-r--r-- | doc/build.md | 99 | ||||
-rw-r--r-- | doc/development.md | 208 | ||||
-rw-r--r-- | doc/index.md | 20 | ||||
-rw-r--r-- | doc/interimap.1.md (renamed from interimap.md) | 31 | ||||
-rw-r--r-- | doc/pullimap.1.md (renamed from pullimap.md) | 32 | ||||
-rw-r--r-- | doc/template.html | 76 | ||||
-rw-r--r-- | lib/Net/IMAP/InterIMAP.pm | 17 | ||||
-rwxr-xr-x | pullimap | 2 | ||||
-rw-r--r-- | pullimap@.service | 2 | ||||
-rw-r--r-- | tests/00-db-exclusive/run | 4 | ||||
-rw-r--r-- | tests/00-db-migration-0-to-1/run | 9 | ||||
-rw-r--r-- | tests/05-repair/run | 2 | ||||
-rw-r--r-- | tests/06-largeint/run | 4 | ||||
-rw-r--r-- | tests/07-sync-live-multi/run | 14 | ||||
-rw-r--r-- | tests/07-sync-live/run | 6 | ||||
-rwxr-xr-x | tests/run | 21 |
21 files changed, 533 insertions, 110 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9dae7e6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*~ +/doc/*.1 +/doc/*.html +!/doc/template.html +/.pc/ @@ -10,7 +10,7 @@ interimap (0.5) upstream; namespace. * libinterimap: in tunnel mode, use a socketpair rather than two pipes for IPC between the interimap and the IMAP server. Also, use - SOCK_CLOEXEC to save a fcntl() call when setting the close-on-exec + SOCK_CLOEXEC to save an fcntl() call when setting the close-on-exec flag on the socket. * interimap: new option 'list-reference' to specify a reference name. This is useful for synchronizing multiple remote servers against @@ -18,7 +18,10 @@ interimap (0.5) upstream; different InterIMAP instance for each local namespace <-> remote synchronization, for instance with the newly provided systemd template unit file). - * Add a small test-suite (requires dovecot-imapd). + * Add a test-suite. (Requires dovecot-imapd, pkill(1) and xxd(1).) + * Completely refactor the documentation. In particular, move manpages + to a new 'doc' directory, and generate HTML documentation with `make + html`. + interimap: write which --target to use in --delete command suggestions. + interimap: avoid caching hierarchy delimiters forever in the @@ -29,12 +32,17 @@ interimap (0.5) upstream; BLOB. + interimap: use the 'user_version' SQLite PRAGMA for database schema version. + + interimap, pullimap: in the manpage, clarify that the tunnel command + is run following Perl's `exec` semantics: it is passed to `/bin/sh -c` + when it contains shell metacharacters; and split into words and passed + to execvp(3) otherwise. - libinterimap: bugfix: hierarchy delimiters in LIST responses were returned as an escaped quoted special, like "\\", not as a single character (backslash in this case). - libinterimap: the parser choked on responses with non-quoted/literal astring containing ']' characters. And LIST responses with - non-quoted/literal list-mailbox names '%', '*' or ']' characters. + non-quoted/literal list-mailbox names containing '%', '*' or ']' + characters. - libinterimap: quote() the empty string as "" instead of a 0-length literal. (This saves 3 bytes + one round-trip on servers not supporting non-synchronizing literals, and 4 bytes otherwise.) @@ -86,7 +94,7 @@ interimap (0.4) upstream; interimap (0.3) upstream; + New script 'pullimap', to pull mails from an IMAP mailbox and - deliver them to a SMTP session. + deliver them to an SMTP session. + Convert manpage format from groff to pandoc. + interimap: Add support for IMAP NOTIFY [RFC 5465]. + 'fingerprint' now only pins the cert's SPKI, not the cert itself diff --git a/INSTALL b/INSTALL deleted file mode 100644 index 69afb26..0000000 --- a/INSTALL +++ /dev/null @@ -1,31 +0,0 @@ -InterIMAP depends on Perl >=5.20 and the following Perl modules: - - - Compress::Raw::Zlib (core module) - - Config::Tiny - - DBI - - DBD::SQLite - - Errno (core module) - - Getopt::Long (core module) - - MIME::Base64 (core module) if authentication is required - - List::Util (core module) - - Net::SSLeay >=1.73 - - POSIX (core module) - - Socket (core module) - - Time::HiRes (core module) if 'logfile' is set - -On Debian GNU/Linux systems, these modules can be installed with the -following command: - - apt-get install libconfig-tiny-perl libdbi-perl libdbd-sqlite3-perl libnet-ssleay-perl - -However Debian GNU/Linux users can also use gbp(1) from git-buildpackage -to build their own package: - - $ git checkout debian - $ AUTO_DEBSIGN=no gbp buildpackage - -Alternatively, for the development version: - - $ git checkout debian - $ git merge master - $ AUTO_DEBSIGN=no gbp buildpackage --git-force-create --git-upstream-tree=BRANCH @@ -1,8 +1,11 @@ -all: pullimap.1 interimap.1 +all: manual + +MANUALS = $(patsubst %.md,%,$(wildcard ./doc/*.[1-9].md)) +manual: $(MANUALS) # upper case the headers and remove the links -%.1: %.md - @pandoc -f markdown -t json "$<" | \ +$(MANUALS): %: %.md + @pandoc -f markdown -t json -- "$<" | \ jq " \ def fixheaders: \ if .t == \"Header\" then \ @@ -29,12 +32,29 @@ all: pullimap.1 interimap.1 }" | \ pandoc -s -f json -t man+smart -o "$@" -install: - test: - @for t in tests/*; do [ -d "$$t" ] || continue; ./tests/run "$$t" || exit 1; done + @for t in tests/*; do if [ -f "$$t/run" ]; then ./tests/run "$$t" || exit 1; fi; done + +HTML_ROOTDIR ?= ./doc +CSS ?= /usr/share/javascript/bootstrap/css/bootstrap.min.css +HTML_TEMPLATE ?= ./doc/template.html + +HTML_FILES = $(addprefix $(HTML_ROOTDIR)/,$(patsubst ./doc/%.md,%.html,$(wildcard ./doc/*.md))) +html: $(HTML_FILES) + +## CSS="https://guilhem.org/static/css/bootstrap.min.css" HTML_ROOTDIR="$XDG_RUNTIME_DIR/Downloads" make html +$(HTML_ROOTDIR)/%.html: ./doc/%.md $(HTML_TEMPLATE) + mtime="$$(git --no-pager log -1 --pretty="format:%ct" -- "$<" 2>/dev/null)"; \ + [ -n "$$mtime" ] || mtime="$$(date +%s -r "$<")"; \ + pandoc -sp -f markdown -t html+smart --css=$(CSS) --template=$(HTML_TEMPLATE) \ + --variable=date:"$$(LC_TIME=C date +"Last modified on %a, %d %b %Y at %T %z" -d @"$$mtime")" \ + --output="$@" -- "$<" + +doc: manual html + +install: clean: - rm -f pullimap.1 interimap.1 + rm -f $(MANUALS) $(HTML_FILES) -.PHONY: all install clean test +.PHONY: all manual html doc test install clean @@ -1,9 +1,7 @@ InterIMAP is a fast bidirectional synchronization program for QRESYNC-capable IMAP4rev1 servers. PullIMAP retrieves messages a remote IMAP mailbox and -deliver them to an SMTP session. Consult the manuals for more information. - - https://guilhem.org/man/interimap.1.html - https://guilhem.org/man/pullimap.1.html +deliver them to an SMTP session. Visit https://guilhem.org/interimap +for more information. _______________________________________________________________________ @@ -39,7 +37,7 @@ IMAP traffic is mostly text (beside message bodies perhaps) hence compresses pretty well: enabling compression can save a great amount of network resources. -However establishing a SSL/TLS connection (type=imaps, or type=imap and +However establishing an SSL/TLS connection (type=imaps, or type=imap and STARTTLS=YES) yields a small overhead due to the SSL/TLS handshake. On the other hand if SSH access is allowed on the remote server, one can diff --git a/doc/build.md b/doc/build.md new file mode 100644 index 0000000..38d1bfb --- /dev/null +++ b/doc/build.md @@ -0,0 +1,99 @@ +% Build instructions +% Guilhem Moulin <guilhem@fripost.org> + +On Debian 9 (codename *Stretch*) and later, installing [`interimap`(1)] +is a single command away: + + $ apt-get install interimap + +This document is for those who are running other systems, and/or who +wish to install from [source](https://git.guilhem.org/interimap). + + +Dependencies +============ + +[`interimap`(1)](interimap.1.html) depends on Perl ≥5.20 and the +following Perl modules: + + * [`Compress::Raw::Zlib`](https://perldoc.perl.org/Compress/Raw/Zlib.html) (*core module*) + * [`Config::Tiny`](https://metacpan.org/pod/Config::Tiny) + * [`DBI`](https://metacpan.org/pod/DBI) + * [`DBD::SQLite`](https://metacpan.org/pod/DBD::SQLite) + * [`Errno`](https://perldoc.perl.org/Errno.html) (*core module*) + * [`Getopt::Long`](https://perldoc.perl.org/Getopt/Long.html) (*core module*) + * [`MIME::Base64`](https://perldoc.perl.org/MIME/Base64.html) (*core module*) — if authentication is required + * [`List::Util`](https://perldoc.perl.org/List/Util.html) (*core module*) + * [`Net::SSLeay`](https://metacpan.org/pod/Net::SSLeay) ≥1.73 + * [`POSIX`](https://perldoc.perl.org/POSIX.html) (*core module*) + * [`Socket`](https://perldoc.perl.org/Socket.html) (*core module*) + * [`Time::HiRes`](https://perldoc.perl.org/Time/HiRes.html) (*core module*) — if `logfile` is set + +On Debian GNU/Linux systems, the dependencies can be installed with the +following command: + + $ apt install libconfig-tiny-perl \ + libdbi-perl \ + libdbd-sqlite3-perl \ + libnet-ssleay-perl + +Additional packages are required in order to run the test suite: + + $ apt install dovecot-imapd procps sqlite3 xxd +<!-- --> + $ make test + + +Generate documentation +====================== + +Yet another set of packages is needed to generate the documentation: + + $ apt install jq pandoc + +Run `` `make manual` `` (or just `` `make` ``) in order to generate the +manpages. You'll find them at `doc/*.[1-9]`. Use for instance `` `man +-l doc/interimap.1` `` in order to read your copy of the [`interimap`(1)] +manpage. + +The HTML documentation can be built with `` `make html` ``. HTML files +are generated alongside their Markdown source by default, but you can +choose another target directory using the `HTML_ROOTDIR` environment +variable (the value of which defaults to `./doc`). Moreover the +[`libjs-bootstrap`](https://tracker.debian.org/libjs-bootstrap) is +needed by default for the local CSS file; this can be controlled with +the `CSS` environment variable (the value of which defaults to +`/usr/share/javascript/bootstrap/css/bootstrap.min.css`). + +For instance, use + + $ CSS="https://guilhem.org/static/css/bootstrap.min.css" \ + HTML_ROOTDIR="$XDG_RUNTIME_DIR/interimap" \ + make html + +to generate the HTML documentation under directory `$XDG_RUNTIME_DIR/interimap` +(which needs to exist) using a remote CSS file. + +The `doc` target generates all documentation, manpages as well as HTML +pages. + + +Build custom Debian package +=========================== + +Debian GNU/Linux users can also use [`gbp`(1)] from +[`git-buildpackage`](https://tracker.debian.org/pkg/git-buildpackage) in +order to build their own package: + + $ git checkout debian + $ gbp buildpackage + +Alternatively, for the development version: + + $ git checkout debian + $ git merge master + $ gbp buildpackage --git-force-create --git-upstream-tree=BRANCH + + +[`interimap`(1)]: interimap.1.html +[`gbp`(1)]: https://manpages.debian.org/git-buildpackage/gbp.1.en.html diff --git a/doc/development.md b/doc/development.md new file mode 100644 index 0000000..406207a --- /dev/null +++ b/doc/development.md @@ -0,0 +1,208 @@ +% Test environment setup for [`interimap`(1)] and [`pullimap`(1)] +% Guilhem Moulin <guilhem@fripost.org> + +Introduction +============ + +This document describes how to create dummy mail storage for +[`interimap`(1)] and/or [`pullimap`(1)] development, using +[Dovecot](https://dovecot.org) as [IMAP4rev1] server. Start by creating +a new temporary directory: + + $ BASEDIR="$(mktemp --tmpdir --directory)" + +(The leading `$ ` in this document are command-line prompt strings, and +are not part of the command themselves.) + + +Dovecot configuration +===================== + +Create a file `$BASEDIR/dovecot.conf`, which will be used as +configuration for the various Dovecot commands (the system configuration +will be skipped). + + $ cat >"$BASEDIR/dovecot.conf" <<-EOF + log_path = "$BASEDIR/dovecot.log" + ssl = no + mail_home = "$BASEDIR/%u" + mail_location = maildir:~/mail + EOF + +Here are some details on the above: + +`log_path` + + : Dovecot [logs to syslog](https://wiki.dovecot.org/Logging) by default. + It's annoying to clutter syslog with test entries, so instead we make it + log to a file under `$BASEDIR`. + +`ssl` + + : Not required, but turned off here so dumping the configuration with + `` `doveconf -c "$BASEDIR/dovecot.conf" -n` `` doesn't spew a warning. + +`mail_home` + + : Dovecot needs the name of the user to (pre-)authenticate. It is shown + in the greeting line, and also used in [`%`-variable] expansion. + Several [`doveadm`(1)] sub-commands have a `-u` (or `-d`) option which + can be used to determine the username. When this option is not set, + the username is taken from the `USER` environment variable. If that + environment variable is unset as well, then the return string of + [`getlogin`(3)](https://pubs.opengroup.org/onlinepubs/9699919799/functions/getlogin.html) + is used. + + Similarly, the user's home directory is used in (`~`- and) + [`%`-variable] expansion. It's taken from the `HOME` environment + variable when the `mail_home` setting is left unset in the Dovecot + configuration (and not overridden by the [user database](https://wiki.dovecot.org/UserDatabase)). + + `mail_home` can therefore be left unset if the `HOME` environment + variable is consistently set to `$BASEDIR/$USER`. However it's + safer to explicitly set it in the configuration file: otherwise a + command run in a non-curated environment might mess up with your own + mail storage… + +`mail_location` + + : The user's mail storage resides — in [Maildir](https://wiki.dovecot.org/MailLocation/Maildir) + format — in a directory `mail` under their home directory. This is + enough if you're fine with the default IMAP hierarchy delimiter + (which depends on the mail format) is used, and if you need a single + [IMAP namespace](https://tools.ietf.org/html/rfc2342). For more + complex setups you'll need one or more + [`namespace {…}` block](https://wiki.dovecot.org/Namespaces). + + +Mail storage access +=================== + +Feel free to point a mail client at the dummy mail storage. To start a +pre-authenticated [IMAP4rev1] in the test environment for username +`testuser`, list mailboxes, and exit, run: + + $ env -i PATH="/usr/bin:/bin" USER="testuser" \ + doveadm -c "$BASEDIR/dovecot.conf" exec imap + * PREAUTH [CAPABILITY IMAP4rev1 …] Logged in as testuser + a LIST "" "*" + * LIST (\HasNoChildren) "." INBOX + a OK List completed (0.002 + 0.000 + 0.001 secs). + q LOGOUT + q OK Logout completed (0.001 + 0.000 secs). + +For mailbox (create, delete, rename) and message (add, flag update) +manipulation you can use your mail client, the relevant [IMAP4rev1] +commands, or simply the [`doveadm`(1)] tools. Here is an example using +the latter to create a mailbox `foo`, add a sample message to it, and +finally mark it as `\Seen`. + + $ env -i PATH="/usr/bin:/bin" USER="testuser" \ + doveadm -c "$BASEDIR/dovecot.conf" mailbox create "foo" +<!-- --> + $ env -i PATH="/usr/bin:/bin" USER="testuser" HOME="$BASEDIR/testuser" \ + doveadm -c "$BASEDIR/dovecot.conf" exec dovecot-lda -e -m "foo" <<-EOF + From: <sender@example.net> + To: <recipient@example.net> + Subject: Hello world! + Date: $(date -R) + Message-ID: <$(</proc/sys/kernel/random/uuid)@example.net> + + Hello world! + EOF +<!-- --> + $ env -i PATH="/usr/bin:/bin" USER="testuser" \ + doveadm -c "$BASEDIR/dovecot.conf" flags add "\\Seen" mailbox "foo" "*" + +Normally [`dovecot-lda`(1)](https://wiki.dovecot.org/LDA) tries to do a +userdb lookup in order to determine the user's home directory. Since we +didn't configure a user database we need to explicitly set the `HOME` +environment variable. + + +InterIMAP configuration and test +================================ + +In this example the peers to synchronize are sharing the same Dovecot +configuration file `$BASEDIR/dovecot.conf`. Of course, it's also +possible to use a different configuration on each “server”, for instance +in order to specify different hierarchy delimiters, namespaces, or mail +storage format. + +Create an [`interimap`(1)] configuration file to synchronize the `local` +and `remote` accounts. + + $ cat >"$BASEDIR/interimap.conf" <<-EOF + database = $BASEDIR/interimap.db + + [local] + type = tunnel + command = env -i PATH="$PATH" USER="local" doveadm -c "$BASEDIR/dovecot.conf" exec imap + + [remote] + type = tunnel + command = env -i PATH="$PATH" USER="remote" doveadm -c "$BASEDIR/dovecot.conf" exec imap + EOF + +Run [`interimap`(1)] without `--watch` in order to create the database. + + $ env -i PATH="$PATH" perl -I./lib -T ./interimap --config="$BASEDIR/interimap.conf" + Creating new schema in database file …/interimap.db + database: Created mailbox INBOX + […] + +You can now run [`interimap`(1)] with `--watch` set, here to one second +to observe synchronisation steps early. + + $ env -i PATH="$PATH" perl -I./lib -T ./interimap --config="$BASEDIR/interimap.conf" \ + --watch=1 --debug + +Use instructions from the [previous section][Mail storage access] +(substituting `testuser` with `local` or `remote`) in order to simulate +activity on either end to synchronize. If you run these commands in +another shell, then make sure to re-set the `BASEDIR` environment +variable! + + +PullIMAP configuration and test +=============================== + +Create a [`pullimap`(1)] configuration file with as section `[foo]`. + + $ cat >"$BASEDIR/pullimap.conf" <<-EOF + [foo] + type = tunnel + command = env -i PATH="$PATH" USER="testuser" doveadm -c "$BASEDIR/dovecot.conf" exec imap + statefile = $BASEDIR/pullimap.foo + EOF + +Run [`pullimap`(1)] without `--idle` in order to create the state file. + + $ env -i PATH="$PATH" perl -I./lib -T ./pullimap --config="$BASEDIR/pullimap.conf" \ + --no-delivery foo + +You can now run [`pullimap`(1)] with `--idle` set. + + $ env -i PATH="$PATH" perl -I./lib -T ./pullimap --config="$BASEDIR/pullimap.conf" \ + --no-delivery --idle --debug foo + +Use instructions from the [previous section][Mail storage access] +in order to simulate activity on the “remote” server (in the relevant +mailbox — `INBOX` by default). If you run these commands in another +shell, then make sure to re-set the `BASEDIR` environment variable! + + +Cleanup +======= + +To remove temporary directories and the message they contain, simply +recursively remove the directory `$BASEDIR`. + + $ rm -rf -- "$BASEDIR" + + +[IMAP4rev1]: https://tools.ietf.org/html/rfc3501 +[`interimap`(1)]: interimap.1.html +[`pullimap`(1)]: pullimap.1.html +[`doveadm`(1)]: https://wiki.dovecot.org/Tools/Doveadm +[`%`-variable]: https://wiki.dovecot.org/Variables diff --git a/doc/index.md b/doc/index.md new file mode 100644 index 0000000..12de956 --- /dev/null +++ b/doc/index.md @@ -0,0 +1,20 @@ +% [`interimap`(1)] and [`pullimap`(1)] documentation +% Guilhem Moulin <guilhem@fripost.org> + +Manuals (HTML versions) +----------------------- + + * [`interimap`(1)] — Fast bidirectional synchronization for + QRESYNC-capable IMAP servers + * [`pullimap`(1)] — Pull mails from an IMAP mailbox and deliver them + to an SMTP session + +Resources for developers +------------------------ + + * [Source-code repository](https://git.guilhem.org/interimap) + * [Build instructions](build.html) + * [Test environment setup](development.html) + +[`interimap`(1)]: interimap.1.html +[`pullimap`(1)]: pullimap.1.html diff --git a/interimap.md b/doc/interimap.1.md index 50c1832..387850a 100644 --- a/interimap.md +++ b/doc/interimap.1.md @@ -1,4 +1,4 @@ -% intermap(1) +% interimap(1) % [Guilhem Moulin](mailto:guilhem@fripost.org) % July 2015 @@ -47,7 +47,7 @@ with the [RFC 7162] (sec. 6) amendments, and works as follows: 2. Propagate these changes onto the other server: get the corresponding UIDs from the database, then: - a. issue an `UID STORE` command, followed by `UID EXPUNGE`, to + a. issue a `UID STORE` command, followed by `UID EXPUNGE`, to remove messages that have not already been deleted on both servers; and b. issue some `UID STORE` commands to propagate flag updates (send @@ -62,9 +62,9 @@ with the [RFC 7162] (sec. 6) amendments, and works as follows: Otherwise, update the `HIGHESTMODSEQ` value in the database. 3. Process new messages (if the current `UIDNEXT` value of the mailbox - differs from the one found in the database) by issuing an `UID - FETCH` command; process each received message on-the-fly by issuing - an `APPEND` command with the message's `RFC822` body, `FLAGS` and + differs from the one found in the database) by issuing a `UID FETCH` + command; process each received message on-the-fly by issuing an + `APPEND` command with the message's `RFC822` body, `FLAGS` and `INTERNALDATE`. Repeat this step if the server received new messages in the meantime. Otherwise, update the `UIDNEXT` value in the database. @@ -111,8 +111,8 @@ other than the default [`QRESYNC`][RFC 7162]-based synchronization. existing UID; and 3/ ensure that both flag lists match. Any message found on a server but not in the database is replicated - on the other server (which in the worst case, might lead to a - message duplicate). + on the other server (which in the worst case, might yield a message + duplicate). Flag conflicts are solved by updating each message to the union of both lists. @@ -224,7 +224,7 @@ Valid options are: to the `perso/` sub-hierarchy on the local server. This is useful for synchronizing multiple remote servers against different namespaces belonging to the same local IMAP server (using a - different InterIMAP instance for each local namespace ↔ remote + different `interimap` instance for each local namespace ↔ remote synchronization). (Note that if the reference name is not a level of mailbox hierarchy @@ -281,10 +281,10 @@ Valid options are: : One of `imap`, `imaps` or `tunnel`. `type=imap` and `type=imaps` are respectively used for IMAP and IMAP - over SSL/TLS connections over a INET socket. + over SSL/TLS connections over an INET socket. `type=tunnel` causes `interimap` to create an unnamed pair of connected sockets for interprocess communication with a *command* - instead of a opening a network socket. + instead of opening a network socket. Note that specifying `type=tunnel` in the `[remote]` section makes the default *database* to be `localhost.db`. (Default: `imaps`.) @@ -314,7 +314,9 @@ Valid options are: : Command to use for `type=tunnel`. Must speak the [IMAP4rev1 protocol][RFC 3501] on its standard output, and understand it on its - standard input. + standard input. The value is passed to `` `/bin/sh -c` `` if it + contains shell metacharacters; otherwise it is split into words and + the resulting list is passed to `execvp`(3). *STARTTLS* @@ -348,7 +350,7 @@ Valid options are: *null-stderr* : Whether to redirect *command*'s standard error to `/dev/null` for - type `type=tunnel`. (Default: `NO`.) + `type=tunnel`. (Default: `NO`.) *SSL_protocols* @@ -374,9 +376,8 @@ Valid options are: Attempting to connect to a server with a non-matching certificate SPKI fingerprint causes `interimap` to abort the connection during the SSL/TLS handshake. - - You can use the following command to compute the SHA-256 digest of - certificate's Subject Public Key Info. + The following command can be used to compute the SHA-256 digest of a + certificate's Subject Public Key Info: openssl x509 -in /path/to/server/certificate.pem -pubkey \ | openssl pkey -pubin -outform DER \ diff --git a/pullimap.md b/doc/pullimap.1.md index a367dd1..1b2e509 100644 --- a/pullimap.md +++ b/doc/pullimap.1.md @@ -5,7 +5,7 @@ Name ==== -PullIMAP - Pull mails from an IMAP mailbox and deliver them to a SMTP session +PullIMAP - Pull mails from an IMAP mailbox and deliver them to an SMTP session Synopsis ======== @@ -16,8 +16,8 @@ Synopsis Description =========== -`pullimap` retrieves messages from an IMAP mailbox and deliver them to a -SMTP or LMTP transmission channel. It can also remove old messages +`pullimap` retrieves messages from an IMAP mailbox and deliver them to +an SMTP or LMTP transmission channel. It can also remove old messages after a configurable retention period. A *statefile* is used to keep track of the mailbox's `UIDVALIDITY` and @@ -130,9 +130,10 @@ Valid options are: : One of `imap`, `imaps` or `tunnel`. `type=imap` and `type=imaps` are respectively used for IMAP and IMAP - over SSL/TLS connections over a INET socket. - `type=tunnel` causes `pullimap` to open a pipe to a *command* - instead of a raw socket. + over SSL/TLS connections over an INET socket. + `type=tunnel` causes `pullimap` to create an unnamed pair of + connected sockets for interprocess communication with a *command* + instead of opening a network socket. (Default: `imaps`.) *host* @@ -160,7 +161,9 @@ Valid options are: : Command to use for `type=tunnel`. Must speak the [IMAP4rev1 protocol][RFC 3501] on its standard output, and understand it on its - standard input. + standard input. The value is passed to `` `/bin/sh -c` `` if it + contains shell metacharacters; otherwise it is split into words and + the resulting list is passed to `execvp`(3). *STARTTLS* @@ -192,7 +195,7 @@ Valid options are: *null-stderr* : Whether to redirect *command*'s standard error to `/dev/null` for - type `type=tunnel`. (Default: `NO`.) + `type=tunnel`. (Default: `NO`.) *SSL_protocols* @@ -218,9 +221,8 @@ Valid options are: Attempting to connect to a server with a non-matching certificate SPKI fingerprint causes `pullimap` to abort the connection during the SSL/TLS handshake. - - You can use the following command to compute the SHA-256 digest of - certificate's Subject Public Key Info. + The following command can be used to compute the SHA-256 digest of a + certificate's Subject Public Key Info: openssl x509 -in /path/to/server/certificate.pem -pubkey \ | openssl pkey -pubin -outform DER \ @@ -263,7 +265,7 @@ Usually there are only two integers: the first is the *mailbox*'s smaller than this `UIDNEXT` value have already been retrieved and delivered). The [IMAP4rev1 specification][RFC 3501] does not guaranty that untagged -`FETCH` responses are sent ordered by UID in response to an `UID FETCH` +`FETCH` responses are sent ordered by UID in response to a `UID FETCH` command. Thus it would be unsafe for `pullimap` to update the `UIDNEXT` value in its *statefile* while the `UID FETCH` command is progress. Instead, for each untagged `FETCH` response received while the `UID @@ -278,7 +280,7 @@ FETCH` command is in progress. In more details, `pullimap` works as follows: - 1. Issue an `UID FETCH` command to retrieve message `ENVELOPE` and + 1. Issue a `UID FETCH` command to retrieve message `ENVELOPE` and `RFC822` (and `UID`) with UID bigger or equal than the `UIDNEXT` value found in the *statefile*. While the `UID FETCH` command is in progress, perform the following @@ -296,10 +298,10 @@ In more details, `pullimap` works as follows: i. append the message UID to the *statefile*. - 2. If a SMTP/LMTP transmission channel was opened, send a `QUIT` command + 2. If an SMTP/LMTP transmission channel was opened, send a `QUIT` command to terminate it gracefully. - 3. Issue an `UID STORE` command to mark all retrieved messages (and + 3. Issue a `UID STORE` command to mark all retrieved messages (and stalled UIDs found in the *statefile* after the eigth byte) as `\Seen`. diff --git a/doc/template.html b/doc/template.html new file mode 100644 index 0000000..e17f0e3 --- /dev/null +++ b/doc/template.html @@ -0,0 +1,76 @@ +<!DOCTYPE html> +<html xmlns="http://www.w3.org/1999/xhtml" lang="$lang$" xml:lang="$lang$"$if(dir)$ dir="$dir$"$endif$> +<head> + <meta charset="utf-8" /> + <meta name="generator" content="pandoc" /> + <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=yes" /> +$for(author-meta)$ + <meta name="author" content="$author-meta$" /> +$endfor$ +$if(date-meta)$ + <meta name="dcterms.date" content="$date-meta$" /> +$endif$ +$if(keywords)$ + <meta name="keywords" content="$for(keywords)$$keywords$$sep$, $endfor$" /> +$endif$ + <title>$if(title-prefix)$$title-prefix$ – $endif$$pagetitle$</title> + <style type="text/css"> + code{white-space: pre-wrap;} + span.smallcaps{font-variant: small-caps;} + span.underline{text-decoration: underline;} + div.column{display: inline-block; vertical-align: top; width: 50%;} +$if(quotes)$ + q { quotes: "“" "”" "‘" "’"; } +$endif$ + </style> +$if(highlighting-css)$ + <style type="text/css"> +$highlighting-css$ + </style> +$endif$ +$for(css)$ + <link rel="stylesheet" href="$css$" /> +$endfor$ + <style type="text/css"> + @media(max-width: 1440px) { .container{ max-width: 1080px; } } + @media(max-width: 1280px) { .container{ max-width: 960px; } } + @media(max-width: 1024px) { .container{ max-width: 768px; } } + </style> +$if(math)$ + $math$ +$endif$ +$for(header-includes)$ + $header-includes$ +$endfor$ +</head> + +<body> +$for(include-before)$ +$include-before$ +$endfor$ +<div class="container text-justify"> + <div class="content"> +$if(title)$ + <div class="page-header"><h1>$title$</h1></div> +$endif$ + +$body$ + </div> + + <footer> + <hr/> + <div class="row"> + <div class="col-md-8 text-muted"> +$if(author)$ + <a href="https://git.guilhem.org/interimap/plain/COPYING">©</a> + $for(author)$$author$$sep$, $endfor$ +$endif$ + </div> + <div class="col-md-4 text-muted text-right small"> + $if(date)$$date$$endif$ + </div> + </div> + </footer> +</div> +</body> +</html> diff --git a/lib/Net/IMAP/InterIMAP.pm b/lib/Net/IMAP/InterIMAP.pm index 1dd54b7..19895c4 100644 --- a/lib/Net/IMAP/InterIMAP.pm +++ b/lib/Net/IMAP/InterIMAP.pm @@ -26,7 +26,8 @@ use Errno qw/EEXIST EINTR/; use Net::SSLeay 1.73 (); use List::Util qw/all first/; use POSIX ':signal_h'; -use Socket qw/SOCK_STREAM SOCK_RAW IPPROTO_TCP AF_UNIX AF_INET AF_INET6 PF_UNSPEC SOCK_CLOEXEC :addrinfo/; +use Socket qw/SOCK_STREAM SOCK_RAW SOCK_CLOEXEC IPPROTO_TCP SHUT_RDWR + AF_UNIX AF_INET AF_INET6 PF_UNSPEC :addrinfo/; use Exporter 'import'; BEGIN { @@ -506,7 +507,7 @@ sub stats($) { } -# Log out when the Net::IMAP::InterIMAP object is destroyed. +# Destroy a Net::IMAP::InterIMAP object. sub DESTROY($) { my $self = shift; $self->{_STATE} = 'LOGOUT'; @@ -514,8 +515,12 @@ sub DESTROY($) { Net::SSLeay::free($self->{_SSL}) if defined $self->{_SSL}; Net::SSLeay::CTX_free($self->{_SSL_CTX}) if defined $self->{_SSL_CTX}; - shutdown($self->{S}, 2) if $self->{type} ne 'tunnel' and defined $self->{S}; - $self->{S}->close() if defined $self->{S} and $self->{S}->opened(); + if (defined (my $s = $self->{S})) { + # for type=tunnel we assume the child won't linger around once + # we close its standard input and output. + shutdown($s, SHUT_RDWR); + $s->close() if $s->opened(); + } $self->stats() unless $self->{quiet}; } @@ -606,7 +611,7 @@ sub incapable($@) { # $self->search($criterion) -# Issue an UID SEARCH command with the given $criterion. For the "normal" +# Issue a UID SEARCH command with the given $criterion. For the "normal" # UID SEARCH command from RFC 3501, return the list of matching UIDs; # for the extended UID SEARCH command from RFC 4731 (ensuring ESEARCH # capability is the caller's responsibility), return an optional "UID" @@ -917,7 +922,7 @@ sub append($$@) { # $self->fetch($set, $flags, [$callback]) -# Issue an UID FETCH command with the given UID $set, $flags, and +# Issue a UID FETCH command with the given UID $set, $flags, and # optional $callback. sub fetch($$$;&) { my ($self, $set, $flags, $callback) = @_; @@ -1,7 +1,7 @@ #!/usr/bin/perl -T #---------------------------------------------------------------------- -# Pull mails from an IMAP mailbox and deliver them to a SMTP session +# Pull mails from an IMAP mailbox and deliver them to an SMTP session # Copyright © 2016-2018 Guilhem Moulin <guilhem@fripost.org> # # This program is free software: you can redistribute it and/or modify diff --git a/pullimap@.service b/pullimap@.service index 53694da..a9ce09a 100644 --- a/pullimap@.service +++ b/pullimap@.service @@ -1,5 +1,5 @@ [Unit] -Description=Pull mails from an IMAP mailbox and deliver them to a SMTP session (instance %i) +Description=Pull mails from an IMAP mailbox and deliver them to an SMTP session (instance %i) Documentation=man:pullimap(1) Wants=network-online.target After=network-online.target diff --git a/tests/00-db-exclusive/run b/tests/00-db-exclusive/run index 1ae27b6..1528b3b 100644 --- a/tests/00-db-exclusive/run +++ b/tests/00-db-exclusive/run @@ -9,8 +9,8 @@ interimap interimap --watch=60 & pid=$! cleanup() { # kill interimap process and its children - pkill -P "$pid" -TERM - kill -TERM "$pid" + pkill -P "$pid" -TERM || true + kill -TERM "$pid" || true wait } trap cleanup EXIT INT TERM diff --git a/tests/00-db-migration-0-to-1/run b/tests/00-db-migration-0-to-1/run index e4eb770..757fe04 100644 --- a/tests/00-db-migration-0-to-1/run +++ b/tests/00-db-migration-0-to-1/run @@ -18,9 +18,16 @@ sqlite3 "$XDG_DATA_HOME/interimap/remote.db" >"$TMPDIR/dump.sql" <<-EOF .dump EOF +# re-import and dump the expected dump to work around SQLite format +# differences across versions +sqlite3 "$XDG_DATA_HOME/interimap/remote2.db" <"$TESTDIR/after.sql" +sqlite3 "$XDG_DATA_HOME/interimap/remote2.db" >"$TMPDIR/dump-expected.sql" <<-EOF + .dump +EOF + # XXX need 'user_version' PRAGMA in the dump for future migrations # http://sqlite.1065341.n5.nabble.com/dump-command-and-user-version-td101228.html diff -u --label="a/dump.sql" --label="b/dump.sql" \ - "$TESTDIR/after.sql" "$TMPDIR/dump.sql" + "$TMPDIR/dump-expected.sql" "$TMPDIR/dump.sql" # vim: set filetype=sh : diff --git a/tests/05-repair/run b/tests/05-repair/run index 747b974..15553e0 100644 --- a/tests/05-repair/run +++ b/tests/05-repair/run @@ -65,7 +65,7 @@ spoof UIDNEXT "local" spoof HIGHESTMODSEQ "local" "remote" # now repair -interimap --repair "baz" "foo.bar" +interimap --repair "baz" "foo.bar" # 6 updates with \Answered (luid 4,8,11:13,16), 2 of which (luid 12,13) vanished from remote # 3 updates with \Seen (ruid 6,8,10), 1 of which (uid 10) vanished from remote diff --git a/tests/06-largeint/run b/tests/06-largeint/run index edcbd31..b08bcfa 100644 --- a/tests/06-largeint/run +++ b/tests/06-largeint/run @@ -7,9 +7,9 @@ doveadm -u "local" mailbox update --uid-validity 2147483648 "bar" # 2^31 doveadm -u "local" mailbox update --uid-validity 4294967295 "baz" # 2^32-1 doveadm -u "remote" mailbox update --uid-validity 4294967295 "INBOX" # 2^32-1 -doveadm -u "remote" mailbox update --uid-validity 2147483648 "foo" # 2^31 +doveadm -u "remote" mailbox update --uid-validity 2147483648 "foo" # 2^31 doveadm -u "remote" mailbox update --uid-validity 2147483647 "bar" # 2^31-1 -doveadm -u "remote" mailbox update --uid-validity 1 "baz" # +doveadm -u "remote" mailbox update --uid-validity 1 "baz" # run() { local u m diff --git a/tests/07-sync-live-multi/run b/tests/07-sync-live-multi/run index bf0d2f5..15a27fd 100644 --- a/tests/07-sync-live-multi/run +++ b/tests/07-sync-live-multi/run @@ -15,12 +15,12 @@ interimap --config="config3" --watch=1 & pid3=$! abort() { # kill interimap process and its children - pkill -P "$pid" -TERM - kill -TERM "$pid" - pkill -P "$pid2" -TERM - kill -TERM "$pid2" - pkill -P "$pid3" -TERM - kill -TERM "$pid3" + pkill -P "$pid" -TERM || true + kill -TERM "$pid" || true + pkill -P "$pid2" -TERM || true + kill -TERM "$pid2" || true + pkill -P "$pid3" -TERM || true + kill -TERM "$pid3" || true wait } trap abort EXIT INT TERM @@ -101,7 +101,7 @@ while [ $(date +%s) -le $timer ]; do done # sleep a little bit - sleep "0.$(shuf -n1 -i1-99)" + sleep "$(printf "0.%03d" "$(shuf -n1 -i1-999)")" done # wait a little longer so interimap has time to run loop() again and diff --git a/tests/07-sync-live/run b/tests/07-sync-live/run index 1950e0b..04d8247 100644 --- a/tests/07-sync-live/run +++ b/tests/07-sync-live/run @@ -6,8 +6,8 @@ interimap --watch=1 & pid=$! abort() { # kill interimap process and its children - pkill -P "$pid" -TERM - kill -TERM "$pid" + pkill -P "$pid" -TERM || true + kill -TERM "$pid" || true wait } trap abort EXIT INT TERM @@ -63,7 +63,7 @@ while [ $(date +%s) -le $timer ]; do done # sleep a little bit - sleep "0.$(shuf -n1 -i1-99)" + sleep "$(printf "0.%03d" "$(shuf -n1 -i1-999)")" done # wait a little longer so interimap has time to run loop() again and @@ -36,7 +36,7 @@ if [ ! -d "$TESTDIR" ]; then exit 1 fi -ROOTDIR="$(mktemp --tmpdir=/dev/shm --directory "$NAME.XXXXXXXXXX")" +ROOTDIR="$(mktemp --tmpdir="${TMPDIR:-/dev/shm}" --directory "$NAME.XXXXXXXXXX")" trap 'rm -rf -- "$ROOTDIR"' EXIT INT TERM STDOUT="$ROOTDIR/stdout" @@ -73,6 +73,7 @@ prepare() { cat >"$home/.config/dovecot/config" <<-EOF log_path = /dev/null mail_home = $ROOTDIR/home/%u + mailbox_list_index = yes ssl = no EOF cat >>"$home/.config/dovecot/config" <"$src" @@ -102,12 +103,12 @@ prepare() { [local] type = tunnel command = exec ${HOME_local@Q}/.local/bin/doveadm exec imap - null-stderr = YES + null-stderr = NO [remote] type = tunnel command = exec ${home@Q}/.local/bin/doveadm exec imap - null-stderr = YES + null-stderr = NO EOF done } @@ -129,14 +130,18 @@ doveadm() { shift 2 "$home/.local/bin/doveadm" "$@" } +sqlite3() { + command sqlite3 -init /dev/null "$@" +} # Sample (random) message sample_message() { + local date="$(date +"%s.%N")" cat <<-EOF From: <sender@example.net> To: <recipient@example.net> - Date: $(date -R) - Message-ID: <$(< /proc/sys/kernel/random/uuid)@example.net> + Date: $(date -R -d@"$date") + Message-ID: <$date@example.net> EOF local len="$(shuf -i1-4096 -n1)" @@ -156,7 +161,7 @@ deliver() { doveadm "${argv[@]}" exec dovecot-lda -e "$@" } -# Dump test results +# Dump test results dump_test_result() { local below=">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>" local above="<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<" @@ -215,7 +220,7 @@ check_mailbox_status2() { read MESSAGES < <( sqlite3 "$XDG_DATA_HOME/interimap/$u.db" <<-EOF .mode csv .separator " " "\\n" - SELECT COUNT(*) + SELECT COUNT(*) FROM mailboxes a JOIN mapping b ON a.idx = b.idx WHERE mailbox = $blob EOF @@ -319,7 +324,7 @@ xcgrep() { declare -a ENVIRON=() environ_set "local" export TMPDIR TESTDIR STDOUT STDERR "${ENVIRON[@]}" -export -f environ_set doveadm interimap sample_message deliver +export -f environ_set doveadm interimap sqlite3 sample_message deliver export -f check_mailbox_status check_mailbox_status_values check_mailbox_status2 export -f check_mailboxes_status check_mailbox_list xgrep xcgrep printf "%s..." "$TEST" |