diff options
| -rw-r--r-- | Changelog | 6 | ||||
| -rw-r--r-- | INSTALL | 7 | ||||
| -rw-r--r-- | README | 59 | ||||
| -rwxr-xr-x | interimap (renamed from imapsync) | 12 | ||||
| -rw-r--r-- | interimap.1 (renamed from imapsync.1) | 53 | ||||
| -rw-r--r-- | interimap.sample (renamed from imapsync.sample) | 0 | ||||
| -rw-r--r-- | interimap.service (renamed from imapsync.service) | 4 | ||||
| -rw-r--r-- | lib/Net/IMAP/InterIMAP.pm (renamed from lib/Net/IMAP/Sync.pm) | 6 |
8 files changed, 89 insertions, 58 deletions
@@ -1,5 +1,5 @@ -imapsync (0.1) upstream; +interimap (0.1) upstream; - * Initial release. + * Initial public release. Development was started in July 2015. - -- Guilhem Moulin <guilhem@guilhem.org> Thu, 23 Jul 2015 04:15:47 +0200 + -- Guilhem Moulin <guilhem@guilhem.org> Mon, 07 Sep 2015 17:14:42 +0200 @@ -1,4 +1,4 @@ -imapsync depends on the following Perl modules: +InterIMAP depends on the following Perl modules: - Config::Tiny - DBI @@ -23,5 +23,10 @@ However Debian GNU/Linux users can also use gbp(1) from git-buildpackage to build their own package: $ git checkout debian + $ AUTO_DEBSIGN=no gbp buildpackage + +Alternatively, for the development version: + + $ git checkout debian $ git merge master $ AUTO_DEBSIGN=no gbp buildpackage --git-force-create --git-upstream-tree=BRANCH @@ -1,17 +1,34 @@ -imapsync performs stateful synchronization between two IMAP4rev1 -servers. Such synchronization is made possible by the QRESYNC extension -from [RFC7162]; for convenience reasons servers must also support -LIST-EXTENDED [RFC5258], LIST-STATUS [RFC5819] and UIDPLUS [RFC4315]. -Furthermore, while imapsync can work with servers lacking support for -LITERAL+ [RFC2088] and MULTIAPPEND [RFC3502], these extensions greatly -improve performance by reducing the number of required round trips hence -are recommended. - -Consult the manual for more information. - -imapsync is Copyright© 2015 Guilhem Moulin ⟨guilhem@fripost.org⟩, and -licensed for use under the GNU General Public License version 3 or -later. See ‘COPYING’ for specific terms and distribution information. +InterIMAP is a fast two-way synchronization program for QRESYNC-capable +IMAP4rev1 servers. Consult the manual for more information. + + +####################################################################### + + +Compared to IMAP-to-Maildir synchronization solutions like OfflineIMAP, +adding an IMAP server between the Maildir storage and the MUA saves +loads of readdir(2) system calls and other File System quirks; moreover +the abstraction layer offered by the IMAP server makes the MUA and +synchronization program agnostic to the storage backend (Maildir, mbox, +dbox,...) in use. + +IMAP synchronization of a mailbox is usually two-folds: 1/ detect and +propagate changes (flag updates and message deletions) to existing +messages, then 2/ copy the new messages. The naive way to perform the +first step is to issue a FETCH command to list all messages in the +mailbox along with their flags and UIDs, causing heavy network usage. +Instead, InterIMAP takes advantage of the QRESYNC extension from +[RFC7162] to perform stateful synchronization: querying changes since +the last synchronization only gives a phenomenal performance boost and +drastically reduces the network traffic. + +For convenience reasons servers must also support LIST-EXTENDED +[RFC5258], LIST-STATUS [RFC5819] and UIDPLUS [RFC4315]. Furthermore, +while InterIMAP can work with servers lacking support for LITERAL+ +[RFC2088] and MULTIAPPEND [RFC3502], these extensions greatly improve +performance by reducing the number of required round trips hence are +recommended. + ####################################################################### @@ -33,14 +50,14 @@ the AUTHENTICATE command. For instance the following configuration snippet saves bandwidth and brings a significant speed gain compared to type=imaps. - local: $XDG_CONFIG_HOME/imapsync: + local: $XDG_CONFIG_HOME/interimap: [remote] type = tunnel command = /usr/bin/ssh user@imap.example.net local: ~/.ssh/config: Host imap.example.net - IdentityFile ~/.ssh/id-imapsync + IdentityFile ~/.ssh/id-interimap IdentitiesOnly yes ControlPath /run/shm/%u@%n ControlMaster auto @@ -52,4 +69,12 @@ type=imaps. Compression yes remote: ~user/.ssh/authorized_keys: - command="/usr/lib/dovecot/imap",no-agent-forwarding,no-port-forwarding,no-pty,no-user-rc,no-X11-forwarding ssh-... id-imapsync + command="/usr/lib/dovecot/imap",no-agent-forwarding,no-port-forwarding,no-pty,no-user-rc,no-X11-forwarding ssh-... id-interimap + + +####################################################################### + + +InterIMAP is Copyright© 2015 Guilhem Moulin ⟨guilhem@fripost.org⟩, and +licensed for use under the GNU General Public License version 3 or +later. See ‘COPYING’ for specific terms and distribution information. @@ -1,7 +1,7 @@ #!/usr/bin/perl -T #---------------------------------------------------------------------- -# IMAP-to-IMAP synchronization program for QRESYNC-capable servers +# Fast two-way synchronization program for QRESYNC-capable IMAP servers # Copyright © 2015 Guilhem Moulin <guilhem@fripost.org> # # This program is free software: you can redistribute it and/or modify @@ -22,14 +22,14 @@ use strict; use warnings; our $VERSION = '0.1'; -my $NAME = 'imapsync'; +my $NAME = 'interimap'; use Getopt::Long qw/:config posix_default no_ignore_case gnu_compat bundling auto_version/; use DBI (); use List::Util 'first'; use lib 'lib'; -use Net::IMAP::Sync qw/read_config compact_set $IMAP_text $IMAP_cond/; +use Net::IMAP::InterIMAP qw/read_config compact_set $IMAP_text $IMAP_cond/; # Clean up PATH $ENV{PATH} = join ':', qw{/usr/local/bin /usr/bin /bin}; @@ -229,7 +229,7 @@ foreach my $name (qw/local remote/) { $config{name} = $name; $config{'logger-fd'} = $LOGGER_FD if defined $LOGGER_FD; - $IMAP->{$name} = { client => Net::IMAP::Sync::->new(%config) }; + $IMAP->{$name} = { client => Net::IMAP::InterIMAP::->new(%config) }; my $client = $IMAP->{$name}->{client}; die "Non $_-capable IMAP server.\n" foreach $client->incapable(qw/LIST-EXTENDED LIST-STATUS UIDPLUS/); @@ -254,8 +254,8 @@ foreach my $name (qw/local remote/) { $list .= (defined $CONF->{_}->{'list-mailbox'} ? '('.$CONF->{_}->{'list-mailbox'}.')' : '*') unless @ARGV; @params = ('SUBSCRIBED', 'STATUS (UIDVALIDITY UIDNEXT HIGHESTMODSEQ)'); } - $list .= $#ARGV == 0 ? Net::IMAP::Sync::quote($ARGV[0]) - : ('('.join(' ',map {Net::IMAP::Sync::quote($_)} @ARGV).')') if @ARGV; + $list .= $#ARGV == 0 ? Net::IMAP::InterIMAP::quote($ARGV[0]) + : ('('.join(' ',map {Net::IMAP::InterIMAP::quote($_)} @ARGV).')') if @ARGV; @{$IMAP->{$name}}{qw/mailboxes delims/} = $client->list($list, @params); } @@ -1,20 +1,21 @@ -.TH IMAPSYNC "1" "JULY 2015" "imapsync" "User Commands" +.TH INTERIMAP "1" "JULY 2015" "InterIMAP" "User Commands" .SH NAME -imapsync \- IMAP-to-IMAP synchronization program for QRESYNC-capable servers +InterIMAP \- Fast two-way synchronization program for QRESYNC-capable +IMAP servers .SH SYNOPSIS -.B imapsync\fR [\fIOPTION\fR ...] [\fICOMMAND\fR] [\fIMAILBOX\fR ...] +.B interimap\fR [\fIOPTION\fR ...] [\fICOMMAND\fR] [\fIMAILBOX\fR ...] .SH DESCRIPTION .PP -.B imapsync\fR performs stateful synchronization between two IMAP4rev1 +.B InterIMAP\fR performs stateful synchronization between two IMAP4rev1 servers. Such synchronization is made possible by the QRESYNC extension from [RFC7162]; for convenience reasons servers must also support LIST\-EXTENDED [RFC5258], LIST\-STATUS [RFC5819] and UIDPLUS [RFC4315]. -Furthermore, while \fBimapsync\fR can work with servers lacking support +Furthermore, while \fBInterIMAP\fR can work with servers lacking support for LITERAL+ [RFC2088] and MULTIAPPEND [RFC3502], these extensions greatly improve performance by reducing the number of required round trips hence are recommended. @@ -23,9 +24,9 @@ trips hence are recommended. Stateful synchronization is only possible for mailboxes supporting persistent message Unique Identifiers (UID) and persistent storage of mod\-sequences (MODSEQ); any non\-compliant mailbox will cause -\fBimapsync\fR to abort. +\fBInterIMAP\fR to abort. Furthermore, because UIDs are allocated not by the client but by the -server, \fBimapsync\fR needs to keep track of associations between local +server, \fBInterIMAP\fR needs to keep track of associations between local and remote UIDs for each mailbox. The synchronization state of a mailbox consists of its UIDNEXT and HIGHESTMODSEQ values on each server; @@ -51,7 +52,7 @@ and b/ issue UID STORE commands to propagate flag updates (send a single command for each flag list in order the reduce the number of round trips). (Conflicts may occur if the metadata of a message has been updated on -both servers with different flag lists; in that case \fBimapsync\fR +both servers with different flag lists; in that case \fBInterIMAP\fR issues a warning and updates the message on each server with the union of both flag lists.) Repeat this step if the server sent some updates in the meantime. @@ -71,13 +72,13 @@ Go back to step 1 to proceed with the next unsynchronized mailbox. .SH COMMANDS .PP -By default \fBimapsync\fR synchronizes each mailbox listed by the +By default \fBInterIMAP\fR synchronizes each mailbox listed by the \(lqLIST "" "*"\(rq IMAP command; the \fIlist-mailbox\fR, \fIlist-select-opts\fR and \fIignore-mailbox\fR options from the configuration file can be used to shrink that list and save bandwidth. However if some extra argument are provided on the command line, -\fBimapsync\fR ignores said options and synchronizes the given +\fBInterIMAP\fR ignores said options and synchronizes the given \fIMAILBOX\fRes instead. Note that each \fIMAILBOX\fR is taken \(lqas is\(rq; in particular, it must be UTF-7 encoded, unquoted, and the list wildcards \(oq*\(cq and \(oq%\(cq are not interpolated. @@ -85,16 +86,16 @@ wildcards \(oq*\(cq and \(oq%\(cq are not interpolated. .PP If the synchronization was interrupted during a previous run while some messages were being replicated (but before the UIDNEXT or HIGHESTMODSEQ -values have been updated), \fBimapsync\fR performs a \(lqfull +values have been updated), \fBInterIMAP\fR performs a \(lqfull synchronization\(rq on theses messages only: downloading the whole UID and flag lists on each servers allows -\fBimapsync\fR to detect messages that have been removed or for which +\fBInterIMAP\fR to detect messages that have been removed or for which their flags have changed in the meantime. Finally, after propagating the offline changes for these messages, -\fBimapsync\fR resumes the synchronization for the rest of the mailbox. +\fBInterIMAP\fR resumes the synchronization for the rest of the mailbox. .PP -Specifying one of the commands below makes \fBimapsync\fR perform an +Specifying one of the commands below makes \fBInterIMAP\fR perform an action other than the default QRESYNC-based synchronization. .TP @@ -125,7 +126,7 @@ children are not deleted. Rename the mailbox \fISOURCE\fR to \fIDEST\fR on each target (by default each server plus the database, unless \fB\-\-target\fR specifies otherwise) where it exists. -\fBimapsync\fR aborts if \fIDEST\fR already exists on either target. +\fBInterIMAP\fR aborts if \fIDEST\fR already exists on either target. Note that per [RFC3501] the renaming is recursive: \fISOURCE\fR's children are moved to become \fIDEST\fR's children instead. @@ -164,8 +165,8 @@ Show the version number and exit. .SH CONFIGURATION FILE Unless told otherwise by the \fB\-\-config=\fR\fIFILE\fR option, -\fBimapsync\fR reads its configuration from -\fI$XDG_CONFIG_HOME/imapsync\fR (or \fI~/.config/imapsync\fR if the +\fBInterIMAP\fR reads its configuration from +\fI$XDG_CONFIG_HOME/interimap\fR (or \fI~/.config/interimap\fR if the XDG_CONFIG_HOME environment variable is unset) as an INI file. The syntax of the configuration file is a serie of \fIOPTION\fR=\fIVALUE\fR lines organized under some \fI[SECTION]\fR; @@ -180,8 +181,8 @@ Valid options are: SQLite version 3 database file to use to keep track of associations between local and remote UIDs, as well as the UIDVALIDITY, UIDNEXT and HIGHESTMODSEQ of each known mailbox on both servers. -Relative paths start from \fI$XDG_DATA_HOME/imapsync\fR, or -\fI~/.local/share/imapsync\fR if the XDG_DATA_HOME environment variable +Relative paths start from \fI$XDG_DATA_HOME/interimap\fR, or +\fI~/.local/share/interimap\fR if the XDG_DATA_HOME environment variable is unset. This option is only available in the default section. (Default: \(lq\fIhost\fR.db\)\(rq, where \fIhost\fR is taken from the @@ -233,7 +234,7 @@ only available in the default section. One of \(lqimap\(rq, \(lqimaps\(rq or \(lqtunnel\(rq. \fItype\fR=imap and \fItype\fR=imaps are respectively used for IMAP and IMAP over SSL/TLS connections over a INET socket. -\fItype\fR=tunnel causes \fBimapsync\fR to open a pipe to a +\fItype\fR=tunnel causes \fBInterIMAP\fR to open a pipe to a \fIcommand\fR instead of a raw socket. Note that specifying \fItype\fR=tunnel in the \(lq[remote]\(rq section makes the default \fIdatabase\fR to be \(lqlocalhost.db\(rq. @@ -259,14 +260,14 @@ on its standard output, and understand it on its standard input. .I STARTTLS Whether to use the \(lqSTARTTLS\(rq directive to upgrade to a secure connection. Setting this to \(lqYES\(rq for a server not advertising -the \(lqSTARTTLS\(rq capability causes \fBimapsync\fR to immediately +the \(lqSTARTTLS\(rq capability causes \fBInterIMAP\fR to immediately abort the connection. (Ignored for \fItype\fRs other than \(lqimap\(rq. Default: \(lqYES\(rq.) .TP .I auth Space\-separated list of preferred authentication mechanisms. -\fBimapsync\fR uses the first mechanism in that list that is also +\fBInterIMAP\fR uses the first mechanism in that list that is also advertised (prefixed with \(lqAUTH=\(rq) in the server's capability list. Supported authentication mechanisms are \(lqPLAIN\(rq and \(lqLOGIN\(rq. (Default: \(lqPLAIN LOGIN\(rq.) @@ -288,7 +289,7 @@ Fingerprint of the server certificate in the form \fIALGO\fR$\fIDIGEST_HEX\fR, where \fIALGO\fR is the used algorithm (default \(lqsha256\(rq). Attempting to connect to a server with a non-matching certificate -fingerprint causes \fBimapsync\fR to abort the connection immediately +fingerprint causes \fBInterIMAP\fR to abort the connection immediately after the SSL/TLS handshake. .TP @@ -307,11 +308,11 @@ Authorities, used for server certificate verification. .SH KNOWN BUGS AND LIMITATIONS .IP \[bu] -Using \fBimapsync\fR on two identical servers with a non-existent or +Using \fBInterIMAP\fR on two identical servers with a non-existent or empty database will duplicate each message due to the absence of local/remote UID association. .IP \[bu] -\fBimapsync\fR is single threaded and doesn't use IMAP command +\fBInterIMAP\fR is single threaded and doesn't use IMAP command pipelining. Synchronization could be boosted up by sending independent commands (such as the initial LIST/STATUS command) to each server in parallel, and for a given server, by sending independent commands (such @@ -320,7 +321,7 @@ as flag updates) in a pipeline. Because the IMAP protocol doesn't have a specific response code for when a message is moved to another mailbox (using the MOVE command from [RFC6851] or COPY + STORE + EXPUNGE), moving a messages causes -\fBimapsync\fR to believe that it was deleted while another one (which +\fBInterIMAP\fR to believe that it was deleted while another one (which is replicated again) was added to the other mailbox in the meantime. .IP \[bu] diff --git a/imapsync.sample b/interimap.sample index 296f766..296f766 100644 --- a/imapsync.sample +++ b/interimap.sample diff --git a/imapsync.service b/interimap.service index 02b4d13..7f2d035 100644 --- a/imapsync.service +++ b/interimap.service @@ -1,10 +1,10 @@ [Unit] -Description=IMAP-to-IMAP Syncronization service +Description=Fast two-way synchronization program for QRESYNC-capable IMAP servers Wants=network-online.target After=network-online.target [Service] -ExecStart=/usr/bin/imapsync +ExecStart=/usr/bin/interimap RestartSec=60s Restart=always diff --git a/lib/Net/IMAP/Sync.pm b/lib/Net/IMAP/InterIMAP.pm index 6561a66..26cfbbd 100644 --- a/lib/Net/IMAP/Sync.pm +++ b/lib/Net/IMAP/InterIMAP.pm @@ -16,7 +16,7 @@ # along with this program. If not, see <http://www.gnu.org/licenses/>. #---------------------------------------------------------------------- -package Net::IMAP::Sync v0.0.1; +package Net::IMAP::InterIMAP v0.0.1; use warnings; use strict; @@ -195,7 +195,7 @@ our $IMAP_cond; our $IMAP_text; -# Create a new Net::IMAP::Sync object. Connect to the server, +# Create a new Net::IMAP::InterIMAP object. Connect to the server, # upgrade to a secure connection (STARTTLS), LOGIN/AUTHENTICATE if needed, and # update the CAPABILITY list. # In addition to the %OPTIONS above, valid parameters include: @@ -384,7 +384,7 @@ sub new($%) { } -# Log out when the Net::IMAP::Sync object is destroyed. +# Log out when the Net::IMAP::InterIMAP object is destroyed. sub DESTROY($) { my $self = shift; foreach (qw/STDIN STDOUT/) { |