diff options
| -rwxr-xr-x | imapsync | 10 | ||||
| -rw-r--r-- | imapsync.1 | 273 | ||||
| -rw-r--r-- | imapsync.sample | 6 | ||||
| -rw-r--r-- | lib/Net/IMAP/Sync.pm | 60 |
4 files changed, 311 insertions, 38 deletions
@@ -1,7 +1,7 @@ #!/usr/bin/perl -T #---------------------------------------------------------------------- -# A minimal IMAP4 client for QRESYNC-capable servers +# IMAP-to-IMAP synchronization program for QRESYNC-capable servers # Copyright © 2015 Guilhem Moulin <guilhem@fripost.org> # # This program is free software: you can redistribute it and/or modify @@ -1059,10 +1059,6 @@ while (defined (my $row = $STH_GET_CACHE->fetchrow_hashref())) { while (@REPAIR) { $MAILBOX = shift @REPAIR; - unless (defined $MAILBOX) { - cleanup(); - exit 0; - } $STH_GET_INDEX->execute($MAILBOX); ($IDX) = $STH_GET_INDEX->fetchrow_array(); @@ -1072,6 +1068,10 @@ while (@REPAIR) { $rIMAP->select($MAILBOX); repair($IDX, $MAILBOX); } +if ($CONFIG{repair}) { + cleanup(); + exit 0; +} while(1) { diff --git a/imapsync.1 b/imapsync.1 new file mode 100644 index 0000000..eda493a --- /dev/null +++ b/imapsync.1 @@ -0,0 +1,273 @@ +.TH IMAPSYNC "1" "JULY 2015" "imapsync" "User Commands" + +.SH NAME +imapsync \- IMAP-to-IMAP synchronization program for QRESYNC-capable servers + +.SH SYNOPSIS +.B imapsync\fR [\fIOPTION\fR ...] [\fIMAILBOX\fR ...] + + +.SH DESCRIPTION +.PP +.B imapsync\fR performs stateful synchronization between two IMAP4rev1 +servers, then (unless the flag \fB\-\-oneshot\fR is set) keeps both +connection open and wait for new changes to arrive. +Such synchronization is made possible by the QRESYNC extension from +[RFC7162]; for convenience reasons support for LIST\-EXTENDED [RFC5258], +LIST\-STATUS [RFC5819] and UIDPLUS [RFC4315] is also required. +Furthermore, support for LITERAL+ [RFC2088] and MULTIAPPEND [RFC3502] +is recommended: while they are not needed for \fBimapsync\fR to work, +these extensions greatly improve performance by reducing the number of +required round trips. + +.PP +Stateful synchronization is only possible for mailboxes supporting +persistent message Unique Identifiers (UID) and persistent storage of +mod\-sequences (MODSEQ); any non\-compliant mailbox will cause +\fBimapsync\fR to abort. +Furthermore, because UIDs are allocated not by the client but by the +server, \fBimapsync\fR needs to keep track of associations between local +and remote UIDs for each mailbox. +The synchronization state of a mailbox consists of its UIDNEXT and +HIGHESTMODSEQ values on each server; +it is then assumed that each message with UID < $UIDNEXT have been +replicated to the other server, and that the metadata (such as flags) of +each message with MODSEQ <= $HIGHESTMODSEQ have been synchronized. +Conceptually, the synchronization algorithm is derived from [RFC4549] +with the [RFC7162, section 6.1] amendments, and works as follows: + +.nr step 1 1 +.IP \n[step]. 8 +SELECT (on both servers) a mailbox the current UIDNEXT or HIGHESTMODSEQ +values of which differ from the values found in the database (for either +server). Use the QRESYNC SELECT parameter from [RFC7162] to list +changes (vanished messages and flag updates) since $HIGHESTMODSEQ to +messages with UID<$UIDNEXT. + +.IP \n+[step]. +Propagate these changes onto the other server: get the corresponding +UIDs from the database, then a/ issue an UID STORE + UID EXPUNGE command +to remove messages that have not already been deleted on both servers, +and /b issue UID STORE commands to propagate flag updates (send a single +command for each flag list in order the reduce the number of round +trips). +(Conflicts may occur if the metadata of a message has been updated on +both servers with different flag lists; in that case \fBimapsync\fR +issues a warning and updates the message on each server with the union +of both flag lists.) +Repeat this step if the server sent some updates in the meantime. +Otherwise, update the HIGHESTMODSEQ values in the database. + +.IP \n+[step]. +Process new messages (if the current UIDNEXT value differ from the one +found in the database) by issuing an UID FETCH command and for each +message RFC822 body received, issue an APPEND command to the other +server on\-the\-fly. +Repeat this step if the server received new messages in the meantime. +Otherwise, update the UIDNEXT values in the database. +Go back to step 2 if the server sent some updates in the meantime. + +.IP \n+[step]. +Go back to step 1 to proceed with the next unsynchronized mailbox. + +.PP +By default \fBimapsync\fR synchronizes each subscribed mailbox; +providing extra arguments limits the synchronization to the given +\fIMAILBOX\fRes only. + +.PP +In its default mode (unless the flag \fB\-\-oneshot\fR or +\fB\-\-repair\fR is set), \fBimapsync\fR does not exit once all +mailboxes have been synchronized. Instead, it keeps both connection +open and uses the NOTIFY command from [RFC5465] to be notified of new +changes (on any mailbox) as soon as they arrive. If no update is sent +in 15 minutes, a NOOP command is issued in order not to trigger the +servers' inactivity timeout and be logged out. + +.PP +If the synchronization was interrupted during a previous run while some +messages were being replicated (but before the UIDNEXT or HIGHESTMODSEQ +values have been updated), \fBimapsync\fR performs a \(lqfull +synchronization\(rq on theses messages only: +downloading the whole UID and flag lists on each servers allows +\fBimapsync\fR to detect messages that have been removed or for which +their flags have changed in the meantime. +Finally, after propagating the offline changes for these messages, +\fBimapsync\fR resumes the synchronization for the rest of the mailbox. + +.SH OPTIONS +.TP +.B \-\-config=\fR\fIFILE\fR +Specify an alternate configuration file. Relative paths start from +\fI$XDG_CONFIG_HOME\fR, or \fI~/.config\fR if the XDG_CONFIG_HOME +environment variable is unset. + +.TP +.B \-1\fR, \fB\-\-oneshot\fR +Exit as soon as all mailboxes are synchronized, instead of passively +waiting for updates from the open connections. +Using \fB\-\-oneshot\fR removes the requirement that IMAP servers must +advertise support the NOTIFY extension [RFC5465]. + +.TP +.B \-\-repair +List the database anomalies and try to repair them. +This is done by performing a so\-called \(lqfull synchronization\(rq, +namely 1/ download all UIDs along with their flags from both the local +and remote servers, 2/ ensure that each entry in the database corresponds +to an existing UID, and 3/ ensure that both flag lists match. +Any message found on a server but not in the database is replicated on +the other server (which in the worst case, might lead to a message +duplicate). +Flag conflicts are solved by updating each message to the union of both +lists. + +.TP +.B \-q\fR, \fB\-\-quiet\fR +Try to be quiet. + +.TP +.B \-\-debug +Turn on debug mode. +Note that all IMAP traffic (excluding literals) is then printed to the +error output. Depending on the chosen authentication mechanism, +this might include authentication credentials. + +.TP +.B \-h\fR, \fB\-\-help\fR +Output a brief help and exit. + +.TP +.B \-\-version +Show the version number and exit. + +.SH CONFIGURATION FILE + +Unless told otherwise by the \fB\-\-config=\fR\fIFILE\fR option, +\fBimapsync\fR reads its configuration from +\fI$XDG_CONFIG_HOME/imapsync\fR (or \fI~/.config/imapsync\fR if the +XDG_CONFIG_HOME environment variable is unset) as an INI file. +The syntax of the configuration file is a serie of +\fIOPTION\fR=\fIVALUE\fR lines organized under some \fI[SECTION]\fR; +lines starting with a \(oq#\(cq or \(oq;\(cq character are ignored as +comments. +The sections \(lq[local]\(rq and \(lq[remote]\(rq define the two IMAP +servers to synchronize. +Valid options are: + +.TP +.I database +SQLite version 3 database file to use to keep track of associations +between local and remote UIDs, as well as the UIDVALIDITY, UIDNEXT and +HIGHESTMODSEQ of each known mailbox on both servers. +Relative paths start from \fI$XDG_DATA_HOME/imapsync\fR, or +\fI~/.local/share/imapsync\fR if the XDG_DATA_HOME environment variable +is unset. +This option is only available in the default section. +(Default: \(lq\fIhost\fR.db\)\(rq, where \fIhost\fR is taken from the +\(lq[remote]\(rq or \(lq[local]\(rq sections, in that order. + +.TP +.I type +One of \(lqimap\(rq, \(lqimaps\(rq or \(lqtunnel\(rq. +\fItype\fR=imap and \fItype\fR=imaps are respectively used for IMAP and +IMAP over SSL/TLS connections over a INET socket. +\fItype\fR=tunnel causes \fBimapsync\fR to open a pipe to a +\fIcommand\fR instead of a raw socket. +(Default: \(lqimaps\(rq.) + +.TP +.I host +Server hostname, for \fItype\fR=imap and \fItype\fR=imaps. +(Default: \(lqlocalhost\(rq.) + +.TP +.I port +Server port. +(Default: \(lq143\(rq for \fItype\fR=imap, \(lq993\(rq for +\fItype\fR=imaps.) + +.TP +.I command +Command to use for \fItype\fR=tunnel. Must speak the IMAP4rev1 protocol +on its standard output, and understand it on its standard input. + +.TP +.I STARTTLS +Whether to use the \(lqSTARTTLS\(rq directive to upgrade a secure +connection. Setting this to \(lqYES\(rq for a server not advertising +the \(lqSTARTTLS\(rq capability causes \fBimapsync\fR to immediately +abort the connection. +(Ignored for \fItype\fRs other than \(lqimap\(rq. Default: \(lqYES\(rq.) + +.TP +.I auth +Space\-separated list of preferred authentication mechanisms. +\fBimapsync\fR uses the first mechanism in that list that is also +advertised (prefixed with \(lqAUTH=\(rq) in the server's capability list. +Supported authentication mechanisms are \(lqPLAIN\(rq and \(lqLOGIN\(rq. +(Default: \(lqPLAIN LOGIN\(rq.) + +.TP +.I username\fR, \fIpassword\fR +Username and password to authenticate with. Can be required for non +pre\-authenticated connections, depending on the chosen authentication +mechanism. + +.TP +.I SSL_cipher_list +Cipher list to use for the connection. +See \fIciphers\fR(1ssl) for the format of such list. + +.TP +.I SSL_fingerprint +Fingerprint of the server certificate in the form +\fIALGO\fR$\fIDIGEST_HEX\fR, where \fIALGO\fR is the used algorithm +(default \(lqsha256\(rq). +Attempting to connect to a server with a non-matching certificate +fingerprint causes \fBimapsync\fR to abort the connection immediately +after the SSL/TLS handshake. + +.TP +.I SSL_verify_trusted_peer +Whether to verify that the peer certificate has been signed by a trusted +Certificate Authority. Note that using \fISSL_fingerprint\fR to specify +the fingerprint of the server certificate is orthogonal and does not +rely on Certificate Authorities. +(Default: \(lqYES\(rq.) + +.TP +.I SSL_ca_path +Directory containing the certificate(s) of the trusted Certificate +Authorities, used for server certificate verification. + +.SH KNOWN BUGS AND LIMITATIONS + +.IP \[bu] 2 +Mailbox deletion and renaming are not very well tested yet. +.IP \[bu] +Detecting whether a mailbox has been renamed or deleted while +\fBimapsync\fR wasn't running is done by looking for a mailbox with same +UIDVALIDITY. [RFC3501] describes the purpose of UIDVALIDITY as to let +clients know when to invalidate their UID cache. In particular, there +is no requirement that two mailboxes can't share same UIDVALIDITY. +However such a possibility would defeat \fBimapsync\fR's heuristic to +detect whether a mailbox has been renamed or deleted offline. +.IP \[bu] +\fBimapsync\fR is single threaded and doesn't use IMAP command +pipelining. Performance improvement could be achieved by sending +independent commands to each server in parallel, and for a given server, +by sending independent commands (such as flag updates) in a pipeline. +.IP \[bu] +Because the IMAP protocol doesn't have a specific response code for when +a message is moved to another mailbox (using the MOVE command from +[RFC6851] or COPY + STORE + EXPUNGE), moving a messages causes +\fBimapsync\fR to believe that it was deleted while another one (which +is replicated again) was added to the other mailbox in the meantime. + +.IP \[bu] +\(lqPLAIN\(rq and \(lqLOGIN\(rq are the only authentication mechanisms +currently supported. + +.SH AUTHOR +Guilhem Moulin <guilhem@fripost.org> diff --git a/imapsync.sample b/imapsync.sample index 51958aa..e563e94 100644 --- a/imapsync.sample +++ b/imapsync.sample @@ -1,7 +1,7 @@ ; database = imap.guilhem.org.db [local] -type = preauth +type = tunnel command = /usr/lib/dovecot/imap [remote] @@ -12,9 +12,9 @@ username = guilhem password = xxxxxxxxxxxxxxxx ; SSL options -;SSL_verify_peer = TRUE -SSL_ca_path = /etc/ssl/certs ;SSL_cipher_list = EECDH+AES:EDH+AES:!MEDIUM:!LOW:!EXP:!aNULL:!eNULL:!SSLv2:!SSLv3:!TLSv1:!TLSv1.1 ;SSL_fingerprint = sha256$62E436BB329C46A628314C49BDA7C2A2E86C57B2021B9A964B8FABB6540D3605 +;SSL_verify_trusted_peer = YES +SSL_ca_path = /etc/ssl/certs ; vim:ft=dosini diff --git a/lib/Net/IMAP/Sync.pm b/lib/Net/IMAP/Sync.pm index 362d436..9db339b 100644 --- a/lib/Net/IMAP/Sync.pm +++ b/lib/Net/IMAP/Sync.pm @@ -39,17 +39,17 @@ my $RE_TEXT_CHAR = qr/[\x01-\x09\x0B\x0C\x0E-\x7F]/; my %OPTIONS = ( host => qr/\A([0-9a-zA-Z:.-]+)\z/, port => qr/\A([0-9]+)\z/, - type => qr/\A(imaps?|preauth)\z/, - STARTTLS => qr/\A(true|false)\z/i, + type => qr/\A(imaps?|tunnel)\z/, + STARTTLS => qr/\A(YES|NO)\z/i, username => qr/\A([\x01-\x7F]+)\z/, password => qr/\A([\x01-\x7F]+)\z/, auth => qr/\A($RE_ATOM_CHAR+(?: $RE_ATOM_CHAR+)*)\z/, - command => qr/\A(\P{Control}+)\z/, - 'read-only' => qr/\A(TRUE|FALSE)\z/i, - SSL_ca_path => qr/\A(\P{Control}+)\z/, - SSL_cipher_list => qr/\A(\P{Control}+)\z/, + command => qr/\A(\/\P{Control}+)\z/, + 'read-only' => qr/\A(YES|NO)\z/i, SSL_fingerprint => qr/\A([A-Za-z0-9]+\$\p{AHex}+)\z/, - SSL_verify_peer => qr/\A(TRUE|FALSE)\z/i, + SSL_cipher_list => qr/\A(\P{Control}+)\z/, + SSL_verify_trusted_peer => qr/\A(YES|NO)\z/i, + SSL_ca_path => qr/\A(\P{Control}+)\z/, ); @@ -87,7 +87,7 @@ sub read_config($$%) { $conf->{host} //= 'localhost'; $conf->{port} //= $conf->{type} eq 'imaps' ? 993 : $conf->{type} eq 'imap' ? 143 : undef; $conf->{auth} //= 'PLAIN LOGIN'; - $conf->{STARTTLS} //= 'TRUE'; + $conf->{STARTTLS} //= 'YES'; # untaint and validate the config foreach my $k (keys %$conf) { @@ -203,7 +203,7 @@ our $IMAP_text; # # - 'enable': An extension or array reference of extensions to ENABLE # (RFC 5161) after entering AUTH state. Croak if the server did not -# advertize "ENABLE" in its CAPABILITY list or does not reply with +# advertise "ENABLE" in its CAPABILITY list or does not reply with # an untagged ENABLED response with all the given extensions. # # - 'STDERR': Where to log debug and informational messages (default: @@ -225,7 +225,7 @@ sub new($%) { bless $self, $class; # whether we're allowed to to use read-write command - $self->{'read-only'} = uc ($self->{'read-only'} // 'FALSE') ne 'TRUE' ? 0 : 1; + $self->{'read-only'} = uc ($self->{'read-only'} // 'NO') ne 'YES' ? 0 : 1; # where to log $self->{STDERR} //= \*STDERR; @@ -234,10 +234,10 @@ sub new($%) { # (cf RFC 3501 section 3) $self->{_STATE} = ''; - if ($self->{type} eq 'preauth') { + if ($self->{type} eq 'tunnel') { require 'IPC/Open2.pm'; - my $command = $self->{command} // $self->fail("Missing preauth command"); - my $pid = IPC::Open2::open2(@$self{qw/STDOUT STDIN/}, split(/ /, $command)) + my $command = $self->{command} // $self->fail("Missing tunnel command"); + my $pid = IPC::Open2::open2(@$self{qw/STDOUT STDIN/}, $command) or $self->panic("Can't fork: $!"); } else { @@ -252,8 +252,8 @@ sub new($%) { } else { require 'IO/Socket/SSL.pm'; - if (defined (my $vrfy = delete $self->{SSL_verify_peer})) { - $args{SSL_verify_mode} = 0 if uc $vrfy eq 'FALSE'; + if (defined (my $vrfy = delete $self->{SSL_verify_trusted_peer})) { + $args{SSL_verify_mode} = 0 if uc $vrfy eq 'NO'; } my $fpr = delete $self->{SSL_fingerprint}; $args{$_} = $self->{$_} foreach grep /^SSL_/, keys %$self; @@ -311,16 +311,16 @@ sub new($%) { $self->{_STATE} = 'UNAUTH'; my @caps = $self->capabilities(); - if ($self->{type} eq 'imap' and uc $self->{STARTTLS} ne 'FALSE') { # RFC 2595 section 5.1 - $self->fail("Server did not advertize STARTTLS capability.") + if ($self->{type} eq 'imap' and uc $self->{STARTTLS} ne 'NO') { # RFC 2595 section 5.1 + $self->fail("Server did not advertise STARTTLS capability.") unless grep {$_ eq 'STARTTLS'} @caps; require 'IO/Socket/SSL.pm'; $self->_send('STARTTLS'); my %sslargs; - if (defined (my $vrfy = delete $self->{SSL_verify_peer})) { - $sslargs{SSL_verify_mode} = 0 if uc $vrfy eq 'FALSE'; + if (defined (my $vrfy = delete $self->{SSL_verify_trusted_peer})) { + $sslargs{SSL_verify_mode} = 0 if uc $vrfy eq 'NO'; } my $fpr = delete $self->{SSL_fingerprint}; $sslargs{$_} = $self->{$_} foreach grep /^SSL_/, keys %$self; @@ -373,7 +373,7 @@ sub new($%) { : ref $self->{enable} eq 'ARRAY' ? @{$self->{enable}} : ($self->{enable}); if (@extensions) { - $self->fail("Server did not advertize ENABLE (RFC 5161) capability.") unless $self->_capable('ENABLE'); + $self->fail("Server did not advertise ENABLE (RFC 5161) capability.") unless $self->_capable('ENABLE'); $self->_send('ENABLE '.join(' ',@extensions)); my @enabled = @{$self->{_ENABLED} // []}; $self->fail("Couldn't ENABLE $_") foreach @@ -451,7 +451,7 @@ sub capabilities($) { # $self->incapable(@capabilities) # In list context, return the list capabilties from @capabilities -# which were NOT advertized by the server. In scalar context, return +# which were NOT advertised by the server. In scalar context, return # the length of said list. sub incapable($@) { my ($self, @caps) = @_; @@ -569,7 +569,7 @@ sub list($$@) { # $self->remove_message($uid, [...]) -# Remove the given $uid list. Croak if the server did not advertize +# Remove the given $uid list. Croak if the server did not advertise # "UIDPLUS" (RFC 4315) in its CAPABILITY list. # Successfully EXPUNGEd UIDs are removed from the pending VANISHED and # MODIFIED lists. @@ -577,7 +577,7 @@ sub list($$@) { sub remove_message($@) { my $self = shift; my @set = @_; - $self->fail("Server did not advertize UIDPLUS (RFC 4315) capability.") + $self->fail("Server did not advertise UIDPLUS (RFC 4315) capability.") if $self->incapable('UIDPLUS'); my $set = compact_set(@set); @@ -609,8 +609,8 @@ sub remove_message($@) { # $self->append($mailbox, $mail, [...]) # Issue an APPEND command with the given mails. Croak if the server -# did not advertize "UIDPLUS" (RFC 4315) in its CAPABILITY list. -# Providing multiple mails is only allowed for servers advertizing +# did not advertise "UIDPLUS" (RFC 4315) in its CAPABILITY list. +# Providing multiple mails is only allowed for servers advertising # "MULTIAPPEND" (RFC 3502) in their CAPABILITY list. # Return the list of UIDs allocated for the new messages. sub append($$@) { @@ -618,7 +618,7 @@ sub append($$@) { my $mailbox = shift; return unless @_; $self->fail("Server is read-only.") if $self->{'read-only'}; - $self->fail("Server did not advertize UIDPLUS (RFC 4315) capability.") + $self->fail("Server did not advertise UIDPLUS (RFC 4315) capability.") if $self->incapable('UIDPLUS'); my @appends; @@ -630,7 +630,7 @@ sub append($$@) { $append .= "{".length($mail->{RFC822})."}\r\n".$mail->{RFC822}; push @appends, $append; } - $self->fail("Server did not advertize MULTIAPPEND (RFC 3502) capability.") + $self->fail("Server did not advertise MULTIAPPEND (RFC 3502) capability.") if $#appends > 0 and $self->incapable('MULTIAPPEND'); # dump the cache before issuing the command if we're appending to the current mailbox @@ -692,10 +692,10 @@ sub fetch($$$$) { # $self->notify(@specifications) # Issue a NOTIFY command with the given mailbox @specifications (cf RFC # 5465 section 6) to be monitored. Croak if the server did not -# advertize "NOTIFY" (RFC 5465) in its CAPABILITY list. +# advertise "NOTIFY" (RFC 5465) in its CAPABILITY list. sub notify($@) { my $self = shift; - $self->fail("Server did not advertize NOTIFY (RFC 5465) capability.") + $self->fail("Server did not advertise NOTIFY (RFC 5465) capability.") if $self->incapable('NOTIFY'); my $events = join ' ', qw/MessageNew MessageExpunge FlagChange MailboxName SubscriptionChange/; # Be notified of new messages with EXISTS/RECENT responses, but @@ -1216,7 +1216,7 @@ sub _select_or_examine($$$) { ($pcache->{HIGHESTMODSEQ} // 0) > 0 and ($pcache->{UIDNEXT} // 1) > 1; if ($self->{_STATE} eq 'SELECTED' and ($self->_capable('CONDSTORE') or $self->_capable('QRESYNC'))) { - # A mailbox is currently selected and the server advertizes + # A mailbox is currently selected and the server advertises # 'CONDSTORE' or 'QRESYNC' (RFC 7162). Delay the mailbox # selection until the [CLOSED] response code has been received: # all responses before the [CLOSED] response code refer to the |