diff options
| -rw-r--r-- | Makefile | 4 | ||||
| -rwxr-xr-x | interimap | 170 | ||||
| -rw-r--r-- | interimap.1 | 403 | ||||
| -rw-r--r-- | interimap.md | 492 | ||||
| -rw-r--r-- | lib/Net/IMAP/InterIMAP.pm | 180 | ||||
| -rwxr-xr-x | pullimap | 9 | ||||
| -rw-r--r-- | pullimap.md | 291 |
7 files changed, 832 insertions, 717 deletions
@@ -1,4 +1,4 @@ -all: pullimap.1 +all: pullimap.1 interimap.1 # upper case the headers and remove the links %.1: %.md @@ -24,6 +24,6 @@ all: pullimap.1 install: clean: - rm -f pullimap.1 + rm -f pullimap.1 interimap.1 .PHONY: all install clean @@ -53,7 +53,7 @@ sub usage(;$) { } my @COMMANDS = qw/repair delete rename/; -usage(1) unless GetOptions(\%CONFIG, qw/config=s quiet|q target=s@ debug help|h watch:i/, @COMMANDS); +usage(1) unless GetOptions(\%CONFIG, qw/config=s quiet|q target=s@ debug help|h watch:i notify/, @COMMANDS); usage(0) if $CONFIG{help}; my $COMMAND = do { my @command = grep {exists $CONFIG{$_}} @COMMANDS; @@ -61,9 +61,9 @@ my $COMMAND = do { $command[0] }; usage(1) if defined $COMMAND and (($COMMAND eq 'delete' and !@ARGV) or ($COMMAND eq 'rename' and $#ARGV != 1)); -usage(1) if defined $COMMAND and defined $CONFIG{watch}; +usage(1) if defined $COMMAND and (defined $CONFIG{watch} or defined $CONFIG{notify}); usage(1) if $CONFIG{target} and !(defined $COMMAND and ($COMMAND eq 'delete'or $COMMAND eq 'rename')); -$CONFIG{watch} = 60 if defined $CONFIG{watch} and $CONFIG{watch} == 0; +$CONFIG{watch} = $CONFIG{notify} ? 900 : 60 unless $CONFIG{watch}; @ARGV = map {uc $_ eq 'INBOX' ? 'INBOX' : $_ } @ARGV; # INBOX is case-insensitive die "Invalid mailbox name $_" foreach grep !/\A([\x01-\x7F]+)\z/, @ARGV; @@ -76,7 +76,7 @@ my $CONF = read_config( delete $CONFIG{config} // $NAME , 'list-select-opts' => qr/\A([\x21\x23\x24\x26\x27\x2B-\x5B\x5E-\x7A\x7C-\x7E]+)\z/ , 'ignore-mailbox' => qr/\A([\x01-\x09\x0B\x0C\x0E-\x7F]+)\z/ ); -my ($DBFILE, $LOCKFILE, $LOGGER_FD); +my ($DBFILE, $LOGGER_FD); { $DBFILE = $CONF->{_}->{database} if defined $CONF->{_}; @@ -94,8 +94,6 @@ my ($DBFILE, $LOCKFILE, $LOGGER_FD); } } - $LOCKFILE = $DBFILE =~ s/([^\/]+)\z/.$1.lck/r; - if (defined $CONF->{_} and defined $CONF->{_}->{logfile}) { require 'POSIX.pm'; require 'Time/HiRes.pm'; @@ -116,7 +114,6 @@ my ($IMAP, $lIMAP, $rIMAP); sub cleanup() { undef $_ foreach grep defined, ($IMAP, $lIMAP, $rIMAP); logger(undef, "Cleaning up...") if $CONFIG{debug}; - unlink $LOCKFILE if defined $LOCKFILE and -f $LOCKFILE; close $LOGGER_FD if defined $LOGGER_FD; $DBH->disconnect() if defined $DBH; } @@ -125,33 +122,16 @@ $SIG{TERM} = sub { cleanup(); exit 0; }; ############################################################################# -# Lock the database -{ - if (-f $LOCKFILE) { - open my $lock, '<', $LOCKFILE or die "Can't open $LOCKFILE: $!\n"; - my $pid = <$lock>; - close $lock; - chomp $pid; - my $msg = "LOCKFILE '$LOCKFILE' exists."; - undef $LOCKFILE; # don't delete the lockfile - $msg .= " (Is PID $pid running?)" if defined $pid and $pid =~ /^[0-9]+$/; - die $msg, "\n"; - } - - open my $lock, '>', $LOCKFILE or die "Can't open $LOCKFILE: $!\n"; - print $lock $$, "\n"; - close $lock; -} - - -############################################################################# # Open the database and create tables $DBH = DBI::->connect("dbi:SQLite:dbname=$DBFILE", undef, undef, { AutoCommit => 0, RaiseError => 1, sqlite_see_if_its_a_number => 1, # see if the bind values are numbers or not + sqlite_use_immediate_transaction => 1, }); +$DBH->sqlite_busy_timeout(250); +$DBH->do('PRAGMA locking_mode = EXCLUSIVE'); $DBH->do('PRAGMA foreign_keys = ON'); @@ -236,10 +216,13 @@ logger(undef, ">>> $NAME $VERSION"); my $LIST = '"" '; my @LIST_PARAMS; +my %LIST_PARAMS_STATUS = (STATUS => [qw/UIDVALIDITY UIDNEXT HIGHESTMODSEQ/]); if (!defined $COMMAND or $COMMAND eq 'repair') { $LIST = '('.uc($CONF->{_}->{'list-select-opts'}).') '.$LIST if defined $CONF->{_}->{'list-select-opts'}; $LIST .= (defined $CONF->{_}->{'list-mailbox'} ? '('.$CONF->{_}->{'list-mailbox'}.')' : '*') unless @ARGV; - @LIST_PARAMS = ('SUBSCRIBED', 'STATUS (UIDVALIDITY UIDNEXT HIGHESTMODSEQ)'); + @LIST_PARAMS = ('SUBSCRIBED'); + push @LIST_PARAMS, map { "$_ (".join(' ', @{$LIST_PARAMS_STATUS{$_}}).")" } keys %LIST_PARAMS_STATUS + unless $CONFIG{notify}; } $LIST .= $#ARGV == 0 ? Net::IMAP::InterIMAP::quote($ARGV[0]) : ('('.join(' ',map {Net::IMAP::InterIMAP::quote($_)} @ARGV).')') if @ARGV; @@ -257,20 +240,8 @@ foreach my $name (qw/local remote/) { $IMAP->{$name} = { client => Net::IMAP::InterIMAP::->new(%config) }; my $client = $IMAP->{$name}->{client}; - die "Non $_-capable IMAP server.\n" foreach $client->incapable(qw/LIST-EXTENDED LIST-STATUS UIDPLUS/); - # XXX We should start by listing all mailboxes matching the user's LIST - # criterion, then issue "SET NOTIFY (mailboxes ... (...))". But this - # crashes the IMAP client: - # http://dovecot.org/pipermail/dovecot/2015-July/101473.html - #my $mailboxes = $client->list((uc $config{'subscribed-only'} eq 'TRUE' ? '(SUBSCRIBED)' : '' ) - # .$config{mailboxes}, 'SUBSCRIBED'); - # $client->notify('SELECTED', 'MAILBOXES ('.join(' ', keys %$mailboxes).')'); - # XXX NOTIFY doesn't work as expected for INBOX - # http://dovecot.org/pipermail/dovecot/2015-July/101514.html - #$client->notify(qw/SELECTED SUBSCRIBED/) if $CONFIG{watch}; - # XXX We shouldn't need to ask for STATUS responses here, and use - # NOTIFY's STATUS indicator instead. However Dovecot violates RFC - # 5464: http://dovecot.org/pipermail/dovecot/2015-July/101474.html + die "Non $_-capable IMAP server.\n" foreach $client->incapable(qw/LIST-EXTENDED UIDPLUS/); + die "Non LIST-STATUS-capable IMAP server.\n" if !$CONFIG{notify} and $client->incapable('LIST-STATUS'); } @{$IMAP->{$_}}{qw/mailboxes delims/} = $IMAP->{$_}->{client}->list($LIST, @LIST_PARAMS) for qw/local remote/; @@ -425,9 +396,8 @@ elsif (defined $COMMAND and $COMMAND eq 'rename') { ############################################################################## # Synchronize mailbox and subscription lists -my @MAILBOXES; sub sync_mailbox_list() { - my %mailboxes; + my (%mailboxes, @mailboxes); $mailboxes{$_} = 1 foreach keys %{$IMAP->{local}->{mailboxes}}; $mailboxes{$_} = 1 foreach keys %{$IMAP->{remote}->{mailboxes}}; my $sth_subscribe = $DBH->prepare(q{UPDATE mailboxes SET subscribed = ? WHERE idx = ?}); @@ -444,7 +414,7 @@ sub sync_mailbox_list() { }; check_delim($mailbox); # ensure that the delimiter match - push @MAILBOXES, $mailbox unless grep {lc $_ eq lc '\NoSelect'} @attrs; + push @mailboxes, $mailbox unless grep {lc $_ eq lc '\NoSelect'} @attrs; $STH_GET_INDEX->execute($mailbox); my ($idx,$subscribed) = $STH_GET_INDEX->fetchrow_array(); @@ -512,9 +482,10 @@ sub sync_mailbox_list() { $DBH->commit(); } } + return @mailboxes; } -sync_mailbox_list(); +my @MAILBOXES = sync_mailbox_list(); ($lIMAP, $rIMAP) = map {$IMAP->{$_}->{client}} qw/local remote/; my $ATTRS = join ' ', qw/MODSEQ FLAGS INTERNALDATE BODY.PEEK[]/; @@ -1066,26 +1037,6 @@ sub sync_messages($$;$$) { } -# Wait up to $timout seconds for notifications on either IMAP server. -# Then issue a NOOP so the connection doesn't terminate for inactivity. -sub wait_notifications(;$) { - my $timeout = shift // 300; - - while ($timeout > 0) { - my $r1 = $lIMAP->slurp(); - my $r2 = $rIMAP->slurp(); - last if $r1 or $r2; # got update! - - sleep 1; - if (--$timeout == 0) { - $lIMAP->noop(); - $rIMAP->noop(); - # might have got updates so exit the loop - } - } -} - - ############################################################################# # Resume interrupted mailbox syncs (before initializing the cache). # @@ -1166,7 +1117,27 @@ if (defined $COMMAND and $COMMAND eq 'repair') { } -while(1) { +if ($CONFIG{notify}) { + # Be notified of new messages with EXISTS/RECENT responses, but don't + # receive unsolicited FETCH responses with a RFC822/BODY[]. It costs us an + # extra roundtrip, but we need to sync FLAG updates and VANISHED responses + # in batch mode, update the HIGHESTMODSEQ, and *then* issue an explicit UID + # FETCH command to get new message, and process each FETCH response with a + # RFC822/BODY[] attribute as they arrive. + my $mailboxes = join(' ', map {Net::IMAP::InterIMAP::quote($_)} @MAILBOXES); + my %mailboxes = map { $_ => [qw/MessageNew MessageExpunge FlagChange/] } + ( "MAILBOXES ($mailboxes)", 'SELECTED' ); + my %personal = ( personal => [qw/MailboxName SubscriptionChange/] ); + + foreach ($lIMAP, $rIMAP) { + # require STATUS responses for our @MAILBOXES only + $_->notify('SET STATUS', %mailboxes); + $_->notify('SET', %mailboxes, %personal); + } +} + + +sub loop() { while(@MAILBOXES) { if (defined $MAILBOX and ($lIMAP->is_dirty($MAILBOX) or $rIMAP->is_dirty($MAILBOX))) { # $MAILBOX is dirty on either the local or remote mailbox @@ -1202,26 +1173,55 @@ while(1) { sync_messages($IDX, $MAILBOX); } } - # clean state! - exit 0 unless $CONFIG{watch}; - - # we need to issue a NOOP command or go back to AUTH state since the - # LIST command may not report the correct HIGHESTMODSEQ value for - # the mailbox currently selected. - if (defined $MAILBOX) { - # Prefer UNSELECT over NOOP commands as it requires a single command per cycle - if ($lIMAP->incapable('UNSELECT') or $rIMAP->incapable('UNSELECT')) { - $_->noop() foreach ($lIMAP, $rIMAP); - } else { - $_->unselect() foreach ($lIMAP, $rIMAP); - undef $MAILBOX; - } +} +sub notify(@) { + # TODO: interpret LIST responses to detect mailbox + # creation/deletion/subcription/unsubscription + # mailbox creation + # * LIST () "/" test + # mailbox subscribtion + # * LIST (\Subscribed) "/" test + # mailbox unsubscribtion + # * LIST () "/" test + # mailbox renaming + # * LIST () "/" test2 ("OLDNAME" (test)) + # mailbox deletion + # * LIST (\NonExistent) "/" test2 + unless (Net::IMAP::InterIMAP::slurp(\@_, $CONFIG{watch}, \&Net::IMAP::InterIMAP::is_dirty)) { + $_->noop() foreach @_; + } +} + +unless (defined $CONFIG{watch}) { + loop(); + exit 0; +} + +while (1) { + loop(); + + if ($CONFIG{notify}) { + notify($lIMAP, $rIMAP); } + else { + # we need to issue a NOOP command or go back to AUTH state since the + # LIST command may not report the correct HIGHESTMODSEQ value for + # the mailbox currently selected + if (defined $MAILBOX) { + # Prefer UNSELECT over NOOP commands as it requires a single command per cycle + if ($lIMAP->incapable('UNSELECT') or $rIMAP->incapable('UNSELECT')) { + $_->noop() foreach ($lIMAP, $rIMAP); + } else { + $_->unselect() foreach ($lIMAP, $rIMAP); + undef $MAILBOX; + } + } - sleep $CONFIG{watch}; - # Refresh the mailbox list and status - @{$IMAP->{$_}}{qw/mailboxes delims/} = $IMAP->{$_}->{client}->list($LIST, @LIST_PARAMS) for qw/local remote/; - sync_mailbox_list(); + sleep $CONFIG{watch}; + # refresh the mailbox list and status + @{$IMAP->{$_}}{qw/mailboxes delims/} = $IMAP->{$_}->{client}->list($LIST, @LIST_PARAMS) for qw/local remote/; + @MAILBOXES = sync_mailbox_list(); + } } END { cleanup(); } diff --git a/interimap.1 b/interimap.1 deleted file mode 100644 index 3aabc3f..0000000 --- a/interimap.1 +++ /dev/null @@ -1,403 +0,0 @@ -.TH INTERIMAP "1" "JULY 2015" "InterIMAP" "User Commands" - -.SH NAME -InterIMAP \- Fast bidirectional synchronization for QRESYNC-capable IMAP servers - -.SH SYNOPSIS -.B interimap\fR [\fIOPTION\fR ...] [\fICOMMAND\fR] [\fIMAILBOX\fR ...] - - -.SH DESCRIPTION -.PP -.B InterIMAP\fR performs stateful synchronization between two IMAP4rev1 -servers. -Such synchronization is made possible by the QRESYNC extension from -[RFC7162]; for convenience reasons servers must also support -LIST\-EXTENDED [RFC5258], LIST\-STATUS [RFC5819] and UIDPLUS [RFC4315]. -See also the \fBSUPPORTED EXTENSIONS\fR section. - -.PP -Stateful synchronization is only possible for mailboxes supporting -persistent message Unique Identifiers (UID) and persistent storage of -mod\-sequences (MODSEQ); any non\-compliant mailbox will cause -\fBInterIMAP\fR to abort. -Furthermore, because UIDs are allocated not by the client but by the -server, \fBInterIMAP\fR needs to keep track of associations between local -and remote UIDs for each mailbox. -The synchronization state of a mailbox consists of its UIDNEXT and -HIGHESTMODSEQ values on each server; -it is then assumed that each message with UID < $UIDNEXT have been -replicated to the other server, and that the metadata (such as flags) of -each message with MODSEQ <= $HIGHESTMODSEQ have been synchronized. -Conceptually, the synchronization algorithm is derived from [RFC4549] -with the [RFC7162, section 6] amendments, and works as follows: - -.nr step 1 1 -.IP \n[step]. 4 -SELECT (on both servers) a mailbox the current UIDNEXT or HIGHESTMODSEQ -values of which differ from the values found in the database (for either -server). Use the QRESYNC SELECT parameter from [RFC7162] to list -changes (vanished messages and flag updates) since $HIGHESTMODSEQ to -messages with UID<$UIDNEXT. - -.IP \n+[step]. -Propagate these changes onto the other server: get the corresponding -UIDs from the database, then a/ issue an UID STORE + UID EXPUNGE command -to remove messages that have not already been deleted on both servers, -and b/ issue UID STORE commands to propagate flag updates (send a single -command for each flag list in order the reduce the number of round -trips). -(Conflicts may occur if the metadata of a message has been updated on -both servers with different flag lists; in that case \fBInterIMAP\fR -issues a warning and updates the message on each server with the union -of both flag lists.) -Repeat this step if the server sent some updates in the meantime. -Otherwise, update the HIGHESTMODSEQ value in the database. - -.IP \n+[step]. -Process new messages (if the current UIDNEXT value differ from the one -found in the database) by issuing an UID FETCH command and for each -message RFC822 body received, issue an APPEND command to the other -server on\-the\-fly. -Repeat this step if the server received new messages in the meantime. -Otherwise, update the UIDNEXT value in the database. -Go back to step 2 if the server sent some updates in the meantime. - -.IP \n+[step]. -Go back to step 1 to proceed with the next unsynchronized mailbox. - -.SH COMMANDS -.PP -By default \fBInterIMAP\fR synchronizes each mailbox listed by the -\(lqLIST "" "*"\(rq IMAP command; -the \fIlist-mailbox\fR, \fIlist-select-opts\fR and \fIignore-mailbox\fR -options from the configuration file can be used to shrink that list and -save bandwidth. -However if some extra argument are provided on the command line, -\fBInterIMAP\fR ignores said options and synchronizes the given -\fIMAILBOX\fRes instead. Note that each \fIMAILBOX\fR is taken \(lqas -is\(rq; in particular, it must be UTF-7 encoded, unquoted, and the list -wildcards \(oq*\(cq and \(oq%\(cq are not interpolated. - -.PP -If the synchronization was interrupted during a previous run while some -messages were being replicated (but before the UIDNEXT or HIGHESTMODSEQ -values have been updated), \fBInterIMAP\fR performs a \(lqfull -synchronization\(rq on theses messages only: -downloading the whole UID and flag lists on each servers allows -\fBInterIMAP\fR to detect messages that have been removed or for which -their flags have changed in the meantime. -Finally, after propagating the offline changes for these messages, -\fBInterIMAP\fR resumes the synchronization for the rest of the mailbox. - -.PP -Specifying one of the commands below makes \fBInterIMAP\fR perform an -action other than the default QRESYNC-based synchronization. - -.TP -.B \-\-repair \fR[\fIMAILBOX\fR ...] -List the database anomalies and try to repair them. -(Consider only the given \fIMAILBOX\fRes if non-optional arguments are -provided.) -This is done by performing a so\-called \(lqfull synchronization\(rq, -namely 1/ download all UIDs along with their flags from both the local -and remote servers, 2/ ensure that each entry in the database corresponds -to an existing UID, and 3/ ensure that both flag lists match. -Any message found on a server but not in the database is replicated on -the other server (which in the worst case, might lead to a message -duplicate). -Flag conflicts are solved by updating each message to the union of both -lists. - -.TP -.B \-\-delete \fIMAILBOX\fR [...] -Delete the given \fIMAILBOX\fRes on each target (by default each server -plus the database, unless \fB\-\-target\fR specifies otherwise) where -it exists. -Note that per [RFC3501] deletion is not recursive: \fIMAILBOX\fR's -children are not deleted. - -.TP -.B \-\-rename \fISOURCE\fR \fIDEST\fR -Rename the mailbox \fISOURCE\fR to \fIDEST\fR on each target (by default -each server plus the database, unless \fB\-\-target\fR specifies -otherwise) where it exists. -\fBInterIMAP\fR aborts if \fIDEST\fR already exists on either target. -Note that per [RFC3501] the renaming is recursive: \fISOURCE\fR's -children are moved to become \fIDEST\fR's children instead. - - -.SH OPTIONS -.TP -.B \-\-config=\fR\fIFILE\fR -Specify an alternate configuration file. Relative paths start from -\fI$XDG_CONFIG_HOME\fR, or \fI~/.config\fR if the XDG_CONFIG_HOME -environment variable is unset. - -.TP -.B \fB\-\-target=\fR{local,remote,database} -Limit the scope of a \fB\-\-delete\fR or \fB\-\-rename\fR command -to the given target. Can be repeated to act on multiple targets. By -default all three targets are considered. - -.TP -.B \fB\-\-watch\fR[\fB=\fR\fIseconds\fR] -Don't exit after a successful synchronization, and keep synchronizing -forever instead. Sleep for the given number of \fIseconds\fR (or -\(lq60\(rq if omitted) between two synchronizations. - -.TP -.B \-q\fR, \fB\-\-quiet\fR -Try to be quiet. - -.TP -.B \-\-debug -Turn on debug mode. Debug messages are written to the given \fIlogfile\fR. -Note that this include all IMAP traffic (except literals). Depending on the -chosen authentication mechanism, this might include authentication credentials. - -.TP -.B \-h\fR, \fB\-\-help\fR -Output a brief help and exit. - -.TP -.B \-\-version -Show the version number and exit. - -.SH CONFIGURATION FILE - -Unless told otherwise by the \fB\-\-config=\fR\fIFILE\fR option, -\fBInterIMAP\fR reads its configuration from -\fI$XDG_CONFIG_HOME/interimap\fR (or \fI~/.config/interimap\fR if the -XDG_CONFIG_HOME environment variable is unset) as an INI file. -The syntax of the configuration file is a series of -\fIOPTION\fR=\fIVALUE\fR lines organized under some \fI[SECTION]\fR; -lines starting with a \(oq#\(cq or \(oq;\(cq character are ignored as -comments. -The sections \(lq[local]\(rq and \(lq[remote]\(rq define the two IMAP -servers to synchronize. -Valid options are: - -.TP -.I database -SQLite version 3 database file to use to keep track of associations -between local and remote UIDs, as well as the UIDVALIDITY, UIDNEXT and -HIGHESTMODSEQ of each known mailbox on both servers. -Relative paths start from \fI$XDG_DATA_HOME/interimap\fR, or -\fI~/.local/share/interimap\fR if the XDG_DATA_HOME environment variable -is unset. -This option is only available in the default section. -(Default: \(lq\fIhost\fR.db\)\(rq, where \fIhost\fR is taken from the -\(lq[remote]\(rq or \(lq[local]\(rq sections, in that order.) - -.TP -.I list-mailbox -A space separated list of mailbox patterns to use when issuing the -initial LIST command (overridden by the \fIMAILBOX\fRes given as -command-line arguments). -Note that each pattern containing special characters such as spaces or -brackets (see [RFC3501] for the exact syntax) must be quoted. -Furthermore, non-ASCII names must be UTF\-7 encoded. -Two wildcards are available: a \(oq*\(cq character matches zero or more -characters, while a \(oq%\(cq character matches zero or more characters -up to the mailbox's hierarchy delimiter. -This option is only available in the default section. -(The default pattern, \(lq*\(rq, matches all visible mailboxes on the -server.) - -.TP -.I list-select-opts -An optional space separated list of selectors for the initial LIST -command. (Requires a server supporting the LIST-EXTENDED [RFC5258] -extension.) Useful values are -\(lqSUBSCRIBED\(rq (to list only subscribed mailboxes), -\(lqREMOTE\(rq (to also list remote mailboxes on a server supporting -mailbox referrals), and \(lqRECURSIVEMATCH\(rq (to list parent mailboxes -with children matching one of the \fIlist-mailbox\fR patterns above). -This option is only available in the default section. - -.TP -.I ignore-mailbox -An optional Perl Compatible Regular Expressions (PCRE) covering -mailboxes to exclude: -any (UTF-7 encoded, unquoted) mailbox listed in the initial LIST -responses is ignored if it matches the given expression. -Note that the \fIMAILBOX\fRes given as command-line arguments bypass the -check and are always considered for synchronization. -This option is only available in the default section. - -.TP -.I logfile -A file name to use to log debug and informational messages. This option is -only available in the default section. - -.TP -.I type -One of \(lqimap\(rq, \(lqimaps\(rq or \(lqtunnel\(rq. -\fItype\fR=imap and \fItype\fR=imaps are respectively used for IMAP and -IMAP over SSL/TLS connections over a INET socket. -\fItype\fR=tunnel causes \fBInterIMAP\fR to open a pipe to a -\fIcommand\fR instead of a raw socket. -Note that specifying \fItype\fR=tunnel in the \(lq[remote]\(rq section -makes the default \fIdatabase\fR to be \(lqlocalhost.db\(rq. -(Default: \(lqimaps\(rq.) - -.TP -.I host -Server hostname, for \fItype\fR=imap and \fItype\fR=imaps. -(Default: \(lqlocalhost\(rq.) - -.TP -.I port -Server port. -(Default: \(lq143\(rq for \fItype\fR=imap, \(lq993\(rq for -\fItype\fR=imaps.) - -.TP -.I proxy -An optional SOCKS proxy to use for TCP connections to the IMAP server -(\fItype\fR=imap and \fItype\fR=imaps only), formatted as -\(lq\fIprotocol\fR://[\fIuser\fR:\fIpassword\fR@]\fIproxyhost\fR[:\fIproxyport\fR]\(rq. -If \fIproxyport\fR is omitted, it is assumed at port 1080. -Only SOCKSv5 is supported, in two flavors: \(lqsocks5://\(rq to resolve -\fIhostname\fR locally, and \(lqsocks5h://\(rq to let the proxy resolve -\fIhostname\fR. - -.TP -.I command -Command to use for \fItype\fR=tunnel. Must speak the IMAP4rev1 protocol -on its standard output, and understand it on its standard input. - -.TP -.I STARTTLS -Whether to use the \(lqSTARTTLS\(rq directive to upgrade to a secure -connection. Setting this to \(lqYES\(rq for a server not advertising -the \(lqSTARTTLS\(rq capability causes \fBInterIMAP\fR to immediately -abort the connection. -(Ignored for \fItype\fRs other than \(lqimap\(rq. Default: \(lqYES\(rq.) - -.TP -.I auth -Space\-separated list of preferred authentication mechanisms. -\fBInterIMAP\fR uses the first mechanism in that list that is also -advertised (prefixed with \(lqAUTH=\(rq) in the server's capability list. -Supported authentication mechanisms are \(lqPLAIN\(rq and \(lqLOGIN\(rq. -(Default: \(lqPLAIN LOGIN\(rq.) - -.TP -.I username\fR, \fIpassword\fR -Username and password to authenticate with. Can be required for non -pre\-authenticated connections, depending on the chosen authentication -mechanism. - -.TP -.I compress -Whether to use the IMAP COMPRESS extension [RFC4978] for servers -advertising it. -(Default: \(lqNO\(rq for the \(lq[local]\(rq section, \(lqYES\(rq for -the \(lq[remote]\(rq section.) - -.TP -.I null\-stderr -Whether to redirect \fIcommand\fR's standard error to \(lq/dev/null\(rq -for type \fItype\fR=tunnel. -(Default: \(lqNO\(rq.) - -.TP -.I SSL_protocols -A space-separated list of SSL protocols to enable or disable (if -prefixed with an exclamation mark \(oq!\(cq). Known protocols are -\(lqSSLv2\(rq, \(lqSSLv3\(rq, \(lqTLSv1\(rq, \(lqTLSv1.1\(rq, and -\(lqTLSv1.2\(rq. Enabling a protocol is a short-hand for disabling all -other protocols. -(Default: \(lq!SSLv2 !SSLv3\(rq, i.e., only enable TLSv1 and above.) - -.TP -.I SSL_cipher_list -The cipher list to send to the server. Although the server determines -which cipher suite is used, it should take the first supported cipher in -the list sent by the client. See \fBciphers\fR(1ssl) for more -information. - -.TP -.I SSL_fingerprint -Fingerprint of the server certificate (or its public key) in the form -\fIALGO\fR$\fIDIGEST_HEX\fR, where \fIALGO\fR is the used algorithm -(default \(lqsha256\(rq). -Attempting to connect to a server with a non-matching certificate -fingerprint causes \fBInterIMAP\fR to abort the connection during the -SSL/TLS handshake. - -.TP -.I SSL_verify -Whether to verify the server certificate chain. -Note that using \fISSL_fingerprint\fR to specify the fingerprint of the -server certificate is an orthogonal authentication measure as it ignores -the CA chain. -(Default: \(lqYES\(rq.) - -.TP -.I SSL_CApath -Directory to use for server certificate verification if -\(lq\fISSL_verify\fR=YES\(rq. -This directory must be in \(lqhash format\(rq, see \fBverify\fR(1ssl) -for more information. - -.TP -.I SSL_CAfile -File containing trusted certificates to use during server certificate -authentication if \(lq\fISSL_verify\fR=YES\(rq. - -.SH SUPPORTED EXTENSIONS - -Performance is better for servers supporting the following extensions to -the IMAP4rev1 [RFC3501] protocol: - -.IP \[bu] 4 -LITERAL+ [RFC2088] non-synchronizing literals (recommended), -.IP \[bu] -MULTIAPPEND [RFC3502] (recommended), -.IP \[bu] -COMPRESS=DEFLATE [RFC4978] (recommended), -.IP \[bu] -SASL-IR [RFC4959] SASL Initial Client Response, and -.IP \[bu] -UNSELECT [RFC3691]. - -.SH KNOWN BUGS AND LIMITATIONS - -.IP \[bu] 4 -Using \fBInterIMAP\fR on two identical servers with a non-existent or -empty database will duplicate each message due to the absence of -local/remote UID association. -.IP \[bu] -\fBInterIMAP\fR is single threaded and doesn't use IMAP command -pipelining. Synchronization could be boosted up by sending independent -commands (such as the initial LIST/STATUS command) to each server in -parallel, and for a given server, by sending independent commands (such -as flag updates) in a pipeline. -.IP \[bu] -Because the IMAP protocol doesn't have a specific response code for when -a message is moved to another mailbox (using the MOVE command from -[RFC6851] or COPY + STORE + EXPUNGE), moving a messages causes -\fBInterIMAP\fR to believe that it was deleted while another one (which -is replicated again) was added to the other mailbox in the meantime. -.IP \[bu] -\(lqPLAIN\(rq and \(lqLOGIN\(rq are the only authentication mechanisms -currently supported. -.IP \[bu] -\fBInterIMAP\fR will probably not work with non RFC-compliant servers. -In particular, no work-around are currently implemented beside the -tunables in the \fBCONFIGURATION FILE\fR. Moreover, few IMAP servers -have been tested so far. - -.SH AUTHOR -.ie \n[www-html] \{\ - Written by -. MTO guilhem@fripost.org "Guilhem Moulin" . -\} -.el \{\ - Written by Guilhem Moulin -. MT guilhem@fripost.org -. ME . -\} diff --git a/interimap.md b/interimap.md new file mode 100644 index 0000000..9515c4a --- /dev/null +++ b/interimap.md @@ -0,0 +1,492 @@ +% intermap(1) +% [Guilhem Moulin](mailto:guilhem@fripost.org) +% July 2015 + +Name +==== + +InterIMAP - Fast bidirectional synchronization for QRESYNC-capable IMAP servers + +Synopsis +======== + +`interimap` [*OPTION* ...] [*COMMAND*] [*MAILBOX* ...] + +Description +=========== + +`interimap` performs stateful synchronization between two IMAP4rev1 +servers. +Such synchronization is made possible by the [`QRESYNC` IMAP +extension][RFC 7162]; for convenience reasons servers must also support +the [`LIST-EXTENDED`][RFC 5258], [`LIST-STATUS`][RFC 5819] (or +[`NOTIFY`][RFC 5465]) and [`UIDPLUS`][RFC 4315] IMAP extensions. +See also the **[supported extensions](#supported-extensions)** section +below. + +Stateful synchronization is only possible for mailboxes supporting +persistent message Unique Identifiers (UID) and persistent storage of +mod-sequences (MODSEQ); any non-compliant mailbox will cause `interimap` +to abort. +Furthermore, because UIDs are allocated not by the client but by the +server, `interimap` needs to keep track of associations between local +and remote UIDs for each mailbox. +The synchronization state of a mailbox consists of its `UIDNEXT` and +`HIGHESTMODSEQ` values on each server; it is then assumed that each +message with UID smaller than `UIDNEXT` have been replicated to the +other server, and that the metadata (such as flags) of each message with +MODSEQ at most `HIGHESTMODSEQ` have been synchronized. +Conceptually, the synchronization algorithm is derived from [RFC 4549] +with the [RFC 7162] (section 6) amendments, and works as follows: + + 1. `SELECT` (on both servers) a mailbox the current `UIDNEXT` or `HIGHESTMODSEQ` + values of which differ from the values found in the database (for + either server). Use the `QRESYNC` `SELECT` parameter from [RFC + 7162] to list changes (vanished messages and flag updates) since + `HIGHESTMODSEQ` to messages with UID smaller than `UIDNEXT`. + + 2. Propagate these changes onto the other server: get the corresponding + UIDs from the database, then: + a. issue an `UID STORE` command, followed by `UID EXPUNGE`, to + remove messages that have not already been deleted on both + servers; and + b. issue some `UID STORE` commands to propagate flag updates (send + a single command for each flag list in order the reduce the + number of round trips). + + (Conflicts may occur if the metadata of a message has been updated + on both servers with different flag lists; in that case, `interimap` + issues a warning and updates the message on each server with the + union of both flag lists.) + Repeat this step if the server sent some updates in the meantime. + Otherwise, update the `HIGHESTMODSEQ` value in the database. + + 3. Process new messages (if the current `UIDNEXT` value of the mailbox + differs from the one found in the database) by issuing an `UID + FETCH` command; process each received message on-the-fly by issuing + an `APPEND` command with the message's `RFC822` body, `FLAGS` and + `INTERNALDATE`. + Repeat this step if the server received new messages in the + meantime. Otherwise, update the `UIDNEXT` value in the database. + Go back to step 2 if the server sent some metadata (such as flag) + updates in the meantime. + + 4. Go back to step 1 to proceed with the next unsynchronized mailbox. + +Commands +======== + +By default, `interimap` synchronizes each mailbox listed by the `LIST "" +"*"` IMAP command; +the *list-mailbox*, *list-select-opts* and *ignore-mailbox* options from +the [configuration file](#configuration-file) can be used to shrink that +list and save bandwidth. +However if some extra argument are provided on the command line, +`interimap` ignores said options and synchronizes the given +*MAILBOX*es instead. Note that each *MAILBOX* is taken “as is”; in +particular, it must be [UTF-7 encoded][RFC 2152], unquoted, and the list +wildcards ‘\*’ and ‘%’ are not interpolated. + +If the synchronization was interrupted during a previous run while some +messages were being replicated (but before the `UIDNEXT` or +`HIGHESTMODSEQ` values have been updated), `interimap` performs a “full +synchronization” on theses messages: downloading the whole UID and flag +lists on each servers allows `interimap` to detect messages that have +been removed or for which their flags have changed in the meantime. +Finally, after propagating the offline changes for these messages, +`interimap` resumes the synchronization for the rest of the mailbox. + +Specifying one of the commands below makes `interimap` perform an action +other than the default [`QRESYNC`][RFC 7162]-based synchronization. + +`--repair` [*MAILBOX* ...] + +: List the database anomalies and try to repair them. (Consider only + the given *MAILBOX*es if non-optional arguments are provided.) + This is done by performing a so-called “full synchronization”, + namely: + 1/ download all UIDs along with their flag list both from the + local and remote servers; + 2/ ensure that each entry in the database corresponds to an + existing UID; and + 3/ ensure that both flag lists match. + Any message found on a server but not in the database is replicated + on the other server (which in the worst case, might lead to a + message duplicate). + Flag conflicts are solved by updating each message to the union of + both lists. + +`--delete` *MAILBOX* [*MAILBOX* ...] + +: Delete the given *MAILBOX*es on each target (by default each server + plus the database, unless `--target` specifies otherwise) where it + exists. + Note that per the [IMAP4rev1 standard][RFC 3501] deletion is not + recursive. Thus *MAILBOX*'s children are not deleted. + +`--rename` *SOURCE* *DEST* + +: Rename the mailbox *SOURCE* to *DEST* on each target (by default + each server plus the database, unless `--target` specifies + otherwise) where it exists. + `interimap` aborts if *DEST* already exists on either target. + Note that per the [IMAP4rev1 standard][RFC 3501] renaming is + recursive. Thus *SOURCE*'s children are moved to become *DEST*'s + children instead. + +Options +======= + +`--config=`*FILE* + +: Specify an alternate [configuration file](#configuration-file). + Relative paths start from *$XDG_CONFIG_HOME*, or *~/.config* if the + `XDG_CONFIG_HOME` environment variable is unset. + +`--target={local,remote,database}` + +: Limit the scope of a `--delete` or `--rename` command to the given + target. Can be repeated to act on multiple targets. By default all + three targets are considered. + +`--watch`[`=`*seconds*] + +: Don't exit after a successful synchronization. Instead, keep + synchronizing forever. Sleep for the given number of *seconds* (by + default 1 minute if `--notify` is unset, and 15 minutes if + `--notify` is set) between two synchronizations. Setting this + options enables `SO_KEEPALIVE` on the socket for *type*s other than + `tunnel`. + +`--notify` + +: Wether to use the [IMAP `NOTIFY` extension][RFC 5465] to instruct + the server to automatically send updates to the client. (Both local + and remote servers must support [RFC 5465] for this to work.) + This greatly reduces IMAP traffic since `interimap` can rely on + server notifications instead of manually polling for updates. + If the connection remains idle for 15 minutes (configurable with + `--watch`), then `interimap` sends a `NOOP` command to avoid being + logged out for inactivity. + +`-q`, `--quiet` + +: Try to be quiet. + +`--debug` + +: Turn on debug mode. Debug messages are written to the given *logfile*. + Note that this include all IMAP traffic (except literals). + Depending on the chosen authentication mechanism, this might include + authentication credentials. + +`-h`, `--help` + +: Output a brief help and exit. + +`--version` + +: Show the version number and exit. + +Configuration file +================== + +Unless told otherwise by the `--config=FILE` command-line option, +`interimap` reads its configuration from *$XDG_CONFIG_HOME/interimap* +(or *~/.config/interimap* if the `XDG_CONFIG_HOME` environment variable +is unset) as an [INI file]. +The syntax of the configuration file is a series of `OPTION=VALUE` +lines organized under some `[SECTION]`; lines starting with a ‘#’ or +‘;’ character are ignored as comments. +The `[local]` and `[remote]` sections define the two IMAP servers to +synchronize. +Valid options are: + +*database* + +: SQLite version 3 database file to use to keep track of associations + between local and remote UIDs, as well as the `UIDVALIDITY`, + `UIDNEXT` and `HIGHESTMODSEQ` of each known mailbox on both servers. + Relative paths start from *$XDG_DATA_HOME/interimap*, or + *~/.local/share/interimap* if the `XDG_DATA_HOME` environment + variable is unset. This option is only available in the default + section. + (Default: `HOST.db`, where *HOST* is taken from the `[remote]` or + `[local]` sections, in that order.) + +*list-mailbox* + +: A space separated list of mailbox patterns to use when issuing the + initial `LIST` command (overridden by the *MAILBOX*es given as + command-line arguments). + Note that each pattern containing special characters such as spaces + or brackets (see [RFC 3501] for the exact syntax) must be quoted. + Furthermore, non-ASCII names must be [UTF-7 encoded][RFC 2152]. + Two wildcards are available: a ‘\*’ character matches zero or more + characters, while a ‘%’ character matches zero or more characters up + to the mailbox's hierarchy delimiter. + This option is only available in the default section. + (The default pattern, `*`, matches all visible mailboxes on the + server.) + +*list-select-opts* + +: An optional space separated list of selectors for the initial `LIST` + command. (Requires a server supporting the [`LIST-EXTENDED` IMAP + extension][RFC 5258].) Useful values are `SUBSCRIBED` (to list only + subscribed mailboxes), `REMOTE` (to also list remote mailboxes on a + server supporting mailbox referrals), and `RECURSIVEMATCH` (to + list parent mailboxes with children matching one of the above + *list-mailbox* patterns). This option is only available in the + default section. + +*ignore-mailbox* + +: An optional Perl Compatible Regular Expressions ([PCRE]) covering + mailboxes to exclude: any ([UTF-7 encoded][RFC 2152] and unquoted) + mailbox listed in the initial `LIST` responses is ignored if it + matches the given expression. + Note that the *MAILBOX*es given as command-line arguments bypass the + check and are always considered for synchronization. This option is + only available in the default section. + +*logfile* + +: A file name to use to log debug and informational messages. (By + default these messages are written to the error output.) This + option is only available in the default section. + +*type* + +: One of `imap`, `imaps` or `tunnel`. + `type=imap` and `type=imaps` are respectively used for IMAP and IMAP + over SSL/TLS connections over a INET socket. + `type=tunnel` causes `interimap` to open a pipe to a *command* + instead of a raw socket. + Note that specifying `type=tunnel` in the `[remote]` section makes + the default *database* to be `localhost.db`. + (Default: `imaps`.) + +*host* + +: Server hostname, for `type=imap` and `type=imaps`. + (Default: `localhost`.) + +*port* + +: Server port. + (Default: `143` for `type=imap`, `993` for `type=imaps`.) + +*proxy* + +: An optional SOCKS proxy to use for TCP connections to the IMAP + server (`type=imap` and `type=imaps` only), formatted as + `PROTOCOL://[USER:PASSWORD@]PROXYHOST[:PROXYPORT]`. + If `PROXYPORT` is omitted, it is assumed at port 1080. + Only [SOCKSv5][RFC 1928] is supported (with optional + [username/password authentication][RFC 1929]), in two flavors: + `socks5://` to resolve *hostname* locally, and `socks5h://` to let + the proxy resolve *hostname*. + +*command* + +: Command to use for `type=tunnel`. Must speak the [IMAP4rev1 + protocol][RFC 3501] on its standard output, and understand it on its + standard input. + +*STARTTLS* + +: Whether to use the [`STARTTLS`][RFC 2595] directive to upgrade to a + secure connection. Setting this to `YES` for a server not + advertising the `STARTTLS` capability causes `interimap` to + immediately abort the connection. + (Ignored for *type*s other than `imap`. Default: `YES`.) + +*auth* + +: Space-separated list of preferred authentication mechanisms. + `interimap` uses the first mechanism in that list that is also + advertised (prefixed with `AUTH=`) in the server's capability list. + Supported authentication mechanisms are `PLAIN` and `LOGIN`. + (Default: `PLAIN LOGIN`.) + +*username*, *password* + +: Username and password to authenticate with. Can be required for non + pre-authenticated connections, depending on the chosen + authentication mechanism. + +*compress* + +: Whether to use the [`IMAP COMPRESS` extension][RFC 4978] for servers + advertising it. + (Default: `NO` for the `[local]` section, `YES` for the `[remote]` + section.) + +*null-stderr* + +: Whether to redirect *command*'s standard error to `/dev/null` for + type `type=tunnel`. (Default: `NO`.) + +*SSL_protocols* + +: A space-separated list of SSL protocols to enable or disable (if + prefixed with an exclamation mark `!`. Known protocols are `SSLv2`, + `SSLv3`, `TLSv1`, `TLSv1.1`, and `TLSv1.2`. Enabling a protocol is + a short-hand for disabling all other protocols. + (Default: `!SSLv2 !SSLv3`, i.e., only enable TLSv1 and above.) + +*SSL_cipher_list* + +: The cipher list to send to the server. Although the server + determines which cipher suite is used, it should take the first + supported cipher in the list sent by the client. See + [`ciphers`(1ssl)] for more information. + +*SSL_fingerprint* + +: Fingerprint of the server certificate (or its public key) in the + form `[ALGO$]DIGEST_HEX`, where `ALGO` is the used algorithm + (by default `sha256`). + Attempting to connect to a server with a non-matching certificate + fingerprint causes `interimap` to abort the connection during the + SSL/TLS handshake. + +*SSL_verify* + +: Whether to verify the server certificate chain. + Note that using *SSL_fingerprint* to specify the fingerprint of the + server certificate is an orthogonal authentication measure as it + ignores the CA chain. + (Default: `YES`.) + +*SSL_CApath* + +: Directory to use for server certificate verification if + `SSL_verify=YES`. + This directory must be in “hash format”, see [`verify`(1ssl)] for + more information. + +*SSL_CAfile* + +: File containing trusted certificates to use during server + certificate authentication if `SSL_verify=YES`. + +Supported extensions +==================== + +Performance is better for servers supporting the following extensions to +the [IMAP4rev1 protocol][RFC 3501]: + + * `LITERAL+` ([RFC 2088], recommended); + * `MULTIAPPEND` ([RFC 3502], recommended); + * `COMPRESS=DEFLATE` ([RFC 4978], recommended); + * `NOTIFY` ([RFC 5465], recommended); + * `SASL-IR` ([RFC 4959]); and + * `UNSELECT` ([RFC 3691]). + +Known bugs and limitations +========================== + + * Using `interimap` on two identical servers with a non-existent or + empty *database* will duplicate each message due to the absence of + local ↔ remote UID association. + + * `interimap` is single threaded and doesn't use IMAP command + pipelining. Synchronization could be boosted up by sending + independent commands (such as the initial `LIST` and `STATUS` + commands) to both servers in parallel, and for a given server, by + sending independent commands (such as flag updates) in a pipeline. + + * Because the [IMAP protocol][RFC 3501] doesn't have a specific + response code for when a message is moved to another mailbox (either + using the `MOVE` command from [RFC 6851], or via `COPY` + `STORE` + + `EXPUNGE`), moving a message causes `interimap` to believe that it + was deleted while another one (which is replicated again) was added + to the other mailbox in the meantime. + + * `PLAIN` and `LOGIN` are the only authentication mechanisms currently + supported. + + * `interimap` will probably not work with non [RFC][RFC 3501]-compliant + servers. In particular, no work-around is currently implemented + beside the tunables in the [configuration file](#configuration-file). + Moreover, few IMAP servers have been tested so far. + +Standards +========= + + * M. Leech, M. Ganis, Y. Lee, R. Kuris, D. Koblas and L. Jones, + _SOCKS Protocol Version 5_, + [RFC 1928], March 1996. + * M. Leech, _Username/Password Authentication for SOCKS V5_, + [RFC 1929], March 1996. + * J. Myers, _IMAP4 non-synchronizing literals_, + [RFC 2088], January 1997. + * D. Goldsmith and M. Davis, + _A Mail-Safe Transformation Format of Unicode_, + [RFC 2152], May 1997. + * C. Newman, _Using TLS with IMAP, POP3 and ACAP_, + [RFC 2595], June 1999. + * M. Crispin, _Internet Message Access Protocol - Version 4rev1_, + [RFC 3501], March 2003. + * M. Crispin, + _Internet Message Access Protocol (IMAP) - `MULTIAPPEND` Extension_, + [RFC 3502], March 2003. + * A. Melnikov, + _Internet Message Access Protocol (IMAP) `UNSELECT` command_, + [RFC 3691], February 2004. + * M. Crispin, + _Internet Message Access Protocol (IMAP) - `UIDPLUS` extension_, + [RFC 4315], December 2005. + * A. Melnikov, + _Synchronization Operations for Disconnected IMAP4 Clients_, + [RFC 4549], June 2006. + * A. Gulbrandsen, _The IMAP `COMPRESS` Extension_, + [RFC 4978], August 2007. + * R. Siemborski and A. Gulbrandsen, _IMAP Extension for Simple + Authentication and Security Layer (SASL) Initial Client Response_, + [RFC 4959], September 2007. + * A. Gulbrandsen and A. Melnikov, + _The IMAP `ENABLE` Extension_, + [RFC 5161], March 2008. + * B. Leiba and A. Melnikov, + _Internet Message Access Protocol version 4 - `LIST` Command Extensions_, + [RFC 5258], June 2008. + * A. Gulbrandsen, C. King and A. Melnikov, + _The IMAP `NOTIFY` Extension_, + [RFC 5465], February 2009. + * A. Melnikov and T. Sirainen, + _IMAP4 Extension for Returning `STATUS` Information in Extended LIST_, + [RFC 5819], March 2010. + * A. Gulbrandsen and N. Freed, + _Internet Message Access Protocol (IMAP) - `MOVE` Extension_, + [RFC 6851], January 2013. + * A. Melnikov and D. Cridland, + _IMAP Extensions: Quick Flag Changes Resynchronization (`CONDSTORE`) + and Quick Mailbox Resynchronization (`QRESYNC`)_, + [RFC 7162], May 2014. + +[RFC 7162]: https://tools.ietf.org/html/rfc7162 +[RFC 5258]: https://tools.ietf.org/html/rfc5258 +[RFC 5819]: https://tools.ietf.org/html/rfc5819 +[RFC 4315]: https://tools.ietf.org/html/rfc4315 +[RFC 4549]: https://tools.ietf.org/html/rfc4549 +[RFC 2152]: https://tools.ietf.org/html/rfc2152 +[RFC 3501]: https://tools.ietf.org/html/rfc3501 +[RFC 1928]: https://tools.ietf.org/html/rfc1928 +[RFC 1929]: https://tools.ietf.org/html/rfc1929 +[RFC 2595]: https://tools.ietf.org/html/rfc2595 +[RFC 4978]: https://tools.ietf.org/html/rfc4978 +[RFC 2088]: https://tools.ietf.org/html/rfc2088 +[RFC 3502]: https://tools.ietf.org/html/rfc3502 +[RFC 4959]: https://tools.ietf.org/html/rfc4959 +[RFC 3691]: https://tools.ietf.org/html/rfc3691 +[RFC 6851]: https://tools.ietf.org/html/rfc6851 +[RFC 5161]: https://tools.ietf.org/html/rfc5161 +[RFC 5465]: https://tools.ietf.org/html/rfc5465 + +[INI file]: https://en.wikipedia.org/wiki/INI_file +[PCRE]: https://en.wikipedia.org/wiki/Perl_Compatible_Regular_Expressions +[`ciphers`(1ssl)]: https://www.openssl.org/docs/manmaster/apps/ciphers.html +[`verify`(1ssl)]: https://www.openssl.org/docs/manmaster/apps/verify.html diff --git a/lib/Net/IMAP/InterIMAP.pm b/lib/Net/IMAP/InterIMAP.pm index 45253c1..a899831 100644 --- a/lib/Net/IMAP/InterIMAP.pm +++ b/lib/Net/IMAP/InterIMAP.pm @@ -35,7 +35,8 @@ BEGIN { Net::SSLeay::SSLeay_add_ssl_algorithms(); Net::SSLeay::randomize(); - our @EXPORT_OK = qw/read_config compact_set $IMAP_text $IMAP_cond/; + our @EXPORT_OK = qw/read_config compact_set $IMAP_text $IMAP_cond + slurp is_dirty has_new_mails/; } @@ -313,6 +314,9 @@ sub new($%) { foreach ($rd, $wd) { close $_ or $self->panic("Can't close: $!"); } + foreach (qw/STDIN STDOUT/) { + binmode($self->{$_}) // $self->panic("binmode: $!") + } } else { foreach (qw/host port/) { @@ -320,28 +324,17 @@ sub new($%) { } my $socket = defined $self->{proxy} ? $self->_proxify(@$self{qw/proxy host port/}) : $self->_tcp_connect(@$self{qw/host port/}); - my ($cnt, $intvl) = (3, 5); if (defined $self->{keepalive}) { - # detect dead peers and drop the connection after 60 secs + $cnt*$intvl setsockopt($socket, Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, 1) or $self->fail("Can't setsockopt SO_KEEPALIVE: $!"); setsockopt($socket, Socket::IPPROTO_TCP, Socket::TCP_KEEPIDLE, 60) or $self->fail("Can't setsockopt TCP_KEEPIDLE: $!"); - setsockopt($socket, Socket::IPPROTO_TCP, Socket::TCP_KEEPCNT, $cnt) - or $self->fail("Can't setsockopt TCP_KEEPCNT: $!"); - setsockopt($socket, Socket::IPPROTO_TCP, Socket::TCP_KEEPINTVL, $intvl) - or $self->fail("Can't setsockopt TCP_KEEPINTVL: $!"); } - # Abort after 15secs if write(2) isn't acknowledged - # XXX Socket::TCP_USER_TIMEOUT isn't defined. - # `grep TCP_USER_TIMEOUT /usr/include/linux/tcp.h` gives 18 - setsockopt($socket, Socket::IPPROTO_TCP, 18, 1000 * $cnt * $intvl) - or $self->fail("Can't setsockopt TCP_USER_TIMEOUT: $!"); + binmode($socket) // $self->panic("binmode: $!"); $self->_start_ssl($socket) if $self->{type} eq 'imaps'; $self->{$_} = $socket for qw/STDOUT STDIN/; } - binmode $self->{$_} foreach qw/STDIN STDOUT/; # command counter $self->{_TAG} = 0; @@ -645,6 +638,7 @@ sub unselect($) { # we'll get back to it $self->{_VANISHED} = []; $self->{_MODIFIED} = {}; + $self->{_NEW} = 0; } @@ -916,91 +910,93 @@ sub fetch($$$;&) { } -# $self->notify(@specifications) -# Issue a NOTIFY command with the given mailbox @specifications (cf RFC -# 5465 section 6) to be monitored. Croak if the server did not -# advertise "NOTIFY" (RFC 5465) in its CAPABILITY list. -sub notify($@) { +# $self->notify($arg, %specifications) +# Issue a NOTIFY command with the given $arg ("SET", "SET STATUS" or +# "NONE") and mailbox %specifications (cf RFC 5465 section 6) to be +# monitored. Croak if the server did not advertise "NOTIFY" (RFC +# 5465) in its CAPABILITY list. +sub notify($$@) { my $self = shift; $self->fail("Server did not advertise NOTIFY (RFC 5465) capability.") unless $self->_capable('NOTIFY'); - my $events = join ' ', qw/MessageNew MessageExpunge FlagChange MailboxName SubscriptionChange/; - # Be notified of new messages with EXISTS/RECENT responses, but - # don't receive unsolicited FETCH responses with a RFC822/BODY[]. - # It costs us an extra roundtrip, but we need to sync FLAG updates - # and VANISHED responses in batch mode, update the HIGHESTMODSEQ, - # and *then* issue an explicit UID FETCH command to get new message, - # and process each FETCH response with a RFC822/BODY[] attribute as - # they arrive. - my $command = 'NOTIFY '; - $command .= @_ ? ('SET '. join(' ', map {"($_ ($events))"} @_)) : 'NONE'; + my $command = 'NOTIFY '.shift; + while (@_) { + $command .= " (".shift." (".join(' ', @{shift()})."))"; + } $self->_send($command); } -# $self->slurp([$callback, $cmd]) -# See if the server has sent some unprocessed data; try to as many -# lines as possible, process them, and return the number of lines -# read. +# slurp($imap, $timeout, $stopwhen) +# Keep reading untagged responses from the @$imap servers until the +# $stopwhen condition becomes true (then return true), or until the +# $timeout expires (then return false). # This is mostly useful when waiting for notifications while no # command is progress, cf. RFC 2177 (IDLE) or RFC 5465 (NOTIFY). -sub slurp($;&$) { - my ($self, $callback, $cmd) = @_; - my $ssl = $self->{_SSL}; - my $read = 0; +sub slurp($$$) { + my ($selfs, $timeout, $stopwhen) = @_; + my $aborted = 0; + + my $rin = ''; + vec($rin, fileno($_->{STDOUT}), 1) = 1 foreach @$selfs; - vec(my $rin, fileno($self->{STDOUT}), 1) = 1; while (1) { - unless ((defined $self->{_OUTBUF} and $self->{_OUTBUF} ne '') or - # Unprocessed data within the current TLS record would - # cause select(2) to block/timeout due to the raw socket - # not being ready. - (defined $ssl and Net::SSLeay::pending($ssl) > 0)) { - my $r = CORE::select($rin, undef, undef, 0); + # first, consider only unprocessed data without our own output + # buffer, or within the current TLS record: these would cause + # select(2) to block/timeout due to the raw socket not being + # ready. + my @ready = grep { (defined $_->{_OUTBUF} and $_->{_OUTBUF} ne '') or + (defined $_->{_SSL} and Net::SSLeay::pending($_->{_SSL}) > 0) + } @$selfs; + unless (@ready) { + my ($r, $timeleft) = CORE::select(my $rout = $rin, undef, undef, $timeout); next if $r == -1 and $! == EINTR; # select(2) was interrupted - $self->panic("Can't select: $!") if $r == -1; - return $read if $r == 0; # nothing more to read + die "select: $!" if $r == -1; + return $aborted if $r == 0; # nothing more to read (timeout reached) + @ready = grep {vec($rout, fileno($_->{STDOUT}), 1)} @$selfs; + $timeout = $timeleft if $timeout > 0; + } + + foreach my $imap (@ready) { + my $x = $imap->_getline(); + $imap->_resp($x, sub($) { + if ($stopwhen->($imap, shift)) { + $aborted = 1; + $timeout = 0; # keep reading the handles while there is pending data + } + }, 'slurp'); } - my $x = $self->_getline(); - $self->_resp($x, $callback, $cmd); - $read++; } } -# $self->idle([$timeout, $stopwhen]) +# $self->idle($timeout, $stopwhen) # Enter IDLE (RFC 2177) for $timout seconds (by default 29 mins), or # when the callback $stopwhen returns true. -# Return false if the timeout was reached, and true if IDLE was -# stopped due the callback. -sub idle($;$&) { +# Return true if the callback returned true (either aborting IDLE, or +# after the $timeout) and false otherwise. +sub idle($$$) { my ($self, $timeout, $stopwhen) = @_; - $timeout //= 1740; # 29 mins - my $callback = sub() {$timeout = -1 if $stopwhen->()}; $self->fail("Server did not advertise IDLE (RFC 2177) capability.") unless $self->_capable('IDLE'); my $tag = $self->_cmd_init('IDLE'); $self->_cmd_flush(); - - for (; $timeout > 0; $timeout--) { - $self->slurp($callback, 'IDLE'); - sleep 1 if $timeout > 0; - } + my $r = slurp([$self], $timeout // 1740, $stopwhen); # 29 mins # done idling $self->_cmd_extend('DONE'); $self->_cmd_flush(); # run the callback again to update the return value if we received # untagged responses between the DONE and the tagged response - $self->_recv($tag, $callback, 'IDLE'); + $self->_recv($tag, sub($) { $r = 1 if $stopwhen->($self, shift) }, 'slurp'); - return $timeout < 0 ? 1 : 0; + return $r; } -# $self->set_cache( $mailbox, STATE ) +# $self->set_cache($mailbox, STATE) # Initialize or update the persistent cache, that is, associate a # known $mailbox with the last known (synced) state: # * UIDVALIDITY @@ -1082,6 +1078,7 @@ sub get_cache($@) { # persistent cache's values. sub is_dirty($$) { my ($self, $mailbox) = @_; + return 1 if $self->{_NEW}; $self->_updated_cache($mailbox, qw/HIGHESTMODSEQ UIDNEXT/); } @@ -1091,6 +1088,7 @@ sub is_dirty($$) { # internal cache's UIDNEXT value differs from its persistent cache's. sub has_new_mails($$) { my ($self, $mailbox) = @_; + return 1 if $self->{_NEW}; $self->_updated_cache($mailbox, 'UIDNEXT'); } @@ -1181,6 +1179,7 @@ sub pull_new_messages($$&@) { my @ignore = sort { $a <=> $b } @_; my $mailbox = $self->{_SELECTED} // $self->panic(); + my $cache = $self->{_CACHE}->{$mailbox}; my $UIDNEXT; do { @@ -1205,19 +1204,20 @@ sub pull_new_messages($$&@) { # 2^32-1: don't use '*' since the highest UID can be known already $range .= "$since:4294967295"; - $UIDNEXT = $self->{_CACHE}->{$mailbox}->{UIDNEXT} // $self->panic(); # sanity check + $UIDNEXT = $cache->{UIDNEXT} // $self->panic(); # sanity check $self->_send("UID FETCH $range ($attrs)", sub($) { my $mail = shift; $UIDNEXT = $mail->{UID} + 1 if $UIDNEXT <= $mail->{UID}; $callback->($mail) if defined $callback; - }) if $first < $UIDNEXT; + }) if $first < $UIDNEXT or $self->{_NEW}; # update the persistent cache for UIDNEXT (not for HIGHESTMODSEQ # since there might be pending updates) $self->set_cache($mailbox, UIDNEXT => $UIDNEXT); + $self->{_NEW} = 0; } # loop if new messages were received in the meantime - while ($UIDNEXT < $self->{_CACHE}->{$mailbox}->{UIDNEXT}); + while ($self->{_NEW} or $UIDNEXT < $cache->{UIDNEXT}); } @@ -1372,12 +1372,21 @@ sub _tcp_connect($$$) { SOCKETS: foreach my $ai (@res) { socket (my $s, $ai->{family}, $ai->{socktype}, $ai->{protocol}) or $self->panic("connect: $!"); - # TODO: add a connection timeout - # http://devpit.org/wiki/Connect%28%29_with_timeout_%28in_Perl%29 + + # timeout connect/read/write/... after 30s + # XXX we need to pack the struct timeval manually: not portable! + # https://stackoverflow.com/questions/8284243/how-do-i-set-so-rcvtimeo-on-a-socket-in-perl + my $timeout = pack('l!l!', 30, 0); + setsockopt($s, Socket::SOL_SOCKET, Socket::SO_RCVTIMEO, $timeout) + or $self->fail("Can't setsockopt SO_RCVTIMEO: $!"); + setsockopt($s, Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, $timeout) + or $self->fail("Can't setsockopt SO_RCVTIMEO: $!"); + until (connect($s, $ai->{addr})) { next if $! == EINTR; # try again if connect(2) was interrupted by a signal next SOCKETS; } + my $flags = fcntl($s, F_GETFD, 0) or $self->panic("fcntl F_GETFD: $!"); fcntl($s, F_SETFD, $flags | FD_CLOEXEC) or $self->panic("fcntl F_SETFD: $!"); return $s; @@ -1908,11 +1917,11 @@ sub _send($$;&) { my $tag = $self->_cmd_init($command); $self->_cmd_flush(); + my $cmd = $$command =~ /\AUID ($RE_ATOM_CHAR+) / ? $1 : $$command =~ /\A($RE_ATOM_CHAR+) / ? $1 : $$command; if (!defined $callback) { - $self->_recv($tag); + $self->_recv($tag, undef, $cmd); } else { - my $cmd = $$command =~ /\AUID ($RE_ATOM_CHAR+) / ? $1 : $$command =~ /\A($RE_ATOM_CHAR+) / ? $1 : $$command; my $set = $$command =~ /\AUID (?:FETCH|STORE) ([0-9:,*]+)/ ? $1 : undef; $self->_recv($tag, $callback, $cmd, $set); } @@ -1993,6 +2002,7 @@ sub _open_mailbox($$) { # we'll get back to it $self->{_VANISHED} = []; $self->{_MODIFIED} = {}; + $self->{_NEW} = 0; $self->{_SELECTED} = $mailbox; $self->{_CACHE}->{$mailbox} //= {}; @@ -2219,6 +2229,7 @@ sub _resp($$;&$$) { } elsif (s/\A(?:OK|NO|BAD) //) { $self->_resp_text($_); + $callback->($self->{_SELECTED}) if defined $self->{_SELECTED} and defined $callback and $cmd eq 'slurp'; } elsif (/\ACAPABILITY((?: $RE_ATOM_CHAR+)+)\z/) { $self->{_CAPABILITIES} = [ split / /, ($1 =~ s/^ //r) ]; @@ -2233,17 +2244,20 @@ sub _resp($$;&$$) { # /!\ $cache->{EXISTS} MUST NOT be defined on SELECT if (defined $cache->{EXISTS}) { $self->panic("Unexpected EXISTS shrink $1 < $cache->{EXISTS}!") if $1 < $cache->{EXISTS}; - # the actual UIDNEXT is *at least* that - $cache->{UIDNEXT} += $1 - $cache->{EXISTS} if defined $cache->{UIDNEXT}; + $self->{_NEW} += $1 - $cache->{EXISTS} if $1 > $cache->{EXISTS}; # new mails } $cache->{EXISTS} = $1; + $callback->($self->{_SELECTED} // $self->panic()) if defined $callback and $cmd eq 'slurp'; } elsif (/\A([0-9]+) EXPUNGE\z/) { + $self->panic() unless defined $cache->{EXISTS}; # sanity check # /!\ No bookkeeping since there is no internal cache mapping sequence numbers to UIDs if ($self->_enabled('QRESYNC')) { $self->panic("$1 <= $cache->{EXISTS}") if $1 <= $cache->{EXISTS}; # sanity check $self->fail("RFC 7162 violation! Got an EXPUNGE response with QRESYNC enabled."); } + # the new message was expunged before it was synced + $self->{_NEW} = 0 if $self->{_NEW} == 1 and $cache->{EXISTS} == $1; $cache->{EXISTS}--; # explicit EXISTS responses are optional } elsif (/\ASEARCH((?: [0-9]+)*)\z/) { @@ -2266,11 +2280,20 @@ sub _resp($$;&$$) { /\A \((\\?$RE_ATOM_CHAR+ [0-9]+(?: \\?$RE_ATOM_CHAR+ [0-9]+)*)?\)\z/ or $self->panic($_); my %status = split / /, $1; $mailbox = 'INBOX' if uc $mailbox eq 'INBOX'; # INBOX is case-insensitive + $self->panic("RFC 5465 violation! Missing HIGHESTMODSEQ data item in STATUS response") + if $self->_enabled('QRESYNC') and !defined $status{HIGHESTMODSEQ} and defined $cmd and + ($cmd eq 'NOTIFY' or $cmd eq 'slurp'); $self->_update_cache_for($mailbox, %status); - $callback->($mailbox, %status) if defined $callback and $cmd eq 'STATUS'; + if (defined $callback) { + if ($cmd eq 'STATUS') { + $callback->($mailbox, %status); + } elsif ($cmd eq 'slurp') { + $callback->($mailbox); + } + } } elsif (s/\A([0-9]+) FETCH \(//) { - $self->panic("$1 <= $cache->{EXISTS}") unless $1 <= $cache->{EXISTS}; # sanity check + $cache->{EXISTS} = $1 if $1 > $cache->{EXISTS}; my ($seq, $first) = ($1, 1); my %mail; while ($_ ne ')') { @@ -2313,8 +2336,13 @@ sub _resp($$;&$$) { my $flags = join ' ', sort(grep {lc $_ ne '\recent'} @{$mail{FLAGS}}) if defined $mail{FLAGS}; $self->{_MODIFIED}->{$uid} = [ $mail{MODSEQ}, $flags ]; } - $callback->(\%mail) if defined $callback and ($cmd eq 'FETCH' or $cmd eq 'STORE') and - defined $uid and in_set($uid, $set); + if (defined $callback) { + if ($cmd eq 'FETCH' or $cmd eq 'STORE') { + $callback->(\%mail) if defined $uid and in_set($uid, $set); + } elsif ($cmd eq 'slurp') { + $callback->($self->{_SELECTED} // $self->panic()) + } + } } elsif (/\AENABLED((?: $RE_ATOM_CHAR+)+)\z/) { # RFC 5161 ENABLE $self->{_ENABLED} //= []; @@ -2338,6 +2366,7 @@ sub _resp($$;&$$) { push @{$self->{_VANISHED}}, ($min .. $max); } } + $callback->($self->{_SELECTED} // $self->panic()) if defined $callback and $cmd eq 'slurp'; } } elsif (s/\A\+// and ($_ eq '' or s/\A //)) { @@ -2351,7 +2380,6 @@ sub _resp($$;&$$) { else { $self->panic("Unexpected response: ", $_); } - $callback->() if defined $callback and $cmd eq 'IDLE'; } @@ -25,7 +25,7 @@ our $VERSION = '0.3'; my $NAME = 'pullimap'; use Errno 'EINTR'; -use Fcntl qw/O_CREAT O_RDWR O_DSYNC LOCK_EX SEEK_SET F_GETFD F_SETFD FD_CLOEXEC/; +use Fcntl qw/O_CREAT O_RDWR O_DSYNC F_SETLK F_WRLCK SEEK_SET F_GETFD F_SETFD FD_CLOEXEC/; use Getopt::Long qw/:config posix_default no_ignore_case gnu_getopt auto_version/; use List::Util 'first'; use Socket qw/PF_INET PF_INET6 SOCK_STREAM/; @@ -82,10 +82,9 @@ do { } sysopen($STATE, $statefile, O_CREAT|O_RDWR|O_DSYNC, 0600) or die "Can't open $statefile: $!"; + fcntl($STATE, F_SETLK, pack('sslll', F_WRLCK, SEEK_SET, 0, 0, $$)) or die "Can't lock $statefile: $!"; my $flags = fcntl($STATE, F_GETFD, 0) or die "fcntl F_GETFD: $!"; fcntl($STATE, F_SETFD, $flags | FD_CLOEXEC) or die "fcntl F_SETFD: $!"; - - flock($STATE, LOCK_EX) or die "Can't flock $statefile: $!"; }; @@ -141,6 +140,7 @@ sub sendmail($$) { next if $! == EINTR; # try again if connect(2) was interrupted by a signal die "connect: $!"; } + binmode($SMTP) // die "binmode: $!"; smtp_resp('220'); my @r = smtp_send($ehlo => '250'); @@ -344,7 +344,6 @@ unless (defined $CONFIG{idle}) { $CONFIG{idle} = 1740 if defined $CONFIG{idle} and $CONFIG{idle} == 0; # 29 mins while(1) { - my $r = $IMAP->idle($CONFIG{idle}, sub() { $IMAP->has_new_mails($MAILBOX) }); - pull() if $r; + pull() if $IMAP->idle($CONFIG{idle}, \&Net::IMAP::InterIMAP::has_new_mails); purge(); } diff --git a/pullimap.md b/pullimap.md index 54c6ce5..244e7ac 100644 --- a/pullimap.md +++ b/pullimap.md @@ -5,8 +5,7 @@ Name ==== -PullIMAP - Pull mails from an IMAP mailbox and deliver them to a SMTP -session +PullIMAP - Pull mails from an IMAP mailbox and deliver them to a SMTP session Synopsis ======== @@ -25,58 +24,57 @@ A *statefile* is used to keep track of the mailbox's `UIDVALIDITY` and `UIDNEXT` values. While `pullimap` is running, the *statefile* is also used to keep track of UIDs being delivered, which avoids duplicate deliveries in case the process is interrupted. -See the [**Control flow**](#control-flow) section below for details. +See the **[control flow](#control-flow)** section below for details. Options ======= `--config=`*FILE* -: Specify an alternate configuration file. Relative paths start from - *$XDG_CONFIG_HOME*, or *~/.config* if the `XDG_CONFIG_HOME` - environment variable is unset. +: Specify an alternate [configuration file](#configuration-file). + Relative paths start from *$XDG_CONFIG_HOME*, or *~/.config* if the + `XDG_CONFIG_HOME` environment variable is unset. `--idle`[`=`*seconds*] -: Don't exit after a successful poll. Instead, keep the connection open - and issue `IDLE` commands (require an IMAP server supporting [RFC - 2177]) to watch for updates in the mailbox. This also enables - `SO_KEEPALIVE` on the socket. Each `IDLE` command is terminated after - at most *seconds* (29 minutes by default) to avoid being logged out - for inactivity. +: Don't exit after a successful poll. Instead, keep the connection open + and issue `IDLE` commands (require an IMAP server supporting [RFC + 2177]) to watch for updates in the mailbox. This also enables + `SO_KEEPALIVE` on the socket. + Each `IDLE` command is terminated after at most *seconds* (29 + minutes by default) to avoid being logged out for inactivity. `--no-delivery` -: Update the *statefile*, but skip SMTP/LMTP delivery. This is mostly - useful for initializing the *statefile* when migrating to - `pullimap` from another similar program such as [`fetchmail`(1)] or - [`getmail`(1)]. +: Update the *statefile*, but skip SMTP/LMTP delivery. This is mostly + useful for initializing the *statefile* when migrating to `pullimap` + from another similar program such as [`fetchmail`(1)] or + [`getmail`(1)]. `-q`, `--quiet` -: Try to be quiet. +: Try to be quiet. `--debug` -: Turn on debug mode. Debug messages are written to the error output. - Note that this include all IMAP traffic (except literals). Depending - on the chosen authentication mechanism, this might include - authentication credentials. +: Turn on debug mode. Debug messages are written to the error output. + Note that this include all IMAP traffic (except literals). + Depending on the chosen authentication mechanism, this might include + authentication credentials. `-h`, `--help` -: Output a brief help and exit. +: Output a brief help and exit. `--version` -: Show the version number and exit. - +: Show the version number and exit. Configuration file ================== -Unless told otherwise by the `--config=FILE` option, `pullimap` reads -its configuration from *$XDG_CONFIG_HOME/pullimap* (or +Unless told otherwise by the `--config=FILE` command-line option, +`pullimap` reads its configuration from *$XDG_CONFIG_HOME/pullimap* (or *~/.config/pullimap* if the `XDG_CONFIG_HOME` environment variable is unset) as an [INI file]. The syntax of the configuration file is a series of `OPTION=VALUE` @@ -86,158 +84,156 @@ Valid options are: *statefile* -: State file to use to keep track of the *mailbox*'s `UIDVALIDITY` and - `UIDNEXT` values. Relative paths start from - *$XDG_DATA_HOME/pullimap*, or *~/.local/share/pullimap* if the - `XDG_DATA_HOME` environment variable is unset. - (Default: the parent section name of the option.) +: State file to use to keep track of the *mailbox*'s `UIDVALIDITY` and + `UIDNEXT` values. Relative paths start from + *$XDG_DATA_HOME/pullimap*, or *~/.local/share/pullimap* if the + `XDG_DATA_HOME` environment variable is unset. + (Default: the parent section name of the option.) *mailbox* -: The IMAP mailbox to pull messages from. - Support for persistent message Unique Identifiers (UID) is required. - (Default: `INBOX`.) +: The IMAP mailbox to pull messages from. Support for persistent + message Unique Identifiers (UID) is required. (Default: `INBOX`.) *deliver-method* -: `PROTOCOL:[ADDRESS]:PORT` where to deliver messages. Both [SMTP][RFC - 5321] and [LMTP][RFC 2033] servers are supported, and [SMTP - pipelining][RFC 2920] is used when possible. - (Default: `smtp:[127.0.0.1]:25`.) +: `PROTOCOL:[ADDRESS]:PORT` where to deliver messages. Both + [SMTP][RFC 5321] and [LMTP][RFC 2033] servers are supported, and + [SMTP pipelining][RFC 2920] is used when possible. + (Default: `smtp:[127.0.0.1]:25`.) *deliver-ehlo* -: Hostname to use in `EHLO` or `LHLO` commands. - (Default: `localhost.localdomain`.) +: Hostname to use in `EHLO` or `LHLO` commands. + (Default: `localhost.localdomain`.) *deliver-rcpt* -: Message recipient. - (Default: the username associated with the effective uid of the - `pullimap` process.) +: Message recipient. + (Default: the username associated with the effective uid of the + `pullimap` process.) *purge-after* -: Retention period (in days), after which messages are removed from the - IMAP server. (The value is at best 24h accurate due to the IMAP - `SEARCH` criterion ignoring time and timezone.) - If *purge-after* is set to `0` then messages are deleted immediately - after delivery. Otherwise `pullimap` issues an IMAP `SEARCH` command - to list old messages; if `--idle` is set then the `SEARCH` command is - issued again every 12 hours. +: Retention period (in days), after which messages are removed from + the IMAP server. (The value is at best 24h accurate due to the IMAP + `SEARCH` criterion ignoring time and timezone.) + If *purge-after* is set to `0` then messages are deleted immediately + after delivery. Otherwise `pullimap` issues an IMAP `SEARCH` + command to list old messages; if `--idle` is set then the `SEARCH` + command is issued again every 12 hours. *type* -: One of `imap`, `imaps` or `tunnel`. - `type=imap` and `type=imaps` are respectively used for IMAP and IMAP - over SSL/TLS connections over a INET socket. - `type=tunnel` causes `pullimap` to open a pipe to a *command* instead - of a raw socket. - (Default: `imaps`.) +: One of `imap`, `imaps` or `tunnel`. + `type=imap` and `type=imaps` are respectively used for IMAP and IMAP + over SSL/TLS connections over a INET socket. + `type=tunnel` causes `pullimap` to open a pipe to a *command* + instead of a raw socket. + (Default: `imaps`.) *host* -: Server hostname, for `type=imap` and `type=imaps`. - (Default: `localhost`.) +: Server hostname, for `type=imap` and `type=imaps`. + (Default: `localhost`.) *port* -: Server port. - (Default: `143` for `type=imap`, `993` for - `type=imaps`.) +: Server port. + (Default: `143` for `type=imap`, `993` for `type=imaps`.) *proxy* -: An optional SOCKS proxy to use for TCP connections to the IMAP server - (`type=imap` and `type=imaps` only), formatted as - `PROTOCOL://[USER:PASSWORD@]PROXYHOST[:PROXYPORT]`. - If `PROXYPORT` is omitted, it is assumed at port 1080. - Only [SOCKSv5][RFC 1928] is supported (with optional - [username/password authentication][RFC 1929]), in two flavors: - `socks5://` to resolve *hostname* locally, and `socks5h://` to let the - proxy resolve *hostname*. +: An optional SOCKS proxy to use for TCP connections to the IMAP + server (`type=imap` and `type=imaps` only), formatted as + `PROTOCOL://[USER:PASSWORD@]PROXYHOST[:PROXYPORT]`. + If `PROXYPORT` is omitted, it is assumed at port 1080. + Only [SOCKSv5][RFC 1928] is supported (with optional + [username/password authentication][RFC 1929]), in two flavors: + `socks5://` to resolve *hostname* locally, and `socks5h://` to let + the proxy resolve *hostname*. *command* -: Command to use for `type=tunnel`. Must speak the [IMAP4rev1 - protocol][RFC 3501] on its standard output, and understand it on its - standard input. +: Command to use for `type=tunnel`. Must speak the [IMAP4rev1 + protocol][RFC 3501] on its standard output, and understand it on its + standard input. *STARTTLS* -: Whether to use the [`STARTTLS`][RFC 2595] directive to upgrade to a - secure connection. Setting this to `YES` for a server not advertising - the `STARTTLS` capability causes `pullimap` to immediately abort the - connection. - (Ignored for *type*s other than `imap`. Default: `YES`.) +: Whether to use the [`STARTTLS`][RFC 2595] directive to upgrade to a + secure connection. Setting this to `YES` for a server not + advertising the `STARTTLS` capability causes `pullimap` to + immediately abort the connection. + (Ignored for *type*s other than `imap`. Default: `YES`.) *auth* -: Space-separated list of preferred authentication mechanisms. - `pullimap` uses the first mechanism in that list that is also - advertised (prefixed with `AUTH=`) in the server's capability list. - Supported authentication mechanisms are `PLAIN` and `LOGIN`. - (Default: `PLAIN LOGIN`.) +: Space-separated list of preferred authentication mechanisms. + `pullimap` uses the first mechanism in that list that is also + advertised (prefixed with `AUTH=`) in the server's capability list. + Supported authentication mechanisms are `PLAIN` and `LOGIN`. + (Default: `PLAIN LOGIN`.) *username*, *password* -: Username and password to authenticate with. Can be required for non - pre-authenticated connections, depending on the chosen authentication - mechanism. +: Username and password to authenticate with. Can be required for non + pre-authenticated connections, depending on the chosen + authentication mechanism. *compress* -: Whether to use the [`IMAP COMPRESS` extension][RFC 4978] for servers - advertising it. (Default: `YES`.) +: Whether to use the [`IMAP COMPRESS` extension][RFC 4978] for servers + advertising it. (Default: `YES`.) *null-stderr* -: Whether to redirect *command*'s standard error to `/dev/null` for type - `type=tunnel`. (Default: `NO`.) +: Whether to redirect *command*'s standard error to `/dev/null` for + type `type=tunnel`. (Default: `NO`.) *SSL_protocols* -: A space-separated list of SSL protocols to enable or disable (if - prefixed with an exclamation mark `!`. Known protocols are - `SSLv2`, `SSLv3`, `TLSv1`, `TLSv1.1`, and `TLSv1.2`. Enabling a - protocol is a short-hand for disabling all other protocols. - (Default: `!SSLv2 !SSLv3`, i.e., only enable TLSv1 and above.) +: A space-separated list of SSL protocols to enable or disable (if + prefixed with an exclamation mark `!`. Known protocols are `SSLv2`, + `SSLv3`, `TLSv1`, `TLSv1.1`, and `TLSv1.2`. Enabling a protocol is + a short-hand for disabling all other protocols. + (Default: `!SSLv2 !SSLv3`, i.e., only enable TLSv1 and above.) *SSL_cipher_list* -: The cipher list to send to the server. Although the server determines - which cipher suite is used, it should take the first supported cipher - in the list sent by the client. See [`ciphers`(1ssl)] for more - information. +: The cipher list to send to the server. Although the server + determines which cipher suite is used, it should take the first + supported cipher in the list sent by the client. See + [`ciphers`(1ssl)] for more information. *SSL_fingerprint* -: Fingerprint of the server certificate (or its public key) in the form - `ALGO$DIGEST_HEX`, where `ALGO` is the used algorithm (default - `sha256`). - Attempting to connect to a server with a non-matching certificate - fingerprint causes `pullimap` to abort the connection during the - SSL/TLS handshake. +: Fingerprint of the server certificate (or its public key) in the + form `[ALGO$]DIGEST_HEX`, where `ALGO` is the used algorithm + (by default `sha256`). + Attempting to connect to a server with a non-matching certificate + fingerprint causes `pullimap` to abort the connection during the + SSL/TLS handshake. *SSL_verify* -: Whether to verify the server certificate chain. - Note that using *SSL_fingerprint* to specify the fingerprint of the - server certificate is an orthogonal authentication measure as it - ignores the CA chain. - (Default: `YES`.) +: Whether to verify the server certificate chain. + Note that using *SSL_fingerprint* to specify the fingerprint of the + server certificate is an orthogonal authentication measure as it + ignores the CA chain. + (Default: `YES`.) *SSL_CApath* -: Directory to use for server certificate verification if - `SSL_verify=YES`. - This directory must be in “hash format”, see [`verify`(1ssl)] for - more information. +: Directory to use for server certificate verification if + `SSL_verify=YES`. + This directory must be in “hash format”, see [`verify`(1ssl)] for + more information. *SSL_CAfile* -: File containing trusted certificates to use during server certificate - authentication if `SSL_verify=YES`. +: File containing trusted certificates to use during server + certificate authentication if `SSL_verify=YES`. Control flow ============ @@ -260,9 +256,9 @@ The [IMAP4rev1 specification][RFC 3501] does not guaranty that untagged command. Thus it would be unsafe for `pullimap` to update the `UIDNEXT` value in its *statefile* while the `UID FETCH` command is progress. Instead, for each untagged `FETCH` response received while the `UID -FETCH` command is in progress, `pullimap` delivers the message `BODY` to -the SMTP or LMTP server (specified with *deliver-method*) then appends -the message UID to the *statefile*. +FETCH` command is in progress, `pullimap` delivers the message `RFC822` +body to the SMTP or LMTP server (specified with *deliver-method*) then +appends the message UID to the *statefile*. When the `UID FETCH` command eventually terminates, `pullimap` updates the `UIDNEXT` value in the *statefile* and truncate the file down to 8 bytes. Keeping track of message UIDs as they are received avoids @@ -272,20 +268,22 @@ FETCH` command is in progress. In more details, `pullimap` works as follows: 1. Issue an `UID FETCH` command to retrieve message `ENVELOPE` and - `BODY` (and `UID`) with UID bigger or equal than the `UIDNEXT` value - found in the *statefile*. + `RFC822` (and `UID`) with UID bigger or equal than the `UIDNEXT` + value found in the *statefile*. While the `UID FETCH` command is in progress, perform the following for each untagged `FETCH` response sent by the server: - i) if no SMTP/LMTP transmission channel was opened, open one to the - server specified with *deliver-method* and send an `EHLO` (or - `LHO`) command with the domain specified by *deliver-ehlo*; + i. if no SMTP/LMTP transmission channel was opened, open one to the + server specified with *deliver-method* and send an `EHLO` (or + `LHO`) command with the domain specified by *deliver-ehlo* (the + channel is kept open and shared for all messages retrieved while + the `UID FETCH` IMAP command is in progress); - ii) perform a mail transaction (using [SMTP pipelining][RFC 2920] if - possible) to send the retrieved message BODY to the SMTP or LMTP - session; and + i. perform a mail transaction (using [SMTP pipelining][RFC 2920] if + possible) to deliver the retrieved message `RFC822` body to the + SMTP or LMTP session; and - ii) append the message UID to the *statefile*. + i. append the message UID to the *statefile*. 2. If a SMTP/LMTP transmission channel was opened, send a `QUIT` command to terminate it gracefully. @@ -296,43 +294,44 @@ In more details, `pullimap` works as follows: 4. Update the *statefile* with the new UIDNEXT value (bytes 5-8). - 5. Truncate the *statefile* down to 8 bytes (so that it contains only - two 32-bits integers, respectively the *mailbox*'s current - `UIDVALIDITY` and `UIDNEXT` values). + 5. Truncate the *statefile* down to 8 bytes (so that it contains only + two 32-bits integers, respectively the *mailbox*'s current + `UIDVALIDITY` and `UIDNEXT` values). 6. If `--idle` was set, issue an `IDLE` command; stop idling and go - back to step 1. whenever a new message is received. + back to step 1 when a new message is received (or when the `IDLE` + timeout expires). Standards ========= * M. Leech, M. Ganis, Y. Lee, R. Kuris, D. Koblas and L. Jones, _SOCKS Protocol Version 5_, - [RFC 1928], March 1996 + [RFC 1928], March 1996. * M. Leech, _Username/Password Authentication for SOCKS V5_, - [RFC 1929], March 1996 + [RFC 1929], March 1996. * J. Myers, _Local Mail Transfer Protocol_, - [RFC 2033], October 1996 + [RFC 2033], October 1996. * J. Myers, _IMAP4 non-synchronizing literals_, - [RFC 2088], January 1997 - * B. Leiba, _IMAP4 IDLE command_, - [RFC 2177], June 1997 + [RFC 2088], January 1997. + * B. Leiba, _IMAP4 `IDLE` command_, + [RFC 2177], June 1997. * C. Newman, _Using TLS with IMAP, POP3 and ACAP_, - [RFC 2595], June 1999 + [RFC 2595], June 1999. * N. Freed, _SMTP Service Extension for Command Pipelining_, - [RFC 2920], September 2000 + [RFC 2920], September 2000. * M. Crispin, _Internet Message Access Protocol - Version 4rev1_, - [RFC 3501], March 2003 + [RFC 3501], March 2003. * M. Crispin, - _Internet Message Access Protocol (IMAP) - UIDPLUS extension_, - [RFC 4315], December 2005 - * A. Gulbrandsen, _The IMAP COMPRESS Extension_, - [RFC 4978], August 2007 + _Internet Message Access Protocol (IMAP) - `UIDPLUS` extension_, + [RFC 4315], December 2005. + * A. Gulbrandsen, _The IMAP `COMPRESS` Extension_, + [RFC 4978], August 2007. * R. Siemborski and A. Gulbrandsen, _IMAP Extension for Simple Authentication and Security Layer (SASL) Initial Client Response_, - [RFC 4959], September 2007 + [RFC 4959], September 2007. * J. Klensin, _Simple Mail Transfer Protocol_, - [RFC 5321], October 2008 + [RFC 5321], October 2008. [RFC 4315]: https://tools.ietf.org/html/rfc4315 [RFC 2177]: https://tools.ietf.org/html/rfc2177 |