aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--Makefile4
-rwxr-xr-xinterimap170
-rw-r--r--interimap.1403
-rw-r--r--interimap.md492
-rw-r--r--lib/Net/IMAP/InterIMAP.pm180
-rwxr-xr-xpullimap9
-rw-r--r--pullimap.md291
7 files changed, 832 insertions, 717 deletions
diff --git a/Makefile b/Makefile
index 7a56a47..c1513c6 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-all: pullimap.1
+all: pullimap.1 interimap.1
# upper case the headers and remove the links
%.1: %.md
@@ -24,6 +24,6 @@ all: pullimap.1
install:
clean:
- rm -f pullimap.1
+ rm -f pullimap.1 interimap.1
.PHONY: all install clean
diff --git a/interimap b/interimap
index 849aa85..a6e2d06 100755
--- a/interimap
+++ b/interimap
@@ -53,7 +53,7 @@ sub usage(;$) {
}
my @COMMANDS = qw/repair delete rename/;
-usage(1) unless GetOptions(\%CONFIG, qw/config=s quiet|q target=s@ debug help|h watch:i/, @COMMANDS);
+usage(1) unless GetOptions(\%CONFIG, qw/config=s quiet|q target=s@ debug help|h watch:i notify/, @COMMANDS);
usage(0) if $CONFIG{help};
my $COMMAND = do {
my @command = grep {exists $CONFIG{$_}} @COMMANDS;
@@ -61,9 +61,9 @@ my $COMMAND = do {
$command[0]
};
usage(1) if defined $COMMAND and (($COMMAND eq 'delete' and !@ARGV) or ($COMMAND eq 'rename' and $#ARGV != 1));
-usage(1) if defined $COMMAND and defined $CONFIG{watch};
+usage(1) if defined $COMMAND and (defined $CONFIG{watch} or defined $CONFIG{notify});
usage(1) if $CONFIG{target} and !(defined $COMMAND and ($COMMAND eq 'delete'or $COMMAND eq 'rename'));
-$CONFIG{watch} = 60 if defined $CONFIG{watch} and $CONFIG{watch} == 0;
+$CONFIG{watch} = $CONFIG{notify} ? 900 : 60 unless $CONFIG{watch};
@ARGV = map {uc $_ eq 'INBOX' ? 'INBOX' : $_ } @ARGV; # INBOX is case-insensitive
die "Invalid mailbox name $_" foreach grep !/\A([\x01-\x7F]+)\z/, @ARGV;
@@ -76,7 +76,7 @@ my $CONF = read_config( delete $CONFIG{config} // $NAME
, 'list-select-opts' => qr/\A([\x21\x23\x24\x26\x27\x2B-\x5B\x5E-\x7A\x7C-\x7E]+)\z/
, 'ignore-mailbox' => qr/\A([\x01-\x09\x0B\x0C\x0E-\x7F]+)\z/
);
-my ($DBFILE, $LOCKFILE, $LOGGER_FD);
+my ($DBFILE, $LOGGER_FD);
{
$DBFILE = $CONF->{_}->{database} if defined $CONF->{_};
@@ -94,8 +94,6 @@ my ($DBFILE, $LOCKFILE, $LOGGER_FD);
}
}
- $LOCKFILE = $DBFILE =~ s/([^\/]+)\z/.$1.lck/r;
-
if (defined $CONF->{_} and defined $CONF->{_}->{logfile}) {
require 'POSIX.pm';
require 'Time/HiRes.pm';
@@ -116,7 +114,6 @@ my ($IMAP, $lIMAP, $rIMAP);
sub cleanup() {
undef $_ foreach grep defined, ($IMAP, $lIMAP, $rIMAP);
logger(undef, "Cleaning up...") if $CONFIG{debug};
- unlink $LOCKFILE if defined $LOCKFILE and -f $LOCKFILE;
close $LOGGER_FD if defined $LOGGER_FD;
$DBH->disconnect() if defined $DBH;
}
@@ -125,33 +122,16 @@ $SIG{TERM} = sub { cleanup(); exit 0; };
#############################################################################
-# Lock the database
-{
- if (-f $LOCKFILE) {
- open my $lock, '<', $LOCKFILE or die "Can't open $LOCKFILE: $!\n";
- my $pid = <$lock>;
- close $lock;
- chomp $pid;
- my $msg = "LOCKFILE '$LOCKFILE' exists.";
- undef $LOCKFILE; # don't delete the lockfile
- $msg .= " (Is PID $pid running?)" if defined $pid and $pid =~ /^[0-9]+$/;
- die $msg, "\n";
- }
-
- open my $lock, '>', $LOCKFILE or die "Can't open $LOCKFILE: $!\n";
- print $lock $$, "\n";
- close $lock;
-}
-
-
-#############################################################################
# Open the database and create tables
$DBH = DBI::->connect("dbi:SQLite:dbname=$DBFILE", undef, undef, {
AutoCommit => 0,
RaiseError => 1,
sqlite_see_if_its_a_number => 1, # see if the bind values are numbers or not
+ sqlite_use_immediate_transaction => 1,
});
+$DBH->sqlite_busy_timeout(250);
+$DBH->do('PRAGMA locking_mode = EXCLUSIVE');
$DBH->do('PRAGMA foreign_keys = ON');
@@ -236,10 +216,13 @@ logger(undef, ">>> $NAME $VERSION");
my $LIST = '"" ';
my @LIST_PARAMS;
+my %LIST_PARAMS_STATUS = (STATUS => [qw/UIDVALIDITY UIDNEXT HIGHESTMODSEQ/]);
if (!defined $COMMAND or $COMMAND eq 'repair') {
$LIST = '('.uc($CONF->{_}->{'list-select-opts'}).') '.$LIST if defined $CONF->{_}->{'list-select-opts'};
$LIST .= (defined $CONF->{_}->{'list-mailbox'} ? '('.$CONF->{_}->{'list-mailbox'}.')' : '*') unless @ARGV;
- @LIST_PARAMS = ('SUBSCRIBED', 'STATUS (UIDVALIDITY UIDNEXT HIGHESTMODSEQ)');
+ @LIST_PARAMS = ('SUBSCRIBED');
+ push @LIST_PARAMS, map { "$_ (".join(' ', @{$LIST_PARAMS_STATUS{$_}}).")" } keys %LIST_PARAMS_STATUS
+ unless $CONFIG{notify};
}
$LIST .= $#ARGV == 0 ? Net::IMAP::InterIMAP::quote($ARGV[0])
: ('('.join(' ',map {Net::IMAP::InterIMAP::quote($_)} @ARGV).')') if @ARGV;
@@ -257,20 +240,8 @@ foreach my $name (qw/local remote/) {
$IMAP->{$name} = { client => Net::IMAP::InterIMAP::->new(%config) };
my $client = $IMAP->{$name}->{client};
- die "Non $_-capable IMAP server.\n" foreach $client->incapable(qw/LIST-EXTENDED LIST-STATUS UIDPLUS/);
- # XXX We should start by listing all mailboxes matching the user's LIST
- # criterion, then issue "SET NOTIFY (mailboxes ... (...))". But this
- # crashes the IMAP client:
- # http://dovecot.org/pipermail/dovecot/2015-July/101473.html
- #my $mailboxes = $client->list((uc $config{'subscribed-only'} eq 'TRUE' ? '(SUBSCRIBED)' : '' )
- # .$config{mailboxes}, 'SUBSCRIBED');
- # $client->notify('SELECTED', 'MAILBOXES ('.join(' ', keys %$mailboxes).')');
- # XXX NOTIFY doesn't work as expected for INBOX
- # http://dovecot.org/pipermail/dovecot/2015-July/101514.html
- #$client->notify(qw/SELECTED SUBSCRIBED/) if $CONFIG{watch};
- # XXX We shouldn't need to ask for STATUS responses here, and use
- # NOTIFY's STATUS indicator instead. However Dovecot violates RFC
- # 5464: http://dovecot.org/pipermail/dovecot/2015-July/101474.html
+ die "Non $_-capable IMAP server.\n" foreach $client->incapable(qw/LIST-EXTENDED UIDPLUS/);
+ die "Non LIST-STATUS-capable IMAP server.\n" if !$CONFIG{notify} and $client->incapable('LIST-STATUS');
}
@{$IMAP->{$_}}{qw/mailboxes delims/} = $IMAP->{$_}->{client}->list($LIST, @LIST_PARAMS) for qw/local remote/;
@@ -425,9 +396,8 @@ elsif (defined $COMMAND and $COMMAND eq 'rename') {
##############################################################################
# Synchronize mailbox and subscription lists
-my @MAILBOXES;
sub sync_mailbox_list() {
- my %mailboxes;
+ my (%mailboxes, @mailboxes);
$mailboxes{$_} = 1 foreach keys %{$IMAP->{local}->{mailboxes}};
$mailboxes{$_} = 1 foreach keys %{$IMAP->{remote}->{mailboxes}};
my $sth_subscribe = $DBH->prepare(q{UPDATE mailboxes SET subscribed = ? WHERE idx = ?});
@@ -444,7 +414,7 @@ sub sync_mailbox_list() {
};
check_delim($mailbox); # ensure that the delimiter match
- push @MAILBOXES, $mailbox unless grep {lc $_ eq lc '\NoSelect'} @attrs;
+ push @mailboxes, $mailbox unless grep {lc $_ eq lc '\NoSelect'} @attrs;
$STH_GET_INDEX->execute($mailbox);
my ($idx,$subscribed) = $STH_GET_INDEX->fetchrow_array();
@@ -512,9 +482,10 @@ sub sync_mailbox_list() {
$DBH->commit();
}
}
+ return @mailboxes;
}
-sync_mailbox_list();
+my @MAILBOXES = sync_mailbox_list();
($lIMAP, $rIMAP) = map {$IMAP->{$_}->{client}} qw/local remote/;
my $ATTRS = join ' ', qw/MODSEQ FLAGS INTERNALDATE BODY.PEEK[]/;
@@ -1066,26 +1037,6 @@ sub sync_messages($$;$$) {
}
-# Wait up to $timout seconds for notifications on either IMAP server.
-# Then issue a NOOP so the connection doesn't terminate for inactivity.
-sub wait_notifications(;$) {
- my $timeout = shift // 300;
-
- while ($timeout > 0) {
- my $r1 = $lIMAP->slurp();
- my $r2 = $rIMAP->slurp();
- last if $r1 or $r2; # got update!
-
- sleep 1;
- if (--$timeout == 0) {
- $lIMAP->noop();
- $rIMAP->noop();
- # might have got updates so exit the loop
- }
- }
-}
-
-
#############################################################################
# Resume interrupted mailbox syncs (before initializing the cache).
#
@@ -1166,7 +1117,27 @@ if (defined $COMMAND and $COMMAND eq 'repair') {
}
-while(1) {
+if ($CONFIG{notify}) {
+ # Be notified of new messages with EXISTS/RECENT responses, but don't
+ # receive unsolicited FETCH responses with a RFC822/BODY[]. It costs us an
+ # extra roundtrip, but we need to sync FLAG updates and VANISHED responses
+ # in batch mode, update the HIGHESTMODSEQ, and *then* issue an explicit UID
+ # FETCH command to get new message, and process each FETCH response with a
+ # RFC822/BODY[] attribute as they arrive.
+ my $mailboxes = join(' ', map {Net::IMAP::InterIMAP::quote($_)} @MAILBOXES);
+ my %mailboxes = map { $_ => [qw/MessageNew MessageExpunge FlagChange/] }
+ ( "MAILBOXES ($mailboxes)", 'SELECTED' );
+ my %personal = ( personal => [qw/MailboxName SubscriptionChange/] );
+
+ foreach ($lIMAP, $rIMAP) {
+ # require STATUS responses for our @MAILBOXES only
+ $_->notify('SET STATUS', %mailboxes);
+ $_->notify('SET', %mailboxes, %personal);
+ }
+}
+
+
+sub loop() {
while(@MAILBOXES) {
if (defined $MAILBOX and ($lIMAP->is_dirty($MAILBOX) or $rIMAP->is_dirty($MAILBOX))) {
# $MAILBOX is dirty on either the local or remote mailbox
@@ -1202,26 +1173,55 @@ while(1) {
sync_messages($IDX, $MAILBOX);
}
}
- # clean state!
- exit 0 unless $CONFIG{watch};
-
- # we need to issue a NOOP command or go back to AUTH state since the
- # LIST command may not report the correct HIGHESTMODSEQ value for
- # the mailbox currently selected.
- if (defined $MAILBOX) {
- # Prefer UNSELECT over NOOP commands as it requires a single command per cycle
- if ($lIMAP->incapable('UNSELECT') or $rIMAP->incapable('UNSELECT')) {
- $_->noop() foreach ($lIMAP, $rIMAP);
- } else {
- $_->unselect() foreach ($lIMAP, $rIMAP);
- undef $MAILBOX;
- }
+}
+sub notify(@) {
+ # TODO: interpret LIST responses to detect mailbox
+ # creation/deletion/subcription/unsubscription
+ # mailbox creation
+ # * LIST () "/" test
+ # mailbox subscribtion
+ # * LIST (\Subscribed) "/" test
+ # mailbox unsubscribtion
+ # * LIST () "/" test
+ # mailbox renaming
+ # * LIST () "/" test2 ("OLDNAME" (test))
+ # mailbox deletion
+ # * LIST (\NonExistent) "/" test2
+ unless (Net::IMAP::InterIMAP::slurp(\@_, $CONFIG{watch}, \&Net::IMAP::InterIMAP::is_dirty)) {
+ $_->noop() foreach @_;
+ }
+}
+
+unless (defined $CONFIG{watch}) {
+ loop();
+ exit 0;
+}
+
+while (1) {
+ loop();
+
+ if ($CONFIG{notify}) {
+ notify($lIMAP, $rIMAP);
}
+ else {
+ # we need to issue a NOOP command or go back to AUTH state since the
+ # LIST command may not report the correct HIGHESTMODSEQ value for
+ # the mailbox currently selected
+ if (defined $MAILBOX) {
+ # Prefer UNSELECT over NOOP commands as it requires a single command per cycle
+ if ($lIMAP->incapable('UNSELECT') or $rIMAP->incapable('UNSELECT')) {
+ $_->noop() foreach ($lIMAP, $rIMAP);
+ } else {
+ $_->unselect() foreach ($lIMAP, $rIMAP);
+ undef $MAILBOX;
+ }
+ }
- sleep $CONFIG{watch};
- # Refresh the mailbox list and status
- @{$IMAP->{$_}}{qw/mailboxes delims/} = $IMAP->{$_}->{client}->list($LIST, @LIST_PARAMS) for qw/local remote/;
- sync_mailbox_list();
+ sleep $CONFIG{watch};
+ # refresh the mailbox list and status
+ @{$IMAP->{$_}}{qw/mailboxes delims/} = $IMAP->{$_}->{client}->list($LIST, @LIST_PARAMS) for qw/local remote/;
+ @MAILBOXES = sync_mailbox_list();
+ }
}
END { cleanup(); }
diff --git a/interimap.1 b/interimap.1
deleted file mode 100644
index 3aabc3f..0000000
--- a/interimap.1
+++ /dev/null
@@ -1,403 +0,0 @@
-.TH INTERIMAP "1" "JULY 2015" "InterIMAP" "User Commands"
-
-.SH NAME
-InterIMAP \- Fast bidirectional synchronization for QRESYNC-capable IMAP servers
-
-.SH SYNOPSIS
-.B interimap\fR [\fIOPTION\fR ...] [\fICOMMAND\fR] [\fIMAILBOX\fR ...]
-
-
-.SH DESCRIPTION
-.PP
-.B InterIMAP\fR performs stateful synchronization between two IMAP4rev1
-servers.
-Such synchronization is made possible by the QRESYNC extension from
-[RFC7162]; for convenience reasons servers must also support
-LIST\-EXTENDED [RFC5258], LIST\-STATUS [RFC5819] and UIDPLUS [RFC4315].
-See also the \fBSUPPORTED EXTENSIONS\fR section.
-
-.PP
-Stateful synchronization is only possible for mailboxes supporting
-persistent message Unique Identifiers (UID) and persistent storage of
-mod\-sequences (MODSEQ); any non\-compliant mailbox will cause
-\fBInterIMAP\fR to abort.
-Furthermore, because UIDs are allocated not by the client but by the
-server, \fBInterIMAP\fR needs to keep track of associations between local
-and remote UIDs for each mailbox.
-The synchronization state of a mailbox consists of its UIDNEXT and
-HIGHESTMODSEQ values on each server;
-it is then assumed that each message with UID < $UIDNEXT have been
-replicated to the other server, and that the metadata (such as flags) of
-each message with MODSEQ <= $HIGHESTMODSEQ have been synchronized.
-Conceptually, the synchronization algorithm is derived from [RFC4549]
-with the [RFC7162, section 6] amendments, and works as follows:
-
-.nr step 1 1
-.IP \n[step]. 4
-SELECT (on both servers) a mailbox the current UIDNEXT or HIGHESTMODSEQ
-values of which differ from the values found in the database (for either
-server). Use the QRESYNC SELECT parameter from [RFC7162] to list
-changes (vanished messages and flag updates) since $HIGHESTMODSEQ to
-messages with UID<$UIDNEXT.
-
-.IP \n+[step].
-Propagate these changes onto the other server: get the corresponding
-UIDs from the database, then a/ issue an UID STORE + UID EXPUNGE command
-to remove messages that have not already been deleted on both servers,
-and b/ issue UID STORE commands to propagate flag updates (send a single
-command for each flag list in order the reduce the number of round
-trips).
-(Conflicts may occur if the metadata of a message has been updated on
-both servers with different flag lists; in that case \fBInterIMAP\fR
-issues a warning and updates the message on each server with the union
-of both flag lists.)
-Repeat this step if the server sent some updates in the meantime.
-Otherwise, update the HIGHESTMODSEQ value in the database.
-
-.IP \n+[step].
-Process new messages (if the current UIDNEXT value differ from the one
-found in the database) by issuing an UID FETCH command and for each
-message RFC822 body received, issue an APPEND command to the other
-server on\-the\-fly.
-Repeat this step if the server received new messages in the meantime.
-Otherwise, update the UIDNEXT value in the database.
-Go back to step 2 if the server sent some updates in the meantime.
-
-.IP \n+[step].
-Go back to step 1 to proceed with the next unsynchronized mailbox.
-
-.SH COMMANDS
-.PP
-By default \fBInterIMAP\fR synchronizes each mailbox listed by the
-\(lqLIST "" "*"\(rq IMAP command;
-the \fIlist-mailbox\fR, \fIlist-select-opts\fR and \fIignore-mailbox\fR
-options from the configuration file can be used to shrink that list and
-save bandwidth.
-However if some extra argument are provided on the command line,
-\fBInterIMAP\fR ignores said options and synchronizes the given
-\fIMAILBOX\fRes instead. Note that each \fIMAILBOX\fR is taken \(lqas
-is\(rq; in particular, it must be UTF-7 encoded, unquoted, and the list
-wildcards \(oq*\(cq and \(oq%\(cq are not interpolated.
-
-.PP
-If the synchronization was interrupted during a previous run while some
-messages were being replicated (but before the UIDNEXT or HIGHESTMODSEQ
-values have been updated), \fBInterIMAP\fR performs a \(lqfull
-synchronization\(rq on theses messages only:
-downloading the whole UID and flag lists on each servers allows
-\fBInterIMAP\fR to detect messages that have been removed or for which
-their flags have changed in the meantime.
-Finally, after propagating the offline changes for these messages,
-\fBInterIMAP\fR resumes the synchronization for the rest of the mailbox.
-
-.PP
-Specifying one of the commands below makes \fBInterIMAP\fR perform an
-action other than the default QRESYNC-based synchronization.
-
-.TP
-.B \-\-repair \fR[\fIMAILBOX\fR ...]
-List the database anomalies and try to repair them.
-(Consider only the given \fIMAILBOX\fRes if non-optional arguments are
-provided.)
-This is done by performing a so\-called \(lqfull synchronization\(rq,
-namely 1/ download all UIDs along with their flags from both the local
-and remote servers, 2/ ensure that each entry in the database corresponds
-to an existing UID, and 3/ ensure that both flag lists match.
-Any message found on a server but not in the database is replicated on
-the other server (which in the worst case, might lead to a message
-duplicate).
-Flag conflicts are solved by updating each message to the union of both
-lists.
-
-.TP
-.B \-\-delete \fIMAILBOX\fR [...]
-Delete the given \fIMAILBOX\fRes on each target (by default each server
-plus the database, unless \fB\-\-target\fR specifies otherwise) where
-it exists.
-Note that per [RFC3501] deletion is not recursive: \fIMAILBOX\fR's
-children are not deleted.
-
-.TP
-.B \-\-rename \fISOURCE\fR \fIDEST\fR
-Rename the mailbox \fISOURCE\fR to \fIDEST\fR on each target (by default
-each server plus the database, unless \fB\-\-target\fR specifies
-otherwise) where it exists.
-\fBInterIMAP\fR aborts if \fIDEST\fR already exists on either target.
-Note that per [RFC3501] the renaming is recursive: \fISOURCE\fR's
-children are moved to become \fIDEST\fR's children instead.
-
-
-.SH OPTIONS
-.TP
-.B \-\-config=\fR\fIFILE\fR
-Specify an alternate configuration file. Relative paths start from
-\fI$XDG_CONFIG_HOME\fR, or \fI~/.config\fR if the XDG_CONFIG_HOME
-environment variable is unset.
-
-.TP
-.B \fB\-\-target=\fR{local,remote,database}
-Limit the scope of a \fB\-\-delete\fR or \fB\-\-rename\fR command
-to the given target. Can be repeated to act on multiple targets. By
-default all three targets are considered.
-
-.TP
-.B \fB\-\-watch\fR[\fB=\fR\fIseconds\fR]
-Don't exit after a successful synchronization, and keep synchronizing
-forever instead. Sleep for the given number of \fIseconds\fR (or
-\(lq60\(rq if omitted) between two synchronizations.
-
-.TP
-.B \-q\fR, \fB\-\-quiet\fR
-Try to be quiet.
-
-.TP
-.B \-\-debug
-Turn on debug mode. Debug messages are written to the given \fIlogfile\fR.
-Note that this include all IMAP traffic (except literals). Depending on the
-chosen authentication mechanism, this might include authentication credentials.
-
-.TP
-.B \-h\fR, \fB\-\-help\fR
-Output a brief help and exit.
-
-.TP
-.B \-\-version
-Show the version number and exit.
-
-.SH CONFIGURATION FILE
-
-Unless told otherwise by the \fB\-\-config=\fR\fIFILE\fR option,
-\fBInterIMAP\fR reads its configuration from
-\fI$XDG_CONFIG_HOME/interimap\fR (or \fI~/.config/interimap\fR if the
-XDG_CONFIG_HOME environment variable is unset) as an INI file.
-The syntax of the configuration file is a series of
-\fIOPTION\fR=\fIVALUE\fR lines organized under some \fI[SECTION]\fR;
-lines starting with a \(oq#\(cq or \(oq;\(cq character are ignored as
-comments.
-The sections \(lq[local]\(rq and \(lq[remote]\(rq define the two IMAP
-servers to synchronize.
-Valid options are:
-
-.TP
-.I database
-SQLite version 3 database file to use to keep track of associations
-between local and remote UIDs, as well as the UIDVALIDITY, UIDNEXT and
-HIGHESTMODSEQ of each known mailbox on both servers.
-Relative paths start from \fI$XDG_DATA_HOME/interimap\fR, or
-\fI~/.local/share/interimap\fR if the XDG_DATA_HOME environment variable
-is unset.
-This option is only available in the default section.
-(Default: \(lq\fIhost\fR.db\)\(rq, where \fIhost\fR is taken from the
-\(lq[remote]\(rq or \(lq[local]\(rq sections, in that order.)
-
-.TP
-.I list-mailbox
-A space separated list of mailbox patterns to use when issuing the
-initial LIST command (overridden by the \fIMAILBOX\fRes given as
-command-line arguments).
-Note that each pattern containing special characters such as spaces or
-brackets (see [RFC3501] for the exact syntax) must be quoted.
-Furthermore, non-ASCII names must be UTF\-7 encoded.
-Two wildcards are available: a \(oq*\(cq character matches zero or more
-characters, while a \(oq%\(cq character matches zero or more characters
-up to the mailbox's hierarchy delimiter.
-This option is only available in the default section.
-(The default pattern, \(lq*\(rq, matches all visible mailboxes on the
-server.)
-
-.TP
-.I list-select-opts
-An optional space separated list of selectors for the initial LIST
-command. (Requires a server supporting the LIST-EXTENDED [RFC5258]
-extension.) Useful values are
-\(lqSUBSCRIBED\(rq (to list only subscribed mailboxes),
-\(lqREMOTE\(rq (to also list remote mailboxes on a server supporting
-mailbox referrals), and \(lqRECURSIVEMATCH\(rq (to list parent mailboxes
-with children matching one of the \fIlist-mailbox\fR patterns above).
-This option is only available in the default section.
-
-.TP
-.I ignore-mailbox
-An optional Perl Compatible Regular Expressions (PCRE) covering
-mailboxes to exclude:
-any (UTF-7 encoded, unquoted) mailbox listed in the initial LIST
-responses is ignored if it matches the given expression.
-Note that the \fIMAILBOX\fRes given as command-line arguments bypass the
-check and are always considered for synchronization.
-This option is only available in the default section.
-
-.TP
-.I logfile
-A file name to use to log debug and informational messages. This option is
-only available in the default section.
-
-.TP
-.I type
-One of \(lqimap\(rq, \(lqimaps\(rq or \(lqtunnel\(rq.
-\fItype\fR=imap and \fItype\fR=imaps are respectively used for IMAP and
-IMAP over SSL/TLS connections over a INET socket.
-\fItype\fR=tunnel causes \fBInterIMAP\fR to open a pipe to a
-\fIcommand\fR instead of a raw socket.
-Note that specifying \fItype\fR=tunnel in the \(lq[remote]\(rq section
-makes the default \fIdatabase\fR to be \(lqlocalhost.db\(rq.
-(Default: \(lqimaps\(rq.)
-
-.TP
-.I host
-Server hostname, for \fItype\fR=imap and \fItype\fR=imaps.
-(Default: \(lqlocalhost\(rq.)
-
-.TP
-.I port
-Server port.
-(Default: \(lq143\(rq for \fItype\fR=imap, \(lq993\(rq for
-\fItype\fR=imaps.)
-
-.TP
-.I proxy
-An optional SOCKS proxy to use for TCP connections to the IMAP server
-(\fItype\fR=imap and \fItype\fR=imaps only), formatted as
-\(lq\fIprotocol\fR://[\fIuser\fR:\fIpassword\fR@]\fIproxyhost\fR[:\fIproxyport\fR]\(rq.
-If \fIproxyport\fR is omitted, it is assumed at port 1080.
-Only SOCKSv5 is supported, in two flavors: \(lqsocks5://\(rq to resolve
-\fIhostname\fR locally, and \(lqsocks5h://\(rq to let the proxy resolve
-\fIhostname\fR.
-
-.TP
-.I command
-Command to use for \fItype\fR=tunnel. Must speak the IMAP4rev1 protocol
-on its standard output, and understand it on its standard input.
-
-.TP
-.I STARTTLS
-Whether to use the \(lqSTARTTLS\(rq directive to upgrade to a secure
-connection. Setting this to \(lqYES\(rq for a server not advertising
-the \(lqSTARTTLS\(rq capability causes \fBInterIMAP\fR to immediately
-abort the connection.
-(Ignored for \fItype\fRs other than \(lqimap\(rq. Default: \(lqYES\(rq.)
-
-.TP
-.I auth
-Space\-separated list of preferred authentication mechanisms.
-\fBInterIMAP\fR uses the first mechanism in that list that is also
-advertised (prefixed with \(lqAUTH=\(rq) in the server's capability list.
-Supported authentication mechanisms are \(lqPLAIN\(rq and \(lqLOGIN\(rq.
-(Default: \(lqPLAIN LOGIN\(rq.)
-
-.TP
-.I username\fR, \fIpassword\fR
-Username and password to authenticate with. Can be required for non
-pre\-authenticated connections, depending on the chosen authentication
-mechanism.
-
-.TP
-.I compress
-Whether to use the IMAP COMPRESS extension [RFC4978] for servers
-advertising it.
-(Default: \(lqNO\(rq for the \(lq[local]\(rq section, \(lqYES\(rq for
-the \(lq[remote]\(rq section.)
-
-.TP
-.I null\-stderr
-Whether to redirect \fIcommand\fR's standard error to \(lq/dev/null\(rq
-for type \fItype\fR=tunnel.
-(Default: \(lqNO\(rq.)
-
-.TP
-.I SSL_protocols
-A space-separated list of SSL protocols to enable or disable (if
-prefixed with an exclamation mark \(oq!\(cq). Known protocols are
-\(lqSSLv2\(rq, \(lqSSLv3\(rq, \(lqTLSv1\(rq, \(lqTLSv1.1\(rq, and
-\(lqTLSv1.2\(rq. Enabling a protocol is a short-hand for disabling all
-other protocols.
-(Default: \(lq!SSLv2 !SSLv3\(rq, i.e., only enable TLSv1 and above.)
-
-.TP
-.I SSL_cipher_list
-The cipher list to send to the server. Although the server determines
-which cipher suite is used, it should take the first supported cipher in
-the list sent by the client. See \fBciphers\fR(1ssl) for more
-information.
-
-.TP
-.I SSL_fingerprint
-Fingerprint of the server certificate (or its public key) in the form
-\fIALGO\fR$\fIDIGEST_HEX\fR, where \fIALGO\fR is the used algorithm
-(default \(lqsha256\(rq).
-Attempting to connect to a server with a non-matching certificate
-fingerprint causes \fBInterIMAP\fR to abort the connection during the
-SSL/TLS handshake.
-
-.TP
-.I SSL_verify
-Whether to verify the server certificate chain.
-Note that using \fISSL_fingerprint\fR to specify the fingerprint of the
-server certificate is an orthogonal authentication measure as it ignores
-the CA chain.
-(Default: \(lqYES\(rq.)
-
-.TP
-.I SSL_CApath
-Directory to use for server certificate verification if
-\(lq\fISSL_verify\fR=YES\(rq.
-This directory must be in \(lqhash format\(rq, see \fBverify\fR(1ssl)
-for more information.
-
-.TP
-.I SSL_CAfile
-File containing trusted certificates to use during server certificate
-authentication if \(lq\fISSL_verify\fR=YES\(rq.
-
-.SH SUPPORTED EXTENSIONS
-
-Performance is better for servers supporting the following extensions to
-the IMAP4rev1 [RFC3501] protocol:
-
-.IP \[bu] 4
-LITERAL+ [RFC2088] non-synchronizing literals (recommended),
-.IP \[bu]
-MULTIAPPEND [RFC3502] (recommended),
-.IP \[bu]
-COMPRESS=DEFLATE [RFC4978] (recommended),
-.IP \[bu]
-SASL-IR [RFC4959] SASL Initial Client Response, and
-.IP \[bu]
-UNSELECT [RFC3691].
-
-.SH KNOWN BUGS AND LIMITATIONS
-
-.IP \[bu] 4
-Using \fBInterIMAP\fR on two identical servers with a non-existent or
-empty database will duplicate each message due to the absence of
-local/remote UID association.
-.IP \[bu]
-\fBInterIMAP\fR is single threaded and doesn't use IMAP command
-pipelining. Synchronization could be boosted up by sending independent
-commands (such as the initial LIST/STATUS command) to each server in
-parallel, and for a given server, by sending independent commands (such
-as flag updates) in a pipeline.
-.IP \[bu]
-Because the IMAP protocol doesn't have a specific response code for when
-a message is moved to another mailbox (using the MOVE command from
-[RFC6851] or COPY + STORE + EXPUNGE), moving a messages causes
-\fBInterIMAP\fR to believe that it was deleted while another one (which
-is replicated again) was added to the other mailbox in the meantime.
-.IP \[bu]
-\(lqPLAIN\(rq and \(lqLOGIN\(rq are the only authentication mechanisms
-currently supported.
-.IP \[bu]
-\fBInterIMAP\fR will probably not work with non RFC-compliant servers.
-In particular, no work-around are currently implemented beside the
-tunables in the \fBCONFIGURATION FILE\fR. Moreover, few IMAP servers
-have been tested so far.
-
-.SH AUTHOR
-.ie \n[www-html] \{\
- Written by
-. MTO guilhem@fripost.org "Guilhem Moulin" .
-\}
-.el \{\
- Written by Guilhem Moulin
-. MT guilhem@fripost.org
-. ME .
-\}
diff --git a/interimap.md b/interimap.md
new file mode 100644
index 0000000..9515c4a
--- /dev/null
+++ b/interimap.md
@@ -0,0 +1,492 @@
+% intermap(1)
+% [Guilhem Moulin](mailto:guilhem@fripost.org)
+% July 2015
+
+Name
+====
+
+InterIMAP - Fast bidirectional synchronization for QRESYNC-capable IMAP servers
+
+Synopsis
+========
+
+`interimap` [*OPTION* ...] [*COMMAND*] [*MAILBOX* ...]
+
+Description
+===========
+
+`interimap` performs stateful synchronization between two IMAP4rev1
+servers.
+Such synchronization is made possible by the [`QRESYNC` IMAP
+extension][RFC 7162]; for convenience reasons servers must also support
+the [`LIST-EXTENDED`][RFC 5258], [`LIST-STATUS`][RFC 5819] (or
+[`NOTIFY`][RFC 5465]) and [`UIDPLUS`][RFC 4315] IMAP extensions.
+See also the **[supported extensions](#supported-extensions)** section
+below.
+
+Stateful synchronization is only possible for mailboxes supporting
+persistent message Unique Identifiers (UID) and persistent storage of
+mod-sequences (MODSEQ); any non-compliant mailbox will cause `interimap`
+to abort.
+Furthermore, because UIDs are allocated not by the client but by the
+server, `interimap` needs to keep track of associations between local
+and remote UIDs for each mailbox.
+The synchronization state of a mailbox consists of its `UIDNEXT` and
+`HIGHESTMODSEQ` values on each server; it is then assumed that each
+message with UID smaller than `UIDNEXT` have been replicated to the
+other server, and that the metadata (such as flags) of each message with
+MODSEQ at most `HIGHESTMODSEQ` have been synchronized.
+Conceptually, the synchronization algorithm is derived from [RFC 4549]
+with the [RFC 7162] (section 6) amendments, and works as follows:
+
+ 1. `SELECT` (on both servers) a mailbox the current `UIDNEXT` or `HIGHESTMODSEQ`
+ values of which differ from the values found in the database (for
+ either server). Use the `QRESYNC` `SELECT` parameter from [RFC
+ 7162] to list changes (vanished messages and flag updates) since
+ `HIGHESTMODSEQ` to messages with UID smaller than `UIDNEXT`.
+
+ 2. Propagate these changes onto the other server: get the corresponding
+ UIDs from the database, then:
+ a. issue an `UID STORE` command, followed by `UID EXPUNGE`, to
+ remove messages that have not already been deleted on both
+ servers; and
+ b. issue some `UID STORE` commands to propagate flag updates (send
+ a single command for each flag list in order the reduce the
+ number of round trips).
+
+ (Conflicts may occur if the metadata of a message has been updated
+ on both servers with different flag lists; in that case, `interimap`
+ issues a warning and updates the message on each server with the
+ union of both flag lists.)
+ Repeat this step if the server sent some updates in the meantime.
+ Otherwise, update the `HIGHESTMODSEQ` value in the database.
+
+ 3. Process new messages (if the current `UIDNEXT` value of the mailbox
+ differs from the one found in the database) by issuing an `UID
+ FETCH` command; process each received message on-the-fly by issuing
+ an `APPEND` command with the message's `RFC822` body, `FLAGS` and
+ `INTERNALDATE`.
+ Repeat this step if the server received new messages in the
+ meantime. Otherwise, update the `UIDNEXT` value in the database.
+ Go back to step 2 if the server sent some metadata (such as flag)
+ updates in the meantime.
+
+ 4. Go back to step 1 to proceed with the next unsynchronized mailbox.
+
+Commands
+========
+
+By default, `interimap` synchronizes each mailbox listed by the `LIST ""
+"*"` IMAP command;
+the *list-mailbox*, *list-select-opts* and *ignore-mailbox* options from
+the [configuration file](#configuration-file) can be used to shrink that
+list and save bandwidth.
+However if some extra argument are provided on the command line,
+`interimap` ignores said options and synchronizes the given
+*MAILBOX*es instead. Note that each *MAILBOX* is taken “as is”; in
+particular, it must be [UTF-7 encoded][RFC 2152], unquoted, and the list
+wildcards ‘\*’ and ‘%’ are not interpolated.
+
+If the synchronization was interrupted during a previous run while some
+messages were being replicated (but before the `UIDNEXT` or
+`HIGHESTMODSEQ` values have been updated), `interimap` performs a “full
+synchronization” on theses messages: downloading the whole UID and flag
+lists on each servers allows `interimap` to detect messages that have
+been removed or for which their flags have changed in the meantime.
+Finally, after propagating the offline changes for these messages,
+`interimap` resumes the synchronization for the rest of the mailbox.
+
+Specifying one of the commands below makes `interimap` perform an action
+other than the default [`QRESYNC`][RFC 7162]-based synchronization.
+
+`--repair` [*MAILBOX* ...]
+
+: List the database anomalies and try to repair them. (Consider only
+ the given *MAILBOX*es if non-optional arguments are provided.)
+ This is done by performing a so-called “full synchronization”,
+ namely:
+ 1/ download all UIDs along with their flag list both from the
+ local and remote servers;
+ 2/ ensure that each entry in the database corresponds to an
+ existing UID; and
+ 3/ ensure that both flag lists match.
+ Any message found on a server but not in the database is replicated
+ on the other server (which in the worst case, might lead to a
+ message duplicate).
+ Flag conflicts are solved by updating each message to the union of
+ both lists.
+
+`--delete` *MAILBOX* [*MAILBOX* ...]
+
+: Delete the given *MAILBOX*es on each target (by default each server
+ plus the database, unless `--target` specifies otherwise) where it
+ exists.
+ Note that per the [IMAP4rev1 standard][RFC 3501] deletion is not
+ recursive. Thus *MAILBOX*'s children are not deleted.
+
+`--rename` *SOURCE* *DEST*
+
+: Rename the mailbox *SOURCE* to *DEST* on each target (by default
+ each server plus the database, unless `--target` specifies
+ otherwise) where it exists.
+ `interimap` aborts if *DEST* already exists on either target.
+ Note that per the [IMAP4rev1 standard][RFC 3501] renaming is
+ recursive. Thus *SOURCE*'s children are moved to become *DEST*'s
+ children instead.
+
+Options
+=======
+
+`--config=`*FILE*
+
+: Specify an alternate [configuration file](#configuration-file).
+ Relative paths start from *$XDG_CONFIG_HOME*, or *~/.config* if the
+ `XDG_CONFIG_HOME` environment variable is unset.
+
+`--target={local,remote,database}`
+
+: Limit the scope of a `--delete` or `--rename` command to the given
+ target. Can be repeated to act on multiple targets. By default all
+ three targets are considered.
+
+`--watch`[`=`*seconds*]
+
+: Don't exit after a successful synchronization. Instead, keep
+ synchronizing forever. Sleep for the given number of *seconds* (by
+ default 1 minute if `--notify` is unset, and 15 minutes if
+ `--notify` is set) between two synchronizations. Setting this
+ options enables `SO_KEEPALIVE` on the socket for *type*s other than
+ `tunnel`.
+
+`--notify`
+
+: Wether to use the [IMAP `NOTIFY` extension][RFC 5465] to instruct
+ the server to automatically send updates to the client. (Both local
+ and remote servers must support [RFC 5465] for this to work.)
+ This greatly reduces IMAP traffic since `interimap` can rely on
+ server notifications instead of manually polling for updates.
+ If the connection remains idle for 15 minutes (configurable with
+ `--watch`), then `interimap` sends a `NOOP` command to avoid being
+ logged out for inactivity.
+
+`-q`, `--quiet`
+
+: Try to be quiet.
+
+`--debug`
+
+: Turn on debug mode. Debug messages are written to the given *logfile*.
+ Note that this include all IMAP traffic (except literals).
+ Depending on the chosen authentication mechanism, this might include
+ authentication credentials.
+
+`-h`, `--help`
+
+: Output a brief help and exit.
+
+`--version`
+
+: Show the version number and exit.
+
+Configuration file
+==================
+
+Unless told otherwise by the `--config=FILE` command-line option,
+`interimap` reads its configuration from *$XDG_CONFIG_HOME/interimap*
+(or *~/.config/interimap* if the `XDG_CONFIG_HOME` environment variable
+is unset) as an [INI file].
+The syntax of the configuration file is a series of `OPTION=VALUE`
+lines organized under some `[SECTION]`; lines starting with a ‘#’ or
+‘;’ character are ignored as comments.
+The `[local]` and `[remote]` sections define the two IMAP servers to
+synchronize.
+Valid options are:
+
+*database*
+
+: SQLite version 3 database file to use to keep track of associations
+ between local and remote UIDs, as well as the `UIDVALIDITY`,
+ `UIDNEXT` and `HIGHESTMODSEQ` of each known mailbox on both servers.
+ Relative paths start from *$XDG_DATA_HOME/interimap*, or
+ *~/.local/share/interimap* if the `XDG_DATA_HOME` environment
+ variable is unset. This option is only available in the default
+ section.
+ (Default: `HOST.db`, where *HOST* is taken from the `[remote]` or
+ `[local]` sections, in that order.)
+
+*list-mailbox*
+
+: A space separated list of mailbox patterns to use when issuing the
+ initial `LIST` command (overridden by the *MAILBOX*es given as
+ command-line arguments).
+ Note that each pattern containing special characters such as spaces
+ or brackets (see [RFC 3501] for the exact syntax) must be quoted.
+ Furthermore, non-ASCII names must be [UTF-7 encoded][RFC 2152].
+ Two wildcards are available: a ‘\*’ character matches zero or more
+ characters, while a ‘%’ character matches zero or more characters up
+ to the mailbox's hierarchy delimiter.
+ This option is only available in the default section.
+ (The default pattern, `*`, matches all visible mailboxes on the
+ server.)
+
+*list-select-opts*
+
+: An optional space separated list of selectors for the initial `LIST`
+ command. (Requires a server supporting the [`LIST-EXTENDED` IMAP
+ extension][RFC 5258].) Useful values are `SUBSCRIBED` (to list only
+ subscribed mailboxes), `REMOTE` (to also list remote mailboxes on a
+ server supporting mailbox referrals), and `RECURSIVEMATCH` (to
+ list parent mailboxes with children matching one of the above
+ *list-mailbox* patterns). This option is only available in the
+ default section.
+
+*ignore-mailbox*
+
+: An optional Perl Compatible Regular Expressions ([PCRE]) covering
+ mailboxes to exclude: any ([UTF-7 encoded][RFC 2152] and unquoted)
+ mailbox listed in the initial `LIST` responses is ignored if it
+ matches the given expression.
+ Note that the *MAILBOX*es given as command-line arguments bypass the
+ check and are always considered for synchronization. This option is
+ only available in the default section.
+
+*logfile*
+
+: A file name to use to log debug and informational messages. (By
+ default these messages are written to the error output.) This
+ option is only available in the default section.
+
+*type*
+
+: One of `imap`, `imaps` or `tunnel`.
+ `type=imap` and `type=imaps` are respectively used for IMAP and IMAP
+ over SSL/TLS connections over a INET socket.
+ `type=tunnel` causes `interimap` to open a pipe to a *command*
+ instead of a raw socket.
+ Note that specifying `type=tunnel` in the `[remote]` section makes
+ the default *database* to be `localhost.db`.
+ (Default: `imaps`.)
+
+*host*
+
+: Server hostname, for `type=imap` and `type=imaps`.
+ (Default: `localhost`.)
+
+*port*
+
+: Server port.
+ (Default: `143` for `type=imap`, `993` for `type=imaps`.)
+
+*proxy*
+
+: An optional SOCKS proxy to use for TCP connections to the IMAP
+ server (`type=imap` and `type=imaps` only), formatted as
+ `PROTOCOL://[USER:PASSWORD@]PROXYHOST[:PROXYPORT]`.
+ If `PROXYPORT` is omitted, it is assumed at port 1080.
+ Only [SOCKSv5][RFC 1928] is supported (with optional
+ [username/password authentication][RFC 1929]), in two flavors:
+ `socks5://` to resolve *hostname* locally, and `socks5h://` to let
+ the proxy resolve *hostname*.
+
+*command*
+
+: Command to use for `type=tunnel`. Must speak the [IMAP4rev1
+ protocol][RFC 3501] on its standard output, and understand it on its
+ standard input.
+
+*STARTTLS*
+
+: Whether to use the [`STARTTLS`][RFC 2595] directive to upgrade to a
+ secure connection. Setting this to `YES` for a server not
+ advertising the `STARTTLS` capability causes `interimap` to
+ immediately abort the connection.
+ (Ignored for *type*s other than `imap`. Default: `YES`.)
+
+*auth*
+
+: Space-separated list of preferred authentication mechanisms.
+ `interimap` uses the first mechanism in that list that is also
+ advertised (prefixed with `AUTH=`) in the server's capability list.
+ Supported authentication mechanisms are `PLAIN` and `LOGIN`.
+ (Default: `PLAIN LOGIN`.)
+
+*username*, *password*
+
+: Username and password to authenticate with. Can be required for non
+ pre-authenticated connections, depending on the chosen
+ authentication mechanism.
+
+*compress*
+
+: Whether to use the [`IMAP COMPRESS` extension][RFC 4978] for servers
+ advertising it.
+ (Default: `NO` for the `[local]` section, `YES` for the `[remote]`
+ section.)
+
+*null-stderr*
+
+: Whether to redirect *command*'s standard error to `/dev/null` for
+ type `type=tunnel`. (Default: `NO`.)
+
+*SSL_protocols*
+
+: A space-separated list of SSL protocols to enable or disable (if
+ prefixed with an exclamation mark `!`. Known protocols are `SSLv2`,
+ `SSLv3`, `TLSv1`, `TLSv1.1`, and `TLSv1.2`. Enabling a protocol is
+ a short-hand for disabling all other protocols.
+ (Default: `!SSLv2 !SSLv3`, i.e., only enable TLSv1 and above.)
+
+*SSL_cipher_list*
+
+: The cipher list to send to the server. Although the server
+ determines which cipher suite is used, it should take the first
+ supported cipher in the list sent by the client. See
+ [`ciphers`(1ssl)] for more information.
+
+*SSL_fingerprint*
+
+: Fingerprint of the server certificate (or its public key) in the
+ form `[ALGO$]DIGEST_HEX`, where `ALGO` is the used algorithm
+ (by default `sha256`).
+ Attempting to connect to a server with a non-matching certificate
+ fingerprint causes `interimap` to abort the connection during the
+ SSL/TLS handshake.
+
+*SSL_verify*
+
+: Whether to verify the server certificate chain.
+ Note that using *SSL_fingerprint* to specify the fingerprint of the
+ server certificate is an orthogonal authentication measure as it
+ ignores the CA chain.
+ (Default: `YES`.)
+
+*SSL_CApath*
+
+: Directory to use for server certificate verification if
+ `SSL_verify=YES`.
+ This directory must be in “hash format”, see [`verify`(1ssl)] for
+ more information.
+
+*SSL_CAfile*
+
+: File containing trusted certificates to use during server
+ certificate authentication if `SSL_verify=YES`.
+
+Supported extensions
+====================
+
+Performance is better for servers supporting the following extensions to
+the [IMAP4rev1 protocol][RFC 3501]:
+
+ * `LITERAL+` ([RFC 2088], recommended);
+ * `MULTIAPPEND` ([RFC 3502], recommended);
+ * `COMPRESS=DEFLATE` ([RFC 4978], recommended);
+ * `NOTIFY` ([RFC 5465], recommended);
+ * `SASL-IR` ([RFC 4959]); and
+ * `UNSELECT` ([RFC 3691]).
+
+Known bugs and limitations
+==========================
+
+ * Using `interimap` on two identical servers with a non-existent or
+ empty *database* will duplicate each message due to the absence of
+ local ↔ remote UID association.
+
+ * `interimap` is single threaded and doesn't use IMAP command
+ pipelining. Synchronization could be boosted up by sending
+ independent commands (such as the initial `LIST` and `STATUS`
+ commands) to both servers in parallel, and for a given server, by
+ sending independent commands (such as flag updates) in a pipeline.
+
+ * Because the [IMAP protocol][RFC 3501] doesn't have a specific
+ response code for when a message is moved to another mailbox (either
+ using the `MOVE` command from [RFC 6851], or via `COPY` + `STORE` +
+ `EXPUNGE`), moving a message causes `interimap` to believe that it
+ was deleted while another one (which is replicated again) was added
+ to the other mailbox in the meantime.
+
+ * `PLAIN` and `LOGIN` are the only authentication mechanisms currently
+ supported.
+
+ * `interimap` will probably not work with non [RFC][RFC 3501]-compliant
+ servers. In particular, no work-around is currently implemented
+ beside the tunables in the [configuration file](#configuration-file).
+ Moreover, few IMAP servers have been tested so far.
+
+Standards
+=========
+
+ * M. Leech, M. Ganis, Y. Lee, R. Kuris, D. Koblas and L. Jones,
+ _SOCKS Protocol Version 5_,
+ [RFC 1928], March 1996.
+ * M. Leech, _Username/Password Authentication for SOCKS V5_,
+ [RFC 1929], March 1996.
+ * J. Myers, _IMAP4 non-synchronizing literals_,
+ [RFC 2088], January 1997.
+ * D. Goldsmith and M. Davis,
+ _A Mail-Safe Transformation Format of Unicode_,
+ [RFC 2152], May 1997.
+ * C. Newman, _Using TLS with IMAP, POP3 and ACAP_,
+ [RFC 2595], June 1999.
+ * M. Crispin, _Internet Message Access Protocol - Version 4rev1_,
+ [RFC 3501], March 2003.
+ * M. Crispin,
+ _Internet Message Access Protocol (IMAP) - `MULTIAPPEND` Extension_,
+ [RFC 3502], March 2003.
+ * A. Melnikov,
+ _Internet Message Access Protocol (IMAP) `UNSELECT` command_,
+ [RFC 3691], February 2004.
+ * M. Crispin,
+ _Internet Message Access Protocol (IMAP) - `UIDPLUS` extension_,
+ [RFC 4315], December 2005.
+ * A. Melnikov,
+ _Synchronization Operations for Disconnected IMAP4 Clients_,
+ [RFC 4549], June 2006.
+ * A. Gulbrandsen, _The IMAP `COMPRESS` Extension_,
+ [RFC 4978], August 2007.
+ * R. Siemborski and A. Gulbrandsen, _IMAP Extension for Simple
+ Authentication and Security Layer (SASL) Initial Client Response_,
+ [RFC 4959], September 2007.
+ * A. Gulbrandsen and A. Melnikov,
+ _The IMAP `ENABLE` Extension_,
+ [RFC 5161], March 2008.
+ * B. Leiba and A. Melnikov,
+ _Internet Message Access Protocol version 4 - `LIST` Command Extensions_,
+ [RFC 5258], June 2008.
+ * A. Gulbrandsen, C. King and A. Melnikov,
+ _The IMAP `NOTIFY` Extension_,
+ [RFC 5465], February 2009.
+ * A. Melnikov and T. Sirainen,
+ _IMAP4 Extension for Returning `STATUS` Information in Extended LIST_,
+ [RFC 5819], March 2010.
+ * A. Gulbrandsen and N. Freed,
+ _Internet Message Access Protocol (IMAP) - `MOVE` Extension_,
+ [RFC 6851], January 2013.
+ * A. Melnikov and D. Cridland,
+ _IMAP Extensions: Quick Flag Changes Resynchronization (`CONDSTORE`)
+ and Quick Mailbox Resynchronization (`QRESYNC`)_,
+ [RFC 7162], May 2014.
+
+[RFC 7162]: https://tools.ietf.org/html/rfc7162
+[RFC 5258]: https://tools.ietf.org/html/rfc5258
+[RFC 5819]: https://tools.ietf.org/html/rfc5819
+[RFC 4315]: https://tools.ietf.org/html/rfc4315
+[RFC 4549]: https://tools.ietf.org/html/rfc4549
+[RFC 2152]: https://tools.ietf.org/html/rfc2152
+[RFC 3501]: https://tools.ietf.org/html/rfc3501
+[RFC 1928]: https://tools.ietf.org/html/rfc1928
+[RFC 1929]: https://tools.ietf.org/html/rfc1929
+[RFC 2595]: https://tools.ietf.org/html/rfc2595
+[RFC 4978]: https://tools.ietf.org/html/rfc4978
+[RFC 2088]: https://tools.ietf.org/html/rfc2088
+[RFC 3502]: https://tools.ietf.org/html/rfc3502
+[RFC 4959]: https://tools.ietf.org/html/rfc4959
+[RFC 3691]: https://tools.ietf.org/html/rfc3691
+[RFC 6851]: https://tools.ietf.org/html/rfc6851
+[RFC 5161]: https://tools.ietf.org/html/rfc5161
+[RFC 5465]: https://tools.ietf.org/html/rfc5465
+
+[INI file]: https://en.wikipedia.org/wiki/INI_file
+[PCRE]: https://en.wikipedia.org/wiki/Perl_Compatible_Regular_Expressions
+[`ciphers`(1ssl)]: https://www.openssl.org/docs/manmaster/apps/ciphers.html
+[`verify`(1ssl)]: https://www.openssl.org/docs/manmaster/apps/verify.html
diff --git a/lib/Net/IMAP/InterIMAP.pm b/lib/Net/IMAP/InterIMAP.pm
index 45253c1..a899831 100644
--- a/lib/Net/IMAP/InterIMAP.pm
+++ b/lib/Net/IMAP/InterIMAP.pm
@@ -35,7 +35,8 @@ BEGIN {
Net::SSLeay::SSLeay_add_ssl_algorithms();
Net::SSLeay::randomize();
- our @EXPORT_OK = qw/read_config compact_set $IMAP_text $IMAP_cond/;
+ our @EXPORT_OK = qw/read_config compact_set $IMAP_text $IMAP_cond
+ slurp is_dirty has_new_mails/;
}
@@ -313,6 +314,9 @@ sub new($%) {
foreach ($rd, $wd) {
close $_ or $self->panic("Can't close: $!");
}
+ foreach (qw/STDIN STDOUT/) {
+ binmode($self->{$_}) // $self->panic("binmode: $!")
+ }
}
else {
foreach (qw/host port/) {
@@ -320,28 +324,17 @@ sub new($%) {
}
my $socket = defined $self->{proxy} ? $self->_proxify(@$self{qw/proxy host port/})
: $self->_tcp_connect(@$self{qw/host port/});
- my ($cnt, $intvl) = (3, 5);
if (defined $self->{keepalive}) {
- # detect dead peers and drop the connection after 60 secs + $cnt*$intvl
setsockopt($socket, Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, 1)
or $self->fail("Can't setsockopt SO_KEEPALIVE: $!");
setsockopt($socket, Socket::IPPROTO_TCP, Socket::TCP_KEEPIDLE, 60)
or $self->fail("Can't setsockopt TCP_KEEPIDLE: $!");
- setsockopt($socket, Socket::IPPROTO_TCP, Socket::TCP_KEEPCNT, $cnt)
- or $self->fail("Can't setsockopt TCP_KEEPCNT: $!");
- setsockopt($socket, Socket::IPPROTO_TCP, Socket::TCP_KEEPINTVL, $intvl)
- or $self->fail("Can't setsockopt TCP_KEEPINTVL: $!");
}
- # Abort after 15secs if write(2) isn't acknowledged
- # XXX Socket::TCP_USER_TIMEOUT isn't defined.
- # `grep TCP_USER_TIMEOUT /usr/include/linux/tcp.h` gives 18
- setsockopt($socket, Socket::IPPROTO_TCP, 18, 1000 * $cnt * $intvl)
- or $self->fail("Can't setsockopt TCP_USER_TIMEOUT: $!");
+ binmode($socket) // $self->panic("binmode: $!");
$self->_start_ssl($socket) if $self->{type} eq 'imaps';
$self->{$_} = $socket for qw/STDOUT STDIN/;
}
- binmode $self->{$_} foreach qw/STDIN STDOUT/;
# command counter
$self->{_TAG} = 0;
@@ -645,6 +638,7 @@ sub unselect($) {
# we'll get back to it
$self->{_VANISHED} = [];
$self->{_MODIFIED} = {};
+ $self->{_NEW} = 0;
}
@@ -916,91 +910,93 @@ sub fetch($$$;&) {
}
-# $self->notify(@specifications)
-# Issue a NOTIFY command with the given mailbox @specifications (cf RFC
-# 5465 section 6) to be monitored. Croak if the server did not
-# advertise "NOTIFY" (RFC 5465) in its CAPABILITY list.
-sub notify($@) {
+# $self->notify($arg, %specifications)
+# Issue a NOTIFY command with the given $arg ("SET", "SET STATUS" or
+# "NONE") and mailbox %specifications (cf RFC 5465 section 6) to be
+# monitored. Croak if the server did not advertise "NOTIFY" (RFC
+# 5465) in its CAPABILITY list.
+sub notify($$@) {
my $self = shift;
$self->fail("Server did not advertise NOTIFY (RFC 5465) capability.")
unless $self->_capable('NOTIFY');
- my $events = join ' ', qw/MessageNew MessageExpunge FlagChange MailboxName SubscriptionChange/;
- # Be notified of new messages with EXISTS/RECENT responses, but
- # don't receive unsolicited FETCH responses with a RFC822/BODY[].
- # It costs us an extra roundtrip, but we need to sync FLAG updates
- # and VANISHED responses in batch mode, update the HIGHESTMODSEQ,
- # and *then* issue an explicit UID FETCH command to get new message,
- # and process each FETCH response with a RFC822/BODY[] attribute as
- # they arrive.
- my $command = 'NOTIFY ';
- $command .= @_ ? ('SET '. join(' ', map {"($_ ($events))"} @_)) : 'NONE';
+ my $command = 'NOTIFY '.shift;
+ while (@_) {
+ $command .= " (".shift." (".join(' ', @{shift()})."))";
+ }
$self->_send($command);
}
-# $self->slurp([$callback, $cmd])
-# See if the server has sent some unprocessed data; try to as many
-# lines as possible, process them, and return the number of lines
-# read.
+# slurp($imap, $timeout, $stopwhen)
+# Keep reading untagged responses from the @$imap servers until the
+# $stopwhen condition becomes true (then return true), or until the
+# $timeout expires (then return false).
# This is mostly useful when waiting for notifications while no
# command is progress, cf. RFC 2177 (IDLE) or RFC 5465 (NOTIFY).
-sub slurp($;&$) {
- my ($self, $callback, $cmd) = @_;
- my $ssl = $self->{_SSL};
- my $read = 0;
+sub slurp($$$) {
+ my ($selfs, $timeout, $stopwhen) = @_;
+ my $aborted = 0;
+
+ my $rin = '';
+ vec($rin, fileno($_->{STDOUT}), 1) = 1 foreach @$selfs;
- vec(my $rin, fileno($self->{STDOUT}), 1) = 1;
while (1) {
- unless ((defined $self->{_OUTBUF} and $self->{_OUTBUF} ne '') or
- # Unprocessed data within the current TLS record would
- # cause select(2) to block/timeout due to the raw socket
- # not being ready.
- (defined $ssl and Net::SSLeay::pending($ssl) > 0)) {
- my $r = CORE::select($rin, undef, undef, 0);
+ # first, consider only unprocessed data without our own output
+ # buffer, or within the current TLS record: these would cause
+ # select(2) to block/timeout due to the raw socket not being
+ # ready.
+ my @ready = grep { (defined $_->{_OUTBUF} and $_->{_OUTBUF} ne '') or
+ (defined $_->{_SSL} and Net::SSLeay::pending($_->{_SSL}) > 0)
+ } @$selfs;
+ unless (@ready) {
+ my ($r, $timeleft) = CORE::select(my $rout = $rin, undef, undef, $timeout);
next if $r == -1 and $! == EINTR; # select(2) was interrupted
- $self->panic("Can't select: $!") if $r == -1;
- return $read if $r == 0; # nothing more to read
+ die "select: $!" if $r == -1;
+ return $aborted if $r == 0; # nothing more to read (timeout reached)
+ @ready = grep {vec($rout, fileno($_->{STDOUT}), 1)} @$selfs;
+ $timeout = $timeleft if $timeout > 0;
+ }
+
+ foreach my $imap (@ready) {
+ my $x = $imap->_getline();
+ $imap->_resp($x, sub($) {
+ if ($stopwhen->($imap, shift)) {
+ $aborted = 1;
+ $timeout = 0; # keep reading the handles while there is pending data
+ }
+ }, 'slurp');
}
- my $x = $self->_getline();
- $self->_resp($x, $callback, $cmd);
- $read++;
}
}
-# $self->idle([$timeout, $stopwhen])
+# $self->idle($timeout, $stopwhen)
# Enter IDLE (RFC 2177) for $timout seconds (by default 29 mins), or
# when the callback $stopwhen returns true.
-# Return false if the timeout was reached, and true if IDLE was
-# stopped due the callback.
-sub idle($;$&) {
+# Return true if the callback returned true (either aborting IDLE, or
+# after the $timeout) and false otherwise.
+sub idle($$$) {
my ($self, $timeout, $stopwhen) = @_;
- $timeout //= 1740; # 29 mins
- my $callback = sub() {$timeout = -1 if $stopwhen->()};
$self->fail("Server did not advertise IDLE (RFC 2177) capability.")
unless $self->_capable('IDLE');
my $tag = $self->_cmd_init('IDLE');
$self->_cmd_flush();
-
- for (; $timeout > 0; $timeout--) {
- $self->slurp($callback, 'IDLE');
- sleep 1 if $timeout > 0;
- }
+ my $r = slurp([$self], $timeout // 1740, $stopwhen); # 29 mins
# done idling
$self->_cmd_extend('DONE');
$self->_cmd_flush();
# run the callback again to update the return value if we received
# untagged responses between the DONE and the tagged response
- $self->_recv($tag, $callback, 'IDLE');
+ $self->_recv($tag, sub($) { $r = 1 if $stopwhen->($self, shift) }, 'slurp');
- return $timeout < 0 ? 1 : 0;
+ return $r;
}
-# $self->set_cache( $mailbox, STATE )
+# $self->set_cache($mailbox, STATE)
# Initialize or update the persistent cache, that is, associate a
# known $mailbox with the last known (synced) state:
# * UIDVALIDITY
@@ -1082,6 +1078,7 @@ sub get_cache($@) {
# persistent cache's values.
sub is_dirty($$) {
my ($self, $mailbox) = @_;
+ return 1 if $self->{_NEW};
$self->_updated_cache($mailbox, qw/HIGHESTMODSEQ UIDNEXT/);
}
@@ -1091,6 +1088,7 @@ sub is_dirty($$) {
# internal cache's UIDNEXT value differs from its persistent cache's.
sub has_new_mails($$) {
my ($self, $mailbox) = @_;
+ return 1 if $self->{_NEW};
$self->_updated_cache($mailbox, 'UIDNEXT');
}
@@ -1181,6 +1179,7 @@ sub pull_new_messages($$&@) {
my @ignore = sort { $a <=> $b } @_;
my $mailbox = $self->{_SELECTED} // $self->panic();
+ my $cache = $self->{_CACHE}->{$mailbox};
my $UIDNEXT;
do {
@@ -1205,19 +1204,20 @@ sub pull_new_messages($$&@) {
# 2^32-1: don't use '*' since the highest UID can be known already
$range .= "$since:4294967295";
- $UIDNEXT = $self->{_CACHE}->{$mailbox}->{UIDNEXT} // $self->panic(); # sanity check
+ $UIDNEXT = $cache->{UIDNEXT} // $self->panic(); # sanity check
$self->_send("UID FETCH $range ($attrs)", sub($) {
my $mail = shift;
$UIDNEXT = $mail->{UID} + 1 if $UIDNEXT <= $mail->{UID};
$callback->($mail) if defined $callback;
- }) if $first < $UIDNEXT;
+ }) if $first < $UIDNEXT or $self->{_NEW};
# update the persistent cache for UIDNEXT (not for HIGHESTMODSEQ
# since there might be pending updates)
$self->set_cache($mailbox, UIDNEXT => $UIDNEXT);
+ $self->{_NEW} = 0;
}
# loop if new messages were received in the meantime
- while ($UIDNEXT < $self->{_CACHE}->{$mailbox}->{UIDNEXT});
+ while ($self->{_NEW} or $UIDNEXT < $cache->{UIDNEXT});
}
@@ -1372,12 +1372,21 @@ sub _tcp_connect($$$) {
SOCKETS:
foreach my $ai (@res) {
socket (my $s, $ai->{family}, $ai->{socktype}, $ai->{protocol}) or $self->panic("connect: $!");
- # TODO: add a connection timeout
- # http://devpit.org/wiki/Connect%28%29_with_timeout_%28in_Perl%29
+
+ # timeout connect/read/write/... after 30s
+ # XXX we need to pack the struct timeval manually: not portable!
+ # https://stackoverflow.com/questions/8284243/how-do-i-set-so-rcvtimeo-on-a-socket-in-perl
+ my $timeout = pack('l!l!', 30, 0);
+ setsockopt($s, Socket::SOL_SOCKET, Socket::SO_RCVTIMEO, $timeout)
+ or $self->fail("Can't setsockopt SO_RCVTIMEO: $!");
+ setsockopt($s, Socket::SOL_SOCKET, Socket::SO_SNDTIMEO, $timeout)
+ or $self->fail("Can't setsockopt SO_RCVTIMEO: $!");
+
until (connect($s, $ai->{addr})) {
next if $! == EINTR; # try again if connect(2) was interrupted by a signal
next SOCKETS;
}
+
my $flags = fcntl($s, F_GETFD, 0) or $self->panic("fcntl F_GETFD: $!");
fcntl($s, F_SETFD, $flags | FD_CLOEXEC) or $self->panic("fcntl F_SETFD: $!");
return $s;
@@ -1908,11 +1917,11 @@ sub _send($$;&) {
my $tag = $self->_cmd_init($command);
$self->_cmd_flush();
+ my $cmd = $$command =~ /\AUID ($RE_ATOM_CHAR+) / ? $1 : $$command =~ /\A($RE_ATOM_CHAR+) / ? $1 : $$command;
if (!defined $callback) {
- $self->_recv($tag);
+ $self->_recv($tag, undef, $cmd);
}
else {
- my $cmd = $$command =~ /\AUID ($RE_ATOM_CHAR+) / ? $1 : $$command =~ /\A($RE_ATOM_CHAR+) / ? $1 : $$command;
my $set = $$command =~ /\AUID (?:FETCH|STORE) ([0-9:,*]+)/ ? $1 : undef;
$self->_recv($tag, $callback, $cmd, $set);
}
@@ -1993,6 +2002,7 @@ sub _open_mailbox($$) {
# we'll get back to it
$self->{_VANISHED} = [];
$self->{_MODIFIED} = {};
+ $self->{_NEW} = 0;
$self->{_SELECTED} = $mailbox;
$self->{_CACHE}->{$mailbox} //= {};
@@ -2219,6 +2229,7 @@ sub _resp($$;&$$) {
}
elsif (s/\A(?:OK|NO|BAD) //) {
$self->_resp_text($_);
+ $callback->($self->{_SELECTED}) if defined $self->{_SELECTED} and defined $callback and $cmd eq 'slurp';
}
elsif (/\ACAPABILITY((?: $RE_ATOM_CHAR+)+)\z/) {
$self->{_CAPABILITIES} = [ split / /, ($1 =~ s/^ //r) ];
@@ -2233,17 +2244,20 @@ sub _resp($$;&$$) {
# /!\ $cache->{EXISTS} MUST NOT be defined on SELECT
if (defined $cache->{EXISTS}) {
$self->panic("Unexpected EXISTS shrink $1 < $cache->{EXISTS}!") if $1 < $cache->{EXISTS};
- # the actual UIDNEXT is *at least* that
- $cache->{UIDNEXT} += $1 - $cache->{EXISTS} if defined $cache->{UIDNEXT};
+ $self->{_NEW} += $1 - $cache->{EXISTS} if $1 > $cache->{EXISTS}; # new mails
}
$cache->{EXISTS} = $1;
+ $callback->($self->{_SELECTED} // $self->panic()) if defined $callback and $cmd eq 'slurp';
}
elsif (/\A([0-9]+) EXPUNGE\z/) {
+ $self->panic() unless defined $cache->{EXISTS}; # sanity check
# /!\ No bookkeeping since there is no internal cache mapping sequence numbers to UIDs
if ($self->_enabled('QRESYNC')) {
$self->panic("$1 <= $cache->{EXISTS}") if $1 <= $cache->{EXISTS}; # sanity check
$self->fail("RFC 7162 violation! Got an EXPUNGE response with QRESYNC enabled.");
}
+ # the new message was expunged before it was synced
+ $self->{_NEW} = 0 if $self->{_NEW} == 1 and $cache->{EXISTS} == $1;
$cache->{EXISTS}--; # explicit EXISTS responses are optional
}
elsif (/\ASEARCH((?: [0-9]+)*)\z/) {
@@ -2266,11 +2280,20 @@ sub _resp($$;&$$) {
/\A \((\\?$RE_ATOM_CHAR+ [0-9]+(?: \\?$RE_ATOM_CHAR+ [0-9]+)*)?\)\z/ or $self->panic($_);
my %status = split / /, $1;
$mailbox = 'INBOX' if uc $mailbox eq 'INBOX'; # INBOX is case-insensitive
+ $self->panic("RFC 5465 violation! Missing HIGHESTMODSEQ data item in STATUS response")
+ if $self->_enabled('QRESYNC') and !defined $status{HIGHESTMODSEQ} and defined $cmd and
+ ($cmd eq 'NOTIFY' or $cmd eq 'slurp');
$self->_update_cache_for($mailbox, %status);
- $callback->($mailbox, %status) if defined $callback and $cmd eq 'STATUS';
+ if (defined $callback) {
+ if ($cmd eq 'STATUS') {
+ $callback->($mailbox, %status);
+ } elsif ($cmd eq 'slurp') {
+ $callback->($mailbox);
+ }
+ }
}
elsif (s/\A([0-9]+) FETCH \(//) {
- $self->panic("$1 <= $cache->{EXISTS}") unless $1 <= $cache->{EXISTS}; # sanity check
+ $cache->{EXISTS} = $1 if $1 > $cache->{EXISTS};
my ($seq, $first) = ($1, 1);
my %mail;
while ($_ ne ')') {
@@ -2313,8 +2336,13 @@ sub _resp($$;&$$) {
my $flags = join ' ', sort(grep {lc $_ ne '\recent'} @{$mail{FLAGS}}) if defined $mail{FLAGS};
$self->{_MODIFIED}->{$uid} = [ $mail{MODSEQ}, $flags ];
}
- $callback->(\%mail) if defined $callback and ($cmd eq 'FETCH' or $cmd eq 'STORE') and
- defined $uid and in_set($uid, $set);
+ if (defined $callback) {
+ if ($cmd eq 'FETCH' or $cmd eq 'STORE') {
+ $callback->(\%mail) if defined $uid and in_set($uid, $set);
+ } elsif ($cmd eq 'slurp') {
+ $callback->($self->{_SELECTED} // $self->panic())
+ }
+ }
}
elsif (/\AENABLED((?: $RE_ATOM_CHAR+)+)\z/) { # RFC 5161 ENABLE
$self->{_ENABLED} //= [];
@@ -2338,6 +2366,7 @@ sub _resp($$;&$$) {
push @{$self->{_VANISHED}}, ($min .. $max);
}
}
+ $callback->($self->{_SELECTED} // $self->panic()) if defined $callback and $cmd eq 'slurp';
}
}
elsif (s/\A\+// and ($_ eq '' or s/\A //)) {
@@ -2351,7 +2380,6 @@ sub _resp($$;&$$) {
else {
$self->panic("Unexpected response: ", $_);
}
- $callback->() if defined $callback and $cmd eq 'IDLE';
}
diff --git a/pullimap b/pullimap
index 623ba1e..49fc837 100755
--- a/pullimap
+++ b/pullimap
@@ -25,7 +25,7 @@ our $VERSION = '0.3';
my $NAME = 'pullimap';
use Errno 'EINTR';
-use Fcntl qw/O_CREAT O_RDWR O_DSYNC LOCK_EX SEEK_SET F_GETFD F_SETFD FD_CLOEXEC/;
+use Fcntl qw/O_CREAT O_RDWR O_DSYNC F_SETLK F_WRLCK SEEK_SET F_GETFD F_SETFD FD_CLOEXEC/;
use Getopt::Long qw/:config posix_default no_ignore_case gnu_getopt auto_version/;
use List::Util 'first';
use Socket qw/PF_INET PF_INET6 SOCK_STREAM/;
@@ -82,10 +82,9 @@ do {
}
sysopen($STATE, $statefile, O_CREAT|O_RDWR|O_DSYNC, 0600) or die "Can't open $statefile: $!";
+ fcntl($STATE, F_SETLK, pack('sslll', F_WRLCK, SEEK_SET, 0, 0, $$)) or die "Can't lock $statefile: $!";
my $flags = fcntl($STATE, F_GETFD, 0) or die "fcntl F_GETFD: $!";
fcntl($STATE, F_SETFD, $flags | FD_CLOEXEC) or die "fcntl F_SETFD: $!";
-
- flock($STATE, LOCK_EX) or die "Can't flock $statefile: $!";
};
@@ -141,6 +140,7 @@ sub sendmail($$) {
next if $! == EINTR; # try again if connect(2) was interrupted by a signal
die "connect: $!";
}
+ binmode($SMTP) // die "binmode: $!";
smtp_resp('220');
my @r = smtp_send($ehlo => '250');
@@ -344,7 +344,6 @@ unless (defined $CONFIG{idle}) {
$CONFIG{idle} = 1740 if defined $CONFIG{idle} and $CONFIG{idle} == 0; # 29 mins
while(1) {
- my $r = $IMAP->idle($CONFIG{idle}, sub() { $IMAP->has_new_mails($MAILBOX) });
- pull() if $r;
+ pull() if $IMAP->idle($CONFIG{idle}, \&Net::IMAP::InterIMAP::has_new_mails);
purge();
}
diff --git a/pullimap.md b/pullimap.md
index 54c6ce5..244e7ac 100644
--- a/pullimap.md
+++ b/pullimap.md
@@ -5,8 +5,7 @@
Name
====
-PullIMAP - Pull mails from an IMAP mailbox and deliver them to a SMTP
-session
+PullIMAP - Pull mails from an IMAP mailbox and deliver them to a SMTP session
Synopsis
========
@@ -25,58 +24,57 @@ A *statefile* is used to keep track of the mailbox's `UIDVALIDITY` and
`UIDNEXT` values. While `pullimap` is running, the *statefile* is also
used to keep track of UIDs being delivered, which avoids duplicate
deliveries in case the process is interrupted.
-See the [**Control flow**](#control-flow) section below for details.
+See the **[control flow](#control-flow)** section below for details.
Options
=======
`--config=`*FILE*
-: Specify an alternate configuration file. Relative paths start from
- *$XDG_CONFIG_HOME*, or *~/.config* if the `XDG_CONFIG_HOME`
- environment variable is unset.
+: Specify an alternate [configuration file](#configuration-file).
+ Relative paths start from *$XDG_CONFIG_HOME*, or *~/.config* if the
+ `XDG_CONFIG_HOME` environment variable is unset.
`--idle`[`=`*seconds*]
-: Don't exit after a successful poll. Instead, keep the connection open
- and issue `IDLE` commands (require an IMAP server supporting [RFC
- 2177]) to watch for updates in the mailbox. This also enables
- `SO_KEEPALIVE` on the socket. Each `IDLE` command is terminated after
- at most *seconds* (29 minutes by default) to avoid being logged out
- for inactivity.
+: Don't exit after a successful poll. Instead, keep the connection open
+ and issue `IDLE` commands (require an IMAP server supporting [RFC
+ 2177]) to watch for updates in the mailbox. This also enables
+ `SO_KEEPALIVE` on the socket.
+ Each `IDLE` command is terminated after at most *seconds* (29
+ minutes by default) to avoid being logged out for inactivity.
`--no-delivery`
-: Update the *statefile*, but skip SMTP/LMTP delivery. This is mostly
- useful for initializing the *statefile* when migrating to
- `pullimap` from another similar program such as [`fetchmail`(1)] or
- [`getmail`(1)].
+: Update the *statefile*, but skip SMTP/LMTP delivery. This is mostly
+ useful for initializing the *statefile* when migrating to `pullimap`
+ from another similar program such as [`fetchmail`(1)] or
+ [`getmail`(1)].
`-q`, `--quiet`
-: Try to be quiet.
+: Try to be quiet.
`--debug`
-: Turn on debug mode. Debug messages are written to the error output.
- Note that this include all IMAP traffic (except literals). Depending
- on the chosen authentication mechanism, this might include
- authentication credentials.
+: Turn on debug mode. Debug messages are written to the error output.
+ Note that this include all IMAP traffic (except literals).
+ Depending on the chosen authentication mechanism, this might include
+ authentication credentials.
`-h`, `--help`
-: Output a brief help and exit.
+: Output a brief help and exit.
`--version`
-: Show the version number and exit.
-
+: Show the version number and exit.
Configuration file
==================
-Unless told otherwise by the `--config=FILE` option, `pullimap` reads
-its configuration from *$XDG_CONFIG_HOME/pullimap* (or
+Unless told otherwise by the `--config=FILE` command-line option,
+`pullimap` reads its configuration from *$XDG_CONFIG_HOME/pullimap* (or
*~/.config/pullimap* if the `XDG_CONFIG_HOME` environment variable is
unset) as an [INI file].
The syntax of the configuration file is a series of `OPTION=VALUE`
@@ -86,158 +84,156 @@ Valid options are:
*statefile*
-: State file to use to keep track of the *mailbox*'s `UIDVALIDITY` and
- `UIDNEXT` values. Relative paths start from
- *$XDG_DATA_HOME/pullimap*, or *~/.local/share/pullimap* if the
- `XDG_DATA_HOME` environment variable is unset.
- (Default: the parent section name of the option.)
+: State file to use to keep track of the *mailbox*'s `UIDVALIDITY` and
+ `UIDNEXT` values. Relative paths start from
+ *$XDG_DATA_HOME/pullimap*, or *~/.local/share/pullimap* if the
+ `XDG_DATA_HOME` environment variable is unset.
+ (Default: the parent section name of the option.)
*mailbox*
-: The IMAP mailbox to pull messages from.
- Support for persistent message Unique Identifiers (UID) is required.
- (Default: `INBOX`.)
+: The IMAP mailbox to pull messages from. Support for persistent
+ message Unique Identifiers (UID) is required. (Default: `INBOX`.)
*deliver-method*
-: `PROTOCOL:[ADDRESS]:PORT` where to deliver messages. Both [SMTP][RFC
- 5321] and [LMTP][RFC 2033] servers are supported, and [SMTP
- pipelining][RFC 2920] is used when possible.
- (Default: `smtp:[127.0.0.1]:25`.)
+: `PROTOCOL:[ADDRESS]:PORT` where to deliver messages. Both
+ [SMTP][RFC 5321] and [LMTP][RFC 2033] servers are supported, and
+ [SMTP pipelining][RFC 2920] is used when possible.
+ (Default: `smtp:[127.0.0.1]:25`.)
*deliver-ehlo*
-: Hostname to use in `EHLO` or `LHLO` commands.
- (Default: `localhost.localdomain`.)
+: Hostname to use in `EHLO` or `LHLO` commands.
+ (Default: `localhost.localdomain`.)
*deliver-rcpt*
-: Message recipient.
- (Default: the username associated with the effective uid of the
- `pullimap` process.)
+: Message recipient.
+ (Default: the username associated with the effective uid of the
+ `pullimap` process.)
*purge-after*
-: Retention period (in days), after which messages are removed from the
- IMAP server. (The value is at best 24h accurate due to the IMAP
- `SEARCH` criterion ignoring time and timezone.)
- If *purge-after* is set to `0` then messages are deleted immediately
- after delivery. Otherwise `pullimap` issues an IMAP `SEARCH` command
- to list old messages; if `--idle` is set then the `SEARCH` command is
- issued again every 12 hours.
+: Retention period (in days), after which messages are removed from
+ the IMAP server. (The value is at best 24h accurate due to the IMAP
+ `SEARCH` criterion ignoring time and timezone.)
+ If *purge-after* is set to `0` then messages are deleted immediately
+ after delivery. Otherwise `pullimap` issues an IMAP `SEARCH`
+ command to list old messages; if `--idle` is set then the `SEARCH`
+ command is issued again every 12 hours.
*type*
-: One of `imap`, `imaps` or `tunnel`.
- `type=imap` and `type=imaps` are respectively used for IMAP and IMAP
- over SSL/TLS connections over a INET socket.
- `type=tunnel` causes `pullimap` to open a pipe to a *command* instead
- of a raw socket.
- (Default: `imaps`.)
+: One of `imap`, `imaps` or `tunnel`.
+ `type=imap` and `type=imaps` are respectively used for IMAP and IMAP
+ over SSL/TLS connections over a INET socket.
+ `type=tunnel` causes `pullimap` to open a pipe to a *command*
+ instead of a raw socket.
+ (Default: `imaps`.)
*host*
-: Server hostname, for `type=imap` and `type=imaps`.
- (Default: `localhost`.)
+: Server hostname, for `type=imap` and `type=imaps`.
+ (Default: `localhost`.)
*port*
-: Server port.
- (Default: `143` for `type=imap`, `993` for
- `type=imaps`.)
+: Server port.
+ (Default: `143` for `type=imap`, `993` for `type=imaps`.)
*proxy*
-: An optional SOCKS proxy to use for TCP connections to the IMAP server
- (`type=imap` and `type=imaps` only), formatted as
- `PROTOCOL://[USER:PASSWORD@]PROXYHOST[:PROXYPORT]`.
- If `PROXYPORT` is omitted, it is assumed at port 1080.
- Only [SOCKSv5][RFC 1928] is supported (with optional
- [username/password authentication][RFC 1929]), in two flavors:
- `socks5://` to resolve *hostname* locally, and `socks5h://` to let the
- proxy resolve *hostname*.
+: An optional SOCKS proxy to use for TCP connections to the IMAP
+ server (`type=imap` and `type=imaps` only), formatted as
+ `PROTOCOL://[USER:PASSWORD@]PROXYHOST[:PROXYPORT]`.
+ If `PROXYPORT` is omitted, it is assumed at port 1080.
+ Only [SOCKSv5][RFC 1928] is supported (with optional
+ [username/password authentication][RFC 1929]), in two flavors:
+ `socks5://` to resolve *hostname* locally, and `socks5h://` to let
+ the proxy resolve *hostname*.
*command*
-: Command to use for `type=tunnel`. Must speak the [IMAP4rev1
- protocol][RFC 3501] on its standard output, and understand it on its
- standard input.
+: Command to use for `type=tunnel`. Must speak the [IMAP4rev1
+ protocol][RFC 3501] on its standard output, and understand it on its
+ standard input.
*STARTTLS*
-: Whether to use the [`STARTTLS`][RFC 2595] directive to upgrade to a
- secure connection. Setting this to `YES` for a server not advertising
- the `STARTTLS` capability causes `pullimap` to immediately abort the
- connection.
- (Ignored for *type*s other than `imap`. Default: `YES`.)
+: Whether to use the [`STARTTLS`][RFC 2595] directive to upgrade to a
+ secure connection. Setting this to `YES` for a server not
+ advertising the `STARTTLS` capability causes `pullimap` to
+ immediately abort the connection.
+ (Ignored for *type*s other than `imap`. Default: `YES`.)
*auth*
-: Space-separated list of preferred authentication mechanisms.
- `pullimap` uses the first mechanism in that list that is also
- advertised (prefixed with `AUTH=`) in the server's capability list.
- Supported authentication mechanisms are `PLAIN` and `LOGIN`.
- (Default: `PLAIN LOGIN`.)
+: Space-separated list of preferred authentication mechanisms.
+ `pullimap` uses the first mechanism in that list that is also
+ advertised (prefixed with `AUTH=`) in the server's capability list.
+ Supported authentication mechanisms are `PLAIN` and `LOGIN`.
+ (Default: `PLAIN LOGIN`.)
*username*, *password*
-: Username and password to authenticate with. Can be required for non
- pre-authenticated connections, depending on the chosen authentication
- mechanism.
+: Username and password to authenticate with. Can be required for non
+ pre-authenticated connections, depending on the chosen
+ authentication mechanism.
*compress*
-: Whether to use the [`IMAP COMPRESS` extension][RFC 4978] for servers
- advertising it. (Default: `YES`.)
+: Whether to use the [`IMAP COMPRESS` extension][RFC 4978] for servers
+ advertising it. (Default: `YES`.)
*null-stderr*
-: Whether to redirect *command*'s standard error to `/dev/null` for type
- `type=tunnel`. (Default: `NO`.)
+: Whether to redirect *command*'s standard error to `/dev/null` for
+ type `type=tunnel`. (Default: `NO`.)
*SSL_protocols*
-: A space-separated list of SSL protocols to enable or disable (if
- prefixed with an exclamation mark `!`. Known protocols are
- `SSLv2`, `SSLv3`, `TLSv1`, `TLSv1.1`, and `TLSv1.2`. Enabling a
- protocol is a short-hand for disabling all other protocols.
- (Default: `!SSLv2 !SSLv3`, i.e., only enable TLSv1 and above.)
+: A space-separated list of SSL protocols to enable or disable (if
+ prefixed with an exclamation mark `!`. Known protocols are `SSLv2`,
+ `SSLv3`, `TLSv1`, `TLSv1.1`, and `TLSv1.2`. Enabling a protocol is
+ a short-hand for disabling all other protocols.
+ (Default: `!SSLv2 !SSLv3`, i.e., only enable TLSv1 and above.)
*SSL_cipher_list*
-: The cipher list to send to the server. Although the server determines
- which cipher suite is used, it should take the first supported cipher
- in the list sent by the client. See [`ciphers`(1ssl)] for more
- information.
+: The cipher list to send to the server. Although the server
+ determines which cipher suite is used, it should take the first
+ supported cipher in the list sent by the client. See
+ [`ciphers`(1ssl)] for more information.
*SSL_fingerprint*
-: Fingerprint of the server certificate (or its public key) in the form
- `ALGO$DIGEST_HEX`, where `ALGO` is the used algorithm (default
- `sha256`).
- Attempting to connect to a server with a non-matching certificate
- fingerprint causes `pullimap` to abort the connection during the
- SSL/TLS handshake.
+: Fingerprint of the server certificate (or its public key) in the
+ form `[ALGO$]DIGEST_HEX`, where `ALGO` is the used algorithm
+ (by default `sha256`).
+ Attempting to connect to a server with a non-matching certificate
+ fingerprint causes `pullimap` to abort the connection during the
+ SSL/TLS handshake.
*SSL_verify*
-: Whether to verify the server certificate chain.
- Note that using *SSL_fingerprint* to specify the fingerprint of the
- server certificate is an orthogonal authentication measure as it
- ignores the CA chain.
- (Default: `YES`.)
+: Whether to verify the server certificate chain.
+ Note that using *SSL_fingerprint* to specify the fingerprint of the
+ server certificate is an orthogonal authentication measure as it
+ ignores the CA chain.
+ (Default: `YES`.)
*SSL_CApath*
-: Directory to use for server certificate verification if
- `SSL_verify=YES`.
- This directory must be in “hash format”, see [`verify`(1ssl)] for
- more information.
+: Directory to use for server certificate verification if
+ `SSL_verify=YES`.
+ This directory must be in “hash format”, see [`verify`(1ssl)] for
+ more information.
*SSL_CAfile*
-: File containing trusted certificates to use during server certificate
- authentication if `SSL_verify=YES`.
+: File containing trusted certificates to use during server
+ certificate authentication if `SSL_verify=YES`.
Control flow
============
@@ -260,9 +256,9 @@ The [IMAP4rev1 specification][RFC 3501] does not guaranty that untagged
command. Thus it would be unsafe for `pullimap` to update the `UIDNEXT`
value in its *statefile* while the `UID FETCH` command is progress.
Instead, for each untagged `FETCH` response received while the `UID
-FETCH` command is in progress, `pullimap` delivers the message `BODY` to
-the SMTP or LMTP server (specified with *deliver-method*) then appends
-the message UID to the *statefile*.
+FETCH` command is in progress, `pullimap` delivers the message `RFC822`
+body to the SMTP or LMTP server (specified with *deliver-method*) then
+appends the message UID to the *statefile*.
When the `UID FETCH` command eventually terminates, `pullimap` updates
the `UIDNEXT` value in the *statefile* and truncate the file down to 8
bytes. Keeping track of message UIDs as they are received avoids
@@ -272,20 +268,22 @@ FETCH` command is in progress.
In more details, `pullimap` works as follows:
1. Issue an `UID FETCH` command to retrieve message `ENVELOPE` and
- `BODY` (and `UID`) with UID bigger or equal than the `UIDNEXT` value
- found in the *statefile*.
+ `RFC822` (and `UID`) with UID bigger or equal than the `UIDNEXT`
+ value found in the *statefile*.
While the `UID FETCH` command is in progress, perform the following
for each untagged `FETCH` response sent by the server:
- i) if no SMTP/LMTP transmission channel was opened, open one to the
- server specified with *deliver-method* and send an `EHLO` (or
- `LHO`) command with the domain specified by *deliver-ehlo*;
+ i. if no SMTP/LMTP transmission channel was opened, open one to the
+ server specified with *deliver-method* and send an `EHLO` (or
+ `LHO`) command with the domain specified by *deliver-ehlo* (the
+ channel is kept open and shared for all messages retrieved while
+ the `UID FETCH` IMAP command is in progress);
- ii) perform a mail transaction (using [SMTP pipelining][RFC 2920] if
- possible) to send the retrieved message BODY to the SMTP or LMTP
- session; and
+ i. perform a mail transaction (using [SMTP pipelining][RFC 2920] if
+ possible) to deliver the retrieved message `RFC822` body to the
+ SMTP or LMTP session; and
- ii) append the message UID to the *statefile*.
+ i. append the message UID to the *statefile*.
2. If a SMTP/LMTP transmission channel was opened, send a `QUIT` command
to terminate it gracefully.
@@ -296,43 +294,44 @@ In more details, `pullimap` works as follows:
4. Update the *statefile* with the new UIDNEXT value (bytes 5-8).
- 5. Truncate the *statefile* down to 8 bytes (so that it contains only
- two 32-bits integers, respectively the *mailbox*'s current
- `UIDVALIDITY` and `UIDNEXT` values).
+ 5. Truncate the *statefile* down to 8 bytes (so that it contains only
+ two 32-bits integers, respectively the *mailbox*'s current
+ `UIDVALIDITY` and `UIDNEXT` values).
6. If `--idle` was set, issue an `IDLE` command; stop idling and go
- back to step 1. whenever a new message is received.
+ back to step 1 when a new message is received (or when the `IDLE`
+ timeout expires).
Standards
=========
* M. Leech, M. Ganis, Y. Lee, R. Kuris, D. Koblas and L. Jones,
_SOCKS Protocol Version 5_,
- [RFC 1928], March 1996
+ [RFC 1928], March 1996.
* M. Leech, _Username/Password Authentication for SOCKS V5_,
- [RFC 1929], March 1996
+ [RFC 1929], March 1996.
* J. Myers, _Local Mail Transfer Protocol_,
- [RFC 2033], October 1996
+ [RFC 2033], October 1996.
* J. Myers, _IMAP4 non-synchronizing literals_,
- [RFC 2088], January 1997
- * B. Leiba, _IMAP4 IDLE command_,
- [RFC 2177], June 1997
+ [RFC 2088], January 1997.
+ * B. Leiba, _IMAP4 `IDLE` command_,
+ [RFC 2177], June 1997.
* C. Newman, _Using TLS with IMAP, POP3 and ACAP_,
- [RFC 2595], June 1999
+ [RFC 2595], June 1999.
* N. Freed, _SMTP Service Extension for Command Pipelining_,
- [RFC 2920], September 2000
+ [RFC 2920], September 2000.
* M. Crispin, _Internet Message Access Protocol - Version 4rev1_,
- [RFC 3501], March 2003
+ [RFC 3501], March 2003.
* M. Crispin,
- _Internet Message Access Protocol (IMAP) - UIDPLUS extension_,
- [RFC 4315], December 2005
- * A. Gulbrandsen, _The IMAP COMPRESS Extension_,
- [RFC 4978], August 2007
+ _Internet Message Access Protocol (IMAP) - `UIDPLUS` extension_,
+ [RFC 4315], December 2005.
+ * A. Gulbrandsen, _The IMAP `COMPRESS` Extension_,
+ [RFC 4978], August 2007.
* R. Siemborski and A. Gulbrandsen, _IMAP Extension for Simple
Authentication and Security Layer (SASL) Initial Client Response_,
- [RFC 4959], September 2007
+ [RFC 4959], September 2007.
* J. Klensin, _Simple Mail Transfer Protocol_,
- [RFC 5321], October 2008
+ [RFC 5321], October 2008.
[RFC 4315]: https://tools.ietf.org/html/rfc4315
[RFC 2177]: https://tools.ietf.org/html/rfc2177