From c5468b4a598afa09d8e4c0d8e46ab44cca1e9c60 Mon Sep 17 00:00:00 2001 From: Guilhem Moulin Date: Wed, 9 Mar 2016 15:04:20 +0100 Subject: pullimap: purge old messages every 12h. --- pullimap | 2 +- pullimap.1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pullimap b/pullimap index c16d6ac..351b4f3 100755 --- a/pullimap +++ b/pullimap @@ -236,7 +236,7 @@ sub purge() { unless ($days == 0) { my $now = time; - return if defined $LAST_PURGED and $now - $LAST_PURGED < 6*3600; # purge every 6h + return if defined $LAST_PURGED and $now - $LAST_PURGED < 43200; # purge every 12h $LAST_PURGED = $now; my @now = gmtime($now - $days*86400); diff --git a/pullimap.1 b/pullimap.1 index 57a4adf..d4ef502 100644 --- a/pullimap.1 +++ b/pullimap.1 @@ -116,7 +116,7 @@ criterion ignoring time and timezone.) If \fIpurge\-after\fR is set to \(lq0\(rq then messages are deleted immediately after delivery. Otherwise \fBPullIMAP\fR issues an IMAP SEARCH command to list old messages; if \fB\-\-idle\fR is set then the -SEARCH command is issued again every 6 hours. +SEARCH command is issued again every 12 hours. .TP .I type -- cgit v1.2.3 From e69c927ac798156e99407b4b00c15f446d4ca4d6 Mon Sep 17 00:00:00 2001 From: Guilhem Moulin Date: Wed, 9 Mar 2016 15:34:39 +0100 Subject: =?UTF-8?q?typo:=20F=5F[GS]ETFL=20=E2=86=92=20F=5F[GS]ETFD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- interimap | 6 +++--- lib/Net/IMAP/InterIMAP.pm | 6 +++--- pullimap | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/interimap b/interimap index d540686..7097f17 100755 --- a/interimap +++ b/interimap @@ -26,7 +26,7 @@ my $NAME = 'interimap'; use Getopt::Long qw/:config posix_default no_ignore_case gnu_compat bundling auto_version/; use DBI (); -use Fcntl qw/F_GETFL F_SETFL FD_CLOEXEC/; +use Fcntl qw/F_GETFD F_SETFD FD_CLOEXEC/; use List::Util 'first'; use lib 'lib'; @@ -103,8 +103,8 @@ my ($DBFILE, $LOCKFILE, $LOGGER_FD); open $LOGGER_FD, '>>', $CONF->{_}->{logfile} or die "Can't open $CONF->{_}->{logfile}: $!\n"; $LOGGER_FD->autoflush(1); - my $flags = fcntl($LOGGER_FD, F_GETFL, 0) or die "fcntl F_GETFL: $!"; - fcntl($LOGGER_FD, F_SETFL, $flags | FD_CLOEXEC) or die "fcntl F_SETFL: $!"; + my $flags = fcntl($LOGGER_FD, F_GETFD, 0) or die "fcntl F_GETFD: $!"; + fcntl($LOGGER_FD, F_SETFD, $flags | FD_CLOEXEC) or die "fcntl F_SETFD: $!"; } elsif ($CONFIG{debug}) { $LOGGER_FD = \*STDERR; diff --git a/lib/Net/IMAP/InterIMAP.pm b/lib/Net/IMAP/InterIMAP.pm index 785de38..5e9028e 100644 --- a/lib/Net/IMAP/InterIMAP.pm +++ b/lib/Net/IMAP/InterIMAP.pm @@ -23,7 +23,7 @@ use strict; use Compress::Raw::Zlib qw/Z_OK Z_FULL_FLUSH Z_SYNC_FLUSH MAX_WBITS/; use Config::Tiny (); use Errno 'EINTR'; -use Fcntl qw/F_GETFL F_SETFL FD_CLOEXEC/; +use Fcntl qw/F_GETFD F_SETFD FD_CLOEXEC/; use Net::SSLeay (); use List::Util qw/all first/; use POSIX ':signal_h'; @@ -1375,8 +1375,8 @@ sub _tcp_connect($$$) { next if $! == EINTR; # try again if connect(2) was interrupted by a signal next SOCKETS; } - my $flags = fcntl($s, F_GETFL, 0) or $self->panic("fcntl F_GETFL: $!"); - fcntl($s, F_SETFL, $flags | FD_CLOEXEC) or $self->panic("fcntl F_SETFL: $!"); + my $flags = fcntl($s, F_GETFD, 0) or $self->panic("fcntl F_GETFD: $!"); + fcntl($s, F_SETFD, $flags | FD_CLOEXEC) or $self->panic("fcntl F_SETFD: $!"); return $s; } $self->fail("Can't connect to $host:$port"); diff --git a/pullimap b/pullimap index 351b4f3..b620891 100755 --- a/pullimap +++ b/pullimap @@ -25,7 +25,7 @@ our $VERSION = '0.3'; my $NAME = 'pullimap'; use Errno 'EINTR'; -use Fcntl qw/O_CREAT O_RDWR O_DSYNC LOCK_EX SEEK_SET F_GETFL F_SETFL FD_CLOEXEC/; +use Fcntl qw/O_CREAT O_RDWR O_DSYNC LOCK_EX SEEK_SET F_GETFD F_SETFD FD_CLOEXEC/; use Getopt::Long qw/:config posix_default no_ignore_case gnu_getopt auto_version/; use List::Util 'first'; use Socket qw/PF_INET PF_INET6 SOCK_STREAM/; @@ -83,8 +83,8 @@ do { } sysopen($STATE, $statefile, O_CREAT|O_RDWR|O_DSYNC, 0600) or die "Can't open $statefile: $!"; - my $flags = fcntl($STATE, F_GETFL, 0) or die "fcntl F_GETFL: $!"; - fcntl($STATE, F_SETFL, $flags | FD_CLOEXEC) or die "fcntl F_SETFL: $!"; + my $flags = fcntl($STATE, F_GETFD, 0) or die "fcntl F_GETFD: $!"; + fcntl($STATE, F_SETFD, $flags | FD_CLOEXEC) or die "fcntl F_SETFD: $!"; flock($STATE, LOCK_EX) or die "Can't flock $statefile: $!"; }; -- cgit v1.2.3 From 2676ef52da48625a2d260169d815717313a0613d Mon Sep 17 00:00:00 2001 From: Guilhem Moulin Date: Wed, 9 Mar 2016 16:02:29 +0100 Subject: wibble --- lib/Net/IMAP/InterIMAP.pm | 2 +- pullimap | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/Net/IMAP/InterIMAP.pm b/lib/Net/IMAP/InterIMAP.pm index 5e9028e..bd69d87 100644 --- a/lib/Net/IMAP/InterIMAP.pm +++ b/lib/Net/IMAP/InterIMAP.pm @@ -887,7 +887,7 @@ sub append($$@) { delete $vanished2{$_} foreach keys %vanished; my $VANISHED = scalar(keys %vanished2); # number of messages VANISHED meanwhile $cache->{EXISTS} += $#_+1 if defined $cache->{EXISTS} and $cache->{EXISTS} + $VANISHED == $EXISTS; - $cache->{UIDNEXT} = $UIDNEXT if ($cache->{UIDNEXT} // 1) < $UIDNEXT; + $cache->{UIDNEXT} = $UIDNEXT if ($cache->{UIDNEXT} // 1) < $UIDNEXT; } unless ($self->{quiet}) { diff --git a/pullimap b/pullimap index b620891..a1f82e2 100755 --- a/pullimap +++ b/pullimap @@ -110,6 +110,7 @@ sub writeUID($) { ; $offset < 4 ; $offset += syswrite($STATE, $uid, 4-$offset, $offset) // die "Can't syswrite: $!" ) {} + # no need to sync (or flush) since $STATE is opened with O_DSYNC } @@ -218,10 +219,10 @@ sub smtp_send(@) { # Initialize the cache from the statefile, then pull new messages from # the remote mailbox # -$CONF->{'logger-fd'} = \*STDERR if $CONFIG{debug}; my $IMAP = do { my %config = (%$CONF, %CONFIG{qw/quiet debug/}, name => $ARGV[0]); $config{keepalive} = 1 if defined $CONFIG{idle}; + $config{'logger-fd'} = \*STDERR if $CONFIG{debug}; Net::IMAP::InterIMAP::->new( %config ); }; @@ -287,7 +288,7 @@ sub pull(;$) { $IMAP->silent_store(compact_set(@$ignore, @uid), '+', '\Seen') if @$ignore or @uid; # update the statefile - sysseek($STATE, 4, SEEK_SET) // die "Can't seek: $!"; + sysseek($STATE, 4, SEEK_SET) or die "Can't seek: $!"; my ($uidnext) = $IMAP->get_cache('UIDNEXT'); writeUID($uidnext); truncate($STATE, 8) // die "Can't truncate"; -- cgit v1.2.3 From 50f79472babc840807bcb183aaab7ddbd9ebfa71 Mon Sep 17 00:00:00 2001 From: Guilhem Moulin Date: Wed, 9 Mar 2016 17:18:07 +0100 Subject: pullimap: add control flow details in the manpage. --- pullimap.1 | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 80 insertions(+), 7 deletions(-) diff --git a/pullimap.1 b/pullimap.1 index d4ef502..8f27d85 100644 --- a/pullimap.1 +++ b/pullimap.1 @@ -15,10 +15,11 @@ to a SMTP or LMTP transmission channel. It can also remove old messages after a configurable retention period. .PP -A statefile is used to keep track of the mailbox's UIDVALIDITY and -UIDNEXT values. While \fBPullIMAP\fR is running, the statefile is also -used to keep track of UIDs being delivered, which avoids duplicate +A \fIstatefile\fR is used to keep track of the mailbox's UIDVALIDITY and +UIDNEXT values. While \fBPullIMAP\fR is running, the \fIstatefile\fR is +also used to keep track of UIDs being delivered, which avoids duplicate deliveries if the process is interrupted. +See the \fBCONTROL FLOW\fR section below. .SH OPTIONS .TP @@ -38,10 +39,10 @@ default) to avoid being logged out for inactivity. .TP .B \fB\-\-no\-delivery -Update the state file, but skip SMTP/LMTP delivery. This is mostly -useful for initializing the statefile when migrating to \fBPullIMAP\fR -from another equivalent program such as \fIgetmail\fR(1) or -\fIfetchmail\fR(1). +Update the \fIstatefile\fR, but skip SMTP/LMTP delivery. This is mostly +useful for initializing the \fIstatefile\fR when migrating to +\fBPullIMAP\fR from another equivalent program such as \fIgetmail\fR(1) +or \fIfetchmail\fR(1). .TP .B \-q\fR, \fB\-\-quiet\fR @@ -234,6 +235,78 @@ for more information. File containing trusted certificates to use during server certificate authentication if \(lq\fISSL_verify\fR=YES\(rq. +.SH CONTROL FLOW +\fBPullIMAP\fR opens the \fIstatefile\fR corresponding to a given +configuration \fISECTION\fR with O_DSYNC to ensure that written data is +flushed to the underlying hardware by the time \fIwrite\fR(2) returns. +Moreover an exclusive lock is placed on the file descriptor immediately +after opening to prevent multiple \fBpullimap\fR processes from +accessing the \fIstatefile\fR concurrently. + +Each \fIstatefile\fR consists of a series of 32-bits big-endian +integers. Usually there are only two integers: +the first is the \fImailbox\fR's UIDVALIDITY value, and the second is +the \fImailbox\fR's last seen UIDNEXT value (\fBPullIMAP\fR then assumes +that all messages with UID smaller than this UIDNEXT value have already +been retrieved and delivered). +The IMAP4rev1 specification [RFC 3501] does not guaranty that untagged +FETCH responses are sent ordered by UID in response of an UID FETCH +command. Thus it would be unsafe for \fBPullIMAP\fR to update the +UIDNEXT value in the \fIstatefile\fR while the UID FETCH command is +progress. +Instead, for each untagged FETCH response received while while the UID +FETCH command is in progress, \fBPullIMAP\fR delivers the message BODY +to the SMTP or LMTP server specified with \fIdeliver\-method\fR then +appends the message UID to the \fIstatefile\fR. When the UID FETCH +command eventually terminates, \fBPullIMAP\fR updates the UIDNEXT value +in the \fIstatefile\fR and truncate the file down to 8 bytes. +Keeping track of message UIDs as they are received avoids duplicate in +the even of a crash or session loss while the UID FETCH command is in +progress. + +In more details, \fBPullIMAP\fR works as follows: + +.nr step 1 1 +.IP \n[step]. 4 +Issue an UID FETCH command to retrieve message ENVELOPE and BODY (and +UID) with UID bigger or equal than the UIDNEXT value found in the +\fIstatefile\fR. +While the UID FETCH command is in progress, perform the following +for each untagged FETCH response sent by the server: +.RS +.nr step 1 1 +.IP \n[step]. 4 +if no SMTP/LMTP transmission channel was opened, open one to the server +specified with \fIdeliver\-method\fR and send an EHLO (or LHO) command +with the domain given by \fIdeliver\-ehlo\fR; +.IP \n+[step]. +perform a mail transaction (using SMTP pipelining [RFC 2920] if +possible) to send the retrieved message BODY to the SMTP or LMTP +session; and +.IP \n+[step]. +append the message UID to the \fIstatefile\fR. +.RE + +.IP \n-[step]. +If a SMTP/LMTP transmission channel was opened, send a QUIT command to +close it gracefully. + +.IP \n+[step]. +Issue an UID STORE command to mark all retrieved messages (and stalled +UIDs found in the \fIstatefile\fR after the UIDNEXT value) as \\Seen. + +.IP \n+[step]. +Update the \fIstatefile\fR with the new UIDNEXT value. + +.IP \n+[step]. +Truncate the \fIstatefile\fR down to 8 bytes (so that it contains only +two 32-bits integers, respectively the \fImailbox\fR's UIDVALIDITY and +UIDNEXT values). + +.IP \n+[step]. +If \fB\-\-idle\fR was set, issue an IDLE command; stop idling and go +back to step 1. whenever a new message is received. + .SH AUTHOR Written by Guilhem Moulin .MT guilhem@fripost.org -- cgit v1.2.3 From 04a259a845a1c9eb9c99db41cd940c89f0bc68e9 Mon Sep 17 00:00:00 2001 From: Guilhem Moulin Date: Wed, 9 Mar 2016 20:12:03 +0100 Subject: Improve mailto: link for groff_www. --- interimap.1 | 12 +++++++++--- pullimap.1 | 12 +++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/interimap.1 b/interimap.1 index 93e463e..3aabc3f 100644 --- a/interimap.1 +++ b/interimap.1 @@ -392,6 +392,12 @@ tunables in the \fBCONFIGURATION FILE\fR. Moreover, few IMAP servers have been tested so far. .SH AUTHOR -Written by Guilhem Moulin -.MT guilhem@fripost.org -.ME . +.ie \n[www-html] \{\ + Written by +. MTO guilhem@fripost.org "Guilhem Moulin" . +\} +.el \{\ + Written by Guilhem Moulin +. MT guilhem@fripost.org +. ME . +\} diff --git a/pullimap.1 b/pullimap.1 index 8f27d85..342afce 100644 --- a/pullimap.1 +++ b/pullimap.1 @@ -308,6 +308,12 @@ If \fB\-\-idle\fR was set, issue an IDLE command; stop idling and go back to step 1. whenever a new message is received. .SH AUTHOR -Written by Guilhem Moulin -.MT guilhem@fripost.org -.ME . +.ie \n[www-html] \{\ + Written by +. MTO guilhem@fripost.org "Guilhem Moulin" . +\} +.el \{\ + Written by Guilhem Moulin +. MT guilhem@fripost.org +. ME . +\} -- cgit v1.2.3 From a657c564c46338bb360690661152cf4a25fe8927 Mon Sep 17 00:00:00 2001 From: Guilhem Moulin Date: Wed, 9 Mar 2016 20:28:31 +0100 Subject: pullimap.1: Work around groff_www rendering bug with nested lists. --- pullimap.1 | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pullimap.1 b/pullimap.1 index 342afce..2d92e9d 100644 --- a/pullimap.1 +++ b/pullimap.1 @@ -274,20 +274,21 @@ UID) with UID bigger or equal than the UIDNEXT value found in the While the UID FETCH command is in progress, perform the following for each untagged FETCH response sent by the server: .RS -.nr step 1 1 -.IP \n[step]. 4 +\(bu if no SMTP/LMTP transmission channel was opened, open one to the server specified with \fIdeliver\-method\fR and send an EHLO (or LHO) command with the domain given by \fIdeliver\-ehlo\fR; -.IP \n+[step]. +.br +\(bu perform a mail transaction (using SMTP pipelining [RFC 2920] if possible) to send the retrieved message BODY to the SMTP or LMTP session; and -.IP \n+[step]. +.br +\(bu append the message UID to the \fIstatefile\fR. .RE -.IP \n-[step]. +.IP \n+[step]. If a SMTP/LMTP transmission channel was opened, send a QUIT command to close it gracefully. -- cgit v1.2.3 From 5c5f8134e5bbedf71a63e377d49c0962fe1ba1ad Mon Sep 17 00:00:00 2001 From: Guilhem Moulin Date: Wed, 9 Mar 2016 20:59:14 +0100 Subject: pullimap: keep trying to pull new messages after issuing any IMAP command. Indeed we might get an untagged EXISTS response, meaning that a new message has been received meanwhile. --- lib/Net/IMAP/InterIMAP.pm | 2 +- pullimap | 58 +++++++++++++++++++++++++++++------------------ 2 files changed, 37 insertions(+), 23 deletions(-) diff --git a/lib/Net/IMAP/InterIMAP.pm b/lib/Net/IMAP/InterIMAP.pm index bd69d87..bad49da 100644 --- a/lib/Net/IMAP/InterIMAP.pm +++ b/lib/Net/IMAP/InterIMAP.pm @@ -1053,7 +1053,7 @@ sub uidvalidity($;$) { } -# $self->set_cache(@attributes) +# $self->get_cache(@attributes) # Return the persistent cache for the mailbox currently selected. If # some @attributes are given, return the list of values corresponding # to these attributes. diff --git a/pullimap b/pullimap index a1f82e2..02aedf6 100755 --- a/pullimap +++ b/pullimap @@ -244,13 +244,18 @@ sub purge() { my @m = qw/Jan Feb Mar Apr May Jun Jul Aug Sep Oct Nov Dec/; # RFC 3501's date-month my $date = sprintf("%02d-%s-%04d", $now[3], $m[$now[4]], $now[5]+1900); my @uid = $IMAP->search("UID $set BEFORE $date"); - return unless @uid; - $set = compact_set(@uid); - $IMAP->log("Removing ".($#uid+1)." UID(s) $set") unless $CONFIG{quiet}; + $set = @uid ? compact_set(@uid) : undef; + $IMAP->log("Removing ".($#uid+1)." UID(s) $set") if defined $set and !$CONFIG{quiet}; } - $IMAP->silent_store($set, '+', '\Deleted'); - $IMAP->expunge($set); + + if (defined $set) { + $IMAP->silent_store($set, '+', '\Deleted'); + $IMAP->expunge($set); + } + + # pull messages that have been received in the meantime + pull() if $IMAP->has_new_mails($MAILBOX); } # Use BODY.PEEK[] so if something gets wrong, unpulled messages @@ -258,37 +263,46 @@ sub purge() { my $ATTRS = "ENVELOPE INTERNALDATE"; $ATTRS .= " BODY.PEEK[]" unless $CONFIG{'no-delivery'}; +sub pull_callback($$) { + my ($uids, $mail) = @_; + return unless exists $mail->{RFC822} or $CONFIG{'no-delivery'}; # not for us + + my $uid = $mail->{UID}; + my $from = first { defined $_ and @$_ } @{$mail->{ENVELOPE}}[2,3,4]; + $from = (defined $from and @$from) ? $from->[0]->[2].'@'.$from->[0]->[3] : ''; + $IMAP->log("UID $uid from <$from> ($mail->{INTERNALDATE})") unless $CONFIG{quiet}; + + sendmail($from, $mail->{RFC822}) unless $CONFIG{'no-delivery'}; + + push @$uids, $uid; + writeUID($uid); +} + # Pull new messages from IMAP and deliver them to SMTP, then update the # statefile sub pull(;$) { my $ignore = shift // []; my @uid; - # invariant: we're at pos 8 + 4*(1+$#ignore + 1+$#uids) in the statefile - $IMAP->pull_new_messages($ATTRS, sub($) { - my $mail = shift; - return unless exists $mail->{RFC822} or $CONFIG{'no-delivery'}; # not for us - - my $uid = $mail->{UID}; - my $from = first { defined $_ and @$_ } @{$mail->{ENVELOPE}}[2,3,4]; - $from = (defined $from and @$from) ? $from->[0]->[2].'@'.$from->[0]->[3] : ''; - $IMAP->log("UID $uid from <$from> ($mail->{INTERNALDATE})") unless $CONFIG{quiet}; + my $callback = sub($) { pull_callback(\@uid, shift) }; - sendmail($from, $mail->{RFC822}) unless $CONFIG{'no-delivery'}; + do { + # invariant: we're at pos 8 + 4*(1+$#ignore + 1+$#uids) in the statefile + $IMAP->pull_new_messages($ATTRS, $callback, @$ignore); - push @uid, $uid; - writeUID($uid); - }, @$ignore); + # now that everything has been deliverd, mark @ignore and @uid as \Seen + $IMAP->silent_store(compact_set(@$ignore, @uid), '+', '\Seen') if @$ignore or @uid; + } + # repeat if we got a message in the meantime + while ($IMAP->has_new_mails($MAILBOX)); # terminate the SMTP transmission channel gracefully, cf RFC 5321 section 4.5.3.2 smtp_send('QUIT' => '221') if defined $SMTP; undef $SMTP; - # now that everything has been deliverd, mark @ignore and @uid as \Seen - $IMAP->silent_store(compact_set(@$ignore, @uid), '+', '\Seen') if @$ignore or @uid; - # update the statefile - sysseek($STATE, 4, SEEK_SET) or die "Can't seek: $!"; + my $p = sysseek($STATE, 4, SEEK_SET) // die "Can't seek: $!"; + die "Couldn't seek to 4" unless $p == 4; # safety check my ($uidnext) = $IMAP->get_cache('UIDNEXT'); writeUID($uidnext); truncate($STATE, 8) // die "Can't truncate"; -- cgit v1.2.3 From ebacf74fdb3a507c6b0d865f4e9a8a6e59dbb10c Mon Sep 17 00:00:00 2001 From: Guilhem Moulin Date: Wed, 9 Mar 2016 21:14:40 +0100 Subject: Net::IMAP::InterIMAP: don't print undefined cache values in debug messages. --- lib/Net/IMAP/InterIMAP.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/Net/IMAP/InterIMAP.pm b/lib/Net/IMAP/InterIMAP.pm index bad49da..73f55e8 100644 --- a/lib/Net/IMAP/InterIMAP.pm +++ b/lib/Net/IMAP/InterIMAP.pm @@ -1027,7 +1027,7 @@ sub set_cache($$%) { } $self->logger("Update last clean state for $mailbox: ". - '('.join(' ', map {"$_ $cache->{$_}"} keys %$cache).')') + '('.join(' ', map {"$_ $cache->{$_}"} grep {defined $cache->{$_}} keys %$cache).')') if $self->{debug}; } -- cgit v1.2.3