diff options
Diffstat (limited to 'imapsync')
-rwxr-xr-x | imapsync | 676 |
1 files changed, 313 insertions, 363 deletions
@@ -39,30 +39,38 @@ delete @ENV{qw/IFS CDPATH ENV BASH_ENV/}; my %CONFIG; sub usage(;$) { my $rv = shift // 0; - print STDERR "$NAME [OPTIONS] [--] [MAILBOX [..]]\n"; if ($rv) { - print STDERR "Try '$NAME --help' or consult the manpage for more information.\n"; + print STDERR "Usage: $NAME [OPTIONS] [COMMAND] [MAILBOX [..]]\n" + ."Try '$NAME --help' or consult the manpage for more information.\n"; } else { - print STDERR "Synchronize the given MAILBOXes between two QRESYNC-capable IMAP4rev1 servers.\n" - ."Options:\n" - ." --config=FILE Specify an alternate configuration file\n" - ." -1, --oneshot Exit as soon as all mailboxes are synchronized\n" - ." --repair List the database anomalies and try to repair them\n" - ." -q, --quiet Try to be quiet\n" - ." --debug Turn on debug mode\n" - ."Consult the manpage for more information.\n"; + print STDERR "Usage: $NAME [OPTIONS] [MAILBOX [..]]\n" + ." or: $NAME [OPTIONS] --repair [MAILBOX [..]]\n" + ." or: $NAME [OPTIONS] --delete MAILBOX [..]\n" + ." or: $NAME [OPTIONS] --rename SOURCE DEST\n" + ."Consult the manpage for more information.\n"; } exit $rv; } -usage(1) unless GetOptions(\%CONFIG, qw/config=s quiet|q oneshot|1 repair debug help|h/); +usage(1) unless GetOptions(\%CONFIG, qw/config=s quiet|q target=s@ debug help|h repair delete rename/); usage(0) if $CONFIG{help}; +my $COMMAND = do { + my @command = grep {exists $CONFIG{$_}} qw/repair delete rename/; + usage(1) if $#command>0; + $command[0] +}; +usage(1) if defined $COMMAND and (($COMMAND eq 'delete' and !@ARGV) or $COMMAND eq 'rename' and $#ARGV != 1); +@ARGV = map {uc $_ eq 'INBOX' ? 'INBOX' : $_ } @ARGV; # INBOX is case-insensitive my $CONF = read_config( delete $CONFIG{config} // $NAME , [qw/_ local remote/] , database => qr/\A(\P{Control}+)\z/ - , logfile => qr/\A(\P{Control}+)\z/ ); + , logfile => qr/\A(\/\P{Control}+)\z/ + , 'list-mailbox' => qr/\A([\x01-\x09\x0B\x0C\x0E-\x7F]+)\z/ + , 'list-select-opts' => qr/\A([\x21\x23\x24\x26\x27\x2B-\x5B\x5E-\x7A\x7C-\x7E]+)\z/ + , 'ignore-mailbox' => qr/\A([\x01-\x09\x0B\x0C\x0E-\x7F]+)\z/ + ); my ($DBFILE, $LOCKFILE, $LOGGER_FD); { @@ -93,13 +101,13 @@ my $DBH; # Clean after us sub cleanup() { - logger("Cleaning up...") if $CONFIG{debug}; + logger(undef, "Cleaning up...") if $CONFIG{debug}; unlink $LOCKFILE if defined $LOCKFILE and -f $LOCKFILE; close $LOGGER_FD if defined $LOGGER_FD; $DBH->disconnect() if defined $DBH; } -$SIG{$_} = sub { cleanup(); msg($!); exit 1; } foreach qw/INT TERM/; -$SIG{$_} = sub { cleanup(); msg($!); exit 0; } foreach qw/HUP/; +$SIG{$_} = sub { msg(undef, $!); cleanup(); exit 1; } foreach qw/INT TERM/; +$SIG{$_} = sub { msg(undef, $!); cleanup(); exit 0; } foreach qw/HUP/; ############################################################################# @@ -208,7 +216,7 @@ logger(undef, ">>> $NAME $VERSION"); my $IMAP; foreach my $name (qw/local remote/) { my %config = %{$CONF->{$name}}; - $config{$_} = $CONFIG{$_} foreach keys %CONFIG; + $config{$_} = $CONFIG{$_} foreach grep {defined $CONFIG{$_}} qw/quiet debug/; $config{enable} = 'QRESYNC'; $config{name} = $name; $config{'logger-fd'} = $LOGGER_FD if defined $LOGGER_FD; @@ -224,324 +232,267 @@ foreach my $name (qw/local remote/) { #my $mailboxes = $client->list((uc $config{'subscribed-only'} eq 'TRUE' ? '(SUBSCRIBED)' : '' ) # .$config{mailboxes}, 'SUBSCRIBED'); # $client->notify('SELECTED', 'MAILBOXES ('.join(' ', keys %$mailboxes).')'); - $client->notify(qw/SELECTED SUBSCRIBED/) unless $CONFIG{oneshot}; + # XXX NOTIFY doesn't work as expected for INBOX + # http://dovecot.org/pipermail/dovecot/2015-July/101514.html + #$client->notify(qw/SELECTED SUBSCRIBED/) if $CONFIG{watch}; # XXX We shouldn't need to ask for STATUS responses here, and use # NOTIFY's STATUS indicator instead. However Dovecot violates RFC # 5464: http://dovecot.org/pipermail/dovecot/2015-July/101474.html - @{$IMAP->{$name}}{qw/mailboxes delims/} = $client->list(q{"" "*"}, 'SUBSCRIBED', 'STATUS (UIDVALIDITY UIDNEXT HIGHESTMODSEQ)' ); + + my $list = '"" '; + my @params; + if (!defined $COMMAND or $COMMAND eq 'repair') { + $list = '('.uc($CONF->{_}->{'list-select-opts'}).') '.$list if defined $CONF->{_}->{'list-select-opts'}; + $list .= (defined $CONF->{_}->{'list-mailbox'} ? '('.$CONF->{_}->{'list-mailbox'}.')' : '*') unless @ARGV; + @params = ('SUBSCRIBED', 'STATUS (UIDVALIDITY UIDNEXT HIGHESTMODSEQ)'); + } + $list .= $#ARGV == 0 ? Net::IMAP::Sync::quote($ARGV[0]) + : ('('.join(' ',map {Net::IMAP::Sync::quote($_)} @ARGV).')') if @ARGV; + @{$IMAP->{$name}}{qw/mailboxes delims/} = $client->list($list, @params); } -############################################################################# -# Synchronize mailbox and subscription lists +############################################################################## +# -sub make_tree(%); -sub print_tree($%); -sub mv_tree($$$%); -sub sync_tree($$%); - -# Take a hash of delimiters, and recursively build a tree out of it. -# For instance ( a => "/", b => "/", "a/c" => ".", "a/c.d" => "/", "a/d" => ".") -# is transformed into the hash reference -# { b => {}, -# { a => { "/c" => { ".d" => {} } } -# , "/d" => {} -# } -# } -sub make_tree(%) { - my %delims = @_; - my @list = sort {length($a) <=> length($b)} keys %delims; - - my %tree; - foreach my $x (@list) { - next unless exists $delims{$x}; # already a children of something - my %children; - foreach (keys %delims) { - next unless defined $delims{$x} and s/\A\Q$x$delims{$x}\E/$delims{$x}/; - $children{$_} = delete $delims{"$x$_"}; - } - delete $delims{$x}; - $tree{$x} = make_tree(%children); +# Add a new mailbox to the database. +my $STH_INSERT_MAILBOX= $DBH->prepare(q{INSERT INTO mailboxes (mailbox,subscribed) VALUES (?,?)}); + +# Get the index associated with a mailbox. +my $STH_GET_INDEX = $DBH->prepare(q{SELECT idx,subscribed FROM mailboxes WHERE mailbox = ?}); + +# Ensure local and remote delimiter match +sub check_delim($) { + my $mbx = shift; + my ($lDelims, $rDelims) = map {$IMAP->{$_}->{delims}} qw/local remote/; + if (exists $lDelims->{$mbx} and exists $rDelims->{$mbx} and + ((defined $lDelims->{$mbx} xor defined $rDelims->{$mbx}) or + (defined $lDelims->{$mbx} and defined $rDelims->{$mbx} and $lDelims->{$mbx} ne $rDelims->{$mbx}))) { + my ($ld, $rd) = ($lDelims->{$mbx}, $rDelims->{$mbx}); + $ld =~ s/([\x22\x5C])/\\$1/g if defined $ld; + $rd =~ s/([\x22\x5C])/\\$1/g if defined $rd; + die "Error: Hierarchy delimiter for $mbx don't match: " + ."local \"". ($ld // '')."\", remote \"".($rd // '')."\"\n" } - return \%tree; + return exists $lDelims->{$mbx} ? $lDelims->{$mbx} : exists $rDelims->{$mbx} ? $rDelims->{$mbx} : undef; } -#sub print_tree($%) { -# my $indent = shift; -# my %tree = @_; -# while (my ($root, $children) = each %tree) { -# print " "x$indent, '|- ', $root, "\n"; -# print_tree($indent+2, %$children); -# } -#} - -# Retrun true if $mailbox exists for $name that is, if doesn't have the -# '\NonExistent' flag set. -sub exists_mbx($$) { - my $name = shift; - my $mailbox = shift; + +# Return true if $mailbox exists on $name +sub mbx_exists($$) { + my ($name, $mailbox) = @_; my $flags = $IMAP->{$name}->{mailboxes}->{$mailbox}; return (defined $flags and !grep {lc $_ eq lc '\NonExistent'} @$flags) ? 1 : 0; } -# Retrun true if $mailbox is subscribed for $name. -sub subscribed_mbx($$) { - my $name = shift; - my $mailbox = shift; + +# Return true if $mailbox is subscribed to on $name +sub mbx_subscribed($$) { + my ($name, $mailbox) = @_; my $flags = $IMAP->{$name}->{mailboxes}->{$mailbox}; return (defined $flags and grep {lc $_ eq lc '\Subscribed'} @$flags) ? 1 : 0; } -# Rename a root recursively in a tree -sub mv_tree($$$%) { - my ($mailboxes, $mbx, $mbx2, %children) = @_; - $mailboxes->{$mbx2} = delete $mailboxes->{$mbx}; - while (my ($root, $children) = each %children) { - mv_tree($mailboxes, $mbx.$root, $mbx2.$root, %children); - } -} -# Syncronize mailbox list -# XXX DELETE and RENAME not tested -sub sync_tree($$%) { - my ($sth, $mbx, %children) = @_; - my %exists = map { $_ => exists_mbx($_,$mbx) } qw/local remote/; - - my $rv = 0; - if ($exists{local} xor $exists{remote}) { - my ($exists,$missing) = $exists{local} ? ('local','remote') : ('remote','local'); - my ($sth_by_mbx, $sth_by_uidvalidity) = @$sth{($missing.'_by_mbx', $exists.'_by_uidvalidity')}; - - # check if there is an entry matching $mbx for $missing in the database - $sth_by_mbx->execute($mbx); - my $row_by_mbx = $sth_by_mbx->fetch(); - die if defined $sth_by_mbx->fetch(); # sanity check - - if (defined $row_by_mbx) { - # $mbx was seen on $missing during the previous round: it - # has either been DELETEd or RENAMEd to another name on - # $missing. - - my %uidvalidities = $IMAP->{$missing}->{client}->uidvalidity(); - my ($idx,$uidvalidity) = @$row_by_mbx; - my @mbx2 = grep { $uidvalidities{$_} == $uidvalidity and !exists_mbx($exists,$_) } - keys %uidvalidities; - - if ($#mbx2 > 0) { - # XXX this is allowed by RFC3501, but we can't guess... - msg($missing, "Multiple mailboxes have same UIDVALIDITY $uidvalidity: ", - join(',',@mbx2), "\n", - "Dunno which one $mbx should be renamed to."); - exit 1; - } - elsif (@mbx2) { - # $mbx's known (from the DB) UIDVALIDITY is that of - # $missing's $mbx2, which is not in the database and - # doesn't exist on $exists - msg($exists, "Rename mailbox $mbx to $mbx2[0]"); - $sth->{rename}->execute($mbx2[0],$idx); - $IMAP->{$exists}->{client}->rename($mbx, $mbx2[0]); - $DBH->commit(); - mv_tree($IMAP->{$exists}->{mailboxes}, $mbx, $mbx2[0], %children); - $mbx = $mbx2[0]; - } - else { - # $mbx's known (from the DB) UIDVALIDITY on $missing - # was not found in any of $missing's mailboxes. - msg($exists, "Delete mailbox $mbx"); - push @{$IMAP->{$exists}->{mailboxes}->{$mbx}}, '\NonExistent'; - $IMAP->{$exists}->{client}->delete($mbx); - } +############################################################################## +# Process --delete command +# +if (defined $COMMAND and $COMMAND eq 'delete') { + my $sth_delete_mailboxes = $DBH->prepare(q{DELETE FROM mailboxes WHERE idx = ?}); + my $sth_delete_local = $DBH->prepare(q{DELETE FROM local WHERE idx = ?}); + my $sth_delete_remote = $DBH->prepare(q{DELETE FROM remote WHERE idx = ?}); + my $sth_delete_mapping = $DBH->prepare(q{DELETE FROM mapping WHERE idx = ?}); + + foreach my $mailbox (@ARGV) { + $STH_GET_INDEX->execute($mailbox); + my ($idx) = $STH_GET_INDEX->fetchrow_array(); + die if defined $STH_GET_INDEX->fetch(); # sanity check + + # delete $mailbox on servers where $mailbox exists. note that + # there is a race condition where the mailbox could have + # appeared meanwhile + foreach my $name (qw/local remote/) { + next if defined $CONFIG{target} and !grep {$_ eq $name} @{$CONFIG{target}}; + $IMAP->{$name}->{client}->delete($mailbox) if mbx_exists($name, $mailbox); } - else { - # $mbx was never seen on $missing: it has either been - # CREATEd or RENAMEd from another name on $exists. - - my ($idx,$mbx2); - if (defined (my $uidvalidity = $IMAP->{$exists}->{client}->uidvalidity($mbx))) { - $sth_by_uidvalidity->execute($uidvalidity); - my $by_uidvalidity = $sth_by_uidvalidity->fetchall_arrayref(); - if (defined $by_uidvalidity and $#$by_uidvalidity > 0) { - # XXX this is allowed by RFC3501, but we can't guess... - my @mbx2 = map {$_->[1]} @$by_uidvalidity; - msg($exists, "Multiple mailboxes have same UIDVALIDITY $uidvalidity: ", - join(',',@mbx2), "\n", - "Dunno which one $mbx should be renamed to."); - exit 1; - } - ($idx,$mbx2) = @{$by_uidvalidity->[0]} if defined $by_uidvalidity and @$by_uidvalidity; - } - if (defined $mbx2) { - # $mbx's UIDVALIDITY on $exists can be found in the - # database as associated with $mbx2, which exists on - # $missing but not on $exists - msg($missing, "Rename mailbox $mbx2 to $mbx"); - $sth->{rename}->execute($mbx,$idx); - $IMAP->{$missing}->{client}->rename($mbx2, $mbx); - $DBH->commit(); - mv_tree($IMAP->{$missing}->{mailboxes}, $mbx2, $mbx, %children); - } - else { - # $mbx's UIDVALIDITY on $exists has never been found in - # the database. - msg($missing, "Create mailbox $mbx"); - $IMAP->{$missing}->{mailboxes}->{$mbx} = - grep {lc $_ ne lc '\NonExistent'} @{$IMAP->{$missing}->{mailboxes}->{$mbx} // []}; - $IMAP->{$missing}->{client}->create($mbx); - } - } - $rv = 1; - } + if (defined $idx and (!defined $CONFIG{target} or grep {$_ eq 'database'} @{$CONFIG{target}})) { + my $r1 = $sth_delete_mapping->execute($idx); + msg('database', "WARNING: `DELETE FROM mapping WHERE idx = $idx` failed") unless $r1; + my $r2 = $sth_delete_local->execute($idx); + msg('database', "WARNING: `DELETE FROM local WHERE idx = $idx` failed") unless $r2; + my $r3 = $sth_delete_remote->execute($idx); + msg('database', "WARNING: `DELETE FROM remote WHERE idx = $idx` failed") unless $r3; + my $r4 = $sth_delete_mailboxes->execute($idx); + msg('database', "WARNING: `DELETE FROM mailboxes WHERE idx = $idx` failed") unless $r4; - while (my ($root, $children) = each %children) { - my $r = sync_tree($sth, $mbx.$root, %$children); - $rv ||= $r; + $DBH->commit(); + msg('database', "Removed mailbox $mailbox") if $r4; + } } - return $rv; + exit 0; } -{ - my %delims; + +############################################################################## +# Process --rename command +# +elsif (defined $COMMAND and $COMMAND eq 'rename') { + my ($from, $to) = @ARGV; + + # get index of the original name + $STH_GET_INDEX->execute($from); + my ($idx) = $STH_GET_INDEX->fetchrow_array(); + die if defined $STH_GET_INDEX->fetch(); # sanity check + + # ensure the local and remote hierarchy delimiter match + my $delim = check_delim($from); + + # ensure the target name doesn't already exist on the servers. there + # is a race condition where the mailbox would be created before we + # issue the RENAME command, then the server would reply with a + # tagged NO response foreach my $name (qw/local remote/) { - while (my ($mbx, $sep) = each %{$IMAP->{$name}->{delims}}) { - if (!exists $delims{$mbx}) { - $delims{$mbx} = $sep; - } else { - die "Hierarchy delimeters for mailbox $mbx don't match!\n" - unless (!defined $sep and !defined $delims{$mbx}) or - (defined $sep and defined $delims{$mbx} and $sep eq $delims{$mbx}); - } + next if defined $CONFIG{target} and !grep {$_ eq $name} @{$CONFIG{target}}; + if (mbx_exists($name, $to)) { + msg($name, "ERROR: Mailbox $to exists. Run `$NAME --delete $to` to delete."); + exit 1; } } - my $tree = make_tree(%delims); - my %sth; - $sth{$_.'_by_mbx'} = $DBH->prepare("SELECT idx,UIDVALIDITY FROM mailboxes NATURAL JOIN $_ WHERE mailbox = ?") - foreach qw/local remote/; - $sth{$_.'_by_uidvalidity'} = $DBH->prepare("SELECT idx,mailbox FROM mailboxes NATURAL JOIN $_ WHERE UIDVALIDITY = ?") - foreach qw/local remote/; - $sth{rename} = $DBH->prepare("UPDATE mailboxes SET mailbox = ? WHERE idx = ?"); - - my $updated = 0; - while (my ($mbx,$children) = each %$tree) { - #print $mbx, "\n"; - #print_tree(0, %$children); - my $u = sync_tree(\%sth, $mbx, %$children); - $updated ||= $u; + # ensure the target name doesn't already exist in the database + $STH_GET_INDEX->execute($to); + if (defined $STH_GET_INDEX->fetch() and + (!defined $CONFIG{target} or grep {$_ eq 'database'} @{$CONFIG{target}})) { + msg('database', "ERROR: Mailbox $to exists. Run `$NAME --delete $to` to delete."); + exit 1; } - if ($updated) { - # refresh the mailbox list - foreach my $name (qw/local remote/) { - @{$IMAP->{$name}}{qw/mailboxes delims/} = $IMAP->{$name}->{client}->list(q{"" "*"}, 'SUBSCRIBED'); - } - my %mailboxes; - $mailboxes{$_} = 1 foreach (keys %{$IMAP->{local}->{mailboxes}}, keys %{$IMAP->{remote}->{mailboxes}}); - foreach my $mbx (keys %mailboxes) { - die "Couldn't sync mailbox list.\n" if exists_mbx('local',$mbx) xor exists_mbx('remote',$mbx); + + # rename $from to $to on servers where $from exists. again there is + # a race condition, but if $to has been created meanwhile the server + # will reply with a tagged NO response + foreach my $name (qw/local remote/) { + next if defined $CONFIG{target} and !grep {$_ eq $name} @{$CONFIG{target}}; + $IMAP->{$name}->{client}->rename($from, $to) if mbx_exists($name, $from); + } + + # rename from to $to in the database + if (defined $idx and (!defined $CONFIG{target} or grep {$_ eq 'database'} @{$CONFIG{target}})) { + my $sth_rename_mailbox = $DBH->prepare(q{UPDATE mailboxes SET mailbox = ? WHERE idx = ?}); + my $r = $sth_rename_mailbox->execute($to, $idx); + msg('database', "WARNING: `UPDATE mailboxes SET mailbox = ".$DBH->quote($to)." WHERE idx = $idx` failed") unless $r; + + # for non-flat mailboxes, rename the children as well + if (defined $delim) { + my $prefix = $from.$delim; + my $sth_rename_children = $DBH->prepare(q{ + UPDATE mailboxes SET mailbox = ? || SUBSTR(mailbox,?) + WHERE SUBSTR(mailbox,1,?) = ? + }); + $sth_rename_children->execute($to, length($prefix), length($prefix), $prefix); } + + $DBH->commit(); + msg('database', "Renamed mailbox $from to $to") if $r; } + exit 0; } -# Syncronize subscription list -my @SUBSCRIPTIONS; -{ - my $sth_search = $DBH->prepare("SELECT idx,subscribed FROM mailboxes WHERE mailbox = ?"); - my $sth_subscribe = $DBH->prepare("UPDATE mailboxes SET subscribed = ? WHERE idx = ?"); +############################################################################## +# Synchronize mailbox and subscription lists + +my @MAILBOXES; +{ my %mailboxes; - $mailboxes{$_} = 1 foreach (keys %{$IMAP->{local}->{mailboxes}}, keys %{$IMAP->{remote}->{mailboxes}}); - - foreach my $mbx (keys %mailboxes) { - if (subscribed_mbx('local',$mbx) xor subscribed_mbx('remote',$mbx)) { - my ($subscribed,$unsubscribed) = subscribed_mbx('local',$mbx) ? ('local','remote') : ('remote','local'); - - $sth_search->execute($mbx); - my $row = $sth_search->fetch(); - die if defined $sth_search->fetch(); # sanity check - - if (defined $row) { - my ($idx,$status) = @$row; - if ($status) { - # $mbx was SUBSCRIBEd before, UNSUBSCRIBE it now - msg($subscribed, "Unsubscribe to mailbox $mbx"); - $sth_subscribe->execute(0,$idx); - $IMAP->{$subscribed}->{client}->unsubscribe($mbx); + $mailboxes{$_} = 1 foreach keys %{$IMAP->{local}->{mailboxes}}; + $mailboxes{$_} = 1 foreach keys %{$IMAP->{remote}->{mailboxes}}; + my $sth_subscribe = $DBH->prepare(q{UPDATE mailboxes SET subscribed = ? WHERE idx = ?}); + + @MAILBOXES = keys %mailboxes; + @MAILBOXES = grep !/$CONF->{_}->{'ignore-mailbox'}/, @MAILBOXES + if defined $CONF->{_}->{'ignore-mailbox'}; + + foreach my $mailbox (@MAILBOXES) { + check_delim($mailbox); # ensure that the delimiter match + my ($lExists, $rExists) = map {mbx_exists($_,$mailbox)} qw/local remote/; + + $STH_GET_INDEX->execute($mailbox); + my ($idx,$subscribed) = $STH_GET_INDEX->fetchrow_array(); + die if defined $STH_GET_INDEX->fetch(); # sanity check + + if ($lExists and $rExists) { + # $mailbox exists on both sides + my ($lSubscribed,$rSubscribed) = map {mbx_subscribed($_, $mailbox)} qw/local remote/; + if (defined $idx) { + if ($lSubscribed xor $rSubscribed) { + # mailbox is subscribed on only one server + if ($subscribed) { # unsubscribe + my $name = $lSubscribed ? 'local' : 'remote'; + $IMAP->{$name}->{client}->unsubscribe($mailbox); + } + else { # subscribe + my $name = $lSubscribed ? 'remote' : 'local'; + $IMAP->{$name}->{client}->subscribe($mailbox); + } + # toggle subscribtion in the database + $subscribed = $subscribed ? 0 : 1; + $sth_subscribe->execute($subscribed, $idx) or + msg('database', "WARNING: `UPDATE mailboxes SET subscribed = $subscribed WHERE idx = $idx` failed"); $DBH->commit(); - $IMAP->{$subscribed}->{mailboxes}->{$mbx} = - grep {lc $_ ne lc '\Subscribed'} @{$IMAP->{$subscribed}->{mailboxes}->{$mbx} // []}; } - else { - # $mbx was UNSUBSCRIBEd before, SUBSCRIBE it now - msg($unsubscribed, "Subscribe to mailbox $mbx"); - $sth_subscribe->execute(1,$idx); - $IMAP->{$unsubscribed}->{client}->subscribe($mbx); + # $mailbox is either subscribed on both servers, or subscribed on both + elsif ($lSubscribed xor $subscribed) { + # update the database if needed + $sth_subscribe->execute($lSubscribed, $idx) or + msg('database', "WARNING: `UPDATE mailboxes SET subscribed = $lSubscribed WHERE idx = $idx` failed"); $DBH->commit(); - $IMAP->{$unsubscribed}->{mailboxes}->{$mbx} //= []; - push @{$IMAP->{$unsubscribed}->{mailboxes}->{$mbx}}, '\Subscribed'; } } else { - # $mbx is unknown; assume the user wants to SUBSCRIBE - msg($unsubscribed, "Subscribe to mailbox $mbx"); - $IMAP->{$unsubscribed}->{client}->subscribe($mbx); - $IMAP->{$unsubscribed}->{mailboxes}->{$mbx} //= []; - push @{$IMAP->{$unsubscribed}->{mailboxes}->{$mbx}}, '\Subscribed'; + # add new mailbox; subscribe on both servers if $mailbox is subscribed on one of them + my $subscribed = ($lSubscribed or $rSubscribed) ? 1 : 0; + $STH_INSERT_MAILBOX->execute($mailbox, $subscribed); + $IMAP->{local}->{client}->subscribe($mailbox) if $subscribed and !$lSubscribed; + $IMAP->{remote}->{client}->subscribe($mailbox) if $subscribed and !$rSubscribed; + $DBH->commit(); } } - else { - $sth_search->execute($mbx); - my $row = $sth_search->fetch(); - die if defined $sth_search->fetch(); # sanity check - - if (defined $row) { - my ($idx,$status) = @$row; - unless (defined $status and $status != 0) { - my $subscribed = subscribed_mbx('local',$mbx) ? 1 : 0; - $sth_subscribe->execute($subscribed, $idx); - $DBH->commit(); - } + elsif ($lExists and !$rExists) { + # $mailbox is on 'local' only + if (defined $idx) { + msg('database', "ERROR: Mailbox $mailbox exists. Run `$NAME --delete $mailbox` to delete."); + exit 1; } + my $subscribed = mbx_subscribed('local', $mailbox); + $STH_INSERT_MAILBOX->execute($mailbox, $subscribed); + $IMAP->{remote}->{client}->create($mailbox); + $IMAP->{remote}->{client}->subscribe($mailbox) if $subscribed; + $DBH->commit(); } - push @SUBSCRIPTIONS, $mbx if subscribed_mbx('local', $mbx) and - subscribed_mbx('remote',$mbx); - } -} - -# Clean database: remove mailboxes that no longer exist -{ - my $sth = $DBH->prepare("SELECT idx,mailbox,subscribed FROM mailboxes"); - my $sth_delete_mailboxes = $DBH->prepare("DELETE FROM mailboxes WHERE idx = ?"); - my $sth_delete_local = $DBH->prepare("DELETE FROM local WHERE idx = ?"); - my $sth_delete_remote = $DBH->prepare("DELETE FROM remote WHERE idx = ?"); - my $sth_delete_mapping = $DBH->prepare("DELETE FROM mapping WHERE idx = ?"); - - my @idx; - $sth->execute(); - while (defined (my $row = $sth->fetch)) { - my ($idx,$mbx,$subscribed) = @$row; - if (!exists_mbx('local',$mbx) and !exists_mbx('remote',$mbx)) { - $_->execute($idx) foreach ($sth_delete_mapping,$sth_delete_local,$sth_delete_remote); - $sth_delete_mailboxes->execute($idx) if - !exists $IMAP->{local}->{mailboxes}->{$mbx} and - !exists $IMAP->{remote}->{mailboxes}->{$mbx}; + elsif (!$lExists and $rExists) { + # $mailbox is on 'remote' only + if (defined $idx) { + msg('database', "ERROR: Mailbox $mailbox exists. Run `$NAME --delete $mailbox` to delete."); + exit 1; + } + my $subscribed = mbx_subscribed('remote', $mailbox); + $STH_INSERT_MAILBOX->execute($mailbox, $subscribed); + $IMAP->{local}->{client}->create($mailbox); + $IMAP->{local}->{client}->subscribe($mailbox) if $subscribed; $DBH->commit(); } } } - +my ($lIMAP, $rIMAP) = map {$IMAP->{$_}->{client}} qw/local remote/; +undef $IMAP; ############################################################################# # Synchronize messages -# Consider only the mailboxes in @ARGV, if the list is non-empty. - -my ($lIMAP, $rIMAP) = map {$IMAP->{$_}->{client}} qw/local remote/; -undef $IMAP; - # Get all cached states from the database. my $STH_GET_CACHE = $DBH->prepare(q{ - SELECT mailbox, + SELECT mailbox, m.idx as idx, l.UIDVALIDITY as lUIDVALIDITY, l.UIDNEXT as lUIDNEXT, l.HIGHESTMODSEQ as lHIGHESTMODSEQ, r.UIDVALIDITY as rUIDVALIDITY, r.UIDNEXT as rUIDNEXT, r.HIGHESTMODSEQ as rHIGHESTMODSEQ FROM mailboxes m JOIN local l ON m.idx = l.idx JOIN remote r ON m.idx = r.idx @@ -554,9 +505,6 @@ my $STH_GET_CACHE_BY_IDX = $DBH->prepare(q{ WHERE m.idx = ? }); -# Get the index associated with a mailbox. -my $STH_GET_INDEX = $DBH->prepare(q{SELECT idx FROM mailboxes WHERE mailbox = ?}); - # Find local/remote UID from the map. my $STH_GET_LOCAL_UID = $DBH->prepare(q{SELECT lUID FROM mapping WHERE idx = ? and rUID = ?}); my $STH_GET_REMOTE_UID = $DBH->prepare(q{SELECT rUID FROM mapping WHERE idx = ? and lUID = ?}); @@ -574,7 +522,6 @@ my $STH_UPDATE_LOCAL = $DBH->prepare(q{UPDATE local SET UIDNEXT = ?, HIGHESTMO my $STH_UPDATE_REMOTE = $DBH->prepare(q{UPDATE remote SET UIDNEXT = ?, HIGHESTMODSEQ = ? WHERE idx = ?}); # Add a new mailbox. -my $STH_INSERT_MAILBOX= $DBH->prepare(q{INSERT INTO mailboxes (mailbox,subscribed) VALUES (?,?)}); my $STH_INSERT_LOCAL = $DBH->prepare(q{INSERT INTO local (idx,UIDVALIDITY,UIDNEXT,HIGHESTMODSEQ) VALUES (?,?,0,0)}); my $STH_INSERT_REMOTE = $DBH->prepare(q{INSERT INTO remote (idx,UIDVALIDITY,UIDNEXT,HIGHESTMODSEQ) VALUES (?,?,0,0)}); @@ -620,7 +567,7 @@ sub download_missing($$$@) { my $uid = $mail->{UID}; my $from = first { defined $_ and @$_ } @{$mail->{ENVELOPE}}[2,3,4]; $from = (defined $from and @$from) ? $from->[0]->[2].'@'.$from->[0]->[3] : ''; - msg("$source($mailbox): UID $uid from <$from> ($mail->{INTERNALDATE})") unless $CONFIG{quiet}; + msg(undef, "$source($mailbox): UID $uid from <$from> ($mail->{INTERNALDATE})") unless $CONFIG{quiet}; callback_new_message($idx, $mailbox, $source, $mail, \@uids, $buff, \$bufflen) }); @@ -631,12 +578,12 @@ sub download_missing($$$@) { # Solve a flag update conflict (by taking the union of the two flag lists). sub flag_conflict($$$$$) { - my ($mailbox, $lUID, $lFlags, $rUID, $rFlags); + my ($mailbox, $lUID, $lFlags, $rUID, $rFlags) = @_; my %flags = map {$_ => 1} (split(/ /, $lFlags), split(/ /, $rFlags)); my $flags = join ' ', sort(keys %flags); - msg("WARNING: Conflicting flag update in $mailbox for local UID $lUID ($lFlags) ". - "and remote UID $rUID ($rFlags). Setting both to the union ($flags)."); + msg(undef, "WARNING: Conflicting flag update in $mailbox for local UID $lUID ($lFlags) ". + "and remote UID $rUID ($rFlags). Setting both to the union ($flags)."); return $flags } @@ -647,15 +594,26 @@ sub delete_mapping($$) { my ($idx, $lUID) = @_; my $r = $STH_DELETE_MAPPING->execute($idx, $lUID); die if $r > 1; # sanity check - msg("WARNING: Can't delete (idx,lUID) = ($idx,$lUID) from the database") if $r == 0; + msg('database', "WARNING: Can't delete (idx,lUID) = ($idx,$lUID)") if $r == 0; } # Check and repair synchronization of a mailbox between the two servers # (in a very crude way, by downloading all existing UID with their flags) -my @REPAIR; -sub repair($$) { - my ($idx, $mailbox) = @_; +sub repair($) { + my $mailbox = shift; + + $STH_GET_INDEX->execute($mailbox); + my ($idx) = $STH_GET_INDEX->fetchrow_array(); + die if defined $STH_GET_INDEX->fetch(); # sanity check + + return unless defined $idx; # not in the database + $lIMAP->select($mailbox); + $rIMAP->select($mailbox); + + $STH_GET_CACHE_BY_IDX->execute($idx); + my $cache = $STH_GET_CACHE_BY_IDX->fetchrow_hashref() // return; # no cache + die if defined $STH_GET_CACHE_BY_IDX->fetch(); # sanity check # get all existing UID with their flags my ($lVanished, $lModified) = $lIMAP->pull_updates(1); @@ -668,10 +626,6 @@ sub repair($$) { my (@rToRemove, %rToUpdate, @rMissing); my @delete_mapping; - $STH_GET_CACHE_BY_IDX->execute($idx); - my $cache = $STH_GET_CACHE_BY_IDX->fetchrow_hashref() // die "Missing cache for index $idx"; - die if defined $STH_GET_CACHE_BY_IDX->fetch(); # sanity check - # process each pair ($lUID,$rUID) found in the mapping table, and # compare with the result from the IMAP servers to detect anomalies @@ -699,7 +653,7 @@ sub repair($$) { } else { # conflict - msg("WARNING: Missed flag update in $mailbox for (lUID,rUID) = ($lUID,$rUID). Repairing.") + msg(undef, "WARNING: Missed flag update in $mailbox for (lUID,rUID) = ($lUID,$rUID). Repairing.") if $lModified->{$lUID}->[0] <= $cache->{lHIGHESTMODSEQ} and $rModified->{$rUID}->[0] <= $cache->{rHIGHESTMODSEQ}; # set both $lUID and $rUID to the union of $lFlags and $rFlags @@ -712,7 +666,7 @@ sub repair($$) { } elsif (!defined $lModified->{$lUID} and !defined $rModified->{$rUID}) { unless ($lVanished{$lUID} and $rVanished{$rUID}) { - msg("WARNING: Pair (lUID,rUID) = ($lUID,$rUID) vanished from $mailbox. Repairing."); + msg(undef, "WARNING: Pair (lUID,rUID) = ($lUID,$rUID) vanished from $mailbox. Repairing."); push @delete_mapping, $lUID; } } @@ -721,7 +675,7 @@ sub repair($$) { if ($lVanished{$lUID}) { push @rToRemove, $rUID; } else { - msg("local($mailbox): WARNING: UID $lUID disappeared. Downloading remote UID $rUID again."); + msg("local($mailbox)", "WARNING: UID $lUID disappeared. Downloading remote UID $rUID again."); push @rMissing, $rUID; } } @@ -730,7 +684,7 @@ sub repair($$) { if ($rVanished{$rUID}) { push @lToRemove, $lUID; } else { - msg("remote($mailbox): WARNING: UID $rUID disappeared. Downloading local UID $lUID again."); + msg("remote($mailbox)", "WARNING: UID $rUID disappeared. Downloading local UID $lUID again."); push @lMissing, $lUID; } } @@ -759,15 +713,15 @@ sub repair($$) { # Process UID found in IMAP but not in the mapping table. - msg("remote($mailbox): WARNING: No match for vanished local UID $_. Ignoring.") foreach keys %lVanished; - msg("local($mailbox): WARNING: No match for vanished remote UID $_. Ignoring.") foreach keys %rVanished; + msg("remote($mailbox)", "WARNING: No match for vanished local UID $_. Ignoring.") foreach keys %lVanished; + msg("local($mailbox)", "WARNING: No match for vanished remote UID $_. Ignoring.") foreach keys %rVanished; foreach my $lUID (keys %$lModified) { - msg("remote($mailbox): WARNING: No match for modified local UID $lUID. Downloading again."); + msg("remote($mailbox)", "WARNING: No match for modified local UID $lUID. Downloading again."); push @lMissing, $lUID; } foreach my $rUID (keys %$rModified) { - msg("local($mailbox): WARNING: No match for modified remote UID $rUID. Downloading again."); + msg("local($mailbox)", "WARNING: No match for modified remote UID $rUID. Downloading again."); push @rMissing, $rUID; } @@ -815,7 +769,7 @@ sub sync_known_messages($$) { my ($rUID) = $STH_GET_REMOTE_UID->fetchrow_array(); die if defined $STH_GET_REMOTE_UID->fetchrow_arrayref(); # sanity check if (!defined $rUID) { - msg("remote($mailbox): WARNING: No match for vanished local UID $lUID. Ignoring."); + msg("remote($mailbox)", "WARNING: No match for vanished local UID $lUID. Ignoring."); } elsif (!exists $rVanished{$rUID}) { push @rToRemove, $rUID; @@ -826,7 +780,7 @@ sub sync_known_messages($$) { my ($lUID) = $STH_GET_LOCAL_UID->fetchrow_array(); die if defined $STH_GET_LOCAL_UID->fetchrow_arrayref(); # sanity check if (!defined $lUID) { - msg("local($mailbox): WARNING: No match for vanished remote UID $rUID. Ignoring."); + msg("local($mailbox)", "WARNING: No match for vanished remote UID $rUID. Ignoring."); } elsif (!exists $lVanished{$lUID}) { push @lToRemove, $lUID; @@ -861,7 +815,7 @@ sub sync_known_messages($$) { my ($rUID) = $STH_GET_REMOTE_UID->fetchrow_array(); die if defined $STH_GET_REMOTE_UID->fetchrow_arrayref(); # sanity check if (!defined $rUID) { - msg("remote($mailbox): WARNING: No match for modified local UID $lUID. Try '--repair'."); + msg("remote($mailbox)", "WARNING: No match for modified local UID $lUID. Try '--repair'."); } elsif (defined (my $rFlags = $rModified->{$rUID})) { unless ($lFlags eq $rFlags) { @@ -882,7 +836,7 @@ sub sync_known_messages($$) { my ($lUID) = $STH_GET_LOCAL_UID->fetchrow_array(); die if defined $STH_GET_LOCAL_UID->fetchrow_arrayref(); # sanity check if (!defined $lUID) { - msg("local($mailbox): WARNING: No match for modified remote UID $rUID. Try '--repair'."); + msg("local($mailbox)", "WARNING: No match for modified remote UID $rUID. Try '--repair'."); } elsif (!exists $lModified->{$lUID}) { # conflicts are taken care of above @@ -915,7 +869,7 @@ sub callback_new_message($$$$;$$$) { my $length = length $mail->{RFC822}; if ($length == 0) { - msg("$name($mailbox): WARNING: Ignoring new 0-length message (UID $mail->{UID})"); + msg("$name($mailbox)", "WARNING: Ignoring new 0-length message (UID $mail->{UID})"); return; } @@ -951,7 +905,8 @@ sub callback_new_message_flush($$$@) { my ($lUIDs, $rUIDs) = $name eq 'local' ? (\@sUID,\@tUID) : (\@tUID,\@sUID); for (my $k=0; $k<=$#messages; $k++) { - logger("Adding mapping (lUID,rUID) = ($lUIDs->[$k],$rUIDs->[$k]) for $mailbox") if $CONFIG{debug}; + logger(undef, "Adding mapping (lUID,rUID) = ($lUIDs->[$k],$rUIDs->[$k]) for $mailbox") + if $CONFIG{debug}; $STH_INSERT_MAPPING->execute($idx, $lUIDs->[$k], $rUIDs->[$k]); } $DBH->commit(); # commit only once per batch @@ -965,26 +920,32 @@ sub callback_new_message_flush($$$@) { # the given UIDs. sub sync_messages($$;$$) { my ($idx, $mailbox, $lIgnore, $rIgnore) = @_; - my ($buff, $bufflen, @lUIDs); - - # get new messages from remote (except @$rIgnore) and APPEND them to local - ($buff, $bufflen) = ([], 0); - undef $buff if $lIMAP->incapable('MULTIAPPEND'); - $rIMAP->pull_new_messages(sub($) { - callback_new_message($idx, $mailbox, 'remote', shift, \@lUIDs, $buff, \$bufflen) - }, @{$rIgnore // []}); - push @lUIDs, callback_new_message_flush($idx, $mailbox, 'remote', @$buff) - if defined $buff and @$buff; - - # get new messages from local (except @$lIgnore and the newly allocated local - # UIDs @lUIDs) and APPEND them to remote - ($buff, $bufflen) = ([], 0); - undef $buff if $rIMAP->incapable('MULTIAPPEND'); - $lIMAP->pull_new_messages(sub($) { - callback_new_message($idx, $mailbox, 'local', shift, undef, $buff, \$bufflen) - }, @{$lIgnore // []}, @lUIDs); - callback_new_message_flush($idx, $mailbox, 'local', @$buff) - if defined $buff and @$buff; + + my %ignore = (local => ($lIgnore // []), remote => ($rIgnore // [])); + my $loop; + do { + # get new messages from $source (except @{$ignore{$source}}) and APPEND them to $target + foreach my $source (qw/remote local/) { # pull remote mails first + my $target = $source eq 'remote' ? 'local' : 'remote'; + my $buff = [] unless ($target eq 'local' ? $lIMAP : $rIMAP)->incapable('MULTIAPPEND'); + my $bufflen = 0; + my @tUIDs; + + ($source eq 'remote' ? $rIMAP : $lIMAP)->pull_new_messages(sub($) { + callback_new_message($idx, $mailbox, $source, shift, \@tUIDs, $buff, \$bufflen) + }, @{$ignore{$source}}); + + push @tUIDs, callback_new_message_flush($idx, $mailbox, $source, @$buff) + if defined $buff and @$buff; + push @{$ignore{$target}}, @tUIDs; + + $loop = @tUIDs ? 1 : 0; + } + # since $source modifies $target's UIDNEXT upon new mails, we + # need to check again the first $source (remote) whenever the + # last one (local) added new messages to it + } + while ($loop); # both local and remote UIDNEXT are now up to date; proceed with # pending flag updates and vanished messages @@ -992,8 +953,10 @@ sub sync_messages($$;$$) { # don't store the new UIDNEXTs before to avoid downloading these # mails again in the event of a crash - $STH_UPDATE_LOCAL->execute($lIMAP->get_cache( qw/UIDNEXT HIGHESTMODSEQ/), $idx); - $STH_UPDATE_REMOTE->execute($rIMAP->get_cache(qw/UIDNEXT HIGHESTMODSEQ/), $idx); + $STH_UPDATE_LOCAL->execute($lIMAP->get_cache( qw/UIDNEXT HIGHESTMODSEQ/), $idx) or + msg('database', "WARNING: Can't update remote UIDNEXT/HIGHESTMODSEQ for $mailbox"); + $STH_UPDATE_REMOTE->execute($rIMAP->get_cache(qw/UIDNEXT HIGHESTMODSEQ/), $idx) or + msg('database', "WARNING: Can't update remote UIDNEXT/HIGHESTMODSEQ for $mailbox"); $DBH->commit(); } @@ -1018,12 +981,15 @@ sub wait_notifications(;$) { } +############################################################################# # Resume interrupted mailbox syncs. +# my ($MAILBOX, $IDX); $STH_LIST_INTERRUPTED->execute(); while (defined (my $row = $STH_LIST_INTERRUPTED->fetchrow_arrayref())) { ($IDX, $MAILBOX) = @$row; - msg("Resuming interrupted sync for $MAILBOX"); + next unless grep { $_ eq $MAILBOX } @MAILBOXES; + msg(undef, "Resuming interrupted sync for $MAILBOX"); my %lUIDs; $STH_GET_INTERRUPTED_BY_IDX->execute($IDX); @@ -1069,8 +1035,10 @@ while (defined (my $row = $STH_LIST_INTERRUPTED->fetchrow_arrayref())) { } - +############################################################################# # Initialize $lIMAP and $rIMAP states to detect mailbox dirtyness. +# +my %KNOWN_INDEXES; $STH_GET_CACHE->execute(); while (defined (my $row = $STH_GET_CACHE->fetchrow_hashref())) { $lIMAP->set_cache($row->{mailbox}, @@ -1083,23 +1051,11 @@ while (defined (my $row = $STH_GET_CACHE->fetchrow_hashref())) { UIDNEXT => $row->{rUIDNEXT}, HIGHESTMODSEQ => $row->{rHIGHESTMODSEQ} ); - push @REPAIR, $row->{mailbox} if $CONFIG{repair} and - (!@ARGV or grep { $_ eq $row->{mailbox} } @ARGV); + $KNOWN_INDEXES{$row->{idx}} = 1; } -while (@REPAIR) { - $MAILBOX = shift @REPAIR; - - $STH_GET_INDEX->execute($MAILBOX); - ($IDX) = $STH_GET_INDEX->fetchrow_array(); - die if defined $STH_GET_INDEX->fetch(); # sanity check - - $lIMAP->select($MAILBOX); - $rIMAP->select($MAILBOX); - repair($IDX, $MAILBOX); -} -if ($CONFIG{repair}) { - cleanup(); +if (defined $COMMAND and $COMMAND eq 'repair') { + repair($_) foreach @MAILBOXES; exit 0; } @@ -1113,44 +1069,38 @@ while(1) { sync_messages($IDX, $MAILBOX); } else { - $MAILBOX = $lIMAP->next_dirty_mailbox(@ARGV) // $rIMAP->next_dirty_mailbox(@ARGV) // last; + $MAILBOX = $lIMAP->next_dirty_mailbox(@MAILBOXES) // $rIMAP->next_dirty_mailbox(@MAILBOXES) // last; $MAILBOX = 'INBOX' if uc $MAILBOX eq 'INBOX'; # INBOX is case insensitive $STH_GET_INDEX->execute($MAILBOX); ($IDX) = $STH_GET_INDEX->fetchrow_array(); die if defined $STH_GET_INDEX->fetch(); # sanity check + die unless defined $IDX; # sanity check; $lIMAP->select($MAILBOX); $rIMAP->select($MAILBOX); - # new mailbox - if (!defined $IDX) { - my $subscribed = (grep { $_ eq $MAILBOX} @SUBSCRIPTIONS) ? 1 : 0; - $STH_INSERT_MAILBOX->execute($MAILBOX, $subscribed); - $STH_GET_INDEX->execute($MAILBOX); - ($IDX) = $STH_GET_INDEX->fetchrow_array(); - die if !defined $IDX or defined $STH_GET_INDEX->fetchrow_arrayref(); # sanity check - + if (!$KNOWN_INDEXES{$IDX}) { $STH_INSERT_LOCAL->execute( $IDX, $lIMAP->uidvalidity($MAILBOX)); $STH_INSERT_REMOTE->execute($IDX, $rIMAP->uidvalidity($MAILBOX)); - # don't commit before the first mapping (lUID,rUID) + # no need to commit before the first mapping (lUID,rUID) + $KNOWN_INDEXES{$IDX} = 1; } elsif (sync_known_messages($IDX, $MAILBOX)) { # sync updates to known messages before fetching new messages # get_cache is safe after pull_update - $STH_UPDATE_LOCAL_HIGHESTMODSEQ->execute( $lIMAP->get_cache('HIGHESTMODSEQ'), $IDX); - $STH_UPDATE_REMOTE_HIGHESTMODSEQ->execute($rIMAP->get_cache('HIGHESTMODSEQ'), $IDX); + $STH_UPDATE_LOCAL_HIGHESTMODSEQ->execute( $lIMAP->get_cache('HIGHESTMODSEQ'), $IDX) or + msg('database', "WARNING: Can't update local HIGHESTMODSEQ for $MAILBOX"); + $STH_UPDATE_REMOTE_HIGHESTMODSEQ->execute($rIMAP->get_cache('HIGHESTMODSEQ'), $IDX) or + msg('database', "WARNING: Can't update remote HIGHESTMODSEQ for $MAILBOX"); $DBH->commit(); } sync_messages($IDX, $MAILBOX); } } # clean state! - if ($CONFIG{oneshot}) { - cleanup(); - exit 0; - } + exit 0 unless defined $COMMAND and $COMMAND eq 'watch'; wait_notifications(900); } |