From 67440844c422ee30b31df9a46a7f99ac0e833add Mon Sep 17 00:00:00 2001 From: Guilhem Moulin Date: Sat, 9 Nov 2019 03:13:45 +0100 Subject: Refactor and improve test suite. --- tests/list | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/list (limited to 'tests/list') diff --git a/tests/list b/tests/list new file mode 100644 index 0000000..21aa3f4 --- /dev/null +++ b/tests/list @@ -0,0 +1,37 @@ +db-no-create--watch `interimap --watch` refuses to create the database +db-exclusive-lock mutually exclusive DB access + +. DB schema upgrade (v0 -> v1) + db-upgrade-0-1 migrate + # may happen if the server(s) software or its configuration changed + db-upgrade-0-1-delim-mismatch abort on hierarchy delimiter mismatch + # foreign key checking was broken until v0.5 + db-migration-0-1-foreign-key-violation abort on foreign key contraint violation + +. Mailbox deletion + ... delete + +. Mailbox renaming + rename-exists-db abort if target exists in the DB + rename-exists-local abort if target exists locally + rename-exists-remote abort if target exists remotely + ... rename-simple + ... rename-inferiors + +# try values beyond the signed integer limit +largeint Large UIDVALIDITY/UIDNEXT/HIGHESTMODSEQ values + +. Mailbox synchronization + ... sync-mailbox-list + list-reference list-reference + list-mailbox list-mailbox = foo "foo bar" "f\\\"o\x21o.*" "f\0o\0o" + list-select-opts list-select-opts = SUBSCRIBED + ignore-mailbox ignore-mailbox = ^virtual(?:\x00|$) + +resume Resume when aborted +repair --repair + +. Live synchronization (60s) + sync-live local/remote simulation + sync-live-crippled local/remote simulation (crippled remote) + sync-live-multi local/remote1+remote2+remote3 simulation (3 local namespaces) -- cgit v1.2.3 From 78522acced782587b3768f3fb57f2f25cb905754 Mon Sep 17 00:00:00 2001 From: Guilhem Moulin Date: Sun, 10 Nov 2019 03:18:26 +0100 Subject: Test suite: add new tests for authentication. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This can't be done with `doveadm exec imap`, so the IMAPd needs to bind to TCP port 10143 on the loopback interface. Also, no longer pass ‘imap_capability’ Dovecot setting explicitely to `doveadm exec imap`; changed tests/sync-live-crippled to use type=imap instead of type=tunnel. --- tests/auth-login/interimap.remote | 5 + tests/auth-login/remote.conf | 2 + tests/auth-login/t | 12 +++ tests/auth-logindisabled/interimap.remote | 1 + tests/auth-logindisabled/remote.conf | 4 + tests/auth-logindisabled/t | 16 +++ tests/auth-noplaintext/interimap.remote | 3 + tests/auth-noplaintext/remote.conf | 1 + tests/auth-noplaintext/t | 15 +++ tests/auth-sasl-plain-no-ir/interimap.remote | 1 + tests/auth-sasl-plain-no-ir/remote.conf | 2 + tests/auth-sasl-plain-no-ir/t | 26 +++++ tests/auth-sasl-plain/interimap.remote | 4 + tests/auth-sasl-plain/remote.conf | 1 + tests/auth-sasl-plain/t | 12 +++ tests/list | 7 ++ tests/run | 109 ++++++++++++++++----- tests/snippets/dovecot/imapd.conf | 12 +++ .../dovecot/interimap-required-capabilities.conf | 3 + tests/sync-live-crippled/interimap.remote | 1 + tests/sync-live-crippled/remote.conf | 4 +- 21 files changed, 213 insertions(+), 28 deletions(-) create mode 100644 tests/auth-login/interimap.remote create mode 100644 tests/auth-login/remote.conf create mode 100644 tests/auth-login/t create mode 120000 tests/auth-logindisabled/interimap.remote create mode 100644 tests/auth-logindisabled/remote.conf create mode 100644 tests/auth-logindisabled/t create mode 100644 tests/auth-noplaintext/interimap.remote create mode 120000 tests/auth-noplaintext/remote.conf create mode 100644 tests/auth-noplaintext/t create mode 120000 tests/auth-sasl-plain-no-ir/interimap.remote create mode 100644 tests/auth-sasl-plain-no-ir/remote.conf create mode 100644 tests/auth-sasl-plain-no-ir/t create mode 100644 tests/auth-sasl-plain/interimap.remote create mode 100644 tests/auth-sasl-plain/remote.conf create mode 100644 tests/auth-sasl-plain/t create mode 100644 tests/snippets/dovecot/imapd.conf create mode 100644 tests/snippets/dovecot/interimap-required-capabilities.conf create mode 120000 tests/sync-live-crippled/interimap.remote (limited to 'tests/list') diff --git a/tests/auth-login/interimap.remote b/tests/auth-login/interimap.remote new file mode 100644 index 0000000..b7d67bf --- /dev/null +++ b/tests/auth-login/interimap.remote @@ -0,0 +1,5 @@ +type = imap +host = localhost +port = 10143 +STARTTLS = NO +auth = login diff --git a/tests/auth-login/remote.conf b/tests/auth-login/remote.conf new file mode 100644 index 0000000..4ab127a --- /dev/null +++ b/tests/auth-login/remote.conf @@ -0,0 +1,2 @@ +!include conf.d/imapd.conf +auth_mechanisms = plain login diff --git a/tests/auth-login/t b/tests/auth-login/t new file mode 100644 index 0000000..7fd83d5 --- /dev/null +++ b/tests/auth-login/t @@ -0,0 +1,12 @@ +for ((i = 0; i < 32; i++)); do + u="$(shuf -n1 -e "local" "remote")" + sample_message | deliver -u "$u" +done + +# check that credentials aren't leaked to the debug output +interimap --debug || error +grep -Fx "remote: C: xxx LOGIN [REDACTED]" <"$STDERR" || error + +check_mailbox_status "INBOX" + +# vim: set filetype=sh : diff --git a/tests/auth-logindisabled/interimap.remote b/tests/auth-logindisabled/interimap.remote new file mode 120000 index 0000000..a4ea3f3 --- /dev/null +++ b/tests/auth-logindisabled/interimap.remote @@ -0,0 +1 @@ +../auth-sasl-plain/interimap.remote \ No newline at end of file diff --git a/tests/auth-logindisabled/remote.conf b/tests/auth-logindisabled/remote.conf new file mode 100644 index 0000000..1f02afe --- /dev/null +++ b/tests/auth-logindisabled/remote.conf @@ -0,0 +1,4 @@ +!include conf.d/imapd.conf + +# trick dovecot into treating local connections as insecure +imap_capability = +LOGINDISABLED diff --git a/tests/auth-logindisabled/t b/tests/auth-logindisabled/t new file mode 100644 index 0000000..0bcd0d6 --- /dev/null +++ b/tests/auth-logindisabled/t @@ -0,0 +1,16 @@ +! interimap --debug || error + +# double check the presence of 'LOGINDISABLED' in the preauth capability list +grep -oE -m1 '^remote: S: \* OK \[CAPABILITY IMAP4rev1( [^]]*)? AUTH=[^]]*\]' <"$STDERR" >"$TMPDIR/capability" + +sed -ri 's/^remote: S: \* OK \[CAPABILITY (.*)\]$/\1/' "$TMPDIR/capability" +tr " " "\\n" <"$TMPDIR/capability" >"$TMPDIR/capabilities" +grep -Fx "IMAP4rev1" <"$TMPDIR/capabilities" || error +grep -Fx "LOGINDISABLED" <"$TMPDIR/capabilities" || error +! grep -Fx "STARTTLS" <"$TMPDIR/capabilities" || error # otherwise we'd try to upgrade the connectionn + +# make sure we didn't send any credentials +grep -Fx "remote: ERROR: Logins are disabled." <"$STDERR" || error +! grep -E "^remote: C: .* (AUTHENTICATE|LOGIN) " <"$STDERR" || error + +# vim: set filetype=sh : diff --git a/tests/auth-noplaintext/interimap.remote b/tests/auth-noplaintext/interimap.remote new file mode 100644 index 0000000..60567e2 --- /dev/null +++ b/tests/auth-noplaintext/interimap.remote @@ -0,0 +1,3 @@ +type = imap +host = localhost +port = 10143 diff --git a/tests/auth-noplaintext/remote.conf b/tests/auth-noplaintext/remote.conf new file mode 120000 index 0000000..dbbb908 --- /dev/null +++ b/tests/auth-noplaintext/remote.conf @@ -0,0 +1 @@ +../auth-sasl-plain/remote.conf \ No newline at end of file diff --git a/tests/auth-noplaintext/t b/tests/auth-noplaintext/t new file mode 100644 index 0000000..11d7d4d --- /dev/null +++ b/tests/auth-noplaintext/t @@ -0,0 +1,15 @@ +! interimap --debug || error + +# double check the presence of 'STARTTLS' in the preauth capability list +grep -oE -m1 '^remote: S: \* OK \[CAPABILITY IMAP4rev1( [^]]*)? AUTH=[^]]*\]' <"$STDERR" >"$TMPDIR/capability" + +sed -ri 's/^remote: S: \* OK \[CAPABILITY (.*)\]$/\1/' "$TMPDIR/capability" +tr " " "\\n" <"$TMPDIR/capability" >"$TMPDIR/capabilities" + grep -Fx "IMAP4rev1" <"$TMPDIR/capabilities" || error +! grep -Fx "STARTTLS" <"$TMPDIR/capabilities" || error + +# make sure we didn't send any credentials +grep -Fx "remote: ERROR: Server did not advertise STARTTLS capability." <"$STDERR" || error +! grep -E "^remote: C: .* (AUTHENTICATE|LOGIN) " <"$STDERR" || error + +# vim: set filetype=sh : diff --git a/tests/auth-sasl-plain-no-ir/interimap.remote b/tests/auth-sasl-plain-no-ir/interimap.remote new file mode 120000 index 0000000..a4ea3f3 --- /dev/null +++ b/tests/auth-sasl-plain-no-ir/interimap.remote @@ -0,0 +1 @@ +../auth-sasl-plain/interimap.remote \ No newline at end of file diff --git a/tests/auth-sasl-plain-no-ir/remote.conf b/tests/auth-sasl-plain-no-ir/remote.conf new file mode 100644 index 0000000..dae9545 --- /dev/null +++ b/tests/auth-sasl-plain-no-ir/remote.conf @@ -0,0 +1,2 @@ +!include conf.d/imapd.conf +!include conf.d/interimap-required-capabilities.conf diff --git a/tests/auth-sasl-plain-no-ir/t b/tests/auth-sasl-plain-no-ir/t new file mode 100644 index 0000000..17aa9e6 --- /dev/null +++ b/tests/auth-sasl-plain-no-ir/t @@ -0,0 +1,26 @@ +n=1 # at least one message to send remotely +sample_message | deliver -u "local" +for ((i = 0; i < 32; i++)); do + u="$(shuf -n1 -e "local" "remote")" + [ "$u" = "remote" ] || n=$(( n+1 )) + sample_message | deliver -u "$u" +done + +# check that credentials aren't leaked to the debug output +interimap --debug || error +grep -Fx "remote: C: xxx AUTHENTICATE PLAIN [REDACTED]" <"$STDERR" || error + +# make sure we didn't use SASL-IR +grep -oE -m1 '^remote: S: \* OK \[CAPABILITY IMAP4rev1( [^]]*)? AUTH=[^]]*\]' <"$STDERR" >"$TMPDIR/capability" + +sed -ri 's/^remote: S: \* OK \[CAPABILITY (.*)\]$/\1/' "$TMPDIR/capability" +tr " " "\\n" <"$TMPDIR/capability" >"$TMPDIR/capabilities" + grep -Fx "IMAP4rev1" <"$TMPDIR/capabilities" || error +! grep -Fx "SASL-IR" <"$TMPDIR/capabilities" || error + +# make sure all literals were synchronizing (and that we didn't use MULTIAPPEND) +xcgrep "$n" -E "^remote(\(INBOX\))?: C: [0-9]+ APPEND INBOX .* \{[0-9]+\}$" <"$STDERR" + +check_mailbox_status "INBOX" + +# vim: set filetype=sh : diff --git a/tests/auth-sasl-plain/interimap.remote b/tests/auth-sasl-plain/interimap.remote new file mode 100644 index 0000000..9c0a623 --- /dev/null +++ b/tests/auth-sasl-plain/interimap.remote @@ -0,0 +1,4 @@ +type = imap +host = localhost +port = 10143 +STARTTLS = NO diff --git a/tests/auth-sasl-plain/remote.conf b/tests/auth-sasl-plain/remote.conf new file mode 100644 index 0000000..3ccbd42 --- /dev/null +++ b/tests/auth-sasl-plain/remote.conf @@ -0,0 +1 @@ +!include conf.d/imapd.conf diff --git a/tests/auth-sasl-plain/t b/tests/auth-sasl-plain/t new file mode 100644 index 0000000..68f71a9 --- /dev/null +++ b/tests/auth-sasl-plain/t @@ -0,0 +1,12 @@ +for ((i = 0; i < 32; i++)); do + u="$(shuf -n1 -e "local" "remote")" + sample_message | deliver -u "$u" +done + +# check that credentials aren't leaked to the debug output +interimap --debug || error +grep -Fx "remote: C: xxx AUTHENTICATE PLAIN [REDACTED]" <"$STDERR" || error + +check_mailbox_status "INBOX" + +# vim: set filetype=sh : diff --git a/tests/list b/tests/list index 21aa3f4..86034ef 100644 --- a/tests/list +++ b/tests/list @@ -31,6 +31,13 @@ largeint Large UIDVALIDITY/UIDNEXT/HIGHESTMODSEQ values resume Resume when aborted repair --repair +. Authentication + auth-sasl-plain AUTHENTICATE (SASL PLAIN) + auth-sasl-plain-no-ir AUTHENTICATE (SASL PLAIN, no SASL-IR) + auth-login LOGIN + auth-logindisabled LOGINDISABLED + auth-noplaintext abort when STARTTLS is not offered + . Live synchronization (60s) sync-live local/remote simulation sync-live-crippled local/remote simulation (crippled remote) diff --git a/tests/run b/tests/run index cb52518..bff9c18 100755 --- a/tests/run +++ b/tests/run @@ -27,7 +27,8 @@ if [ $# -eq 0 ] || [ $# -gt 2 ]; then exit 1 fi -TESTDIR="$(dirname -- "$0")/$1" +BASEDIR="$(dirname -- "$0")" +TESTDIR="$BASEDIR/$1" TESTNAME="${2-$1}" if [ ! -d "$TESTDIR" ]; then printf "ERROR: Not a directory: %s\\n" "$TESTDIR" >&2 @@ -35,7 +36,18 @@ if [ ! -d "$TESTDIR" ]; then fi ROOTDIR="$(mktemp --tmpdir="${TMPDIR:-/dev/shm}" --directory "$1.XXXXXXXXXX")" -trap 'rm -rf -- "$ROOTDIR"' EXIT INT TERM +declare -a DOVECOT_SERVER=() +trap cleanup EXIT INT TERM +cleanup() { + local pid c + for c in "${DOVECOT_SERVER[@]}"; do + if [ ! -f "$c" ] || ! env -i PATH="/usr/bin:/bin" doveadm -c "$c" stop; then + pid="$(< "${c%/*}/run/master.pid")" + kill -TERM "$pid" || printf "kill(1) exited with status %d\\n" "$?" >&2 + fi + done + rm -rf -- "$ROOTDIR" +} _STDOUT="$ROOTDIR/stdout" _STDERR="$ROOTDIR/stderr" @@ -60,7 +72,7 @@ environ_set() { # Prepare the test harness prepare() { declare -a ENVIRON=() - local src cfg target u home n capability + local src cfg target u home n proto if [ -f "$TESTDIR/remotes" ]; then for cfg in $(seq 1 "$(< "$TESTDIR/remotes")"); do REMOTES+=( "remote$cfg" ) @@ -72,27 +84,64 @@ prepare() { for u in "local" "${REMOTES[@]}"; do home="$ROOTDIR/$u/home" export "HOME_$u"="$home" - mkdir -pm0700 -- "$home/.config/dovecot" - cat >"$home/.config/dovecot/config" <<-EOF + environ_set "$u" + + mkdir -pm0700 -- "$home/.dovecot" + cat >"$home/.dovecot/config" <<-EOF log_path = $HOME_local/mail.log mail_home = $home mail_location = dbox:~/inbox:LAYOUT=index mailbox_list_index = yes ssl = no + listen = 127.0.0.1, ::1 namespace inbox { inbox = yes } EOF if [ -f "$TESTDIR/$u.conf" ]; then - cat >>"$home/.config/dovecot/config" <"$TESTDIR/$u.conf" + cat >>"$home/.dovecot/config" <"$TESTDIR/$u.conf" + fi + cp -aT -- "$BASEDIR/snippets/dovecot" "$home/.dovecot/conf.d" + + proto="$(env -i "${ENVIRON[@]}" doveconf -c "$home/.dovecot/config" -h protocols)" + if [ -n "$proto" ]; then + cat >>"$home/.dovecot/config" <<-EOF + # https://wiki.dovecot.org/HowTo/Rootless + base_dir = $home/.dovecot/run + default_internal_user = $(id -un) + default_internal_group = $(id -gn) + default_login_user = $(id -un) + + service anvil { + chroot = + } + service imap-login { + chroot = + } + service stats { + chroot = + } + + passdb { + args = scheme=PLAIN username_format=%u $home/.dovecot/users + driver = passwd-file + } + userdb { + args = username_format=%u $home/.dovecot/users + driver = passwd-file + } + EOF + + env -i PATH="/usr/bin:/bin" /usr/sbin/dovecot -c "$home/.dovecot/config" + DOVECOT_SERVER+=( "$home/.dovecot/config" ) + printf "%s:%s:::::\\n" "$u" "$(xxd -l16 -p "$home/.dovecot/users" fi - environ_set "$u" mkdir -pm0755 -- "$home/.local/bin" cat >"$home/.local/bin/doveadm" <<-EOF #!/bin/sh exec env -i ${ENVIRON[@]@Q} \\ - doveadm -c ${home@Q}/.config/dovecot/config "\$@" + doveadm -c ${home@Q}/.dovecot/config "\$@" EOF chmod +x -- "$home/.local/bin/doveadm" done @@ -123,15 +172,19 @@ prepare() { cat <"$TESTDIR/interimap$n.local" >>"$HOME_local/.config/interimap/config$n" fi - # `doveadm exec imap` ignores 'imap_capability' from doveconf/config - capability="$(doveconf -c "$home/.config/dovecot/config" -h imap_capability)" - cat >>"$HOME_local/.config/interimap/config$n" <<-EOF - - [remote] - type = tunnel - command = exec ${home@Q}/.local/bin/doveadm exec imap ${capability:+-oimap_capability=${capability@Q}} - null-stderr = NO - EOF + printf "\\n[remote]\\n" >>"$HOME_local/.config/interimap/config$n" + if [ -s "$home/.dovecot/users" ]; then + cat <<-EOF + username = $u + password = $(awk -F: -vu="$u" '$1 == u {print $2}' <"$home/.dovecot/users") + EOF + else + cat <<-EOF + type = tunnel + command = exec ${home@Q}/.local/bin/doveadm exec imap + null-stderr = NO + EOF + fi >>"$HOME_local/.config/interimap/config$n" if [ -f "$TESTDIR/interimap$n.remote" ]; then cat <"$TESTDIR/interimap$n.remote" >>"$HOME_local/.config/interimap/config$n" fi @@ -208,7 +261,7 @@ dump_test_result() { environ_set "$u" eval home="\$HOME_$u" printf "%s dovecot configuration:\\n%s\\n" "$u" "$below" - env -i "${ENVIRON[@]}" doveconf -c "$home/.config/dovecot/config" -n + env -i "${ENVIRON[@]}" doveconf -c "$home/.dovecot/config" -n printf "%s\\n\\n" "$above" done @@ -218,6 +271,10 @@ dump_test_result() { printf "%s\\n\\n" "$above" done + printf "mail.log:\\n%s\\n" "$below" + cat -- "$HOME_local/mail.log" 2>/dev/null || true + printf "%s\\n\\n" "$above" + printf "standard output:\\n%s\\n" "$below" cat <"$_STDOUT" printf "%s\\n\\n" "$above" @@ -230,10 +287,10 @@ dump_test_result() { # Check mailbox consistency between the local/remote server and interimap's database check_mailbox_status() { local mailbox="$1" lns="inbox" lsep lprefix rns="inbox" rsep rprefix - lsep="$(doveconf -c "$HOME_local/.config/dovecot/config" -h "namespace/$lns/separator")" - lprefix="$(doveconf -c "$HOME_local/.config/dovecot/config" -h "namespace/$lns/prefix")" - rsep="$(doveconf -c "$HOME_remote/.config/dovecot/config" -h "namespace/$lns/separator")" - rprefix="$(doveconf -c "$HOME_remote/.config/dovecot/config" -h "namespace/$lns/prefix")" + lsep="$(doveconf -c "$HOME_local/.dovecot/config" -h "namespace/$lns/separator")" + lprefix="$(doveconf -c "$HOME_local/.dovecot/config" -h "namespace/$lns/prefix")" + rsep="$(doveconf -c "$HOME_remote/.dovecot/config" -h "namespace/$lns/separator")" + rprefix="$(doveconf -c "$HOME_remote/.dovecot/config" -h "namespace/$lns/prefix")" local blob="x'$(printf "%s" "$mailbox" | tr "$lsep" "\\0" | xxd -c256 -ps)'" local rmailbox="$(printf "%s" "$mailbox" | tr "$lsep" "$rsep")" @@ -305,10 +362,10 @@ check_mailboxes_status() { # Check mailbox list constency between the local and remote servers check_mailbox_list() { local m i lns="inbox" lsep lprefix rns="inbox" rsep rprefix sub= - lsep="$(doveconf -c "$HOME_local/.config/dovecot/config" -h "namespace/$lns/separator")" - lprefix="$(doveconf -c "$HOME_local/.config/dovecot/config" -h "namespace/$lns/prefix")" - rsep="$(doveconf -c "$HOME_remote/.config/dovecot/config" -h "namespace/$lns/separator")" - rprefix="$(doveconf -c "$HOME_remote/.config/dovecot/config" -h "namespace/$lns/prefix")" + lsep="$(doveconf -c "$HOME_local/.dovecot/config" -h "namespace/$lns/separator")" + lprefix="$(doveconf -c "$HOME_local/.dovecot/config" -h "namespace/$lns/prefix")" + rsep="$(doveconf -c "$HOME_remote/.dovecot/config" -h "namespace/$lns/separator")" + rprefix="$(doveconf -c "$HOME_remote/.dovecot/config" -h "namespace/$lns/prefix")" if [ $# -gt 0 ] && [ "$1" = "-s" ]; then sub="-s" shift diff --git a/tests/snippets/dovecot/imapd.conf b/tests/snippets/dovecot/imapd.conf new file mode 100644 index 0000000..c9926ce --- /dev/null +++ b/tests/snippets/dovecot/imapd.conf @@ -0,0 +1,12 @@ +protocols = $protocols imap + +mail_plugins = $mail_plugins zlib +protocol imap { + mail_plugins = $mail_plugins imap_zlib +} + +service imap-login { + inet_listener imap { + port = 10143 + } +} diff --git a/tests/snippets/dovecot/interimap-required-capabilities.conf b/tests/snippets/dovecot/interimap-required-capabilities.conf new file mode 100644 index 0000000..10dd8e1 --- /dev/null +++ b/tests/snippets/dovecot/interimap-required-capabilities.conf @@ -0,0 +1,3 @@ +# strict minimum of IMAP capabilities required for interimap to work +# (in particular, no LITERAL+, MULTIAPPEND, COMPRESS=DEFLATE, SASL-IR) +imap_capability = IMAP4rev1 ENABLE UIDPLUS LIST-EXTENDED QRESYNC LIST-STATUS diff --git a/tests/sync-live-crippled/interimap.remote b/tests/sync-live-crippled/interimap.remote new file mode 120000 index 0000000..a4ea3f3 --- /dev/null +++ b/tests/sync-live-crippled/interimap.remote @@ -0,0 +1 @@ +../auth-sasl-plain/interimap.remote \ No newline at end of file diff --git a/tests/sync-live-crippled/remote.conf b/tests/sync-live-crippled/remote.conf index 76c08e0..ee22c5f 100644 --- a/tests/sync-live-crippled/remote.conf +++ b/tests/sync-live-crippled/remote.conf @@ -2,5 +2,5 @@ namespace inbox { separator = ^ } -# strict minimum of IMAP capabilities required for interimap to work -imap_capability = IMAP4rev1 ENABLE UIDPLUS LIST-EXTENDED QRESYNC LIST-STATUS +!include conf.d/imapd.conf +!include conf.d/interimap-required-capabilities.conf -- cgit v1.2.3 From a7c364bf90a4593cfbc7911b1b7536dc66b1c879 Mon Sep 17 00:00:00 2001 From: Guilhem Moulin Date: Sun, 10 Nov 2019 05:39:41 +0100 Subject: Test suite: add new tests for SSL/TLS. MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SSL connections are accepted on TCP port 10993. Also, fix STARTTLS directive, broken since fba1c36… --- Changelog | 1 + lib/Net/IMAP/InterIMAP.pm | 11 ++-- tests/list | 9 +++ tests/snippets/dovecot/dhparams.pem | 8 +++ tests/snippets/dovecot/dovecot.key | 5 ++ tests/snippets/dovecot/dovecot.pem | 11 ++++ tests/snippets/dovecot/imapd.conf | 4 ++ tests/snippets/dovecot/ssl.conf | 4 ++ tests/starttls-logindisabled/interimap.remote | 4 ++ tests/starttls-logindisabled/remote.conf | 5 ++ tests/starttls-logindisabled/t | 19 +++++++ tests/starttls/interimap.remote | 4 ++ tests/starttls/remote.conf | 2 + tests/starttls/t | 27 +++++++++ tests/sync-live-tls/interimap.remote | 1 + tests/sync-live-tls/local.conf | 1 + tests/sync-live-tls/remote.conf | 6 ++ tests/sync-live-tls/t | 1 + tests/tls-pin-fingerprint/interimap.remote | 1 + tests/tls-pin-fingerprint/remote.conf | 1 + tests/tls-pin-fingerprint/t | 33 +++++++++++ tests/tls-protocols/interimap.remote | 1 + tests/tls-protocols/remote.conf | 1 + tests/tls-protocols/t | 39 +++++++++++++ tests/tls-verify-peer/interimap.remote | 2 + tests/tls-verify-peer/remote.conf | 1 + tests/tls-verify-peer/t | 80 +++++++++++++++++++++++++++ tests/tls/interimap.remote | 3 + tests/tls/remote.conf | 2 + tests/tls/t | 14 +++++ 30 files changed, 296 insertions(+), 5 deletions(-) create mode 100644 tests/snippets/dovecot/dhparams.pem create mode 100644 tests/snippets/dovecot/dovecot.key create mode 100644 tests/snippets/dovecot/dovecot.pem create mode 100644 tests/snippets/dovecot/ssl.conf create mode 100644 tests/starttls-logindisabled/interimap.remote create mode 100644 tests/starttls-logindisabled/remote.conf create mode 100644 tests/starttls-logindisabled/t create mode 100644 tests/starttls/interimap.remote create mode 100644 tests/starttls/remote.conf create mode 100644 tests/starttls/t create mode 120000 tests/sync-live-tls/interimap.remote create mode 120000 tests/sync-live-tls/local.conf create mode 100644 tests/sync-live-tls/remote.conf create mode 120000 tests/sync-live-tls/t create mode 120000 tests/tls-pin-fingerprint/interimap.remote create mode 120000 tests/tls-pin-fingerprint/remote.conf create mode 100644 tests/tls-pin-fingerprint/t create mode 120000 tests/tls-protocols/interimap.remote create mode 120000 tests/tls-protocols/remote.conf create mode 100644 tests/tls-protocols/t create mode 100644 tests/tls-verify-peer/interimap.remote create mode 120000 tests/tls-verify-peer/remote.conf create mode 100644 tests/tls-verify-peer/t create mode 100644 tests/tls/interimap.remote create mode 100644 tests/tls/remote.conf create mode 100644 tests/tls/t (limited to 'tests/list') diff --git a/Changelog b/Changelog index 5b43c6f..c08de80 100644 --- a/Changelog +++ b/Changelog @@ -85,6 +85,7 @@ interimap (0.5) upstream; - libinterimap: don't panic() when inflate() reports the end of the compression stream is reached. - libinterimap: the 'compress' boolean wasn't honored. + - libinterimap: fix STARTTLS directive, broken since 0.2. -- Guilhem Moulin Fri, 10 May 2019 00:58:14 +0200 diff --git a/lib/Net/IMAP/InterIMAP.pm b/lib/Net/IMAP/InterIMAP.pm index 77c1b14..b4d8bec 100644 --- a/lib/Net/IMAP/InterIMAP.pm +++ b/lib/Net/IMAP/InterIMAP.pm @@ -215,7 +215,7 @@ sub in_set($$) { return 1 if $x == $1; } elsif ($r eq '*' or $r eq '*:*') { - warn "Assuming $x belongs to set $set! (Dunno what \"*\" means.)"; + warn "Assuming $x belongs to set $set! (Dunno what \"*\" means.)"; return 1; } elsif ($r =~ /\A([0-9]+):\*\z/ or $r =~ /\A\*:([0-9]+)\z/) { @@ -398,7 +398,8 @@ sub new($%) { if ($self->{type} eq 'imap' and $self->{STARTTLS}) { # RFC 2595 section 5.1 $self->fail("Server did not advertise STARTTLS capability.") unless grep {$_ eq 'STARTTLS'} @caps; - $self->_start_ssl($self->{S}) if $self->{type} eq 'imaps'; + $self->_send('STARTTLS'); + $self->_start_ssl($self->{S}); # refresh the previous CAPABILITY list since the previous one could have been spoofed delete $self->{_CAPABILITIES}; @@ -1610,7 +1611,7 @@ sub _ssl_verify($$$) { my $pkey = Net::SSLeay::X509_get_X509_PUBKEY($cert); unless (defined $pkey and Net::SSLeay::EVP_Digest($pkey, $type) eq $digest) { - $self->warn("Fingerprint doesn't match! MiTM in action?"); + $self->warn("Fingerprint doesn't match! MiTM in action?"); $ok = 0; } } @@ -2355,7 +2356,7 @@ sub _resp($$;&$$) { # /!\ No bookkeeping since there is no internal cache mapping sequence numbers to UIDs if ($self->_enabled('QRESYNC')) { $self->panic("$1 <= $cache->{EXISTS}") if $1 <= $cache->{EXISTS}; # sanity check - $self->fail("RFC 7162 violation! Got an EXPUNGE response with QRESYNC enabled."); + $self->fail("RFC 7162 violation! Got an EXPUNGE response with QRESYNC enabled."); } # the new message was expunged before it was synced $self->{_NEW} = 0 if $self->{_NEW} == 1 and $cache->{EXISTS} == $1; @@ -2406,7 +2407,7 @@ sub _resp($$;&$$) { /\A \((\\?$RE_ATOM_CHAR+ [0-9]+(?: \\?$RE_ATOM_CHAR+ [0-9]+)*)?\)\z/ or $self->panic($_); my %status = split / /, $1; $mailbox = 'INBOX' if uc $mailbox eq 'INBOX'; # INBOX is case-insensitive - $self->panic("RFC 5465 violation! Missing HIGHESTMODSEQ data item in STATUS response") + $self->panic("RFC 5465 violation! Missing HIGHESTMODSEQ data item in STATUS response") if $self->_enabled('QRESYNC') and !defined $status{HIGHESTMODSEQ} and defined $cmd and ($cmd eq 'NOTIFY' or $cmd eq 'slurp'); $self->_update_cache_for($mailbox, %status); diff --git a/tests/list b/tests/list index 86034ef..bc8d144 100644 --- a/tests/list +++ b/tests/list @@ -38,7 +38,16 @@ repair --repair auth-logindisabled LOGINDISABLED auth-noplaintext abort when STARTTLS is not offered +. SSL/TLS + starttls-logindisabled LOGINDISABLED STARTTLS + starttls STARTTLS + tls SSL/TLS handshake + ... tls-verify-peer + tls-pin-fingerprint pubkey fingerprint pinning + tls-protocols force TLS protocol versions + . Live synchronization (60s) sync-live local/remote simulation sync-live-crippled local/remote simulation (crippled remote) + sync-live-tls local/remote simulation (TLS remote) sync-live-multi local/remote1+remote2+remote3 simulation (3 local namespaces) diff --git a/tests/snippets/dovecot/dhparams.pem b/tests/snippets/dovecot/dhparams.pem new file mode 100644 index 0000000..7734d2a --- /dev/null +++ b/tests/snippets/dovecot/dhparams.pem @@ -0,0 +1,8 @@ +-----BEGIN DH PARAMETERS----- +MIIBCAKCAQEA0J1dU8erRgIk4bMCBMLezjx32pcQpXrdNgl04dxZVxnJ5Ik2gGhA +uQRbbZhAlHNHtFtp9s4TdQ3Ddrv9SuWXYul8U5BWbcxs4nOtwFU8912SfiuVr/kc +4ok2zQ1hdMODtaqWS2ZKBmwcuk4QM6e7fMEAkuZX+Dtf2u8bG5G9B7OL5LggYtrP +cFVNQDtfhs64D+sUKJLWkgeg5NH6nbf+0Gs5a8v3/urHKvoxdVScGmKzF+LsFsBm +ycQjYeVtA9gLr41mo80rrFysUQqZtNkbdkaXOIA2r9JGTYex1l/XaediR8J94ck9 +dwAe2ubRqWcPjmoLJYQIPKiCbvXuJAd0wwIBAg== +-----END DH PARAMETERS----- diff --git a/tests/snippets/dovecot/dovecot.key b/tests/snippets/dovecot/dovecot.key new file mode 100644 index 0000000..95c9846 --- /dev/null +++ b/tests/snippets/dovecot/dovecot.key @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIGkqkKq69zVeF17S3y2U2HkQWh8z9M/xeblCztkKIfzJoAoGCCqGSM49 +AwEHoUQDQgAE1LLppulKw8KjINrDhOjEd0NTax5iDCds+vpA2PwsvvtGoprNAjQM +zX+40u30N3CE0r591txqohSBQ/X+nvG2ug== +-----END EC PRIVATE KEY----- diff --git a/tests/snippets/dovecot/dovecot.pem b/tests/snippets/dovecot/dovecot.pem new file mode 100644 index 0000000..7e53d90 --- /dev/null +++ b/tests/snippets/dovecot/dovecot.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBkzCCATmgAwIBAgIUQ+3hBMsPJcl59xDDujDDfexurOswCgYIKoZIzj0EAwIw +HzEdMBsGA1UEAwwUSW50ZXJJTUFQIHRlc3Qgc3VpdGUwHhcNMTkxMTEwMTM1NDAw +WhcNMjkxMTA3MTM1NDAwWjAfMR0wGwYDVQQDDBRJbnRlcklNQVAgdGVzdCBzdWl0 +ZTBZMBMGByqGSM49AgEGCCqGSM49AwEHA0IABNSy6abpSsPCoyDaw4ToxHdDU2se +YgwnbPr6QNj8LL77RqKazQI0DM1/uNLt9DdwhNK+fdbcaqIUgUP1/p7xtrqjUzBR +MB0GA1UdDgQWBBRlh8nSwyX+VlhwuhV7RKYwvKLyDzAfBgNVHSMEGDAWgBRlh8nS +wyX+VlhwuhV7RKYwvKLyDzAPBgNVHRMBAf8EBTADAQH/MAoGCCqGSM49BAMCA0gA +MEUCIQDK8xPPHTbYW5JnZ1Siy8ChZ6GOu2sRwQu7OgtGYGZRSQIgFKn1oAhnq2Oi +aIPqxjvBPMsK/sjrdI/rNsr2XgaulU4= +-----END CERTIFICATE----- diff --git a/tests/snippets/dovecot/imapd.conf b/tests/snippets/dovecot/imapd.conf index c9926ce..18c60f8 100644 --- a/tests/snippets/dovecot/imapd.conf +++ b/tests/snippets/dovecot/imapd.conf @@ -9,4 +9,8 @@ service imap-login { inet_listener imap { port = 10143 } + inet_listener imaps { + port = 10993 + ssl = yes + } } diff --git a/tests/snippets/dovecot/ssl.conf b/tests/snippets/dovecot/ssl.conf new file mode 100644 index 0000000..240f24b --- /dev/null +++ b/tests/snippets/dovecot/ssl.conf @@ -0,0 +1,4 @@ +ssl = required +ssl_cert = "$TMPDIR/capability" + +sed -ri 's/^remote: S: \* OK \[CAPABILITY (.*)\]$/\1/' "$TMPDIR/capability" +tr " " "\\n" <"$TMPDIR/capability" >"$TMPDIR/capabilities" +grep -Fx "IMAP4rev1" <"$TMPDIR/capabilities" || error +grep -Fx "LOGINDISABLED" <"$TMPDIR/capabilities" || error + +# make sure we upgraded the connection and check the capability again +grep -Fx "STARTTLS" <"$TMPDIR/capabilities" || error +grep -Fx "remote: C: 000000 STARTTLS" <"$STDERR" || error +grep -Fx "remote: C: 000001 CAPABILITY" <"$STDERR" || error + +# can't go further as the capability string still has the manually +# enforced 'LOGINDISABLED' + +# vim: set filetype=sh : diff --git a/tests/starttls/interimap.remote b/tests/starttls/interimap.remote new file mode 100644 index 0000000..5d7571d --- /dev/null +++ b/tests/starttls/interimap.remote @@ -0,0 +1,4 @@ +type = imap +host = 127.0.0.1 +port = 10143 +SSL_verify = no diff --git a/tests/starttls/remote.conf b/tests/starttls/remote.conf new file mode 100644 index 0000000..3d07ea9 --- /dev/null +++ b/tests/starttls/remote.conf @@ -0,0 +1,2 @@ +!include conf.d/imapd.conf +!include conf.d/ssl.conf diff --git a/tests/starttls/t b/tests/starttls/t new file mode 100644 index 0000000..99a39c2 --- /dev/null +++ b/tests/starttls/t @@ -0,0 +1,27 @@ +for ((i = 0; i < 32; i++)); do + u="$(shuf -n1 -e "local" "remote")" + sample_message | deliver -u "$u" +done + +interimap --debug || error + +# double check the presence of 'STARTTLS' in the preauth capability list +grep -oE -m1 '^remote: S: \* OK \[CAPABILITY IMAP4rev1( [^]]*)? AUTH=[^]]*\]' <"$STDERR" >"$TMPDIR/capability" + +sed -ri 's/^remote: S: \* OK \[CAPABILITY (.*)\]$/\1/' "$TMPDIR/capability" +tr " " "\\n" <"$TMPDIR/capability" >"$TMPDIR/capabilities" +grep -Fx "IMAP4rev1" <"$TMPDIR/capabilities" || error +grep -Fx "STARTTLS" <"$TMPDIR/capabilities" || error + +# make sure we upgraded the connection and check the capability again +grep -Fx "remote: C: 000000 STARTTLS" <"$STDERR" || error +grep -Fx "remote: C: 000001 CAPABILITY" <"$STDERR" || error + +grep -Fx "remote: Disabling SSL protocols: SSLv3, TLSv1, TLSv1.1" <"$STDERR" || error +grep -Fx "remote: Peer certificate fingerprint: sha256\$35944e3bd3300d3ac310bb497a32cc1eef6931482a587ddbc95343740cdf1323" <"$STDERR" || error +grep "^remote: SSL protocol: TLSv1\.[23] " <"$STDERR" || error +grep "^remote: SSL cipher: " <"$STDERR" || error + +check_mailbox_status "INBOX" + +# vim: set filetype=sh : diff --git a/tests/sync-live-tls/interimap.remote b/tests/sync-live-tls/interimap.remote new file mode 120000 index 0000000..daf3741 --- /dev/null +++ b/tests/sync-live-tls/interimap.remote @@ -0,0 +1 @@ +../tls/interimap.remote \ No newline at end of file diff --git a/tests/sync-live-tls/local.conf b/tests/sync-live-tls/local.conf new file mode 120000 index 0000000..ad27dd1 --- /dev/null +++ b/tests/sync-live-tls/local.conf @@ -0,0 +1 @@ +../sync-live/local.conf \ No newline at end of file diff --git a/tests/sync-live-tls/remote.conf b/tests/sync-live-tls/remote.conf new file mode 100644 index 0000000..b2af2d2 --- /dev/null +++ b/tests/sync-live-tls/remote.conf @@ -0,0 +1,6 @@ +namespace inbox { + separator = ^ +} + +!include conf.d/imapd.conf +!include conf.d/ssl.conf diff --git a/tests/sync-live-tls/t b/tests/sync-live-tls/t new file mode 120000 index 0000000..189360e --- /dev/null +++ b/tests/sync-live-tls/t @@ -0,0 +1 @@ +../sync-live/t \ No newline at end of file diff --git a/tests/tls-pin-fingerprint/interimap.remote b/tests/tls-pin-fingerprint/interimap.remote new file mode 120000 index 0000000..daf3741 --- /dev/null +++ b/tests/tls-pin-fingerprint/interimap.remote @@ -0,0 +1 @@ +../tls/interimap.remote \ No newline at end of file diff --git a/tests/tls-pin-fingerprint/remote.conf b/tests/tls-pin-fingerprint/remote.conf new file mode 120000 index 0000000..6029749 --- /dev/null +++ b/tests/tls-pin-fingerprint/remote.conf @@ -0,0 +1 @@ +../tls/remote.conf \ No newline at end of file diff --git a/tests/tls-pin-fingerprint/t b/tests/tls-pin-fingerprint/t new file mode 100644 index 0000000..1b84390 --- /dev/null +++ b/tests/tls-pin-fingerprint/t @@ -0,0 +1,33 @@ +# backup config +install -m0600 "$XDG_CONFIG_HOME/interimap/config" "$XDG_CONFIG_HOME/interimap/config~" +with_remote_config() { + install -m0600 "$XDG_CONFIG_HOME/interimap/config~" "$XDG_CONFIG_HOME/interimap/config" + cat >>"$XDG_CONFIG_HOME/interimap/config" +} + +# pinned valid fingerprint +with_remote_config <<-EOF + SSL_fingerprint = sha256\$e8fc8d03ffe75e03897136a2f1c5647bf8c36be7136a6883a732a8c4961c1614 +EOF + +for ((i = 0; i < 32; i++)); do + u="$(shuf -n1 -e "local" "remote")" + sample_message | deliver -u "$u" +done +interimap_init +check_mailbox_status "INBOX" + + +# and now an invalid one +with_remote_config <<-EOF + SSL_fingerprint = sha256\$deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef +EOF +! interimap --debug || error + +grep -Fx "remote: ERROR: Can't initiate TLS/SSL handshake" <"$STDERR" || error +grep -Fx "remote: WARNING: Fingerprint doesn't match! MiTM in action?" <"$STDERR" || error +grep -Fx "remote: ERROR: Can't initiate TLS/SSL handshake" <"$STDERR" || error +# make sure we didn't send any credentials +! grep -E "^remote: C: .* (AUTHENTICATE|LOGIN) " <"$STDERR" || error + +# vim: set filetype=sh : diff --git a/tests/tls-protocols/interimap.remote b/tests/tls-protocols/interimap.remote new file mode 120000 index 0000000..daf3741 --- /dev/null +++ b/tests/tls-protocols/interimap.remote @@ -0,0 +1 @@ +../tls/interimap.remote \ No newline at end of file diff --git a/tests/tls-protocols/remote.conf b/tests/tls-protocols/remote.conf new file mode 120000 index 0000000..6029749 --- /dev/null +++ b/tests/tls-protocols/remote.conf @@ -0,0 +1 @@ +../tls/remote.conf \ No newline at end of file diff --git a/tests/tls-protocols/t b/tests/tls-protocols/t new file mode 100644 index 0000000..f34a95b --- /dev/null +++ b/tests/tls-protocols/t @@ -0,0 +1,39 @@ +# backup config +install -m0600 "$XDG_CONFIG_HOME/interimap/config" "$XDG_CONFIG_HOME/interimap/config~" +with_remote_tls_protocols() { + install -m0600 "$XDG_CONFIG_HOME/interimap/config~" "$XDG_CONFIG_HOME/interimap/config" + printf "SSL_protocols = %s\\n" "$*" >>"$XDG_CONFIG_HOME/interimap/config" +} + +# default +interimap --debug || error +grep -Fx "remote: Disabling SSL protocols: SSLv3, TLSv1, TLSv1.1" <"$STDERR" || error +grep -E "^remote: SSL protocol: TLSv1\.[23] " <"$STDERR" || error + +# also disable TLSv1.2 +with_remote_tls_protocols "!SSLv2" "!SSLv3" "!TLSv1" "!TLSv1.1" "!TLSv1.2" +interimap --debug || error +grep -Fx "remote: Disabling SSL protocols: SSLv3, TLSv1, TLSv1.1, TLSv1.2" <"$STDERR" || error +grep -E "^remote: SSL protocol: TLSv1\.3 " <"$STDERR" || error + +# force TLSv1.2 +with_remote_tls_protocols "TLSv1.2" +interimap --debug || error +grep -Fx "remote: Disabling SSL protocols: SSLv3, TLSv1, TLSv1.1, TLSv1.3" <"$STDERR" || error +grep -E "^remote: SSL protocol: TLSv1\.2 " <"$STDERR" || error + +# force TLSv1 to TLSv1.2 +with_remote_tls_protocols "TLSv1" "TLSv1.1" "TLSv1.2" +interimap --debug || error +grep -Fx "remote: Disabling SSL protocols: SSLv3, TLSv1.3" <"$STDERR" || error +grep -E "^remote: SSL protocol: TLSv(1\.[12])? " <"$STDERR" || error + +# force SSLv2 and SSLv3, fails as it's disabled server side +with_remote_tls_protocols "SSLv2" "SSLv3" +! interimap --debug || error +grep -Fx "remote: Disabling SSL protocols: TLSv1, TLSv1.1, TLSv1.2, TLSv1.3" <"$STDERR" || error +grep -Fx "remote: ERROR: Can't initiate TLS/SSL handshake" <"$STDERR" || error +# make sure we didn't send any credentials +! grep -E "^remote: C: .* (AUTHENTICATE|LOGIN) " <"$STDERR" || error + +# vim: set filetype=sh : diff --git a/tests/tls-verify-peer/interimap.remote b/tests/tls-verify-peer/interimap.remote new file mode 100644 index 0000000..b02fcd0 --- /dev/null +++ b/tests/tls-verify-peer/interimap.remote @@ -0,0 +1,2 @@ +host = ::1 +port = 10993 diff --git a/tests/tls-verify-peer/remote.conf b/tests/tls-verify-peer/remote.conf new file mode 120000 index 0000000..6029749 --- /dev/null +++ b/tests/tls-verify-peer/remote.conf @@ -0,0 +1 @@ +../tls/remote.conf \ No newline at end of file diff --git a/tests/tls-verify-peer/t b/tests/tls-verify-peer/t new file mode 100644 index 0000000..d84328a --- /dev/null +++ b/tests/tls-verify-peer/t @@ -0,0 +1,80 @@ +CERT=~/.dovecot/conf.d/dovecot.pem + +unverified_peer() { + ! interimap --debug || error + + grep -Fx "remote: ERROR: Can't initiate TLS/SSL handshake" <"$STDERR" || error + sed -nr "s/remote: \[[0-9]+\] (preverify=[0-9]+)$/\1/p" <"$STDERR" >"$TMPDIR/preverify" + [ -s "$TMPDIR/preverify" ] || error + ! grep -Fvx "preverify=0" <"$TMPDIR/preverify" || error + + # make sure we didn't send any credentials + ! grep -E "^remote: C: .* (AUTHENTICATE|LOGIN) " <"$STDERR" || error +} +verified_peer() { + local i u + for ((i = 0; i < 32; i++)); do + u="$(shuf -n1 -e "local" "remote")" + sample_message | deliver -u "$u" + done + interimap --debug || error + + sed -nr "s/remote: \[[0-9]+\] (preverify=[0-9]+)$/\1/p" <"$STDERR" >"$TMPDIR/preverify" + [ -s "$TMPDIR/preverify" ] || error + ! grep -Fvx "preverify=1" <"$TMPDIR/preverify" || error + + grep "^remote: SSL protocol: TLSv1\.[23] " <"$STDERR" || error + grep "^remote: SSL cipher: " <"$STDERR" || error + + check_mailbox_status "INBOX" +} + +# backup config +install -m0600 "$XDG_CONFIG_HOME/interimap/config" "$XDG_CONFIG_HOME/interimap/config~" +with_remote_config() { + install -m0600 "$XDG_CONFIG_HOME/interimap/config~" "$XDG_CONFIG_HOME/interimap/config" + cat >>"$XDG_CONFIG_HOME/interimap/config" +} + +step_start "peer verification enabled by default" +unverified_peer +step_done + +step_start "peer verification result honored when pinned pubkey matches" +pkey_sha256="$(openssl x509 -pubkey <"$CERT" | openssl pkey -pubin -outform DER \ + | openssl dgst -sha256 | sed -rn "/^.*=\\s*/ {s///p;q}")" +with_remote_config <<-EOF + SSL_fingerprint = sha256\$$pkey_sha256 +EOF +unverified_peer +! grep -Fx "remote: WARNING: Fingerprint doesn't match! MiTM in action?" <"$STDERR" || error +step_done + + +step_start "SSL_CAfile" +if [ -f "/etc/ssl/certs/ca-certificates.crt" ]; then + # the self-signed cert should not be in there + with_remote_config <<<"SSL_CAfile = /etc/ssl/certs/ca-certificates.crt" + unverified_peer +fi +with_remote_config <<<"SSL_CAfile = $CERT" +verified_peer +step_done + + +step_start "SSL_CApath" +if [ -d "/etc/ssl/certs" ]; then + # the self-signed cert should not be in there + with_remote_config <<<"SSL_CApath = /etc/ssl/certs" + unverified_peer +fi + +capath=$(mktemp --tmpdir="$TMPDIR" --directory capath.XXXXXX) +cp -t"$capath" "$CERT" +c_rehash "$capath" + +with_remote_config <<<"SSL_CApath = $capath" +verified_peer +step_done + +# vim: set filetype=sh : diff --git a/tests/tls/interimap.remote b/tests/tls/interimap.remote new file mode 100644 index 0000000..2c0e37e --- /dev/null +++ b/tests/tls/interimap.remote @@ -0,0 +1,3 @@ +host = ::1 +port = 10993 +SSL_verify = no diff --git a/tests/tls/remote.conf b/tests/tls/remote.conf new file mode 100644 index 0000000..3d07ea9 --- /dev/null +++ b/tests/tls/remote.conf @@ -0,0 +1,2 @@ +!include conf.d/imapd.conf +!include conf.d/ssl.conf diff --git a/tests/tls/t b/tests/tls/t new file mode 100644 index 0000000..dd6d955 --- /dev/null +++ b/tests/tls/t @@ -0,0 +1,14 @@ +for ((i = 0; i < 32; i++)); do + u="$(shuf -n1 -e "local" "remote")" + sample_message | deliver -u "$u" +done + +interimap --debug || error +grep -Fx "remote: Disabling SSL protocols: SSLv3, TLSv1, TLSv1.1" <"$STDERR" || error +grep -Fx "remote: Peer certificate fingerprint: sha256\$35944e3bd3300d3ac310bb497a32cc1eef6931482a587ddbc95343740cdf1323" <"$STDERR" || error +grep "^remote: SSL protocol: TLSv1\.[23] " <"$STDERR" || error +grep "^remote: SSL cipher: " <"$STDERR" || error + +check_mailbox_status "INBOX" + +# vim: set filetype=sh : -- cgit v1.2.3 From 7bf1b61dcf4eacc4e9d7a23fbe3050495990f8d9 Mon Sep 17 00:00:00 2001 From: Guilhem Moulin Date: Sun, 10 Nov 2019 20:17:45 +0100 Subject: Test suite: add new test for COMPRESS=DEFLATE. An imapd is required as `doveadm exec imap` won't offer COMPRESS=DEFLATE in its capability list. --- tests/compress/interimap.remote | 1 + tests/compress/remote.conf | 1 + tests/compress/t | 19 +++++++++++++++++++ tests/list | 2 ++ 4 files changed, 23 insertions(+) create mode 120000 tests/compress/interimap.remote create mode 120000 tests/compress/remote.conf create mode 100644 tests/compress/t (limited to 'tests/list') diff --git a/tests/compress/interimap.remote b/tests/compress/interimap.remote new file mode 120000 index 0000000..a4ea3f3 --- /dev/null +++ b/tests/compress/interimap.remote @@ -0,0 +1 @@ +../auth-sasl-plain/interimap.remote \ No newline at end of file diff --git a/tests/compress/remote.conf b/tests/compress/remote.conf new file mode 120000 index 0000000..dbbb908 --- /dev/null +++ b/tests/compress/remote.conf @@ -0,0 +1 @@ +../auth-sasl-plain/remote.conf \ No newline at end of file diff --git a/tests/compress/t b/tests/compress/t new file mode 100644 index 0000000..5625761 --- /dev/null +++ b/tests/compress/t @@ -0,0 +1,19 @@ +for ((i = 0; i < 32; i++)); do + u="$(shuf -n1 -e "local" "remote")" + sample_message | deliver -u "$u" +done + +# compression enabled by default +interimap --debug || error +grep -Fx "remote: C: 000001 COMPRESS DEFLATE" <"$STDERR" || error +grep -E "^remote: S: 000001 OK( |$)" <"$STDERR" || error + +check_mailbox_status "INBOX" + + +# can be disabled +echo "compress = no" >>"$XDG_CONFIG_HOME/interimap/config" +interimap --debug || error +! grep -E "^remote: C: [^[:blank:]]+ COMPRESS DEFLATE$" <"$STDERR" || error + +# vim: set filetype=sh : diff --git a/tests/list b/tests/list index bc8d144..9ce0c06 100644 --- a/tests/list +++ b/tests/list @@ -38,6 +38,8 @@ repair --repair auth-logindisabled LOGINDISABLED auth-noplaintext abort when STARTTLS is not offered +compress COMPRESS=DEFLATE + . SSL/TLS starttls-logindisabled LOGINDISABLED STARTTLS starttls STARTTLS -- cgit v1.2.3 From 3aa5593af18bd4925235d1820fd0fe7c646843aa Mon Sep 17 00:00:00 2001 From: Guilhem Moulin Date: Tue, 12 Nov 2019 01:39:29 +0100 Subject: Net::IMAP::InterIMAP::push_flag_updates() bugfixes. The UNCHANGEDSINCE test from the CONDSTORE extension was incorrectly placed after the flag list in UID STORE commands. In practice this meant the server didn't add the MODIFIED code when needed. The server won't send an untagged FETCH command (and won't increase the message's MODSEQ) if no change was made to the flag list. A panic() was incorrectly triggered in that case. When the flag list was set (by another client) to a superset of the UID STORE command currently processed, the extra flags were not synchronized. Cf. RFC 7162 sec. 3.1.3 ex. 10. --- Changelog | 7 ++++++ lib/Net/IMAP/InterIMAP.pm | 58 ++++++++++++++++++++--------------------------- tests/condstore/t | 50 ++++++++++++++++++++++++++++++++++++++++ tests/list | 1 + 4 files changed, 82 insertions(+), 34 deletions(-) create mode 100644 tests/condstore/t (limited to 'tests/list') diff --git a/Changelog b/Changelog index f90c28c..763a7a1 100644 --- a/Changelog +++ b/Changelog @@ -87,6 +87,13 @@ interimap (0.5) upstream; compression stream is reached. - libinterimap: the 'compress' boolean wasn't honored. - libinterimap: fix STARTTLS directive, broken since 0.2. + - libinterimap: push_flag_updates(): the UNCHANGEDSINCE test from + the CONDSTORE extension was incorrectly placed after the flag list in + UID STORE commands. + - libinterimap: push_flag_updates(): ignore UIDs for which no untagged + FETCH response was received. + - libinterimap: push_flag_updates(): don't ignores received updates (by + another client) to a superset of the desigred flag list. -- Guilhem Moulin Fri, 10 May 2019 00:58:14 +0200 diff --git a/lib/Net/IMAP/InterIMAP.pm b/lib/Net/IMAP/InterIMAP.pm index c25df27..a838dd0 100644 --- a/lib/Net/IMAP/InterIMAP.pm +++ b/lib/Net/IMAP/InterIMAP.pm @@ -1197,16 +1197,15 @@ sub pull_updates($;$) { my $mailbox = $self->{_SELECTED} // $self->panic(); my $pcache = $self->{_PCACHE}->{$mailbox}; - my %modified; $self->_send("UID FETCH 1:".($pcache->{UIDNEXT}-1)." (MODSEQ FLAGS)") if $full and ($pcache->{UIDNEXT} // 1) > 1; - my @missing; + my %modified; while (%{$self->{_MODIFIED}}) { + my @missing; while (my ($uid,$v) = each %{$self->{_MODIFIED}}) { - # don't filter on the fly (during FETCH responses) because - # FLAG updates can arrive while processing pull_new_messages - # for instance + # don't filter on the fly (during FETCH responses) because FLAG updates + # can arrive while processing pull_new_messages() for instance if (defined $v->[1] and $v->[0] > 0) { # setting the MODSEQ to 0 forces a FETCH next unless $uid < ($pcache->{UIDNEXT} // 1) # out of bounds and ($full or $v->[0] > ($pcache->{HIGHESTMODSEQ} // 0)); # already seen @@ -1216,8 +1215,9 @@ sub pull_updates($;$) { } } $self->{_MODIFIED} = {}; + # non-empty @missing indicates a discouraged (but allowed) CONDSTORE server behavior, + # cf. RFC 7162 sec. 3.1.3 ex. 8 and the comment in push_flag_updates() below $self->_send("UID FETCH ".compact_set(@missing)." (MODSEQ FLAGS)") if @missing; - @missing = (); } # do that afterwards since the UID FETCH command above can produce VANISHED responses @@ -1307,22 +1307,18 @@ sub push_flag_updates($$@) { my $mailbox = $self->{_SELECTED} // $self->panic(); my $modseq = $self->{_PCACHE}->{$mailbox}->{HIGHESTMODSEQ} // $self->panic(); - my $command = "UID STORE ".compact_set(@set)." FLAGS.SILENT ($flags) (UNCHANGEDSINCE $modseq)"; - - my %listed; - $self->_send($command, sub($){ $listed{shift->{UID}}++; }); + my $command = "UID STORE ".compact_set(@set)." (UNCHANGEDSINCE $modseq) FLAGS.SILENT ($flags)"; my %failed; + $self->_send($command); if ($IMAP_text =~ /\A\Q$IMAP_cond\E \[MODIFIED ([0-9,:]+)\] $RE_TEXT_CHAR+\z/) { foreach (split /,/, $1) { if (/\A([0-9]+)\z/) { $failed{$1} = 1; - } - elsif (/\A([0-9]+):([0-9]+)\z/) { + } elsif (/\A([0-9]+):([0-9]+)\z/) { my ($min, $max) = $1 < $2 ? ($1,$2) : ($2,$1); $failed{$_} = 1 foreach ($min .. $max); - } - else { + } else { $self->panic($_); } } @@ -1330,34 +1326,28 @@ sub push_flag_updates($$@) { my @ok; foreach my $uid (@set) { + my $modified = $self->{_MODIFIED}; if ($failed{$uid}) { - # $uid was listed in the MODIFIED response code - $self->{_MODIFIED}->{$uid} //= [ 0, undef ]; # will be downloaded again in pull_updates - delete $self->{_MODIFIED}->{$uid} if - # got a FLAG update for $uid; ignore it if it's $flags - defined $self->{_MODIFIED}->{$uid}->[1] and - $self->{_MODIFIED}->{$uid}->[1] eq $flags; - } - else { - # $uid wasn't listed in the MODIFIED response code - next unless defined $self->{_MODIFIED}->{$uid}; # already stored - $self->panic() unless defined $listed{$uid} and $listed{$uid} > 0; # sanity check - if ($listed{$uid} == 1) { - # ignore succesful update - delete $self->{_MODIFIED}->{$uid}; - } - elsif ($self->{_MODIFIED}->{$uid}->[1] and $self->{_MODIFIED}->{$uid}->[1] eq $flags) { - # got multiple FETCH responses for $uid, the last one with $flags - delete $self->{_MODIFIED}->{$uid}; + # $uid was listed in the MODIFIED response code from RFC 7162; will FETCH + # again in pull_updates(); per RFC 7162 sec. 3.1.3 $modified->{$uid} might not + # be defined ("nice" servers send an untagged FETCH response, cf. example 10, + # but they might omit it - allowed but discouraged CONDSTORE server behavior - + # cf. example 8) + $modified->{$uid} //= [ 0, undef ]; + } elsif (defined (my $m = $modified->{$uid})) { + # received an untagged FETCH response, remove from the list of pending changes + # if the flag list was up to date (either implicitely or explicitely) + if (!defined $m->[1] or $m->[1] eq $flags) { + delete $modified->{$uid}; + push @ok, $uid; } - push @ok, $uid; } } unless ($self->{quiet}) { $self->log("Updated flags ($flags) for UID ".compact_set(@ok)) if @ok; $self->log("Couldn't update flags ($flags) for UID ".compact_set(keys %failed).', '. - "trying again later") if %failed; + "will try again later") if %failed; } return keys %failed; } diff --git a/tests/condstore/t b/tests/condstore/t new file mode 100644 index 0000000..d4da50f --- /dev/null +++ b/tests/condstore/t @@ -0,0 +1,50 @@ +TIMEOUT=60 +N=4096 + +# test CONDSTORE/QRESYNC (behavior) in UID STORE commands, in particular +# the UNCHANGEDSINCE test: populate, keep assiging keywords at random, +# and make sure interimap is able to reconciliate the changes + +# populate (with dummy messages to speed things up) only one server +# before initializing interimap, so UIDs concide with sequence numbers +# and are identical on both servers +for ((i = 0; i < N; i++)); do + deliver -u "local" <<< . +done + +interimap_init + +# assign a set of 16 tags; not more because in order to maximize the +# likelyhood of conflicts we want UID STORE commands to use large sets +declare -a FLAGS=(0 1 2 3 4 5 6 7 8 9 a b c d e f) + +# start a long-lived interimap process +interimap --watch=1 & PID=$! +trap "ptree_abort $PID" EXIT INT TERM + +timer=$(( $(date +%s) + TIMEOUT )) +while [ $(date +%s) -le $timer ]; do + a="$(shuf -n1 -e "add" "remove" "replace")" + u="$(shuf -n1 -e "local" "remote")" + f="$(shuf -n1 -e "${FLAGS[@]}")" + seqs="$(shuf -n$((N/8)) -i1-$N)" # trigger changes on 1/8 of all messages + doveadm -u "$u" flags "$a" "$f" mailbox "INBOX" "${seqs//$'\n'/,}" + sleep "0.0$(shuf -n1 -i10-99)" # 10 to 99ms +done +sleep 2 + +ptree_abort $PID +trap - EXIT INT TERM + +# make sure the list of uids for a given tag match +flagged_uids() { + local u="$1" f="$2" + doveadm -u "$u" search mailbox "INBOX" keyword "$f" | cut -d" " -f2 | sort -n +} +for f in "${FLAGS[@]}"; do + diff --label="local/$f" --label="remote/$f" -u -- \ + <(flagged_uids "local" "$f") <(flagged_uids "remote" "$f") || + error "UID list differs for keyword '$f'" +done + +# vim: set filetype=sh : diff --git a/tests/list b/tests/list index 9ce0c06..8bb4478 100644 --- a/tests/list +++ b/tests/list @@ -39,6 +39,7 @@ repair --repair auth-noplaintext abort when STARTTLS is not offered compress COMPRESS=DEFLATE +condstore CONDSTORE . SSL/TLS starttls-logindisabled LOGINDISABLED STARTTLS -- cgit v1.2.3 From 0a2558aabfefd6800fe74c24e5aff2b0d47cc5e2 Mon Sep 17 00:00:00 2001 From: Guilhem Moulin Date: Mon, 11 Nov 2019 00:39:09 +0100 Subject: Avoid sending large UID EXPUNGE|FETCH|STORE and APPEND commands. UID EXPUNGE|FETCH|STORE commands are now split into multiple (sequential) commands when their set representation exceeds 4096 bytes in size. Without splitting logic set representations could grow arbitrarily large, and exceed the server's maximum command size. This adds roundtrips which could be eliminated by pipelining, but it's unlikely to make any difference in typical synchronization work. While set representations seem to remain small in practice, they might grow significantly if many non-contiguous UIDs were flagged and/or expunged, and later synchronized at once. Furthermore, for MULTIAPPEND-capable servers, the number of messages is limited to 128 per APPEND command (also subject to a combined literal size of 1MiB like before). These numbers are currently not configurable. They're intentionally lower than Dovecot's default maximum command size (64k) in order to avoid a deadlock situation after sending 8k-long commands under COMPRESS=DEFLATE: https://dovecot.org/pipermail/dovecot/2019-November/117522.html . --- Changelog | 11 +++++++ interimap | 4 +-- lib/Net/IMAP/InterIMAP.pm | 64 ++++++++++++++++++++++++++------------- tests/list | 1 + tests/snippets/dovecot/imapd.conf | 3 ++ tests/split-set/interimap.remote | 1 + tests/split-set/remote.conf | 1 + tests/split-set/t | 43 ++++++++++++++++++++++++++ 8 files changed, 105 insertions(+), 23 deletions(-) create mode 120000 tests/split-set/interimap.remote create mode 120000 tests/split-set/remote.conf create mode 100644 tests/split-set/t (limited to 'tests/list') diff --git a/Changelog b/Changelog index 763a7a1..f0c9e50 100644 --- a/Changelog +++ b/Changelog @@ -94,6 +94,17 @@ interimap (0.5) upstream; FETCH response was received. - libinterimap: push_flag_updates(): don't ignores received updates (by another client) to a superset of the desigred flag list. + - libinterimap: avoid sending large UID EXPUNGE|FETCH|STORE commands as + they might exceed the server's max acceptable command size; these + commands are now split into multiple (sequential) commands when their + set representation exceeds 4096 bytes in size. Performance could be + improved by pipelining but given the scope of this software + (synchronization) it's unlikely to make any difference in practice. + This is a also a workaround for a bug in Dovecot 2.3.4: + https://dovecot.org/pipermail/dovecot/2019-November/117522.html + - interimap: for the reason explained above, limit number of messages + to 128 per APPEND command (only on servers advertizing MULTIAPPEND, + for other servers the number remains 1). -- Guilhem Moulin Fri, 10 May 2019 00:58:14 +0200 diff --git a/interimap b/interimap index 9a1df0b..386492e 100755 --- a/interimap +++ b/interimap @@ -1166,8 +1166,8 @@ sub callback_new_message($$$$;$$$) { } else { # use MULTIAPPEND (RFC 3502) - # proceed by 1MiB batches to save roundtrips without blowing up the memory - if (@$buff and $$bufflen + $length > 1048576) { + # proceed by batches of 128/1MiB to save roundtrips without blowing up the memory + if ($#$buff >= 127 or (@$buff and $$bufflen + $length > 1048576)) { @UIDs = callback_new_message_flush($idx, $mailbox, $name, @$buff); @$buff = (); $$bufflen = 0; diff --git a/lib/Net/IMAP/InterIMAP.pm b/lib/Net/IMAP/InterIMAP.pm index e595060..02ae65f 100644 --- a/lib/Net/IMAP/InterIMAP.pm +++ b/lib/Net/IMAP/InterIMAP.pm @@ -203,6 +203,21 @@ sub compact_list(@) { return $set; } +# with_set($set, $cmd) +# Split long commands over multiple subsets to avoid exceeding the server limit +sub with_set($&) { + my ($set, $cmd) = @_; + my $max_length = 4096; + for (my $length = length($set); $length > $max_length;) { + my $l = rindex($set, ',', $max_length); + die unless $l > 0; # sanity check + $cmd->(substr($set, 0, $l)); + $set = substr($set, ++$l); + $length -= $l; + } + return $cmd->($set); +} + # in_set($x, $set) # Return true if the UID or sequence number $x belongs to the set $set. @@ -841,9 +856,10 @@ sub remove_message($@) { $self->fail("Server did not advertise UIDPLUS (RFC 4315) capability.") unless $self->_capable('UIDPLUS'); - my $set = compact_set(@set); - $self->_send("UID STORE $set +FLAGS.SILENT (\\Deleted)"); - $self->_send("UID EXPUNGE $set"); # RFC 4315 UIDPLUS + with_set(compact_set(@set), sub($) { + $self->_send("UID STORE $_[0] +FLAGS.SILENT (\\Deleted)"); + $self->_send("UID EXPUNGE $_[0]"); # RFC 4315 UIDPLUS + }); my %vanished = map {$_ => 1} @{$self->{_VANISHED}}; @@ -960,7 +976,9 @@ sub append($$@) { # optional $callback. sub fetch($$$;&) { my ($self, $set, $flags, $callback) = @_; - $self->_send("UID FETCH $set $flags", $callback); + return with_set($set, sub($) { + $self->_send("UID FETCH $_[0] $flags", $callback); + }); } @@ -1217,7 +1235,9 @@ sub pull_updates($;$) { $self->{_MODIFIED} = {}; # non-empty @missing indicates a discouraged (but allowed) CONDSTORE server behavior, # cf. RFC 7162 sec. 3.1.3 ex. 8 and the comment in push_flag_updates() below - $self->_send("UID FETCH ".compact_set(@missing)." (MODSEQ FLAGS)") if @missing; + with_set(compact_set(@missing), sub($) { + $self->_send("UID FETCH $_[0] (MODSEQ FLAGS)") + }) if @missing; } # do that afterwards since the UID FETCH command above can produce VANISHED responses @@ -1279,7 +1299,7 @@ sub pull_new_messages($$&@) { $range .= "$since:4294967295"; $UIDNEXT = $cache->{UIDNEXT} // $self->panic(); # sanity check - $self->_send("UID FETCH $range ($attrs)", sub($) { + $self->fetch($range, "($attrs)", sub($) { my $mail = shift; $UIDNEXT = $mail->{UID} + 1 if $UIDNEXT <= $mail->{UID}; $callback->($mail) if defined $callback; @@ -1307,22 +1327,23 @@ sub push_flag_updates($$@) { my $mailbox = $self->{_SELECTED} // $self->panic(); my $modseq = $self->{_PCACHE}->{$mailbox}->{HIGHESTMODSEQ} // $self->panic(); - my $command = "UID STORE ".compact_set(@set)." (UNCHANGEDSINCE $modseq) FLAGS.SILENT ($flags)"; my %failed; - $self->_send($command); - if ($IMAP_text =~ /\A\Q$IMAP_cond\E \[MODIFIED ([0-9,:]+)\] $RE_TEXT_CHAR+\z/) { - foreach (split /,/, $1) { - if (/\A([0-9]+)\z/) { - $failed{$1} = 1; - } elsif (/\A([0-9]+):([0-9]+)\z/) { - my ($min, $max) = $1 < $2 ? ($1,$2) : ($2,$1); - $failed{$_} = 1 foreach ($min .. $max); - } else { - $self->panic($_); + with_set(compact_set(@set), sub($) { + $self->_send("UID STORE $_[0] (UNCHANGEDSINCE $modseq) FLAGS.SILENT ($flags)"); + if ($IMAP_text =~ /\A\Q$IMAP_cond\E \[MODIFIED ([0-9,:]+)\] $RE_TEXT_CHAR+\z/) { + foreach (split /,/, $1) { + if (/\A([0-9]+)\z/) { + $failed{$1} = 1; + } elsif (/\A([0-9]+):([0-9]+)\z/) { + my ($min, $max) = $1 < $2 ? ($1,$2) : ($2,$1); + $failed{$_} = 1 foreach ($min .. $max); + } else { + $self->panic($_); + } } } - } + }); my @ok; foreach my $uid (@set) { @@ -1360,8 +1381,9 @@ sub push_flag_updates($$@) { sub silent_store($$$@) { my $self = shift; my $set = shift; - my $mod = shift; - $self->_send("UID STORE $set ${mod}FLAGS.SILENT (".join(' ', @_).")"); + my $subcmd = shift . "FLAGS.SILENT"; + my $flags = join(' ', @_); + with_set($set, sub($) { $self->_send("UID STORE $_[0] $subcmd ($flags)") }); } @@ -1374,7 +1396,7 @@ sub expunge($$) { $self->fail("Server did not advertise UIDPLUS (RFC 4315) capability.") unless $self->_capable('UIDPLUS'); - $self->_send("UID EXPUNGE $set"); + with_set($set, sub($) { $self->_send("UID EXPUNGE $_[0]") }); } diff --git a/tests/list b/tests/list index 8bb4478..a18cb29 100644 --- a/tests/list +++ b/tests/list @@ -40,6 +40,7 @@ repair --repair compress COMPRESS=DEFLATE condstore CONDSTORE +split-set Split large sets to avoid extra-long command lines . SSL/TLS starttls-logindisabled LOGINDISABLED STARTTLS diff --git a/tests/snippets/dovecot/imapd.conf b/tests/snippets/dovecot/imapd.conf index 18c60f8..2b26451 100644 --- a/tests/snippets/dovecot/imapd.conf +++ b/tests/snippets/dovecot/imapd.conf @@ -14,3 +14,6 @@ service imap-login { ssl = yes } } + +# we should avoid sending command lines that are too long +imap_max_line_length = 8192 diff --git a/tests/split-set/interimap.remote b/tests/split-set/interimap.remote new file mode 120000 index 0000000..a4ea3f3 --- /dev/null +++ b/tests/split-set/interimap.remote @@ -0,0 +1 @@ +../auth-sasl-plain/interimap.remote \ No newline at end of file diff --git a/tests/split-set/remote.conf b/tests/split-set/remote.conf new file mode 120000 index 0000000..dbbb908 --- /dev/null +++ b/tests/split-set/remote.conf @@ -0,0 +1 @@ +../auth-sasl-plain/remote.conf \ No newline at end of file diff --git a/tests/split-set/t b/tests/split-set/t new file mode 100644 index 0000000..5e8ea52 --- /dev/null +++ b/tests/split-set/t @@ -0,0 +1,43 @@ +N=2048 + +# XXX with COMPRESS=DEFLATE dovecot-imapd 2.3.4 hangs when the command +# line exceeds 'imap_max_line_length' (or 8192, whichever is smaller) +# bytes, instead of returning a tagged BAD response. +# https://dovecot.org/pipermail/dovecot/2019-November/117522.html + +# set UIDNEXT to 10^9 so all uids are 10 chars long, otherwise we'd need +# to add many more messages to obtain large sets +doveadm -u "local" mailbox update --min-next-uid 1000000000 "INBOX" +doveadm -u "remote" mailbox update --min-next-uid 1000000000 "INBOX" + +for ((i = 0; i < N; i++)); do + u="$(shuf -n1 -e "local" "remote")" + sample_message | deliver -u "$u" +done + +interimap_init +check_mailbox_status "INBOX" + +# mark every other message as \Seen on the local server +for ((i = 0; i < N; i+=2)); do + doveadm -u "local" flags add "\\Seen" mailbox "INBOX" $((N-i)) +done + +# send the changes to the remote; this results into an UID STORE set +# representation of size 11*N/2-1, which exceeds $imap_max_line_length +interimap +check_mailbox_status "INBOX" + +# now expunge every other message on the remote server; this results +# into large UID STORE and UID EXPUNGE set representation +for ((i = 0; i < N; i+=2)); do + doveadm -u "local" expunge mailbox "INBOX" $((N-i)) + # add some more messages + u="$(shuf -n1 -e "local" "remote")" + sample_message | deliver -u "$u" +done + +interimap || error +check_mailbox_status "INBOX" + +# vim: set filetype=sh : -- cgit v1.2.3 From dac4ab1c9306bf2035bc1547d2ed27ab09850120 Mon Sep 17 00:00:00 2001 From: Guilhem Moulin Date: Wed, 13 Nov 2019 04:16:48 +0100 Subject: Test suite: add new test for pullimap(1). This adds a dependency on Dovecot's LMTPd, which will bind to to TCP port 10024 on the loopback interface. --- pullimap | 5 +- tests/list | 3 ++ tests/pullimap/interimap.remote | 1 + tests/pullimap/local.conf | 1 + tests/pullimap/pullimap.conf | 1 + tests/pullimap/remote.conf | 1 + tests/pullimap/t | 96 +++++++++++++++++++++++++++++++++++++++ tests/run | 38 ++++++++++------ tests/snippets/dovecot/lmtpd.conf | 7 +++ 9 files changed, 136 insertions(+), 17 deletions(-) create mode 120000 tests/pullimap/interimap.remote create mode 100644 tests/pullimap/local.conf create mode 100644 tests/pullimap/pullimap.conf create mode 120000 tests/pullimap/remote.conf create mode 100644 tests/pullimap/t create mode 100644 tests/snippets/dovecot/lmtpd.conf (limited to 'tests/list') diff --git a/pullimap b/pullimap index 3d1a0ec..1dc4b9e 100755 --- a/pullimap +++ b/pullimap @@ -29,7 +29,7 @@ use Errno 'EINTR'; use Fcntl qw/O_CREAT O_RDWR O_DSYNC F_SETLK F_WRLCK SEEK_SET F_GETFD F_SETFD FD_CLOEXEC/; use Getopt::Long qw/:config posix_default no_ignore_case gnu_getopt auto_version/; use List::Util 'first'; -use Socket qw/PF_INET PF_INET6 SOCK_STREAM/; +use Socket qw/PF_INET PF_INET6 SOCK_STREAM IPPROTO_TCP/; use lib 'lib'; use Net::IMAP::InterIMAP 0.0.5 qw/xdg_basedir read_config compact_set/; @@ -146,8 +146,7 @@ sub sendmail($$) { : $fam == PF_INET6 ? Socket::pack_sockaddr_in6($port, $addr) : die; - my $proto = getprotobyname("tcp") // die; - socket($SMTP, $fam, SOCK_STREAM, $proto) or die "socket: $!"; + socket($SMTP, $fam, SOCK_STREAM, IPPROTO_TCP) or die "socket: $!"; until (connect($SMTP, $sockaddr)) { next if $! == EINTR; # try again if connect(2) was interrupted by a signal die "connect: $!"; diff --git a/tests/list b/tests/list index a18cb29..52417c1 100644 --- a/tests/list +++ b/tests/list @@ -55,3 +55,6 @@ split-set Split large sets to avoid extra-long command lines sync-live-crippled local/remote simulation (crippled remote) sync-live-tls local/remote simulation (TLS remote) sync-live-multi local/remote1+remote2+remote3 simulation (3 local namespaces) + +. pullimap + ... pullimap diff --git a/tests/pullimap/interimap.remote b/tests/pullimap/interimap.remote new file mode 120000 index 0000000..daf3741 --- /dev/null +++ b/tests/pullimap/interimap.remote @@ -0,0 +1 @@ +../tls/interimap.remote \ No newline at end of file diff --git a/tests/pullimap/local.conf b/tests/pullimap/local.conf new file mode 100644 index 0000000..b67641f --- /dev/null +++ b/tests/pullimap/local.conf @@ -0,0 +1 @@ +!include conf.d/lmtpd.conf diff --git a/tests/pullimap/pullimap.conf b/tests/pullimap/pullimap.conf new file mode 100644 index 0000000..3f6c2e1 --- /dev/null +++ b/tests/pullimap/pullimap.conf @@ -0,0 +1 @@ +deliver-method = lmtp:[127.0.0.1]:10024 diff --git a/tests/pullimap/remote.conf b/tests/pullimap/remote.conf new file mode 120000 index 0000000..6029749 --- /dev/null +++ b/tests/pullimap/remote.conf @@ -0,0 +1 @@ +../tls/remote.conf \ No newline at end of file diff --git a/tests/pullimap/t b/tests/pullimap/t new file mode 100644 index 0000000..7ae0c5f --- /dev/null +++ b/tests/pullimap/t @@ -0,0 +1,96 @@ +MAILBOX="INBOX" +TIMEOUT=60 +N=2048 + +step_start "\`pullimap --idle\` refuses to create the state file" +! pullimap --idle "remote" || error +step_done + +# make sure remote UIDs are 11-bytes long +doveadm -u "remote" mailbox update --min-next-uid 1000000000 "$MAILBOX" + +# compare mailboxes; can't compare the RFC 3501 TEXT as LMTP adds a +# Received: header. +# TODO unset lmtp_add_received_header once avaisable in Sid: +# https://doc.dovecot.org/settings/dovecot_core_settings/#lmtp-add-received-header +list_mails_sha256() { + local u="$1" guid uid + while read guid uid; do + doveadm -u "$u" -f "flow" fetch body mailbox-guid "$guid" uid "$uid" \ + | sed "1s/body=//" | sha256sum + done < <(doveadm -u "$u" search mailbox "$MAILBOX") | sort -f +} +check() { + diff -u --label="local/mails" --label="remote/mails" \ + <( list_mails_sha256 "local" ) \ + <( list_mails_sha256 "remote" ) \ + || error "mailboxes differ" +} + + +# Add some messages and sync +step_start "Fetching messages" +for ((i = 0; i < 32; i++)); do + sample_message | deliver -u "remote" -- -m "$MAILBOX" +done + +pullimap "remote" || error +check + +# same thing, but with some missing messages +for ((i = 0; i < N; i+=2)); do + sample_message | deliver -u "remote" -- -m "$MAILBOX" + deliver -u "remote" -- -m "$MAILBOX" "$TMPDIR/unseen" +[ ! -s "$TMPDIR/unseen" ] || error "\\Unseen messages left" +step_done + + +step_start "--idle (${TIMEOUT}s)" + +pullimap --idle "remote" & PID=$! +trap "ptree_abort $PID" EXIT INT TERM + +timer=$(( $(date +%s) + TIMEOUT )) +while [ $(date +%s) -le $timer ]; do + n="$(shuf -n1 -i1-5)" + for (( i=0; i < n; i++)); do + sample_message | deliver -u "remote" -- -m "$MAILBOX" + done + + s=$(shuf -n1 -i1-1500) + [ $s -ge 1000 ] && s="$(printf "1.%03d" $((s-1000)))" || s="$(printf "0.%03d" $s)" + sleep "$s" +done + +sleep 2 +ptree_abort $PID +trap - EXIT INT TERM + +check +step_done + + +step_start "Purging" +echo "purge-after = 0" >>"$XDG_CONFIG_HOME/pullimap/config" +for ((i = 0; i < 32; i++)); do + sample_message | deliver -u "remote" -- -m "$MAILBOX" +done +pullimap "remote" + +doveadm -u "remote" search mailbox "$MAILBOX" all >"$TMPDIR/messages" +[ ! -s "$TMPDIR/messages" ] || error "messages left" +step_done + +# vim: set filetype=sh : diff --git a/tests/run b/tests/run index 2903938..30d20f9 100755 --- a/tests/run +++ b/tests/run @@ -146,9 +146,10 @@ prepare() { chmod +x -- "$home/.local/bin/doveadm" done - # copy interimap config - mkdir -pm0700 -- "$HOME_local/.local/share/interimap" - mkdir -pm0700 -- "$HOME_local/.config/interimap" + # copy interimap and pullimap configuration + mkdir -pm0700 -- "$HOME_local/.local/share/interimap" "$HOME_local/.local/share/pullimap" + mkdir -pm0700 -- "$HOME_local/.config/interimap" "$HOME_local/.config/pullimap" + echo "deliver-rcpt = local" >>"$HOME_local/.config/pullimap/config" for u in "${REMOTES[@]}"; do n="${u#remote}" eval home="\$HOME_$u" @@ -160,6 +161,9 @@ prepare() { if [ -f "$TESTDIR/interimap$n.conf" ] || [ -L "$TESTDIR/interimap$n.conf" ]; then cat <"$TESTDIR/interimap$n.conf" >>"$HOME_local/.config/interimap/config$n" fi + if [ -f "$TESTDIR/pullimap.conf" ] || [ -L "$TESTDIR/pullimap.conf" ]; then + cat <"$TESTDIR/pullimap.conf" >>"$HOME_local/.config/pullimap/config" + fi cat >>"$HOME_local/.config/interimap/config$n" <<-EOF @@ -172,7 +176,6 @@ prepare() { cat <"$TESTDIR/interimap$n.local" >>"$HOME_local/.config/interimap/config$n" fi - printf "\\n[remote]\\n" >>"$HOME_local/.config/interimap/config$n" if [ -s "$home/.dovecot/users" ]; then cat <<-EOF username = $u @@ -184,21 +187,26 @@ prepare() { command = exec ${home@Q}/.local/bin/doveadm exec imap null-stderr = NO EOF - fi >>"$HOME_local/.config/interimap/config$n" + fi >"$HOME_local/.$u.conf" if [ -f "$TESTDIR/interimap$n.remote" ] || [ -L "$TESTDIR/interimap$n.remote" ]; then - cat <"$TESTDIR/interimap$n.remote" >>"$HOME_local/.config/interimap/config$n" + cat <"$TESTDIR/interimap$n.remote" >>"$HOME_local/.$u.conf" fi + + { printf "\\n[remote]\\n"; cat <"$HOME_local/.$u.conf"; } >>"$HOME_local/.config/interimap/config$n" + { printf "\\n[%s]\\n" "$u"; cat <"$HOME_local/.$u.conf"; } >>"$HOME_local/.config/pullimap/config" done } prepare # Wrappers for interimap(1) and doveadm(1) -interimap() { +interimap() { _interimap_cmd "interimap" "$@"; } +pullimap() { _interimap_cmd "pullimap" "$@"; } +_interimap_cmd() { declare -a ENVIRON=() r=0 + local script="$1" + shift environ_set "local" - env -i "${ENVIRON[@]}" perl -I./lib -T ./interimap "$@" 2>"$STDERR" || r=$? - cat "$STDERR" >&2 - return $r + env -i "${ENVIRON[@]}" perl -I./lib -T "./$script" "$@" 2> >(tee "$STDERR" >&2) } interimap_init() { local u="${1-remote}" @@ -227,8 +235,10 @@ sqlite3() { # Sample (random) message sample_message() { local date="$(date +"%s.%N")" + # also try non-conventional addresses for pullimap + local sender="$(shuf -n1 -e "sender" "first.last" "foo-bar" \"\" "\"x\\\" #&\\\\y\"" )" cat <<-EOF - From: + From: <$sender@example.net> To: Date: $(date -R -d@"$date") Message-ID: <$date@example.net> @@ -240,7 +250,7 @@ sample_message() { # Wrapper for dovecot-lda(1) deliver() { - local -a argv + declare -a argv while [ $# -gt 0 ] && [ "$1" != "--" ]; do argv+=( "$1" ) shift @@ -433,8 +443,8 @@ passed() { declare -a ENVIRON=() environ_set "local" export TMPDIR TESTDIR STDERR "${ENVIRON[@]}" -export -f environ_set doveadm interimap sqlite3 sample_message deliver -export -f interimap_init ptree_abort step_start step_done passed +export -f environ_set doveadm interimap interimap_init pullimap _interimap_cmd +export -f sqlite3 sample_message deliver ptree_abort step_start step_done passed export -f check_mailbox_status check_mailbox_status_values check_mailbox_status2 export -f check_mailboxes_status check_mailbox_list xcgrep error [ "$TESTNAME" = "..." ] || printf "%s%s..." "${INDENT-}" "$TESTNAME" diff --git a/tests/snippets/dovecot/lmtpd.conf b/tests/snippets/dovecot/lmtpd.conf new file mode 100644 index 0000000..6aa8365 --- /dev/null +++ b/tests/snippets/dovecot/lmtpd.conf @@ -0,0 +1,7 @@ +protocols = $protocols lmtp + +service lmtp { + inet_listener lmtp { + port = 10024 + } +} -- cgit v1.2.3