diff options
| -rwxr-xr-x | acme-slave | 251 | ||||
| -rwxr-xr-x | acme-webserver | 32 | ||||
| -rwxr-xr-x | letsencrypt | 281 | 
3 files changed, 564 insertions, 0 deletions
| diff --git a/acme-slave b/acme-slave new file mode 100755 index 0000000..ee39f8d --- /dev/null +++ b/acme-slave @@ -0,0 +1,251 @@ +#!/usr/bin/perl -T + +use strict; +use warnings; + +use LWP::UserAgent (); +use Crypt::OpenSSL::RSA (); +use Crypt::OpenSSL::Bignum (); +use MIME::Base64 qw/encode_base64 encode_base64url/; +use JSON (); +use Digest::SHA qw/sha256 sha256_hex/; + +# Clean up PATH +$ENV{PATH} = join ':', qw{/usr/bin /bin}; +delete @ENV{qw/IFS CDPATH ENV BASH_ENV/}; + +my $COMMAND = shift @ARGV // die; +my $PUBKEY = shift @ARGV // die; +die unless grep {$COMMAND eq $_} qw/new-reg new-cert revoke-cert/; +my $TIMEOUT = 10; + + +# Read the public key and build the JSON Web Key (RFC 7517) +my $JWK = do { +    open my $fh, '<', $PUBKEY or die "Can't open $PUBKEY: $!"; +    my $str = do { local $/ = undef; <$fh> }; +    my $pubkey = Crypt::OpenSSL::RSA->new_public_key($str) or die; +    close $fh; + +    my ($n, $e) = $pubkey->get_key_parameters(); +    $_ = encode_base64url($_->to_bin()) foreach ($n, $e); + +    { kty => 'RSA', n => $n, e => $e } +}; +my $JSON = JSON::->new->utf8->canonical(); # breaks hashes otherwise +my $JWK_dgst64 = encode_base64url(sha256($JSON->encode($JWK))); +my $NONCE; + + +# Send an HTTP request to the ACME server +my $UA = LWP::UserAgent::->new( ssl_opts => { +    verify_hostname => 1, +    SSL_version => 'SSLv23:!TLSv1_1:!TLSv1:!SSLv3:!SSLv2', +    SSL_cipher_list => 'EECDH+AESGCM:!MEDIUM:!LOW:!EXP:!aNULL:!eNULL' +}); +sub request($$;$) { +    my ($method, $uri, $json) = @_; +    print STDERR ">>> $method $uri <<<\n" if $ENV{DEBUG}; + +    my $req = HTTP::Request::->new($method => $uri) or die "Can't $method $uri"; +    if (defined $json) { +        $req->header('Content-Type' => 'application/json'); +        $req->content($JSON->encode($json)); +    } +    my $r = $UA->request($req) or die "Can't $method $uri"; +    print STDERR ">>> ", $r->status_line, "\n", $r->headers->as_string, "\n" if $ENV{DEBUG}; +    $NONCE = $r->header('Replay-Nonce') // die; +    my $t = $r->header('Content-Type'); + +    my $content = $r->decoded_content(); +    if (defined $t and $t =~ /\Aapplication\/(?:[a-z]+\+)?json\z/) { +        $content = $JSON->decode($content); +        print STDERR $JSON->pretty->encode($content), "\n" if $ENV{DEBUG}; +    } +    elsif (defined $t and $t eq 'application/pkix-cert') { +        print STDERR encode_base64($content), "\n" if $ENV{DEBUG}; +    } +    else { +        print STDERR $content, "\n" if $ENV{DEBUG}; +    } +    unless ($r->is_success) { +        my $msg = $r->status_line; +        $msg .= " (".$content->{detail}.")" if ref $content and defined $content->{detail}; +        die $msg, "\n"; +    } + +    return $content; +} + + +# ACME client +# https://github.com/letsencrypt/acme-spec/blob/master/draft-barnes-acme.md +sub acme($$) { +    my ($uri, $h) = @_; + +    # Produce the JSON Web Signature: RFC 7515 section 5 +    my $payload = encode_base64url($JSON->encode($h)); +    my %header = ( alg => 'RS256', jwk => $JWK ); +    my $protected = encode_base64url($JSON->encode({ %header, nonce => $NONCE })); +    my $data = $protected .'.'. $payload; +    print STDERR "Requesting a SHA-256 signature for ", $data, "\n" if $ENV{DEBUG}; +    STDOUT->printflush($data, "\n"); + +    # Ask for an (hex) sig +    my $sig = do { local $_ = <STDIN>; chomp; $_ }; +    $sig = encode_base64url(pack('H*', $sig)); +    print STDERR "Got SHA-256 signature ", $sig, "\n" if $ENV{DEBUG}; + +    # Flattened JSON Serialization, RFC 7515 section 7.2.2 +    request(POST => $uri, { +        payload => $payload, +        protected => $protected, +        header => \%header, +        signature => $sig +    }); +} + + +# Query the root ACME directory to initialize the nonce and get the resources URIs +my %RES = %{ request(GET => "https://acme-v01.api.letsencrypt.org/directory") }; + + +if ($COMMAND eq 'new-reg') { +    print STDERR "Requesting new registration ".(@ARGV ? ("for ".join(', ', @ARGV)) : "")."\n"; +    my $uri = "https://letsencrypt.org/documents/LE-SA-v1.0.1-July-27-2015.pdf"; +    my $dgst = sha256_hex($UA->get($uri)->decoded_content()); +    die "Error: The CA's subscriber agreement (URL $uri) has changed!\n" if +        $dgst ne '33d233c8ab558ba6c8ebc370a509acdded8b80e5d587aa5d192193f35226540f'; + +    acme($RES{'new-reg'}, { +        resource => 'new-reg', +        contact => [ map {"mailto:$_"} split(',', @ARGV) ], +        agreement => $uri, +    }); +    exit; +} + + +if ($COMMAND eq 'revoke-cert') { +    print STDERR "Requesting revocation for\n"; +    for my $cert (@ARGV) { +        open my $fh1, '-|', qw/openssl x509 -noout -subject -serial -fingerprint -sha256/, '-in', $cert +            or die "Can't run x509(1ssl): $!"; +        my ($subject, $serial, $fingerprint) = map { s/[^=]+=\s*//; chomp; $_ } <$fh1>; +        close $fh1; + +        print STDERR "\n\tSubject:             $subject\n", +                       "\tSerial:              $serial\n", +                       "\tSHA-256 fingerprint: $fingerprint\n"; + +        open my $fh2, '-|', qw/openssl x509 -outform DER/, '-in', $cert or die "Can't run x509(1ssl): $!"; +        my $der = do { local $/ = undef; <$fh2> }; +        close $fh2; + +        acme($RES{'revoke-cert'}, { +            resource => 'revoke-cert', +            certificate => encode_base64url($der) +        }); +    } +    exit; +} + + +# $COMMAND eq 'new-cert' +my ($CSR, $CHALLENGE_DIR, $X509) = @ARGV; +$CHALLENGE_DIR = $CHALLENGE_DIR =~ /\A(\/\p{Print}+)\z/ ? $1 : +    die "Error: Challenge directory is not absolute: $CHALLENGE_DIR"; + +# Parse the Certificate Signing Request +# XXX use a library instead, perhaps Crypt::OpenSSL::PKCS10 +my @domains = do { +    my @req = (qw/openssl req -noout/, '-in', $CSR); + +    my $RE_label = qr/[0-9a-z](?:[0-9a-z\x2D]{0,61}[0-9a-z])?/aai; +    my $RE_domain = qr/$RE_label(?:\.$RE_label)+/; +    my %domains; + +    open my $fh1, '-|', @req, '-subject' or die "Can't run req(1ssl): $!"; +    my $subject = <$fh1>; +    close $fh1; +    $domains{$1} = 1 if $subject =~ /\Asubject=\/CN=($RE_domain)(?:,.*)?\n\z/o; + +    open my $fh2, '-|', @req, '-text', '-reqopt', 'no_header,no_version,no_subject,no_pubkey,no_sigdump' +        or die "Can't run req(1ssl): $!"; +    while (<$fh2>) { +        /\A\s+X509v3 Subject Alternative Name:/ or next; +        my $san = <$fh2>; +        foreach (split /,/, $san) { +            chomp; +            s/\A\s*//; +            next unless s/\ADNS://; +            if (/\A$RE_domain\z/o) { +                $domains{$_} = 1; +            } +            else { +                warn "WARNING: Ignoring invalid domain $_\n"; +            } +        } +        last; +    } +    close $fh2; + +    keys %domains; +}; +print STDERR "Found domain(s): ".join(", ", @domains), "\n" if $ENV{DEBUG}; + + +# Process DNS Authorizations +foreach my $domain (@domains) { +    print STDERR "Processing new DNS authz for $domain\n" if $ENV{DEBUG}; +    my $challenges = acme($RES{'new-authz'}, { +        resource => 'new-authz', +        identifier => { type => 'dns', value => $domain } +    }); +    die "No challenge in server response" unless defined $challenges->{challenges}; +    my ($challenge) = grep {$_->{type} eq 'http-01'} @{$challenges->{challenges}}; +    my $keyAuthorization = $challenge->{token}.'.'.$JWK_dgst64; + +    # serve $keyAuthorization at http://$domain/.well-known/acme-challenge/$challenge->{token} +    my $filename = $CHALLENGE_DIR.'/'.$challenge->{token}; +    if (-e $filename) { +        warn "WARNING: File exists: $filename\n"; +    } +    else { +        open my $fh, '>', $filename or die "Can't open $filename: $!"; +        print $fh $keyAuthorization; +        close $fh; +    } + +    acme($challenge->{uri}, { +        resource => 'challenge', +        keyAuthorization => $keyAuthorization +    }); + +    for (my $i=0;; $i++) { +        my $status = request('GET' => $challenge->{uri})->{status} // 'pending'; +        die "Invalid challenge for $domain" if $status eq 'invalid'; +        last if $status eq 'valid'; +        die "Timeout exceeded while waiting for challenge to pass ($domain)\n" if $i >= $TIMEOUT; +        sleep 1; +    } +} + + +do { +    print STDERR "Processing new CSR\n" if $ENV{DEBUG}; +    open my $fh1, '-|', qw/openssl req -outform DER/, '-in', $CSR or die "Can't run req(1ssl): $!"; +    my $req = do { local $/ = undef; <$fh1> }; +    close $fh1; + +    # The server also gives the cert URI in its 'Location' header in +    # https://acme-v01.api.letsencrypt.org/acme/cert/$serial +    my $x509 = acme($RES{'new-cert'}, { +        resource => 'new-cert', +        csr => encode_base64url($req) +    }); + +    open my $fh2, '|-', qw/openssl x509 -inform DER/, '-out', $X509 or die "Can't run x509(1ssl): $!"; +    print $fh2 $x509; +    close $fh2; +}; diff --git a/acme-webserver b/acme-webserver new file mode 100755 index 0000000..57ea789 --- /dev/null +++ b/acme-webserver @@ -0,0 +1,32 @@ +#!/usr/bin/perl -T + +use strict; +use warnings; + +my $ROOT = '/.well-known/acme-challenge'; + +$_ = <STDIN> // exit; +my $proto  = s/ HTTP\/(1\.[01])\r\n\z// ? $1 : die "Error: Bad request\n"; +my $method = s/\A(GET|HEAD) //          ? $1 : die "Error: Bad request\n"; + +# Consume the headers (and ignore them) +while (defined (my $h = <STDIN>)) { last if $h eq "\r\n" }; + +my ($status_line, $content_type, $content); +if (/\A\Q$ROOT\E\/([A-Za-z0-9_-]+)\z/ and -f $1) { +    if (open my $fh, '<', $1) { +        ($status_line, $content_type) = ('200 OK', 'application/jose+json'); +        $content = do { local $/ = undef; <$fh> }; +        close $fh; +    } +    else { +        $status_line = '403 Forbidden'; +    } +} + +print "HTTP/$proto ", ($status_line // '404 Not Found'), "\r\n"; +print "Content-Type: $content_type\r\n" if defined $content_type; +print "Content-Length: ".length($content)."\r\n" if defined $content; +print "Connection: close\r\n"; +print "\r\n"; +print $content if defined $content and $method eq 'GET'; diff --git a/letsencrypt b/letsencrypt new file mode 100755 index 0000000..50406f7 --- /dev/null +++ b/letsencrypt @@ -0,0 +1,281 @@ +#!/bin/bash + +# Dependencies nc(1), openssl(1), socat(1) + +set -ue +set -o pipefail +PATH=/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +NAME=$(basename $0) + +WWW_USER=www-data +WWW_GROUP=www-data +ACME_WEBSERVER=acme-webserver +ACME_CLIENT=acme-slave + +declare COMMAND ACCOUNTKEY +declare -l GENKEY +declare RUNAS QUIET= DEBUG= + +declare SRVCERT= CHAIN= CSR SRVKEY +declare -l HASH= +declare SUBJECT=/ +declare SAN= +# https://security.stackexchange.com/questions/24106/which-key-usages-are-required-by-each-key-exchange-method +declare KEYUSAGE='digitalSignature, keyEncipherment, keyCertSign' +declare -a NOTIFY=() + +usage() { +    local msg="${1:-}" +    if [ "$msg" ]; then +        echo "$NAME: $msg" >&2 +        echo "Try '$NAME --help' for more information." >&2 +        exit 1 +    fi +    cat <<- EOF +		Usage: $NAME [OPTIONS] new-reg ACCOUNTKEY [EMAIL ..] +		  or: $NAME [OPTIONS] new-cert ACCOUNTKEY --output=CERT {--csr=CSR | CSR Options } +		  or: $NAME [OPTIONS] revoke-cert ACCOUNTKEY CERT [CERT ..] + +		ACCOUNTKEY is the private key file of the user's account.  Generic options are: +		    --genkey[=ALGO[:BITS]]  For 'new-*' commands, generate key pairs (with mode 0600) if they don't +		                            exist already. (Default: "RSA".)  RSA is the only algorithm curently supported. +		    --runas=USERNAME        Username to run the ACME client as. (This user doesn't need access to +		                            any private key material.) +		    --help, -?              Display this help text and exit +		    --quiet, -q             Be quiet +		    --debug                 Turn on debug mode + +		$NAME new-reg ACCOUNTKEY [EMAIL ..] +		  Register a new ACCOUNTKEY; an optional list of EMAIL addresses can be supplied as contact information. + +		$NAME new-cert ACCOUNTKEY --output=CERT --csr=FILE +		$NAME new-cert ACCOUNTKEY --output=CERT --key=FILE [--hash=ALGO] [--subject=STRING] [--san=STRING] [--keyusage=STRING] +		  Request a new Certificate Issuance.  The Certificate Signing Request can be supplied directly, or +		  generated from the server key. + +		    --csr=FILE        Certificate Signing Request to send (alternatively, use --key to generate it) +		    --key=FILE        Server private key (use --genkey to generate it) +		    --hash=DGST       Message digest to sign the CSR with (in PEM format) +		    --subject=STRING  Subject name, formatted as "/type0=value0/type1=value1/type2=..." (default: "/") +		    --san=STRING      Comma-separated list of Subject Alternative Names formatted as "type:value" +		    --keyusage=STRING Comma-separated list of Key Usages, see x509v3_config(5ssl) +		                      (default: "digitalSignature,keyEncipherment,keyCertSign") +		    --chain           Store not only the server certificate in the file specified with --output, but +		                      also the CA's +		    --output=FILE     Where to store the issued (signed) X.509 certificate +		    --notify=COMMAND  Command to run upon success.  (This option can be repeated.) + +		$NAME revoke-cert {ACCOUNTKEY|SVRKEY} FILE [FILE ..] +		  Request that the given certificate(s) FILE(s) be revoked.  The first argument can be either the account +		  key file or the server's private key. +	EOF +    exit 0 +} + +# Generate a key pair with mode 0600 +genkey() { +    local key="$1" genkey= bits= +    case "$GENKEY" in +        rsa) genkey=genrsa;; +        rsa:*) genkey=genrsa; bits=${GENKEY#*:};; +        *) echo "Error: invalid key type ${GENKEY%%:*}" >&2; exit 1;; +    esac +    install --mode=0600 /dev/null "$key" # atomic creation with mode 0600 +    openssl "$genkey" $bits >"$key" +} + + +# Parse options +declare -a ARGV=() +while [ $# -gt 0 ]; do +    case "$1" in +        --genkey) GENKEY=RSA;; +        --genkey=*) GENKEY="${1#*=}";; +        --runas=*) RUNAS="${1#*=}";; +        --help|-\?) usage;; +        --quiet|-q) QUIET=1;; +        --debug) DEBUG=1;; + +        --output=*) SRVCERT="${1#*=}";; +        --chain) CHAIN=1;; +        --csr=*) CSR="${1#*=}";; +        --key=*) SRVKEY="${1#*=}";; +        --hash=*) HASH="${1#*=}";; +        --subject=*) SUBJECT="${1#*=}";; +        --san=*) SAN="${1#*=}";; +        --keyusage=*) KEYUSAGE="${1#*=}";; +        --notify=*) NOTIFY+=( "${1#*=}" );; + +        --) shift; ARGV+=( "$@" ); break ;; +        -*) usage "unknown option '${1%%=*}'";; +        *) if [ ${ACCOUNTKEY+x} ]; then +               ARGV+=( "$1" ) +           else +               [ ${COMMAND+x} ] && ACCOUNTKEY="$1" || COMMAND="$1" +           fi +    esac +    shift +done + +[ "${COMMAND:-}" ] || usage "missing command" +[ "$COMMAND" = 'new-reg' -o "$COMMAND" = 'new-cert' -o "$COMMAND" = 'revoke-cert' ] || +    usage "invalid command $COMMAND" +[ ${#ARGV[@]} -eq 0 -o "$COMMAND" = 'new-reg' -o "$COMMAND" = 'revoke-cert' ] || +    usage "invalid argument ${ARGV[0]}" + +[ "${ACCOUNTKEY:-}" ] || usage "missing account key" +if ! [ -f "$ACCOUNTKEY" -a -s "$ACCOUNTKEY" ]; then +    if [ "$COMMAND" = 'new-reg' -a "${GENKEY+x}" ]; then +        genkey "$ACCOUNTKEY" +    else +        echo "Error: keyfile '$ACCOUNTKEY' does not exist." >&2 +        [[ ! "$COMMAND" =~ ^new- ]] || echo "Use 'new-reg --genkey' to generate and register an RSA key with mode 0600." >&2 +        exit 1 +    fi +fi + + +declare -a TMPFILES=() +declare CHALLENGE_DIR IPTABLES_SAVE +cleanup() { +    set +e +    [ ! ${IPTABLES_SAVE+x} ] || iptables-restore -c <"$IPTABLES_SAVE" +    pkill -TERM -P $$ +    [ ${#TMPFILES[@]} -eq 0 ] || rm  -${DEBUG:+v}f "${TMPFILES[@]}" +    [ ! ${CHALLENGE_DIR+x} ] || rm -${DEBUG:+v}rf "$CHALLENGE_DIR" +} +trap cleanup EXIT SIGINT + + +# Extract the public part of the user key +accountpub=$(mktemp --tmpdir XXXXXX.pub) +TMPFILES+=( "$accountpub" ) +openssl pkey -pubout <"$ACCOUNTKEY" >"$accountpub" +chmod 0644 "$accountpub" + + +if [ "$COMMAND" = 'revoke-cert' ]; then +    if [ ${#ARGV[@]} -eq 0 ]; then +        echo "Error: Nothing to revoke" >&2 +        exit 1 +    fi +elif [ "$COMMAND" = 'new-cert' ]; then +    if [ ! "${SRVCERT:-}" ]; then +        echo "Error: Missing --output" >&2 +        exit 1 +    fi + +    # Generate a Certificate Signing Request if need be +    if [ ${CSR+x} ]; then +        if [ -z "$CSR" -o ! -s "$CSR" ]; then +            echo "Error: Missing Certificate Signing Request $CSR.  (One of --csr or --key must be set.)" >&2 +            exit 1 +        fi +    else +        CSR=$(mktemp --tmpdir XXXXXX.csr) +        config=$(mktemp --tmpdir XXXXXX.cnf) +        TMPFILES+=( "$CSR" "$config" ) +        chmod 0644 "$CSR" + +        if [ "$HASH" ]; then +            case "$HASH" in +                md5|rmd160|sha1|sha224|sha256|sha384|sha512) HASH="-$HASH";; +                *) echo "Invalid digest algorithm: $HASH" >&2; exit 1;; +            esac +        fi +        if [ ! "${SRVKEY:-}" ] || [ ! -s "$SRVKEY" ]; then +            if [ "${SRVKEY:-}" -a "${GENKEY+x}" ]; then +                genkey "$SRVKEY" +            else +                echo "Error: Missing private server key.  Use new-cert --genkey --key=... to generate" >&2 +                exit 1 +            fi +        fi + +        [ ! "$DEBUG" ] || echo 'Generating Certificate Signing Request...' >&2 +        cat >"$config" <<- EOF +			[ req ] +			distinguished_name = req_distinguished_name +			req_extensions     = v3_req + +			[ req_distinguished_name ] + +			[ v3_req ] +			${SAN:+subjectAltName= $SAN} +			# XXX Golang errors on extensions marked critical +			# https://github.com/letsencrypt/boulder/issues/565 +			#basicConstraints     = critical, CA:FALSE +			#keyUsage             = critical${KEYUSAGE:+, $KEYUSAGE} +			basicConstraints     = CA:FALSE +			keyUsage             = $KEYUSAGE +			subjectKeyIdentifier = hash +		EOF +        openssl req -new -batch -key "$SRVKEY" $HASH -subj "$SUBJECT" -config "$config" -reqexts v3_req >"$CSR" +    fi +    [ ! "$DEBUG" ] || openssl req -noout -text <"$CSR" + +    CHALLENGE_DIR=$(mktemp --tmpdir -d acme-challenge.XXXXXX) +    x509=$(mktemp --tmpdir XXXXXX.pem) +    TMPFILES+=( "$x509" ) + +    [ ! "${RUNAS:-}" ] || chown "$RUNAS" "$CHALLENGE_DIR" "$x509" +    chgrp "$WWW_GROUP" "$CHALLENGE_DIR" +    chmod 0750 "$CHALLENGE_DIR" + +    # Make sure a webserver is configured to server ACME challenges +    if nc -z 127.0.0.1 80; then +        [ ! "$DEBUG" ] || echo "Using existing webserver" >&2 +        ln -${DEBUG:+v}Ts "$CHALLENGE_DIR" /var/www/acme-challenge +        TMPFILES+=( "/var/www/acme-challenge" ) +    else +        temp=$(mktemp --tmpdir) +        TMPFILES+=( "$temp" ) +        iptables-save -c >"$temp" +        IPTABLES_SAVE="$temp" +        iptables -I INPUT  -p tcp -m tcp --dport 80 -m state --state NEW,ESTABLISHED -j ACCEPT +        iptables -I OUTPUT -p tcp -m tcp --sport 80 -m state --state     ESTABLISHED -j ACCEPT +        ( +            [ ! "$DEBUG" ] || echo "Starting ACME webserver in $CHALLENGE_DIR" >&2 +            cd "$CHALLENGE_DIR" || exit 1 +            exec socat \ +                TCP-LISTEN:80,setgid="$WWW_GROUP",setuid="$WWW_USER",reuseaddr,fork,max-children=5 \ +                EXEC:"$ACME_WEBSERVER" +        )& +    fi + +    ARGV=( "$CSR" "$CHALLENGE_DIR" "$x509" ) +fi + + +pipe=$(mktemp --tmpdir -u XXXXXX.fifo) +mkfifo -m0600 "$pipe" +TMPFILES+=( "$pipe" ) + +# Wait for signing requests from the ACME slave +acme_client() { +    if [ "${RUNAS:-}" ]; then +         sudo -u "$RUNAS" ${DEBUG:+DEBUG="$DEBUG"} -- $ACME_CLIENT "$@" +    else +        [ ${RUNAS+x} ] || echo "WARNING: Use --runas=USER to specify a user to drop permissions to." >&2 +        DEBUG="$DEBUG" $ACME_CLIENT "$@" +    fi +} + +acme_client "$COMMAND" "$accountpub" ${ARGV[0]+"${ARGV[@]}"} <"$pipe" | +while read data; do +    echo -n "$data" | openssl dgst -sha256 -sign "$ACCOUNTKEY" -hex | sed 's/.*=\s*//' +done >"$pipe" + +if [ "$COMMAND" = 'new-cert' ]; then +    # TODO +    # Verify: dump and compare public keys +    # Valid cert, signed by the right CA + +    # Copy "$x509" to "$SRVCERT", possibly chained +    # https://crt.sh/?q=cse-fresti.cse.chalmers.se&iCAID=7395 +    cp "$x509" "$SRVCERT" + +    for (( i=0; i<${#NOTIFY[@]}; i++ )); do +        ${NOTIFY[$i]} +    done +fi | 
