Merge branch 'sa/send-email-smtp-batch-data-limit'
authorJunio C Hamano <gitster@pobox.com>
Thu, 15 Oct 2015 22:43:41 +0000 (15:43 -0700)
committerJunio C Hamano <gitster@pobox.com>
Thu, 15 Oct 2015 22:43:41 +0000 (15:43 -0700)
When "git send-email" wanted to talk over Net::SMTP::SSL,
Net::Cmd::datasend() did not like to be fed too many bytes at the
same time and failed to send messages.  Send the payload one line
at a time to work around the problem.

* sa/send-email-smtp-batch-data-limit:
  git-send-email.perl: Fixed sending of many/huge changes/patches

1  2 
git-send-email.perl

diff --combined git-send-email.perl
@@@ -54,12 -54,10 +54,12 @@@ git send-email [options] <file | direct
      --[no-]bcc              <str>  * Email Bcc:
      --subject               <str>  * Email "Subject:"
      --in-reply-to           <str>  * Email "In-Reply-To:"
 +    --[no-]xmailer                 * Add "X-Mailer:" header (default).
      --[no-]annotate                * Review each patch that will be sent in an editor.
      --compose                      * Open an editor for introduction.
      --compose-encoding      <str>  * Encoding to assume for introduction.
      --8bit-encoding         <str>  * Encoding to assume 8bit mails if undeclared
 +    --transfer-encoding     <str>  * Transfer encoding to use (quoted-printable, 8bit, base64)
  
    Sending:
      --envelope-sender       <str>  * Email envelope sender.
@@@ -75,8 -73,6 +75,8 @@@
                                       Pass an empty string to disable certificate
                                       verification.
      --smtp-domain           <str>  * The domain name sent to HELO/EHLO handshake
 +    --smtp-auth             <str>  * Space-separated list of allowed AUTH mechanisms.
 +                                     This setting forces to use one of the listed mechanisms.
      --smtp-debug            <0|1>  * Disable, enable Net::SMTP debug.
  
    Automating:
@@@ -149,15 -145,10 +149,15 @@@ my $have_mail_address = eval { require 
  my $smtp;
  my $auth;
  
 +# Regexes for RFC 2047 productions.
 +my $re_token = qr/[^][()<>@,;:\\"\/?.= \000-\037\177-\377]+/;
 +my $re_encoded_text = qr/[^? \000-\037\177-\377]+/;
 +my $re_encoded_word = qr/=\?($re_token)\?($re_token)\?($re_encoded_text)\?=/;
 +
  # Variables we fill in automatically, or via prompting:
  my (@to,$no_to,@initial_to,@cc,$no_cc,@initial_cc,@bcclist,$no_bcc,@xh,
        $initial_reply_to,$initial_subject,@files,
 -      $author,$sender,$smtp_authpass,$annotate,$compose,$time);
 +      $author,$sender,$smtp_authpass,$annotate,$use_xmailer,$compose,$time);
  
  my $envelope_sender;
  
@@@ -210,12 -201,11 +210,12 @@@ my ($cover_cc, $cover_to)
  my ($to_cmd, $cc_cmd);
  my ($smtp_server, $smtp_server_port, @smtp_server_options);
  my ($smtp_authuser, $smtp_encryption, $smtp_ssl_cert_path);
 -my ($identity, $aliasfiletype, @alias_files, $smtp_domain);
 +my ($identity, $aliasfiletype, @alias_files, $smtp_domain, $smtp_auth);
  my ($validate, $confirm);
  my (@suppress_cc);
  my ($auto_8bit_encoding);
  my ($compose_encoding);
 +my ($target_xfer_encoding);
  
  my ($debug_net_smtp) = 0;             # Net::SMTP, see send_message()
  
@@@ -229,8 -219,7 +229,8 @@@ my %config_bool_settings = 
      "signedoffcc" => [\$signed_off_by_cc, undef],      # Deprecated
      "validate" => [\$validate, 1],
      "multiedit" => [\$multiedit, undef],
 -    "annotate" => [\$annotate, undef]
 +    "annotate" => [\$annotate, undef],
 +    "xmailer" => [\$use_xmailer, 1]
  );
  
  my %config_settings = (
      "smtppass" => \$smtp_authpass,
      "smtpsslcertpath" => \$smtp_ssl_cert_path,
      "smtpdomain" => \$smtp_domain,
 +    "smtpauth" => \$smtp_auth,
      "to" => \@initial_to,
      "tocmd" => \$to_cmd,
      "cc" => \@initial_cc,
      "from" => \$sender,
      "assume8bitencoding" => \$auto_8bit_encoding,
      "composeencoding" => \$compose_encoding,
 +    "transferencoding" => \$target_xfer_encoding,
  );
  
  my %config_path_settings = (
@@@ -302,7 -289,6 +302,7 @@@ my $rc = GetOptions("h" => \$help
                    "bcc=s" => \@bcclist,
                    "no-bcc" => \$no_bcc,
                    "chain-reply-to!" => \$chain_reply_to,
 +                  "no-chain-reply-to" => sub {$chain_reply_to = 0},
                    "smtp-server=s" => \$smtp_server,
                    "smtp-server-option=s" => \@smtp_server_options,
                    "smtp-server-port=s" => \$smtp_server_port,
                    "smtp-ssl-cert-path=s" => \$smtp_ssl_cert_path,
                    "smtp-debug:i" => \$debug_net_smtp,
                    "smtp-domain:s" => \$smtp_domain,
 +                  "smtp-auth=s" => \$smtp_auth,
                    "identity=s" => \$identity,
                    "annotate!" => \$annotate,
 +                  "no-annotate" => sub {$annotate = 0},
                    "compose" => \$compose,
                    "quiet" => \$quiet,
                    "cc-cmd=s" => \$cc_cmd,
                    "suppress-from!" => \$suppress_from,
 +                  "no-suppress-from" => sub {$suppress_from = 0},
                    "suppress-cc=s" => \@suppress_cc,
                    "signed-off-cc|signed-off-by-cc!" => \$signed_off_by_cc,
 +                  "no-signed-off-cc|no-signed-off-by-cc" => sub {$signed_off_by_cc = 0},
                    "cc-cover|cc-cover!" => \$cover_cc,
 +                  "no-cc-cover" => sub {$cover_cc = 0},
                    "to-cover|to-cover!" => \$cover_to,
 +                  "no-to-cover" => sub {$cover_to = 0},
                    "confirm=s" => \$confirm,
                    "dry-run" => \$dry_run,
                    "envelope-sender=s" => \$envelope_sender,
                    "thread!" => \$thread,
 +                  "no-thread" => sub {$thread = 0},
                    "validate!" => \$validate,
 +                  "no-validate" => sub {$validate = 0},
 +                  "transfer-encoding=s" => \$target_xfer_encoding,
                    "format-patch!" => \$format_patch,
 +                  "no-format-patch" => sub {$format_patch = 0},
                    "8bit-encoding=s" => \$auto_8bit_encoding,
                    "compose-encoding=s" => \$compose_encoding,
                    "force" => \$force,
 +                  "xmailer!" => \$use_xmailer,
 +                  "no-xmailer" => sub {$use_xmailer = 0},
         );
  
  usage() if $help;
@@@ -464,11 -438,25 +464,11 @@@ my ($repoauthor, $repocommitter)
  ($repoauthor) = Git::ident_person(@repo, 'author');
  ($repocommitter) = Git::ident_person(@repo, 'committer');
  
 -# Verify the user input
 -
 -foreach my $entry (@initial_to) {
 -      die "Comma in --to entry: $entry'\n" unless $entry !~ m/,/;
 -}
 -
 -foreach my $entry (@initial_cc) {
 -      die "Comma in --cc entry: $entry'\n" unless $entry !~ m/,/;
 -}
 -
 -foreach my $entry (@bcclist) {
 -      die "Comma in --bcclist entry: $entry'\n" unless $entry !~ m/,/;
 -}
 -
  sub parse_address_line {
        if ($have_mail_address) {
                return map { $_->format } Mail::Address->parse($_[0]);
        } else {
 -              return split_addrs($_[0]);
 +              return Git::parse_mailboxes($_[0]);
        }
  }
  
@@@ -477,37 -465,6 +477,37 @@@ sub split_addrs 
  }
  
  my %aliases;
 +
 +sub parse_sendmail_alias {
 +      local $_ = shift;
 +      if (/"/) {
 +              print STDERR "warning: sendmail alias with quotes is not supported: $_\n";
 +      } elsif (/:include:/) {
 +              print STDERR "warning: `:include:` not supported: $_\n";
 +      } elsif (/[\/|]/) {
 +              print STDERR "warning: `/file` or `|pipe` redirection not supported: $_\n";
 +      } elsif (/^(\S+?)\s*:\s*(.+)$/) {
 +              my ($alias, $addr) = ($1, $2);
 +              $aliases{$alias} = [ split_addrs($addr) ];
 +      } else {
 +              print STDERR "warning: sendmail line is not recognized: $_\n";
 +      }
 +}
 +
 +sub parse_sendmail_aliases {
 +      my $fh = shift;
 +      my $s = '';
 +      while (<$fh>) {
 +              chomp;
 +              next if /^\s*$/ || /^\s*#/;
 +              $s .= $_, next if $s =~ s/\\$// || s/^\s+//;
 +              parse_sendmail_alias($s) if $s;
 +              $s = $_;
 +      }
 +      $s =~ s/\\$//; # silently tolerate stray '\' on last line
 +      parse_sendmail_alias($s) if $s;
 +}
 +
  my %parse_alias = (
        # multiline formats can be supported in the future
        mutt => sub { my $fh = shift; while (<$fh>) {
                               $aliases{$alias} = [ split_addrs($addr) ];
                          }
                      } },
 -
 +      sendmail => \&parse_sendmail_aliases,
        gnus => sub { my $fh = shift; while (<$fh>) {
                if (/\(define-mail-alias\s+"(\S+?)"\s+"(\S+?)"\)/) {
                        $aliases{$1} = [ $2 ];
@@@ -551,6 -508,8 +551,6 @@@ if (@alias_files and $aliasfiletype an
        }
  }
  
 -($sender) = expand_aliases($sender) if defined $sender;
 -
  # is_format_patch_arg($f) returns 0 if $f names a patch, or 1 if
  # $f is a revision list specification to be passed to format-patch.
  sub is_format_patch_arg {
@@@ -781,7 -740,6 +781,7 @@@ if (!defined $auto_8bit_encoding && sca
                print "    $f\n";
        }
        $auto_8bit_encoding = ask("Which 8bit encoding should I declare [UTF-8]? ",
 +                                valid_re => qr/.{4}/, confirm_only => 1,
                                  default => "UTF-8");
  }
  
@@@ -795,10 -753,7 +795,10 @@@ if (!$force) 
        }
  }
  
 -if (!defined $sender) {
 +if (defined $sender) {
 +      $sender =~ s/^\s+|\s+$//g;
 +      ($sender) = expand_aliases($sender);
 +} else {
        $sender = $repoauthor || $repocommitter || '';
  }
  
@@@ -830,9 -785,12 +830,9 @@@ sub expand_one_alias 
        return $aliases{$alias} ? expand_aliases(@{$aliases{$alias}}) : $alias;
  }
  
 -@initial_to = expand_aliases(@initial_to);
 -@initial_to = validate_address_list(sanitize_address_list(@initial_to));
 -@initial_cc = expand_aliases(@initial_cc);
 -@initial_cc = validate_address_list(sanitize_address_list(@initial_cc));
 -@bcclist = expand_aliases(@bcclist);
 -@bcclist = validate_address_list(sanitize_address_list(@bcclist));
 +@initial_to = process_address_list(@initial_to);
 +@initial_cc = process_address_list(@initial_cc);
 +@bcclist = process_address_list(@bcclist);
  
  if ($thread && !defined $initial_reply_to && $prompting) {
        $initial_reply_to = ask(
@@@ -955,26 -913,15 +955,26 @@@ $time = time - scalar $#files
  
  sub unquote_rfc2047 {
        local ($_) = @_;
 -      my $encoding;
 -      s{=\?([^?]+)\?q\?(.*?)\?=}{
 -              $encoding = $1;
 -              my $e = $2;
 -              $e =~ s/_/ /g;
 -              $e =~ s/=([0-9A-F]{2})/chr(hex($1))/eg;
 -              $e;
 +      my $charset;
 +      my $sep = qr/[ \t]+/;
 +      s{$re_encoded_word(?:$sep$re_encoded_word)*}{
 +              my @words = split $sep, $&;
 +              foreach (@words) {
 +                      m/$re_encoded_word/;
 +                      $charset = $1;
 +                      my $encoding = $2;
 +                      my $text = $3;
 +                      if ($encoding eq 'q' || $encoding eq 'Q') {
 +                              $_ = $text;
 +                              s/_/ /g;
 +                              s/=([0-9A-F]{2})/chr(hex($1))/egi;
 +                      } else {
 +                              # other encodings not supported yet
 +                      }
 +              }
 +              join '', @words;
        }eg;
 -      return wantarray ? ($_, $encoding) : $_;
 +      return wantarray ? ($_, $charset) : $_;
  }
  
  sub quote_rfc2047 {
  
  sub is_rfc2047_quoted {
        my $s = shift;
 -      my $token = qr/[^][()<>@,;:"\/?.= \000-\037\177-\377]+/;
 -      my $encoded_text = qr/[!->@-~]+/;
        length($s) <= 75 &&
 -      $s =~ m/^(?:"[[:ascii:]]*"|=\?$token\?$token\?$encoded_text\?=)$/o;
 +      $s =~ m/^(?:"[[:ascii:]]*"|$re_encoded_word)$/o;
  }
  
  sub subject_needs_rfc2047_quoting {
@@@ -1025,17 -974,15 +1025,17 @@@ sub sanitize_address 
                return $recipient;
        }
  
 +      # remove non-escaped quotes
 +      $recipient_name =~ s/(^|[^\\])"/$1/g;
 +
        # rfc2047 is needed if a non-ascii char is included
        if ($recipient_name =~ /[^[:ascii:]]/) {
 -              $recipient_name =~ s/^"(.*)"$/$1/;
                $recipient_name = quote_rfc2047($recipient_name);
        }
  
        # double quotes are needed if specials or CTLs are included
        elsif ($recipient_name =~ /[][()<>@,;:\\".\000-\037\177]/) {
 -              $recipient_name =~ s/(["\\\r])/\\$1/g;
 +              $recipient_name =~ s/([\\\r])/\\$1/g;
                $recipient_name = qq["$recipient_name"];
        }
  
@@@ -1047,14 -994,6 +1047,14 @@@ sub sanitize_address_list 
        return (map { sanitize_address($_) } @_);
  }
  
 +sub process_address_list {
 +      my @addr_list = map { parse_address_line($_) } @_;
 +      @addr_list = expand_aliases(@addr_list);
 +      @addr_list = sanitize_address_list(@addr_list);
 +      @addr_list = validate_address_list(@addr_list);
 +      return @addr_list;
 +}
 +
  # Returns the local Fully Qualified Domain Name (FQDN) if available.
  #
  # Tightly configured MTAa require that a caller sends a real DNS
@@@ -1134,12 -1073,6 +1134,12 @@@ sub smtp_auth_maybe 
                Authen::SASL->import(qw(Perl));
        };
  
 +      # Check mechanism naming as defined in:
 +      # https://tools.ietf.org/html/rfc4422#page-8
 +      if ($smtp_auth && $smtp_auth !~ /^(\b[A-Z0-9-_]{1,20}\s*)*$/) {
 +              die "invalid smtp auth: '${smtp_auth}'";
 +      }
 +
        # TODO: Authentication may fail not because credentials were
        # invalid but due to other reasons, in which we should not
        # reject credentials.
                'password' => $smtp_authpass
        }, sub {
                my $cred = shift;
 +
 +              if ($smtp_auth) {
 +                      my $sasl = Authen::SASL->new(
 +                              mechanism => $smtp_auth,
 +                              callback => {
 +                                      user => $cred->{'username'},
 +                                      pass => $cred->{'password'},
 +                                      authname => $cred->{'username'},
 +                              }
 +                      );
 +
 +                      return !!$smtp->auth($sasl);
 +              }
 +
                return !!$smtp->auth($cred->{'username'}, $cred->{'password'});
        });
  
@@@ -1244,10 -1163,8 +1244,10 @@@ To: $to${ccline
  Subject: $subject
  Date: $date
  Message-Id: $message_id
 -X-Mailer: git-send-email $gitversion
  ";
 +      if ($use_xmailer) {
 +              $header .= "X-Mailer: git-send-email $gitversion\n";
 +      }
        if ($reply_to) {
  
                $header .= "In-Reply-To: $reply_to\n";
                $smtp->mail( $raw_from ) or die $smtp->message;
                $smtp->to( @recipients ) or die $smtp->message;
                $smtp->data or die $smtp->message;
-               $smtp->datasend("$header\n$message") or die $smtp->message;
+               $smtp->datasend("$header\n") or die $smtp->message;
+               my @lines = split /^/, $message;
+               foreach my $line (@lines) {
+                       $smtp->datasend("$line") or die $smtp->message;
+               }
                $smtp->dataend() or die $smtp->message;
                $smtp->code =~ /250|200/ or die "Failed to send $subject\n".$smtp->message;
        }
@@@ -1407,8 -1328,6 +1411,8 @@@ foreach my $t (@files) 
        my $author_encoding;
        my $has_content_type;
        my $body_encoding;
 +      my $xfer_encoding;
 +      my $has_mime_version;
        @to = ();
        @cc = ();
        @xh = ();
                                }
                                push @xh, $_;
                        }
 +                      elsif (/^MIME-Version/i) {
 +                              $has_mime_version = 1;
 +                              push @xh, $_;
 +                      }
                        elsif (/^Message-Id: (.*)/i) {
                                $message_id = $1;
                        }
 +                      elsif (/^Content-Transfer-Encoding: (.*)/i) {
 +                              $xfer_encoding = $1 if not defined $xfer_encoding;
 +                      }
                        elsif (!/^Date:\s/i && /^[-A-Za-z]+:\s+\S/) {
                                push @xh, $_;
                        }
                if defined $cc_cmd && !$suppress_cc{'cccmd'};
  
        if ($broken_encoding{$t} && !$has_content_type) {
 +              $xfer_encoding = '8bit' if not defined $xfer_encoding;
                $has_content_type = 1;
 -              push @xh, "MIME-Version: 1.0",
 -                      "Content-Type: text/plain; charset=$auto_8bit_encoding",
 -                      "Content-Transfer-Encoding: 8bit";
 +              push @xh, "Content-Type: text/plain; charset=$auto_8bit_encoding";
                $body_encoding = $auto_8bit_encoding;
        }
  
                                }
                        }
                        else {
 +                              $xfer_encoding = '8bit' if not defined $xfer_encoding;
                                $has_content_type = 1;
                                push @xh,
 -                                'MIME-Version: 1.0',
 -                                "Content-Type: text/plain; charset=$author_encoding",
 -                                'Content-Transfer-Encoding: 8bit';
 +                                "Content-Type: text/plain; charset=$author_encoding";
                        }
                }
        }
 +      if (defined $target_xfer_encoding) {
 +              $xfer_encoding = '8bit' if not defined $xfer_encoding;
 +              $message = apply_transfer_encoding(
 +                      $message, $xfer_encoding, $target_xfer_encoding);
 +              $xfer_encoding = $target_xfer_encoding;
 +      }
 +      if (defined $xfer_encoding) {
 +              push @xh, "Content-Transfer-Encoding: $xfer_encoding";
 +      }
 +      if (defined $xfer_encoding or $has_content_type) {
 +              unshift @xh, 'MIME-Version: 1.0' unless $has_mime_version;
 +      }
  
        $needs_confirm = (
                $confirm eq "always" or
                ($confirm =~ /^(?:auto|compose)$/ && $compose && $message_num == 1));
        $needs_confirm = "inform" if ($needs_confirm && $confirm_unconfigured && @cc);
  
 -      @to = validate_address_list(sanitize_address_list(@to));
 -      @cc = validate_address_list(sanitize_address_list(@cc));
 +      @to = process_address_list(@to);
 +      @cc = process_address_list(@cc);
  
        @to = (@initial_to, @to);
        @cc = (@initial_cc, @cc);
@@@ -1645,32 -1547,6 +1649,32 @@@ sub cleanup_compose_files 
  
  $smtp->quit if $smtp;
  
 +sub apply_transfer_encoding {
 +      my $message = shift;
 +      my $from = shift;
 +      my $to = shift;
 +
 +      return $message if ($from eq $to and $from ne '7bit');
 +
 +      require MIME::QuotedPrint;
 +      require MIME::Base64;
 +
 +      $message = MIME::QuotedPrint::decode($message)
 +              if ($from eq 'quoted-printable');
 +      $message = MIME::Base64::decode($message)
 +              if ($from eq 'base64');
 +
 +      die "cannot send message as 7bit"
 +              if ($to eq '7bit' and $message =~ /[^[:ascii:]]/);
 +      return $message
 +              if ($to eq '7bit' or $to eq '8bit');
 +      return MIME::QuotedPrint::encode($message, "\n", 0)
 +              if ($to eq 'quoted-printable');
 +      return MIME::Base64::encode($message, "\n")
 +              if ($to eq 'base64');
 +      die "invalid transfer encoding";
 +}
 +
  sub unique_email_list {
        my %seen;
        my @emails;