add ability to configure TLS and Authentication for sending PyPI mail

Also updates the configuration template to match the kinds of things you'd see in production.

Configuring for starttls and using authentication will allow us to send mail from transient PyPI web nodes without having to ask for the mail admins to whitelist the sending IP address.

This is obviously dependent on a change to the configuration (so deploy to production cautiously).

It also depends on the mail provider (mail.python.org in this case) supporting TLS and Authentication, so we may need to follow up with them.

Author: ewdurbin

MailingLogger.py

@@ -30,8 +30,8 @@ def format(self,record):
 
 class MailingLogger(SMTPHandler):
 
-    def __init__(self, mailhost, fromaddr, toaddrs, subject, send_empty_entries,flood_level=None):
-        SMTPHandler.__init__(self,mailhost,fromaddr,toaddrs,subject)
+    def __init__(self, mailhost, fromaddr, toaddrs, subject, credentials=None, secure=None, send_empty_entries=False, flood_level=None):
+        SMTPHandler.__init__(self, mailhost, fromaddr, toaddrs, subject, credentials=credentials, secure=secure)
         self.subject_formatter = SubjectFormatter(subject)
         self.send_empty_entries = send_empty_entries
         self.flood_level = flood_level
@@ -42,8 +42,6 @@ def getSubject(self,record):
         return self.subject_formatter.format(record)
 
     def emit(self,record):
-        if not self.send_empty_entries and not record.msg.strip():
-            return
         current_time = now()
         current_hour = current_time.hour
         if current_hour > self.hour:
@@ -69,6 +67,8 @@ def emit(self,record):
 """ % (self.sent,current_time.strftime('%H:%M:%S'),current_hour+1),
                 args = (),
                 exc_info = None)
+        if not self.send_empty_entries and not record.msg.strip():
+            return
         elif self.sent > self.flood_level:
             # do nothing, we've sent too many emails already
             return
@@ -87,6 +87,10 @@ def emit(self,record):
             email['From']=self.fromaddr
             email['To']=', '.join(self.toaddrs)
             email['X-Mailer']='MailingLogger'
+            if self.username:
+                if self.secure is not None:
+                    smtp.starttls(*self.secure)
+                    smtp.login(self.username, self.password)
             smtp.sendmail(self.fromaddr, self.toaddrs, email.as_string())
             smtp.quit()
         except:

config.ini.template

@@ -1,33 +1,57 @@
 [database]
-driver = postgresql2
+
+;Postgres Database
+;host = hostname
+;port = 5432
 name = packages
 user = pypi
-# host = hostname
-# port = 5432
+
+; Redis
+redis_url = redis://localhost:6379/0
+
+; Storage Directories
 files_dir = /MacDev/svn.python.org/pypi-pep345/files
 docs_dir = /MacDev/svn.python.org/pypi-pep345/docs
-package_docs_url = http://pythonhosted.org/
-redis_url = redis://localhost:6379/0
+
+; Third-Party
+pubsubhubbub = http://pubsubhubbub.appspot.com/
 
 [webui]
-mailhost = mail.python.org
-adminemail = richard@python.org
-replyto = richard@python.org
-url =  http://localhost:8000/pypi
-pydotorg = http://www.python.org/
 
-simple_script = /simple
-files_url = http://localhost/pypi_files
+; PyPI config
+debug_mode = yes
 rss_file = /tmp/pypi_rss.xml
 packages_rss_file = /tmp/pypi_packages_rss.xml
-debug_mode = yes
-cheesecake_password = secret
+
+; Email
+adminemail = richard@python.org
+replyto = richard@python.org
+
+; Secrets
+;sshkeys_update = /opt/devpypi/src/sshkeys_update
 key_dir = .
-simple_sign_script = /serversig
-raw_package_prefix = /raw-packages
+cheesecake_password = secret
 ; this is the secret used to sign password reset efforts - keep it secret!
 ; ''.join(random.choice(string.letters + string.digits) for n in range(64))
-reset_secret = secret
+;reset_secret = secret
+
+; URI Paths
+simple_script = /simple
+raw_package_prefix = /raw-packages
+simple_sign_script = /serversig
+
+; URLs
+url =  http://localhost:8000/pypi
+files_url = http://localhost/pypi_files
+pydotorg = http://www.python.org/
+package_docs_url = http://pythonhosted.org/
+
+[smtp]
+hostname = localhost:25
+starttls = off
+auth = off
+;login = postmaster@localhost
+;password = muchsecret
 
 [passlib]
 ; The first listed schemed will automatically be the default, see passlib
@@ -43,26 +67,37 @@ schemes = bcrypt, hex_sha1
 
 [logging]
 file =
-mailhost =
+mail_logger = off
 fromaddr =
 toaddrs =
 
-[mirrors]
-folder = mirrors
-local-stats = local-stats
-global-stats = global-stats
+; Not seeing this used in production
+;[mirrors]
+;folder = mirrors
+;local-stats = local-stats
+;global-stats = global-stats
 
 [sentry]
 dsn =
 
 [uwsgi]
+;uid=pypi
+;gid=pypi
 wsgi-file = pypi.wsgi
 socket = /tmp/pypi.sock
+;pidfile = /var/run/devpypi/pypi.pid 
+;daemonize = 127.0.0.1:8224
+;processes = 2
 harakiri = 60
+;reload-on-as = 400
+;max-requests = 10000
 master = 1
 post-buffering = 8192
 chmod-socket = 666
+;disable-logging = true
+;log-5xx = true
 
+; CDN API
 [fastly]
 api_domain = https://api.fastly.com/
 api_key =

config.py

@@ -43,7 +43,6 @@ def __init__(self, configfile):
             self.package_docs_url = c.get('webui', 'package_docs_url')
         else:
             self.package_docs_url = 'http://pythonhosted.org'
-        self.mailhost = c.get('webui', 'mailhost')
         self.adminemail = c.get('webui', 'adminemail')
         self.replyto = c.get('webui', 'replyto')
         self.url = c.get('webui', 'url')
@@ -67,7 +66,7 @@ def __init__(self, configfile):
         self.reset_secret = c.get('webui', 'reset_secret')
 
         self.logfile = c.get('logging', 'file')
-        self.logging_mailhost = c.get('logging', 'mailhost')
+        self.mail_logger = c.get('logging', 'mail_logger') 
         self.fromaddr = c.get('logging', 'fromaddr')
         self.toaddrs = c.get('logging', 'toaddrs').split(',')
 
@@ -89,6 +88,14 @@ def __init__(self, configfile):
         self.fastly_api_key = c.get("fastly", "api_key")
         self.fastly_service_id = c.get("fastly", "service_id")
 
+        # Get the smtp configuration
+        self.smtp_hostname = c.get("smtp", "hostname")
+        self.smtp_auth = c.get("smtp", "auth")
+        self.smtp_starttls = c.get("smtp", "starttls")
+        if self.smtp_auth:
+            self.smtp_login = c.get("smtp", "login")
+            self.smtp_password = c.get("smtp", "password")
+
     def make_https(self):
         if self.url.startswith("http:"):
             self.url = "https"+self.url[4:]

tools/email_renamed_users.py

@@ -75,7 +75,11 @@
 sent = []
 
 # Email each user
-server = smtplib.SMTP(config.mailhost)
+server = smtplib.SMTP(config.mailgun_hostname)
+if config.smtp_starttls:
+    server.starttls()
+if config.smtp_auth:
+    server.login(config.smtp_login, config.smtp_password)
 for username, packages in users.iteritems():
     packages = sorted(set(packages))
 

tools/hosting_mode_migration.py

@@ -102,7 +102,11 @@
 sent = []
 
 # Email each user
-server = smtplib.SMTP(config.mailhost)
+server = smtplib.SMTP(config.smtp_hostname)
+if config.smtp_starttls:
+    server.starttls()
+if config.smtp_auth:
+    server.login(config.smtp_login, config.smtp_password)
 for i, (package, users) in enumerate(package_users.iteritems()):
     fpackage = store.find_package(package)
 

webui.py

@@ -289,21 +289,30 @@ def __init__(self, handler, env):
         self.url_path = path
 
         # configure logging
-        if self.config.logfile or self.config.mailhost:
+        if self.config.logfile or self.config.mail_logger:
             root = logging.getLogger()
-            hdlrs = []
             if self.config.logfile:
                 hdlr = logging.FileHandler(self.config.logfile)
                 formatter = logging.Formatter(
                     '%(asctime)s %(name)s:%(levelname)s %(message)s')
                 hdlr.setFormatter(formatter)
-                hdlrs.append(hdlr)
-            if self.config.logging_mailhost:
-                hdlr = MailingLogger.MailingLogger(self.config.logging_mailhost,
-                    self.config.fromaddr, self.config.toaddrs,
-                    '[PyPI] %(line)s', False, flood_level=10)
-                hdlrs.append(hdlr)
-            root.handlers = hdlrs
+                root.handlers.append(hdlr)
+            if self.config.mail_logger:
+                smtp_starttls = None
+                if self.config.smtp_starttls:
+                    smtp_starttls = ()
+                smtp_credentials = None
+                if self.config.smtp_auth:
+                    smtp_credentials = (self.config.smtp_login, self.config.smtp_password)
+                hdlr = MailingLogger.MailingLogger(self.config.smtp_hostname,
+                                                   self.config.fromaddr,
+                                                   self.config.toaddrs,
+                                                   '[PyPI] %(line)s',
+                                                   credentials=smtp_credentials,
+                                                   secure=smtp_starttls,
+                                                   send_empty_entries=False,
+                                                   flood_level=10)
+                root.handlers.append(hdlr)
 
     def run(self):
         ''' Run the request, handling all uncaught errors and finishing off
@@ -3172,7 +3181,11 @@ def delete_user(self):
     def send_email(self, recipient, message):
         ''' Send an administrative email to the recipient
         '''
-        smtp = smtplib.SMTP(self.config.mailhost)
+        smtp = smtplib.SMTP(self.config.smtp_hostname)
+        if self.config.smtp_starttls:
+            smtp.starttls()
+        if self.config.smtp_auth:
+            smtp.login(self.config.smtp_login, self.config.smtp_password)
         smtp.sendmail(self.config.adminemail, recipient, message)
 
     def packageURL(self, name, version):