SMTP STARTTLS format

Question

Is the EHLO message required after the TLS connection has been established? I'm using an acorn ltl-6511M wildlife camera that doesn't seem to send an EHLO message after establishing the TLS connection, causing a 503 error in my aiosmtpd-based SMTP server. It works with gmail SMTP though. Is the camera following the protocol or is my server not robust enough?

The code I'm using is:

import email
from email.header import decode_header
from email import message_from_bytes
from email.policy import default
from aiosmtpd.controller import Controller
from aiosmtpd.smtp import LoginPassword, AuthResult
import os
import sys
import time
import signal 
import logging
import ssl

##setting timezone
os.environ['TZ'] = "Europe/London"
time.tzset()

def onExit( sig, func=None):
    print("*************Stopping program*****************")
    controller.stop()
    exit()
 
signal.signal(signal.SIGTERM, onExit)

# removes the spaces and replaces with _ so they're valid folder names
def clean(text):
    return "".join(c if c.isalnum() else "_" for c in text)


log = logging.getLogger('mail.log')

auth_db = {
    b"TestCamera1@gmail.com": b"password1",
    b"user2": b"password2",
    b"TestCamera1": b"password1",
}

def authenticator_func(server, session, envelope, mechanism, auth_data):
    #this deliberately lets everything through
    assert isinstance(auth_data, LoginPassword)
    username = auth_data.login
    password = auth_data.password
    return AuthResult(success=True)


def configure_logging():
    file_handler = logging.FileHandler("aiosmtpd.log", "a")
    stderr_handler = logging.StreamHandler(sys.stderr)
    logger = logging.getLogger("mail.log")
    fmt = "[%(asctime)s %(levelname)s] %(message)s"
    datefmt = None
    formatter = logging.Formatter(fmt, datefmt, "%")
    stderr_handler.setFormatter(formatter)
    logger.addHandler(stderr_handler)
    file_handler.setFormatter(formatter)
    logger.addHandler(file_handler)
    logger.setLevel(logging.DEBUG)

class CustomHandler:
    def handle_exception(self, error):
        print("exception occured")
        print(error)
        return '542 Internal Server Error'

    async def handle_DATA(self, server, session, envelope):
        peer = session.peer
        data = envelope.content         # type: bytes
        msg = message_from_bytes(envelope.content, policy=default)
        # decode the email subject
        print("Msg:{}".format(msg))
        print("Data:{}".format(data))
        print("All of the relevant data has been extracted from the email")
        return '250 OK'


if __name__ == '__main__':
    configure_logging()
    handler = CustomHandler()
    #update hostname to your IP
    context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
    context.load_cert_chain('cert.pem', 'key.pem')    
    controller = Controller(handler, hostname='0.0.0.0', port=587, authenticator=authenticator_func, auth_required=True,auth_require_tls=True,tls_context=context)    
    # Run the event loop in a separate thread.
    controller.start()
    while True:
        time.sleep(10)

The code after trying to integrate is:

import email
from email.header import decode_header
from email import message_from_bytes
from email.policy import default
from aiosmtpd.controller import Controller
from aiosmtpd.smtp import LoginPassword, AuthResult, SMTP
import os
import json
import re
import sys
import time
import signal 
import logging
import ssl

from datetime import datetime
import configparser

##setting timezone
os.environ['TZ'] = "Europe/London"
time.tzset()

spacer = "*"*100

def onExit( sig, func=None):
    print("*************Stopping program*****************",3)
    controller.stop()
    exit()
 
signal.signal(signal.SIGTERM, onExit)

# removes the spaces and replaces with _ so they're valid folder names
def clean(text):
    return "".join(c if c.isalnum() else "_" for c in text)

log = logging.getLogger('mail.log')

auth_db = {
    b"TestCamera1@gmail.com": b"password1",
    b"user2": b"password2",
    b"TestCamera1": b"password1",
}

def authenticator_func(server, session, envelope, mechanism, auth_data):
    # Simple auth - is only being used because of the reolink cam
    assert isinstance(auth_data, LoginPassword)
    username = auth_data.login
    password = auth_data.password
    log.warning("Authenticator is being used")
    return AuthResult(success=True)

def configure_logging():
    file_handler = logging.FileHandler("aiosmtpd.log", "a")
    stderr_handler = logging.StreamHandler(sys.stderr)
    logger = logging.getLogger("mail.log")
    fmt = "[%(asctime)s %(levelname)s] %(message)s"
    datefmt = None
    formatter = logging.Formatter(fmt, datefmt, "%")
    stderr_handler.setFormatter(formatter)
    logger.addHandler(stderr_handler)
    file_handler.setFormatter(formatter)
    logger.addHandler(file_handler)
    logger.setLevel(logging.DEBUG)

class SMTPNoEhloAfterStarttls(SMTP):
    async def smtp_STARTTLS(self, arg: str):
        print(spacer)
        print("using starttls")
        host_name = self.session.host_name
        extended_smtp = self.session.extended_smtp
        await super().smtp_STARTTLS(arg)
        if host_name and extended_smtp and not self.session.host_name:
            # There was an EHLO before the STARTTLS.
            # RFC3207 says that we MUST reset the state
            # and forget the EHLO, but unfortunately
            # the client doesn't re-send the EHLO after STARTTLS,
            # so we need to pretend as if an EHLO has been sent.
            self.session.host_name = host_name
            self.session.extended_smtp = True

class ControllerNoEhloAfterStarttls(Controller):
    def factory(self):
        print(spacer)
        print("updating default settings")
        return SMTPNoEhloAfterStarttls(self.handler, **self.SMTP_kwargs)

class CustomHandler:
    def handle_exception(self, error):
        print("exception occured",3)
        print(error)
        return '542 Internal Server Error'

    async def handle_DATA(self, server, session, envelope):
        peer = session.peer
        data = envelope.content         # type: bytes
        msg = message_from_bytes(envelope.content, policy=default)
        # decode the email subject
        print("Msg:{}".format(msg),3)
        print("Data:{}".format(data),3)
        print("All of the relevant data has been extracted from the email",3)
        print(spacer,3)
        return '250 OK'

if __name__ == '__main__':
    configure_logging()
    handler = CustomHandler()
    # controller = Controller(handler, hostname='10.200.68.132', port=587)
    context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
    context.load_cert_chain('cert.pem', 'key.pem')
    controller = Controller(handler, hostname='10.200.68.133', port=587, authenticator=authenticator_func, auth_required=True,auth_require_tls=True,tls_context=context)    
    
    # Run the event loop in a separate thread.
    controller.start()
    #Confirmed that this is needed to keep the SMTP server running constantly
    while True:
        time.sleep(10)

However, this hasn't made any difference to the error logs.

Answer 1

Yes, EHLO is required after STARTTLS, see RFC3207 Section 4.2 (which specifically mentions forgetting the EHLO line - emphasis mine):

Upon completion of the TLS handshake, the SMTP protocol is reset to the initial state (the state in SMTP after a server issues a 220 service ready greeting). The server MUST discard any knowledge obtained from the client, such as the argument to the EHLO command, which was not obtained from the TLS negotiation itself.

This means that unfortunately your camera is not following the SMTP protocol. It is also unfortunate that GMail SMTP does not follow the protocol (it doesn't require EHLO in-between STARTTLS and AUTH LOGIN).

aiosmtpd is quite insistent on following the SMTP protocol and duly forgets the EHLO data before the STARTTLS; the EHLO hostname is stored in self.session.host_name on the aiosmtpd.smtp.SMTP object, and self.session is reset in SMTP.connection_made(), which is invoked after STARTTLS.

It is possible to make aiosmtpd break the SMTP specification and act in a highly non-conforming way. Obviously this is something you MUST NOT do in production. Use the ControllerNoEhloAfterStarttls defined below instead of the standard aiosmtpd Controller and then it should work.

from aiosmtpd.smtp import SMTP
from aiosmtpd.controller import Controller

class SMTPNoEhloAfterStarttls(SMTP):
    async def smtp_STARTTLS(self, arg: str):
        host_name = self.session.host_name
        extended_smtp = self.session.extended_smtp
        await super().smtp_STARTTLS(arg)
        if host_name and extended_smtp and not self.session.host_name:
            # There was an EHLO before the STARTTLS.
            # RFC3207 says that we MUST reset the state
            # and forget the EHLO, but unfortunately
            # the client doesn't re-send the EHLO after STARTTLS,
            # so we need to pretend as if an EHLO has been sent.
            self.session.host_name = host_name
            self.session.extended_smtp = True

class ControllerNoEhloAfterStarttls(Controller):
    def factory(self):
        return SMTPNoEhloAfterStarttls(self.handler, **self.SMTP_kwargs)

...and then down in if __name__ == "__main__":, instantiate the custom controller class instead of the default Controller:

controller = ControllerNoEhloAfterStarttls(handler, hostname='10.200.68.133', port=587, ......)