New set of workarounds for hyperkitty.

README.md

@@ -57,21 +57,27 @@ the listener does not become available after starting or restarting Mailman.
 The optional compatibility patcher addresses three failures observed with the
 legacy Python 3.6 web stack: malformed `From`/`Subject` headers, nested
 `message/rfc822` parts without `Content-Transfer-Encoding`, and unreadable
-gettext catalogs. Enable it only for the known package versions:
+gettext catalogs. It also normalizes non-ASCII sender addresses and backports
+the byte-oriented delivery used by newer `mailman-hyperkitty` releases, so
+surrogate-escaped message bytes do not pass through Requests as Unicode text.
+Enable it only for the known package versions:
 
 ```yaml
 mailman_enable_legacy_hyperkitty_compatibility_patches: true
 mailman_legacy_hyperkitty_version: '1.3.3'
 mailman_legacy_django_mailman3_version: '1.3.4'
+mailman_legacy_mailman_hyperkitty_version: '1.1.0'
 ```
 
 The Python source patches are idempotent, compiled before any file is replaced,
-and installed atomically. The task fails without changing either source file if
+and installed atomically. The task fails without changing any source file if
 the installed versions or expected source fragments do not match. Invalid
 `django_extensions` `.mo` files are preserved beside the original name with a
 `.disabled-by-ansible` suffix, allowing Django to fall back to another locale.
-The uWSGI service is restarted only when a source file or catalog actually
+The uWSGI and Mailman services are restarted only when a source file or catalog
-changes.
+actually changes. After restarting Mailman, the handler requires the configured
+LMTP listener to become available.
 
-Disable this option before upgrading HyperKitty or django-mailman3, then review
+Disable this option before upgrading HyperKitty, django-mailman3, or
-whether the compatibility patches are still needed with the new versions.
+mailman-hyperkitty, then review whether the compatibility patches are still
+needed with the new versions.

defaults/main.yml

@@ -49,6 +49,7 @@ mailman_weekly_verified_restart_start_timeout: 120
 mailman_enable_legacy_hyperkitty_compatibility_patches: false
 mailman_legacy_hyperkitty_version: '1.3.3'
 mailman_legacy_django_mailman3_version: '1.3.4'
+mailman_legacy_mailman_hyperkitty_version: '1.1.0'
 mailman_legacy_hyperkitty_patch_script: '/usr/local/sbin/mailman-legacy-hyperkitty-compatibility'
 
   # Documentation that must be followed to configure the social auth providers

handlers/main.yml

@@ -4,3 +4,12 @@
 
 - name: Restart mailman
   service: name=mailman state=restarted
+
+- name: Wait for Mailman after restart
+  ansible.builtin.wait_for:
+    host: '{{ mailman_lmtp_host }}'
+    port: '{{ mailman_lmtp_port }}'
+    state: started
+    sleep: 1
+    timeout: '{{ mailman_weekly_verified_restart_start_timeout }}'
+  listen: Restart mailman

tasks/mailman.yml

@@ -72,7 +72,9 @@
       when:
         - mailman_enable_legacy_hyperkitty_compatibility_patches | bool
         - not ansible_check_mode
-      notify: Restart mailmansuite
+      notify:
+        - Restart mailmansuite
+        - Restart mailman
       tags: [ 'mailman', 'mailman_conf', 'mailman_hyperkitty_compatibility' ]
 
     - name: Install the mailman and postfix configuration files

templates/mailman-legacy-hyperkitty-compatibility.py.j2

@@ -18,16 +18,23 @@ import pkg_resources
 EXPECTED_VERSIONS = {
     'HyperKitty': {{ mailman_legacy_hyperkitty_version | to_json }},
     'django-mailman3': {{ mailman_legacy_django_mailman3_version | to_json }},
+    'mailman-hyperkitty': {{ mailman_legacy_mailman_hyperkitty_version | to_json }},
 }
 
 HYPERKITTY_IMPORT_OLD = "import re\nfrom email.message import EmailMessage\n"
+HYPERKITTY_IMPORT_V1 = (
+    "import re\n"
+    "from email.header import decode_header, make_header\n"
+    "from email.message import EmailMessage\n"
+)
 HYPERKITTY_IMPORT_NEW = (
+    "import hashlib\n"
     "import re\n"
     "from email.header import decode_header, make_header\n"
     "from email.message import EmailMessage\n"
 )
 HYPERKITTY_CLASS_MARKER = "class DuplicateMessage(Exception):\n"
-HYPERKITTY_HELPER = '''def _header_to_unicode_safely(message, name):
+HYPERKITTY_HELPER_V1 = '''def _header_to_unicode_safely(message, name):
     """Decode a header even when Python's structured parser rejects it."""
     try:
         return header_to_unicode(message[name])
@@ -44,11 +51,51 @@ HYPERKITTY_HELPER = '''def _header_to_unicode_safely(message, name):
             return raw_value
 
 
+'''
+HYPERKITTY_HELPER = HYPERKITTY_HELPER_V1 + '''def _ascii_sender_address_safely(address, original_header):
+    """Return an ASCII sender key without rejecting the whole message."""
+    address = address.strip()
+    if not address:
+        return ''
+    try:
+        return address.encode('ascii').decode('ascii')
+    except UnicodeEncodeError:
+        pass
+
+    local_part, separator, domain = address.rpartition('@')
+    if separator:
+        try:
+            local_part = local_part.encode('ascii').decode('ascii')
+            domain = domain.encode('idna').decode('ascii')
+            return '{}@{}'.format(local_part, domain)
+        except UnicodeError:
+            pass
+
+    if not isinstance(original_header, str):
+        original_header = repr(original_header)
+    digest = hashlib.sha256(
+        original_header.encode('utf-8', 'surrogatepass')).hexdigest()[:16]
+    return 'unknown-{}@example.invalid'.format(digest)
+
+
 '''
 HYPERKITTY_FROM_OLD = "from_str = header_to_unicode(message['From'])"
 HYPERKITTY_FROM_NEW = "from_str = _header_to_unicode_safely(message, 'From')"
 HYPERKITTY_SUBJECT_OLD = "email.subject = header_to_unicode(message.get('Subject'))"
 HYPERKITTY_SUBJECT_NEW = "email.subject = _header_to_unicode_safely(message, 'Subject')"
+HYPERKITTY_SENDER_OLD = '''    try:
+        from_str = _header_to_unicode_safely(message, 'From')
+        from_name, from_email = parseaddr(from_str)
+        from_name = from_name.strip()
+        sender_address = from_email.encode('ascii').decode("ascii").strip()
+    except (UnicodeDecodeError, UnicodeEncodeError):
+        raise ValueError("Non-ascii sender address", message)
+'''
+HYPERKITTY_SENDER_NEW = '''    from_str = _header_to_unicode_safely(message, 'From')
+    from_name, from_email = parseaddr(from_str)
+    from_name = from_name.strip()
+    sender_address = _ascii_sender_address_safely(from_email, from_str)
+'''
 
 SCRUB_OLD = '''        if ctype == 'message/rfc822':
             # Return message/rfc822 parts as a string.
@@ -66,6 +113,13 @@ SCRUB_NEW = '''        if ctype == 'message/rfc822':
                 decodedpayload = str(payload)
 '''
 
+MAILMAN_HYPERKITTY_SEND_OLD = '''            message_text = msg.as_string()
+        except (MessageError, KeyError) as error:
+'''
+MAILMAN_HYPERKITTY_SEND_NEW = '''            message_text = msg.as_bytes()
+        except (MessageError, KeyError, UnicodeEncodeError) as error:
+'''
+
 
 def distribution(name):
     dist = pkg_resources.get_distribution(name)
@@ -92,28 +146,41 @@ def patch_hyperkitty(source):
         HYPERKITTY_IMPORT_NEW in source and
         HYPERKITTY_FROM_NEW in source and
         HYPERKITTY_SUBJECT_NEW in source and
+        HYPERKITTY_SENDER_NEW in source and
         HYPERKITTY_IMPORT_OLD not in source and
         HYPERKITTY_FROM_OLD not in source and
-        HYPERKITTY_SUBJECT_OLD not in source
+        HYPERKITTY_SUBJECT_OLD not in source and
+        HYPERKITTY_SENDER_OLD not in source
     )
     if patched:
         if not completed:
             raise RuntimeError('HyperKitty compatibility patch is incomplete')
         return source, False
 
+    if HYPERKITTY_HELPER_V1 in source:
+        updated = replace_once(
+            source, HYPERKITTY_IMPORT_V1, HYPERKITTY_IMPORT_NEW,
+            'HyperKitty v1 imports')
+        updated = replace_once(
+            updated, HYPERKITTY_HELPER_V1, HYPERKITTY_HELPER,
+            'HyperKitty v1 helper')
+    else:
+        updated = replace_once(
+            source, HYPERKITTY_IMPORT_OLD, HYPERKITTY_IMPORT_NEW,
+            'HyperKitty imports')
+        updated = replace_once(
+            updated, HYPERKITTY_CLASS_MARKER,
+            HYPERKITTY_HELPER + HYPERKITTY_CLASS_MARKER,
+            'HyperKitty helper insertion point')
+        updated = replace_once(
+            updated, HYPERKITTY_FROM_OLD, HYPERKITTY_FROM_NEW,
+            'HyperKitty From handling')
+        updated = replace_once(
+            updated, HYPERKITTY_SUBJECT_OLD, HYPERKITTY_SUBJECT_NEW,
+            'HyperKitty Subject handling')
     updated = replace_once(
-        source, HYPERKITTY_IMPORT_OLD, HYPERKITTY_IMPORT_NEW,
+        updated, HYPERKITTY_SENDER_OLD, HYPERKITTY_SENDER_NEW,
-        'HyperKitty imports')
+        'HyperKitty sender address handling')
-    updated = replace_once(
-        updated, HYPERKITTY_CLASS_MARKER,
-        HYPERKITTY_HELPER + HYPERKITTY_CLASS_MARKER,
-        'HyperKitty helper insertion point')
-    updated = replace_once(
-        updated, HYPERKITTY_FROM_OLD, HYPERKITTY_FROM_NEW,
-        'HyperKitty From handling')
-    updated = replace_once(
-        updated, HYPERKITTY_SUBJECT_OLD, HYPERKITTY_SUBJECT_NEW,
-        'HyperKitty Subject handling')
     return updated, True
 
 
@@ -127,6 +194,16 @@ def patch_scrubber(source):
         'django-mailman3 message/rfc822 handling'), True
 
 
+def patch_mailman_hyperkitty(source):
+    if MAILMAN_HYPERKITTY_SEND_NEW in source:
+        if MAILMAN_HYPERKITTY_SEND_OLD in source:
+            raise RuntimeError('mailman-hyperkitty compatibility patch is ambiguous')
+        return source, False
+    return replace_once(
+        source, MAILMAN_HYPERKITTY_SEND_OLD, MAILMAN_HYPERKITTY_SEND_NEW,
+        'mailman-hyperkitty byte delivery'), True
+
+
 def load_source(path):
     with open(path, 'rb') as source_file:
         return source_file.read().decode('utf-8')
@@ -178,17 +255,23 @@ def quarantine_catalog(path):
 def main():
     hyperkitty = distribution('HyperKitty')
     django_mailman3 = distribution('django-mailman3')
+    mailman_hyperkitty = distribution('mailman-hyperkitty')
 
     incoming_path = os.path.join(hyperkitty.location, 'hyperkitty', 'lib', 'incoming.py')
     scrub_path = os.path.join(
         django_mailman3.location, 'django_mailman3', 'lib', 'scrub.py')
+    mailman_hyperkitty_path = os.path.join(
+        mailman_hyperkitty.location, 'mailman_hyperkitty', '__init__.py')
 
     incoming_source, incoming_changed = patch_hyperkitty(load_source(incoming_path))
     scrub_source, scrub_changed = patch_scrubber(load_source(scrub_path))
+    mailman_hyperkitty_source, mailman_hyperkitty_changed = (
+        patch_mailman_hyperkitty(load_source(mailman_hyperkitty_path)))
 
-    # Compile both transformed files before replacing either one.
+    # Compile all transformed files before replacing any one of them.
     compile(incoming_source, incoming_path, 'exec')
     compile(scrub_source, scrub_path, 'exec')
+    compile(mailman_hyperkitty_source, mailman_hyperkitty_path, 'exec')
 
     try:
         import django_extensions
@@ -203,12 +286,16 @@ def main():
     if scrub_changed:
         atomic_write(scrub_path, scrub_source)
         print('CHANGED: patched {}'.format(scrub_path))
+    if mailman_hyperkitty_changed:
+        atomic_write(mailman_hyperkitty_path, mailman_hyperkitty_source)
+        print('CHANGED: patched {}'.format(mailman_hyperkitty_path))
     for catalog_path, error in catalogs:
         destination = quarantine_catalog(catalog_path)
         print('CHANGED: quarantined {} as {} ({})'.format(
             catalog_path, destination, error))
 
-    if not incoming_changed and not scrub_changed and not catalogs:
+    if (not incoming_changed and not scrub_changed and
+            not mailman_hyperkitty_changed and not catalogs):
         print('OK: compatibility fixes are already applied')