From f2eecf213e780125e212cddbbd22b940d6fcb0a1 Mon Sep 17 00:00:00 2001 From: Andrea Dell'Amico Date: Tue, 25 Aug 2026 18:24:32 +0200 Subject: [PATCH] New set of workarounds for hyperkitty. --- README.md | 18 ++- defaults/main.yml | 1 + handlers/main.yml | 9 ++ tasks/mailman.yml | 4 +- ...lman-legacy-hyperkitty-compatibility.py.j2 | 119 +++++++++++++++--- 5 files changed, 128 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index b2b181f..5fe9678 100644 --- a/README.md +++ b/README.md @@ -57,21 +57,27 @@ the listener does not become available after starting or restarting Mailman. The optional compatibility patcher addresses three failures observed with the legacy Python 3.6 web stack: malformed `From`/`Subject` headers, nested `message/rfc822` parts without `Content-Transfer-Encoding`, and unreadable -gettext catalogs. Enable it only for the known package versions: +gettext catalogs. It also normalizes non-ASCII sender addresses and backports +the byte-oriented delivery used by newer `mailman-hyperkitty` releases, so +surrogate-escaped message bytes do not pass through Requests as Unicode text. +Enable it only for the known package versions: ```yaml mailman_enable_legacy_hyperkitty_compatibility_patches: true mailman_legacy_hyperkitty_version: '1.3.3' mailman_legacy_django_mailman3_version: '1.3.4' +mailman_legacy_mailman_hyperkitty_version: '1.1.0' ``` The Python source patches are idempotent, compiled before any file is replaced, -and installed atomically. The task fails without changing either source file if +and installed atomically. The task fails without changing any source file if the installed versions or expected source fragments do not match. Invalid `django_extensions` `.mo` files are preserved beside the original name with a `.disabled-by-ansible` suffix, allowing Django to fall back to another locale. -The uWSGI service is restarted only when a source file or catalog actually -changes. +The uWSGI and Mailman services are restarted only when a source file or catalog +actually changes. After restarting Mailman, the handler requires the configured +LMTP listener to become available. -Disable this option before upgrading HyperKitty or django-mailman3, then review -whether the compatibility patches are still needed with the new versions. +Disable this option before upgrading HyperKitty, django-mailman3, or +mailman-hyperkitty, then review whether the compatibility patches are still +needed with the new versions. diff --git a/defaults/main.yml b/defaults/main.yml index 40f9114..e7d9235 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -49,6 +49,7 @@ mailman_weekly_verified_restart_start_timeout: 120 mailman_enable_legacy_hyperkitty_compatibility_patches: false mailman_legacy_hyperkitty_version: '1.3.3' mailman_legacy_django_mailman3_version: '1.3.4' +mailman_legacy_mailman_hyperkitty_version: '1.1.0' mailman_legacy_hyperkitty_patch_script: '/usr/local/sbin/mailman-legacy-hyperkitty-compatibility' # Documentation that must be followed to configure the social auth providers diff --git a/handlers/main.yml b/handlers/main.yml index 1fe40b1..16f0372 100644 --- a/handlers/main.yml +++ b/handlers/main.yml @@ -4,3 +4,12 @@ - name: Restart mailman service: name=mailman state=restarted + +- name: Wait for Mailman after restart + ansible.builtin.wait_for: + host: '{{ mailman_lmtp_host }}' + port: '{{ mailman_lmtp_port }}' + state: started + sleep: 1 + timeout: '{{ mailman_weekly_verified_restart_start_timeout }}' + listen: Restart mailman diff --git a/tasks/mailman.yml b/tasks/mailman.yml index cc7050c..ba28cbe 100644 --- a/tasks/mailman.yml +++ b/tasks/mailman.yml @@ -72,7 +72,9 @@ when: - mailman_enable_legacy_hyperkitty_compatibility_patches | bool - not ansible_check_mode - notify: Restart mailmansuite + notify: + - Restart mailmansuite + - Restart mailman tags: [ 'mailman', 'mailman_conf', 'mailman_hyperkitty_compatibility' ] - name: Install the mailman and postfix configuration files diff --git a/templates/mailman-legacy-hyperkitty-compatibility.py.j2 b/templates/mailman-legacy-hyperkitty-compatibility.py.j2 index acda7b6..a355932 100644 --- a/templates/mailman-legacy-hyperkitty-compatibility.py.j2 +++ b/templates/mailman-legacy-hyperkitty-compatibility.py.j2 @@ -18,16 +18,23 @@ import pkg_resources EXPECTED_VERSIONS = { 'HyperKitty': {{ mailman_legacy_hyperkitty_version | to_json }}, 'django-mailman3': {{ mailman_legacy_django_mailman3_version | to_json }}, + 'mailman-hyperkitty': {{ mailman_legacy_mailman_hyperkitty_version | to_json }}, } HYPERKITTY_IMPORT_OLD = "import re\nfrom email.message import EmailMessage\n" +HYPERKITTY_IMPORT_V1 = ( + "import re\n" + "from email.header import decode_header, make_header\n" + "from email.message import EmailMessage\n" +) HYPERKITTY_IMPORT_NEW = ( + "import hashlib\n" "import re\n" "from email.header import decode_header, make_header\n" "from email.message import EmailMessage\n" ) HYPERKITTY_CLASS_MARKER = "class DuplicateMessage(Exception):\n" -HYPERKITTY_HELPER = '''def _header_to_unicode_safely(message, name): +HYPERKITTY_HELPER_V1 = '''def _header_to_unicode_safely(message, name): """Decode a header even when Python's structured parser rejects it.""" try: return header_to_unicode(message[name]) @@ -44,11 +51,51 @@ HYPERKITTY_HELPER = '''def _header_to_unicode_safely(message, name): return raw_value +''' +HYPERKITTY_HELPER = HYPERKITTY_HELPER_V1 + '''def _ascii_sender_address_safely(address, original_header): + """Return an ASCII sender key without rejecting the whole message.""" + address = address.strip() + if not address: + return '' + try: + return address.encode('ascii').decode('ascii') + except UnicodeEncodeError: + pass + + local_part, separator, domain = address.rpartition('@') + if separator: + try: + local_part = local_part.encode('ascii').decode('ascii') + domain = domain.encode('idna').decode('ascii') + return '{}@{}'.format(local_part, domain) + except UnicodeError: + pass + + if not isinstance(original_header, str): + original_header = repr(original_header) + digest = hashlib.sha256( + original_header.encode('utf-8', 'surrogatepass')).hexdigest()[:16] + return 'unknown-{}@example.invalid'.format(digest) + + ''' HYPERKITTY_FROM_OLD = "from_str = header_to_unicode(message['From'])" HYPERKITTY_FROM_NEW = "from_str = _header_to_unicode_safely(message, 'From')" HYPERKITTY_SUBJECT_OLD = "email.subject = header_to_unicode(message.get('Subject'))" HYPERKITTY_SUBJECT_NEW = "email.subject = _header_to_unicode_safely(message, 'Subject')" +HYPERKITTY_SENDER_OLD = ''' try: + from_str = _header_to_unicode_safely(message, 'From') + from_name, from_email = parseaddr(from_str) + from_name = from_name.strip() + sender_address = from_email.encode('ascii').decode("ascii").strip() + except (UnicodeDecodeError, UnicodeEncodeError): + raise ValueError("Non-ascii sender address", message) +''' +HYPERKITTY_SENDER_NEW = ''' from_str = _header_to_unicode_safely(message, 'From') + from_name, from_email = parseaddr(from_str) + from_name = from_name.strip() + sender_address = _ascii_sender_address_safely(from_email, from_str) +''' SCRUB_OLD = ''' if ctype == 'message/rfc822': # Return message/rfc822 parts as a string. @@ -66,6 +113,13 @@ SCRUB_NEW = ''' if ctype == 'message/rfc822': decodedpayload = str(payload) ''' +MAILMAN_HYPERKITTY_SEND_OLD = ''' message_text = msg.as_string() + except (MessageError, KeyError) as error: +''' +MAILMAN_HYPERKITTY_SEND_NEW = ''' message_text = msg.as_bytes() + except (MessageError, KeyError, UnicodeEncodeError) as error: +''' + def distribution(name): dist = pkg_resources.get_distribution(name) @@ -92,28 +146,41 @@ def patch_hyperkitty(source): HYPERKITTY_IMPORT_NEW in source and HYPERKITTY_FROM_NEW in source and HYPERKITTY_SUBJECT_NEW in source and + HYPERKITTY_SENDER_NEW in source and HYPERKITTY_IMPORT_OLD not in source and HYPERKITTY_FROM_OLD not in source and - HYPERKITTY_SUBJECT_OLD not in source + HYPERKITTY_SUBJECT_OLD not in source and + HYPERKITTY_SENDER_OLD not in source ) if patched: if not completed: raise RuntimeError('HyperKitty compatibility patch is incomplete') return source, False + if HYPERKITTY_HELPER_V1 in source: + updated = replace_once( + source, HYPERKITTY_IMPORT_V1, HYPERKITTY_IMPORT_NEW, + 'HyperKitty v1 imports') + updated = replace_once( + updated, HYPERKITTY_HELPER_V1, HYPERKITTY_HELPER, + 'HyperKitty v1 helper') + else: + updated = replace_once( + source, HYPERKITTY_IMPORT_OLD, HYPERKITTY_IMPORT_NEW, + 'HyperKitty imports') + updated = replace_once( + updated, HYPERKITTY_CLASS_MARKER, + HYPERKITTY_HELPER + HYPERKITTY_CLASS_MARKER, + 'HyperKitty helper insertion point') + updated = replace_once( + updated, HYPERKITTY_FROM_OLD, HYPERKITTY_FROM_NEW, + 'HyperKitty From handling') + updated = replace_once( + updated, HYPERKITTY_SUBJECT_OLD, HYPERKITTY_SUBJECT_NEW, + 'HyperKitty Subject handling') updated = replace_once( - source, HYPERKITTY_IMPORT_OLD, HYPERKITTY_IMPORT_NEW, - 'HyperKitty imports') - updated = replace_once( - updated, HYPERKITTY_CLASS_MARKER, - HYPERKITTY_HELPER + HYPERKITTY_CLASS_MARKER, - 'HyperKitty helper insertion point') - updated = replace_once( - updated, HYPERKITTY_FROM_OLD, HYPERKITTY_FROM_NEW, - 'HyperKitty From handling') - updated = replace_once( - updated, HYPERKITTY_SUBJECT_OLD, HYPERKITTY_SUBJECT_NEW, - 'HyperKitty Subject handling') + updated, HYPERKITTY_SENDER_OLD, HYPERKITTY_SENDER_NEW, + 'HyperKitty sender address handling') return updated, True @@ -127,6 +194,16 @@ def patch_scrubber(source): 'django-mailman3 message/rfc822 handling'), True +def patch_mailman_hyperkitty(source): + if MAILMAN_HYPERKITTY_SEND_NEW in source: + if MAILMAN_HYPERKITTY_SEND_OLD in source: + raise RuntimeError('mailman-hyperkitty compatibility patch is ambiguous') + return source, False + return replace_once( + source, MAILMAN_HYPERKITTY_SEND_OLD, MAILMAN_HYPERKITTY_SEND_NEW, + 'mailman-hyperkitty byte delivery'), True + + def load_source(path): with open(path, 'rb') as source_file: return source_file.read().decode('utf-8') @@ -178,17 +255,23 @@ def quarantine_catalog(path): def main(): hyperkitty = distribution('HyperKitty') django_mailman3 = distribution('django-mailman3') + mailman_hyperkitty = distribution('mailman-hyperkitty') incoming_path = os.path.join(hyperkitty.location, 'hyperkitty', 'lib', 'incoming.py') scrub_path = os.path.join( django_mailman3.location, 'django_mailman3', 'lib', 'scrub.py') + mailman_hyperkitty_path = os.path.join( + mailman_hyperkitty.location, 'mailman_hyperkitty', '__init__.py') incoming_source, incoming_changed = patch_hyperkitty(load_source(incoming_path)) scrub_source, scrub_changed = patch_scrubber(load_source(scrub_path)) + mailman_hyperkitty_source, mailman_hyperkitty_changed = ( + patch_mailman_hyperkitty(load_source(mailman_hyperkitty_path))) - # Compile both transformed files before replacing either one. + # Compile all transformed files before replacing any one of them. compile(incoming_source, incoming_path, 'exec') compile(scrub_source, scrub_path, 'exec') + compile(mailman_hyperkitty_source, mailman_hyperkitty_path, 'exec') try: import django_extensions @@ -203,12 +286,16 @@ def main(): if scrub_changed: atomic_write(scrub_path, scrub_source) print('CHANGED: patched {}'.format(scrub_path)) + if mailman_hyperkitty_changed: + atomic_write(mailman_hyperkitty_path, mailman_hyperkitty_source) + print('CHANGED: patched {}'.format(mailman_hyperkitty_path)) for catalog_path, error in catalogs: destination = quarantine_catalog(catalog_path) print('CHANGED: quarantined {} as {} ({})'.format( catalog_path, destination, error)) - if not incoming_changed and not scrub_changed and not catalogs: + if (not incoming_changed and not scrub_changed and + not mailman_hyperkitty_changed and not catalogs): print('OK: compatibility fixes are already applied')