Temporary fixes for bugs in hyperkitty.

README.md

@@ -51,3 +51,27 @@ verifies the real stopped state and does not start a second instance until the
 master and LMTP listener are gone.
 The normal role execution also waits for the configured LMTP port and fails if
 the listener does not become available after starting or restarting Mailman.
+
+## Legacy HyperKitty compatibility fixes
+
+The optional compatibility patcher addresses three failures observed with the
+legacy Python 3.6 web stack: malformed `From`/`Subject` headers, nested
+`message/rfc822` parts without `Content-Transfer-Encoding`, and unreadable
+gettext catalogs. Enable it only for the known package versions:
+
+```yaml
+mailman_enable_legacy_hyperkitty_compatibility_patches: true
+mailman_legacy_hyperkitty_version: '1.3.3'
+mailman_legacy_django_mailman3_version: '1.3.4'
+```
+
+The Python source patches are idempotent, compiled before any file is replaced,
+and installed atomically. The task fails without changing either source file if
+the installed versions or expected source fragments do not match. Invalid
+`django_extensions` `.mo` files are preserved beside the original name with a
+`.disabled-by-ansible` suffix, allowing Django to fall back to another locale.
+The uWSGI service is restarted only when a source file or catalog actually
+changes.
+
+Disable this option before upgrading HyperKitty or django-mailman3, then review
+whether the compatibility patches are still needed with the new versions.

defaults/main.yml

@@ -44,6 +44,13 @@ mailman_service_stop_timeout: 30
 mailman_weekly_verified_restart_stop_timeout: 120
 mailman_weekly_verified_restart_start_timeout: 120
 
+# Compatibility fixes for the legacy HyperKitty stack. These are deliberately
+# opt-in and pinned: a package upgrade must be reviewed before patching sources.
+mailman_enable_legacy_hyperkitty_compatibility_patches: false
+mailman_legacy_hyperkitty_version: '1.3.3'
+mailman_legacy_django_mailman3_version: '1.3.4'
+mailman_legacy_hyperkitty_patch_script: '/usr/local/sbin/mailman-legacy-hyperkitty-compatibility'
+
   # Documentation that must be followed to configure the social auth providers
 # https://django-allauth.readthedocs.io/en/latest/installation.html
 mailman_use_social_account_providers: False

tasks/mailman.yml

@@ -52,6 +52,29 @@
           editable: no
       with_items: '{{ mailman_pip_packages }}'
 
+    - name: Install the legacy HyperKitty compatibility patcher
+      template:
+        src: mailman-legacy-hyperkitty-compatibility.py.j2
+        dest: '{{ mailman_legacy_hyperkitty_patch_script }}'
+        owner: root
+        group: root
+        mode: '0750'
+      when: mailman_enable_legacy_hyperkitty_compatibility_patches | bool
+      tags: [ 'mailman', 'mailman_conf', 'mailman_hyperkitty_compatibility' ]
+
+    - name: Apply the legacy HyperKitty compatibility patches
+      command:
+        argv:
+          - '{{ mailman_bindir }}/python'
+          - '{{ mailman_legacy_hyperkitty_patch_script }}'
+      register: mailman_legacy_hyperkitty_patch_result
+      changed_when: "'CHANGED:' in mailman_legacy_hyperkitty_patch_result.stdout"
+      when:
+        - mailman_enable_legacy_hyperkitty_compatibility_patches | bool
+        - not ansible_check_mode
+      notify: Restart mailmansuite
+      tags: [ 'mailman', 'mailman_conf', 'mailman_hyperkitty_compatibility' ]
+
     - name: Install the mailman and postfix configuration files
       template: src={{ item }}.j2 dest={{ mailman_conf_dir }}/{{ item }} owner=root group={{ mailman_user }} mode=0440
       with_items:

templates/mailman-legacy-hyperkitty-compatibility.py.j2

@@ -0,0 +1,220 @@
+#!/usr/bin/env python3
+"""Apply narrowly scoped compatibility fixes to the legacy Mailman web stack."""
+
+from __future__ import print_function
+
+import gettext
+import glob
+import hashlib
+import os
+import stat
+import struct
+import sys
+import tempfile
+
+import pkg_resources
+
+
+EXPECTED_VERSIONS = {
+    'HyperKitty': {{ mailman_legacy_hyperkitty_version | to_json }},
+    'django-mailman3': {{ mailman_legacy_django_mailman3_version | to_json }},
+}
+
+HYPERKITTY_IMPORT_OLD = "import re\nfrom email.message import EmailMessage\n"
+HYPERKITTY_IMPORT_NEW = (
+    "import re\n"
+    "from email.header import decode_header, make_header\n"
+    "from email.message import EmailMessage\n"
+)
+HYPERKITTY_CLASS_MARKER = "class DuplicateMessage(Exception):\n"
+HYPERKITTY_HELPER = '''def _header_to_unicode_safely(message, name):
+    """Decode a header even when Python's structured parser rejects it."""
+    try:
+        return header_to_unicode(message[name])
+    except (AttributeError, IndexError, TypeError, ValueError):
+        raw_value = next(
+            (value for key, value in message.raw_items()
+             if key.lower() == name.lower()),
+            None)
+        if raw_value is None:
+            return None
+        try:
+            return str(make_header(decode_header(raw_value)))
+        except (LookupError, UnicodeError):
+            return raw_value
+
+
+'''
+HYPERKITTY_FROM_OLD = "from_str = header_to_unicode(message['From'])"
+HYPERKITTY_FROM_NEW = "from_str = _header_to_unicode_safely(message, 'From')"
+HYPERKITTY_SUBJECT_OLD = "email.subject = header_to_unicode(message.get('Subject'))"
+HYPERKITTY_SUBJECT_NEW = "email.subject = _header_to_unicode_safely(message, 'Subject')"
+
+SCRUB_OLD = '''        if ctype == 'message/rfc822':
+            # Return message/rfc822 parts as a string.
+            decodedpayload = str(payload)
+'''
+SCRUB_NEW = '''        if ctype == 'message/rfc822':
+            # Python 3.6 can fail to serialize malformed nested messages that
+            # do not declare a Content-Transfer-Encoding header.
+            try:
+                decodedpayload = str(payload)
+            except KeyError as error:
+                if error.args != ('content-transfer-encoding',):
+                    raise
+                payload['Content-Transfer-Encoding'] = '8bit'
+                decodedpayload = str(payload)
+'''
+
+
+def distribution(name):
+    dist = pkg_resources.get_distribution(name)
+    expected = EXPECTED_VERSIONS[name]
+    if dist.version != expected:
+        raise RuntimeError(
+            '{} {} is installed; compatibility patch requires {}'.format(
+                name, dist.version, expected))
+    return dist
+
+
+def replace_once(source, old, new, description):
+    count = source.count(old)
+    if count != 1:
+        raise RuntimeError(
+            '{}: expected exactly one unpatched source fragment, found {}'.format(
+                description, count))
+    return source.replace(old, new, 1)
+
+
+def patch_hyperkitty(source):
+    patched = HYPERKITTY_HELPER in source
+    completed = (
+        HYPERKITTY_IMPORT_NEW in source and
+        HYPERKITTY_FROM_NEW in source and
+        HYPERKITTY_SUBJECT_NEW in source and
+        HYPERKITTY_IMPORT_OLD not in source and
+        HYPERKITTY_FROM_OLD not in source and
+        HYPERKITTY_SUBJECT_OLD not in source
+    )
+    if patched:
+        if not completed:
+            raise RuntimeError('HyperKitty compatibility patch is incomplete')
+        return source, False
+
+    updated = replace_once(
+        source, HYPERKITTY_IMPORT_OLD, HYPERKITTY_IMPORT_NEW,
+        'HyperKitty imports')
+    updated = replace_once(
+        updated, HYPERKITTY_CLASS_MARKER,
+        HYPERKITTY_HELPER + HYPERKITTY_CLASS_MARKER,
+        'HyperKitty helper insertion point')
+    updated = replace_once(
+        updated, HYPERKITTY_FROM_OLD, HYPERKITTY_FROM_NEW,
+        'HyperKitty From handling')
+    updated = replace_once(
+        updated, HYPERKITTY_SUBJECT_OLD, HYPERKITTY_SUBJECT_NEW,
+        'HyperKitty Subject handling')
+    return updated, True
+
+
+def patch_scrubber(source):
+    if SCRUB_NEW in source:
+        if SCRUB_OLD in source:
+            raise RuntimeError('django-mailman3 compatibility patch is ambiguous')
+        return source, False
+    return replace_once(
+        source, SCRUB_OLD, SCRUB_NEW,
+        'django-mailman3 message/rfc822 handling'), True
+
+
+def load_source(path):
+    with open(path, 'rb') as source_file:
+        return source_file.read().decode('utf-8')
+
+
+def atomic_write(path, content):
+    current = os.stat(path)
+    directory = os.path.dirname(path)
+    descriptor, temporary = tempfile.mkstemp(prefix='.mailman-patch-', dir=directory)
+    try:
+        with os.fdopen(descriptor, 'wb') as output:
+            output.write(content.encode('utf-8'))
+            output.flush()
+            os.fsync(output.fileno())
+        os.chmod(temporary, stat.S_IMODE(current.st_mode))
+        os.chown(temporary, current.st_uid, current.st_gid)
+        os.replace(temporary, path)
+    finally:
+        if os.path.exists(temporary):
+            os.unlink(temporary)
+
+
+def invalid_catalogs(django_extensions_dir):
+    invalid = []
+    pattern = os.path.join(django_extensions_dir, 'locale', '**', '*.mo')
+    for path in glob.iglob(pattern, recursive=True):
+        try:
+            with open(path, 'rb') as catalog:
+                gettext.GNUTranslations(catalog)
+        except (EOFError, OSError, UnicodeError, struct.error, ValueError) as error:
+            invalid.append((path, error))
+    return invalid
+
+
+def quarantine_catalog(path):
+    with open(path, 'rb') as catalog:
+        digest = hashlib.sha256(catalog.read()).hexdigest()[:12]
+    destination = path + '.disabled-by-ansible'
+    if os.path.exists(destination):
+        destination = destination + '.' + digest
+    if os.path.exists(destination):
+        raise RuntimeError(
+            'invalid gettext catalog is still active and already quarantined: '
+            + path)
+    os.rename(path, destination)
+    return destination
+
+
+def main():
+    hyperkitty = distribution('HyperKitty')
+    django_mailman3 = distribution('django-mailman3')
+
+    incoming_path = os.path.join(hyperkitty.location, 'hyperkitty', 'lib', 'incoming.py')
+    scrub_path = os.path.join(
+        django_mailman3.location, 'django_mailman3', 'lib', 'scrub.py')
+
+    incoming_source, incoming_changed = patch_hyperkitty(load_source(incoming_path))
+    scrub_source, scrub_changed = patch_scrubber(load_source(scrub_path))
+
+    # Compile both transformed files before replacing either one.
+    compile(incoming_source, incoming_path, 'exec')
+    compile(scrub_source, scrub_path, 'exec')
+
+    try:
+        import django_extensions
+    except ImportError:
+        django_extensions = None
+    catalogs = [] if django_extensions is None else invalid_catalogs(
+        os.path.dirname(django_extensions.__file__))
+
+    if incoming_changed:
+        atomic_write(incoming_path, incoming_source)
+        print('CHANGED: patched {}'.format(incoming_path))
+    if scrub_changed:
+        atomic_write(scrub_path, scrub_source)
+        print('CHANGED: patched {}'.format(scrub_path))
+    for catalog_path, error in catalogs:
+        destination = quarantine_catalog(catalog_path)
+        print('CHANGED: quarantined {} as {} ({})'.format(
+            catalog_path, destination, error))
+
+    if not incoming_changed and not scrub_changed and not catalogs:
+        print('OK: compatibility fixes are already applied')
+
+
+if __name__ == '__main__':
+    try:
+        main()
+    except Exception as error:
+        print('ERROR: {}'.format(error), file=sys.stderr)
+        sys.exit(1)