mailman/templates/mailman-legacy-hyperkitty-compatibility.py.j2

#!/usr/bin/env python3
"""Apply narrowly scoped compatibility fixes to the legacy Mailman web stack."""

from __future__ import print_function

import gettext
import glob
import hashlib
import os
import stat
import struct
import sys
import tempfile

import pkg_resources


EXPECTED_VERSIONS = {
    'HyperKitty': {{ mailman_legacy_hyperkitty_version | to_json }},
    'django-mailman3': {{ mailman_legacy_django_mailman3_version | to_json }},
    'mailman-hyperkitty': {{ mailman_legacy_mailman_hyperkitty_version | to_json }},
}

HYPERKITTY_IMPORT_OLD = "import re\nfrom email.message import EmailMessage\n"
HYPERKITTY_IMPORT_V1 = (
    "import re\n"
    "from email.header import decode_header, make_header\n"
    "from email.message import EmailMessage\n"
)
HYPERKITTY_IMPORT_NEW = (
    "import hashlib\n"
    "import re\n"
    "from email.header import decode_header, make_header\n"
    "from email.message import EmailMessage\n"
)
HYPERKITTY_CLASS_MARKER = "class DuplicateMessage(Exception):\n"
HYPERKITTY_HELPER_V1 = '''def _header_to_unicode_safely(message, name):
    """Decode a header even when Python's structured parser rejects it."""
    try:
        return header_to_unicode(message[name])
    except (AttributeError, IndexError, TypeError, ValueError):
        raw_value = next(
            (value for key, value in message.raw_items()
             if key.lower() == name.lower()),
            None)
        if raw_value is None:
            return None
        try:
            return str(make_header(decode_header(raw_value)))
        except (LookupError, UnicodeError):
            return raw_value


'''
HYPERKITTY_HELPER = HYPERKITTY_HELPER_V1 + '''def _ascii_sender_address_safely(address, original_header):
    """Return an ASCII sender key without rejecting the whole message."""
    address = address.strip()
    if not address:
        return ''
    try:
        return address.encode('ascii').decode('ascii')
    except UnicodeEncodeError:
        pass

    local_part, separator, domain = address.rpartition('@')
    if separator:
        try:
            local_part = local_part.encode('ascii').decode('ascii')
            domain = domain.encode('idna').decode('ascii')
            return '{}@{}'.format(local_part, domain)
        except UnicodeError:
            pass

    if not isinstance(original_header, str):
        original_header = repr(original_header)
    digest = hashlib.sha256(
        original_header.encode('utf-8', 'surrogatepass')).hexdigest()[:16]
    return 'unknown-{}@example.invalid'.format(digest)


'''
HYPERKITTY_FROM_OLD = "from_str = header_to_unicode(message['From'])"
HYPERKITTY_FROM_NEW = "from_str = _header_to_unicode_safely(message, 'From')"
HYPERKITTY_SUBJECT_OLD = "email.subject = header_to_unicode(message.get('Subject'))"
HYPERKITTY_SUBJECT_NEW = "email.subject = _header_to_unicode_safely(message, 'Subject')"
HYPERKITTY_SENDER_OLD = '''    try:
        from_str = _header_to_unicode_safely(message, 'From')
        from_name, from_email = parseaddr(from_str)
        from_name = from_name.strip()
        sender_address = from_email.encode('ascii').decode("ascii").strip()
    except (UnicodeDecodeError, UnicodeEncodeError):
        raise ValueError("Non-ascii sender address", message)
'''
HYPERKITTY_SENDER_NEW = '''    from_str = _header_to_unicode_safely(message, 'From')
    from_name, from_email = parseaddr(from_str)
    from_name = from_name.strip()
    sender_address = _ascii_sender_address_safely(from_email, from_str)
'''

SCRUB_OLD = '''        if ctype == 'message/rfc822':
            # Return message/rfc822 parts as a string.
            decodedpayload = str(payload)
'''
SCRUB_NEW = '''        if ctype == 'message/rfc822':
            # Python 3.6 can fail to serialize malformed nested messages that
            # do not declare a Content-Transfer-Encoding header.
            try:
                decodedpayload = str(payload)
            except KeyError as error:
                if error.args != ('content-transfer-encoding',):
                    raise
                payload['Content-Transfer-Encoding'] = '8bit'
                decodedpayload = str(payload)
'''

SCRUB_NAMED_TEXT_ATTACHMENT_OLD = '''            if ctype == 'text/plain':
                if part.is_attachment():
                    attachments.append(self._parse_attachment(part, part_num))
                    part.set_content('\\n')
'''
SCRUB_NAMED_TEXT_ATTACHMENT_NEW = '''            if ctype == 'text/plain':
                filename = part.get_filename()
                if part.is_attachment() or filename:
                    # A missing or empty MIME type defaults to text/plain. If
                    # a filename is present, keep binary data out of the body.
                    raw_content_type = part.get('Content-Type', '')
                    declared_type = str(raw_content_type).split(';', 1)[0].strip()
                    if filename and not declared_type:
                        part.set_type('application/octet-stream')
                    attachments.append(self._parse_attachment(part, part_num))
                    part.set_content('\\n')
'''

SCRUB_NUL_ONE_PART_OLD = '''        if next_part_match:
            result = result[0:next_part_match.start(0)]
        return result
'''
SCRUB_NUL_ONE_PART_NEW = '''        if next_part_match:
            result = result[0:next_part_match.start(0)]
        # Backport from django-mailman3 1.3.6: PostgreSQL text fields cannot
        # contain NUL characters.
        return re.sub('\\x00', '', result)
'''

SCRUB_NUL_MULTIPART_OLD = '''            return '\\n'.join(text)
'''
SCRUB_NUL_MULTIPART_NEW = '''            # Backport from django-mailman3 1.3.6: PostgreSQL text fields
            # cannot contain NUL characters.
            return re.sub('\\x00', '', '\\n'.join(text))
'''

MAILMAN_HYPERKITTY_SEND_OLD = '''            message_text = msg.as_string()
        except (MessageError, KeyError) as error:
'''
MAILMAN_HYPERKITTY_SEND_NEW = '''            message_text = msg.as_bytes()
        except (MessageError, KeyError, UnicodeEncodeError) as error:
'''


def distribution(name):
    dist = pkg_resources.get_distribution(name)
    expected = EXPECTED_VERSIONS[name]
    if dist.version != expected:
        raise RuntimeError(
            '{} {} is installed; compatibility patch requires {}'.format(
                name, dist.version, expected))
    return dist


def replace_once(source, old, new, description):
    count = source.count(old)
    if count != 1:
        raise RuntimeError(
            '{}: expected exactly one unpatched source fragment, found {}'.format(
                description, count))
    return source.replace(old, new, 1)


def patch_hyperkitty(source):
    patched = HYPERKITTY_HELPER in source
    completed = (
        HYPERKITTY_IMPORT_NEW in source and
        HYPERKITTY_FROM_NEW in source and
        HYPERKITTY_SUBJECT_NEW in source and
        HYPERKITTY_SENDER_NEW in source and
        HYPERKITTY_IMPORT_OLD not in source and
        HYPERKITTY_FROM_OLD not in source and
        HYPERKITTY_SUBJECT_OLD not in source and
        HYPERKITTY_SENDER_OLD not in source
    )
    if patched:
        if not completed:
            raise RuntimeError('HyperKitty compatibility patch is incomplete')
        return source, False

    if HYPERKITTY_HELPER_V1 in source:
        updated = replace_once(
            source, HYPERKITTY_IMPORT_V1, HYPERKITTY_IMPORT_NEW,
            'HyperKitty v1 imports')
        updated = replace_once(
            updated, HYPERKITTY_HELPER_V1, HYPERKITTY_HELPER,
            'HyperKitty v1 helper')
    else:
        updated = replace_once(
            source, HYPERKITTY_IMPORT_OLD, HYPERKITTY_IMPORT_NEW,
            'HyperKitty imports')
        updated = replace_once(
            updated, HYPERKITTY_CLASS_MARKER,
            HYPERKITTY_HELPER + HYPERKITTY_CLASS_MARKER,
            'HyperKitty helper insertion point')
        updated = replace_once(
            updated, HYPERKITTY_FROM_OLD, HYPERKITTY_FROM_NEW,
            'HyperKitty From handling')
        updated = replace_once(
            updated, HYPERKITTY_SUBJECT_OLD, HYPERKITTY_SUBJECT_NEW,
            'HyperKitty Subject handling')
    updated = replace_once(
        updated, HYPERKITTY_SENDER_OLD, HYPERKITTY_SENDER_NEW,
        'HyperKitty sender address handling')
    return updated, True


def patch_scrubber(source):
    updated = source
    changed = False
    replacements = (
        (SCRUB_OLD, SCRUB_NEW, 'django-mailman3 message/rfc822 handling'),
        (SCRUB_NAMED_TEXT_ATTACHMENT_OLD, SCRUB_NAMED_TEXT_ATTACHMENT_NEW,
         'django-mailman3 named text attachment handling'),
        (SCRUB_NUL_ONE_PART_OLD, SCRUB_NUL_ONE_PART_NEW,
         'django-mailman3 single-part NUL handling'),
        (SCRUB_NUL_MULTIPART_OLD, SCRUB_NUL_MULTIPART_NEW,
         'django-mailman3 multipart NUL handling'),
    )
    for old, new, description in replacements:
        if new in updated:
            if old in updated:
                raise RuntimeError(
                    '{} compatibility patch is ambiguous'.format(description))
            continue
        updated = replace_once(updated, old, new, description)
        changed = True
    return updated, changed


def patch_mailman_hyperkitty(source):
    if MAILMAN_HYPERKITTY_SEND_NEW in source:
        if MAILMAN_HYPERKITTY_SEND_OLD in source:
            raise RuntimeError('mailman-hyperkitty compatibility patch is ambiguous')
        return source, False
    return replace_once(
        source, MAILMAN_HYPERKITTY_SEND_OLD, MAILMAN_HYPERKITTY_SEND_NEW,
        'mailman-hyperkitty byte delivery'), True


def load_source(path):
    with open(path, 'rb') as source_file:
        return source_file.read().decode('utf-8')


def atomic_write(path, content):
    current = os.stat(path)
    directory = os.path.dirname(path)
    descriptor, temporary = tempfile.mkstemp(prefix='.mailman-patch-', dir=directory)
    try:
        with os.fdopen(descriptor, 'wb') as output:
            output.write(content.encode('utf-8'))
            output.flush()
            os.fsync(output.fileno())
        os.chmod(temporary, stat.S_IMODE(current.st_mode))
        os.chown(temporary, current.st_uid, current.st_gid)
        os.replace(temporary, path)
    finally:
        if os.path.exists(temporary):
            os.unlink(temporary)


def invalid_catalogs(django_extensions_dir):
    invalid = []
    pattern = os.path.join(django_extensions_dir, 'locale', '**', '*.mo')
    for path in glob.iglob(pattern, recursive=True):
        try:
            with open(path, 'rb') as catalog:
                gettext.GNUTranslations(catalog)
        except (EOFError, OSError, UnicodeError, struct.error, ValueError) as error:
            invalid.append((path, error))
    return invalid


def quarantine_catalog(path):
    with open(path, 'rb') as catalog:
        digest = hashlib.sha256(catalog.read()).hexdigest()[:12]
    destination = path + '.disabled-by-ansible'
    if os.path.exists(destination):
        destination = destination + '.' + digest
    if os.path.exists(destination):
        raise RuntimeError(
            'invalid gettext catalog is still active and already quarantined: '
            + path)
    os.rename(path, destination)
    return destination


def main():
    hyperkitty = distribution('HyperKitty')
    django_mailman3 = distribution('django-mailman3')
    mailman_hyperkitty = distribution('mailman-hyperkitty')

    incoming_path = os.path.join(hyperkitty.location, 'hyperkitty', 'lib', 'incoming.py')
    scrub_path = os.path.join(
        django_mailman3.location, 'django_mailman3', 'lib', 'scrub.py')
    mailman_hyperkitty_path = os.path.join(
        mailman_hyperkitty.location, 'mailman_hyperkitty', '__init__.py')

    incoming_source, incoming_changed = patch_hyperkitty(load_source(incoming_path))
    scrub_source, scrub_changed = patch_scrubber(load_source(scrub_path))
    mailman_hyperkitty_source, mailman_hyperkitty_changed = (
        patch_mailman_hyperkitty(load_source(mailman_hyperkitty_path)))

    # Compile all transformed files before replacing any one of them.
    compile(incoming_source, incoming_path, 'exec')
    compile(scrub_source, scrub_path, 'exec')
    compile(mailman_hyperkitty_source, mailman_hyperkitty_path, 'exec')

    try:
        import django_extensions
    except ImportError:
        django_extensions = None
    catalogs = [] if django_extensions is None else invalid_catalogs(
        os.path.dirname(django_extensions.__file__))

    if incoming_changed:
        atomic_write(incoming_path, incoming_source)
        print('CHANGED: patched {}'.format(incoming_path))
    if scrub_changed:
        atomic_write(scrub_path, scrub_source)
        print('CHANGED: patched {}'.format(scrub_path))
    if mailman_hyperkitty_changed:
        atomic_write(mailman_hyperkitty_path, mailman_hyperkitty_source)
        print('CHANGED: patched {}'.format(mailman_hyperkitty_path))
    for catalog_path, error in catalogs:
        destination = quarantine_catalog(catalog_path)
        print('CHANGED: quarantined {} as {} ({})'.format(
            catalog_path, destination, error))

    if (not incoming_changed and not scrub_changed and
            not mailman_hyperkitty_changed and not catalogs):
        print('OK: compatibility fixes are already applied')


if __name__ == '__main__':
    try:
        main()
    except Exception as error:
        print('ERROR: {}'.format(error), file=sys.stderr)
        sys.exit(1)