#!{{ mailman_bindir }}/python3 """Delete HyperKitty archives that Mailman Core no longer archives.""" import argparse import fcntl import logging import os import sys from urllib.error import HTTPError PROJECT_DIR = {{ mailman_postorius_dir | to_json }} LOCK_FILE = {{ mailman_hyperkitty_archive_cleanup_lock | to_json }} DELETE_DELETED_LISTS = {{ mailman_hyperkitty_archive_cleanup_deleted_lists | bool | ternary('True', 'False') }} DELETE_DISABLED_LISTS = {{ mailman_hyperkitty_archive_cleanup_disabled_lists | bool | ternary('True', 'False') }} MAX_DELETIONS = {{ mailman_hyperkitty_archive_cleanup_max_deletions | int }} # HyperKitty 1.3.3 itself uses pages of 10 with the legacy Core client. PAGE_SIZE = 10 logging.basicConfig( level=logging.INFO, format='hyperkitty-archive-cleanup: %(levelname)s: %(message)s') LOG = logging.getLogger(__name__) def normalize_name(value): return value.strip().casefold() def normalize_policy(value): if hasattr(value, 'name'): value = value.name if isinstance(value, int): return {0: 'never', 1: 'private', 2: 'public'}.get(value, str(value)) return str(value).strip().lower() def get_archive_policy(core_list): try: value = core_list.archive_policy except AttributeError: value = core_list.settings['archive_policy'] return normalize_policy(value) def load_core_lists(client): """Fetch a complete Core snapshot before deciding what may be deleted.""" result = {} page_number = 1 while True: page = list(client.get_list_page(count=PAGE_SIZE, page=page_number)) if not page: break previous_count = len(result) for core_list in page: result[normalize_name(core_list.fqdn_listname)] = get_archive_policy(core_list) if len(result) == previous_count: raise RuntimeError( 'Mailman Core pagination returned no new lists on page {}'.format( page_number)) if len(page) < PAGE_SIZE: break page_number += 1 if not result: raise RuntimeError( 'Mailman Core returned no lists; refusing to consider every archive orphaned') return result def recheck_candidate(client, name): """Return the current Core policy, or None only for a confirmed 404.""" try: core_list = client.get_list(name) except HTTPError as error: if error.code == 404: return None raise if core_list is None: return None return get_archive_policy(core_list) def find_candidates(core_lists, archive_lists): candidates = [] for archive in archive_lists: policy = core_lists.get(normalize_name(archive.name)) if policy is None and DELETE_DELETED_LISTS: candidates.append((archive, 'deleted from Mailman Core')) elif policy == 'never' and DELETE_DISABLED_LISTS: candidates.append((archive, 'archive policy is never')) return candidates def confirm_candidates(client, candidates): confirmed = [] for archive, original_reason in candidates: current_policy = recheck_candidate(client, archive.name) if current_policy is None and DELETE_DELETED_LISTS: confirmed.append((archive, 'deleted from Mailman Core')) elif current_policy == 'never' and DELETE_DISABLED_LISTS: confirmed.append((archive, 'archive policy is never')) else: LOG.warning( 'skipping %s: Core changed since the initial snapshot (%s)', archive.name, original_reason) return confirmed def delete_archives(candidates, dry_run): from django.core.cache import cache from django.db import transaction from django.db.models.signals import post_delete, pre_delete from hyperkitty.models import Email, MailingList, Thread, Vote from hyperkitty.signals import ( Email_on_post_delete, Email_on_pre_delete, Thread_on_post_delete, Vote_on_post_delete, ) signal_bindings = ( (pre_delete, Email_on_pre_delete, Email), (post_delete, Email_on_post_delete, Email), (post_delete, Thread_on_post_delete, Thread), (post_delete, Vote_on_post_delete, Vote), ) if dry_run: for archive, reason in candidates: LOG.info( 'would delete %s (%s; %d threads, %d messages)', archive.name, reason, archive.threads.count(), archive.emails.count()) return 0 disconnected = [] try: # Bulk archive deletion must not enqueue cache rebuilds for objects that # cease to exist before django-q executes the tasks (HyperKitty #440). for signal, receiver, sender in signal_bindings: if signal.disconnect(receiver, sender=sender): disconnected.append((signal, receiver, sender)) deleted = 0 for archive, reason in candidates: with transaction.atomic(): try: locked_archive = MailingList.objects.select_for_update().get( pk=archive.pk) except MailingList.DoesNotExist: LOG.warning('archive %s disappeared before deletion', archive.name) continue thread_count = locked_archive.threads.count() message_count = locked_archive.emails.count() # Match HyperKitty's administrative delete view: deleting # threads explicitly also works around backend FK constraints. locked_archive.threads.all().delete() locked_archive.delete() deleted += 1 LOG.info( 'deleted %s (%s; %d threads, %d messages)', archive.name, reason, thread_count, message_count) finally: for signal, receiver, sender in disconnected: signal.connect(receiver, sender=sender) if deleted: cache.clear() return deleted def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( '--dry-run', action='store_true', help='report candidates without deleting archives') arguments = parser.parse_args() os.chdir(PROJECT_DIR) sys.path.insert(0, PROJECT_DIR) os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings') import django django.setup() from django_mailman3.lib.mailman import get_mailman_client from hyperkitty.models import MailingList from mailmanclient import MailmanConnectionError try: with open(LOCK_FILE, 'w') as lock_handle: try: fcntl.flock(lock_handle, fcntl.LOCK_EX | fcntl.LOCK_NB) except OSError: LOG.error('another cleanup process holds %s', LOCK_FILE) return 1 try: client = get_mailman_client() core_lists = load_core_lists(client) archives = list(MailingList.objects.all().order_by('name')) candidates = find_candidates(core_lists, archives) candidates = confirm_candidates(client, candidates) except (HTTPError, MailmanConnectionError, RuntimeError) as error: LOG.error('cannot obtain a reliable Mailman Core view: %s', error) return 1 if MAX_DELETIONS > 0 and len(candidates) > MAX_DELETIONS: LOG.error( 'refusing to delete %d archives; safety limit is %d', len(candidates), MAX_DELETIONS) for archive, reason in candidates: LOG.error('candidate %s (%s)', archive.name, reason) return 1 deleted = delete_archives(candidates, arguments.dry_run) LOG.info( '%d Core lists, %d HyperKitty archives, %d candidates, %d deleted', len(core_lists), len(archives), len(candidates), deleted) return 0 except OSError as error: LOG.error('cannot use cleanup lock %s: %s', LOCK_FILE, error) return 1 if __name__ == '__main__': sys.exit(main())