Fix the uwsgi behaviour. Separate service for qcluster

defaults/main.yml

@@ -120,6 +120,9 @@ mailman_postorius_log_dest_dir: mailmansuite
 mailman_postorius_log_dir: '/{{ mailman_postorius_var_dir }}/{{ mailman_postorius_log_base_dir }}/{{ mailman_postorius_log_dest_dir }}'
 mailman_postorius_dir: '{{ mailman_home }}/mailman-suite/mailman-suite_project'
 mailman_postorius_http_port: 8000
+mailman_postorius_uwsgi_servicename: 'mailmansuite-uwsgi'
+mailman_postorius_uwsgi_config_file: /etc/mailmansuite-uwsgi.ini
+mailman_postorius_uwsgi_rundir: /run/mailmansuite
 # 'systemd_logger,logfile,python36'
 mailman_postorius_uwsgi_plugins: 'systemd_logger,python36'
 # 1 is the predefined one, that must be deleted
@@ -128,6 +131,8 @@ mailman_postorius_allowed_hosts:
   - 'localhost'
   - '{{ ansible_fqdn }}'
 mailman_postorius_hyperkitty_acl: "'127.0.0.1', '::1', '{{ ansible_default_ipv4.address }}'"
+# The qcluster async activity is started by the uwsgi service. No need of a separate one
+mailman_hyperkitty_qcluster_standalone_service: True
 
 mailman_postorius_settings_files:
   - { name: 'manage.py', perms: '0750' }

handlers/main.yml

@@ -1,6 +1,6 @@
 ---
-- name: Restart postorius
-  service: name=uwsgi state=restarted
+- name: Restart mailmansuite
+  service: name={{ mailman_postorius_uwsgi_servicename }} state=restarted
 
 - name: Restart mailman
   service: name=mailman state=restarted

tasks/postorius-hyperkitty.yml

@@ -6,6 +6,7 @@
         name: httpd_can_network_connect
         state: yes
         persistent: yes
+      when: ansible_distribution_file_variety == "RedHat"
 
     - name: Create the postorius log directory
       file: dest={{ item }} state=directory owner={{ mailman_user }} group={{ mailman_user }}
@@ -31,24 +32,12 @@
       with_items: '{{ mailman_postorius_settings_files }}'
       register: postorius_conf
 
-    - name: Install the UWSGI configuration
-      template: src=postorius_{{ item }}.j2 dest=/etc/{{ item }} mode=0640
-      with_items:
-        - 'uwsgi.ini'
-
     - name: Setup postorius and hyperkitty
       become_user: '{{ mailman_user }}'
       shell: cd '{{ mailman_postorius_dir }}' && mkdir -p locale  && {{ mailman_home }}/{{ mailman_virtualenv_name }}/bin/python3 manage.py migrate && {{ mailman_home }}/{{ mailman_virtualenv_name }}/bin/python3 manage.py compilemessages && {{ mailman_home }}/{{ mailman_virtualenv_name }}/bin/python3 manage.py collectstatic
       args:
         creates: '{{ mailman_postorius_dir }}/static/admin/js/actions.js'
 
-    - name: Ensure that the UWSGI postorius service is started and enabled
-      service: name=uwsgi state=started enabled=yes
-
-    - name: Restart the UWSGI service if needed
-      service: name=uwsgi state=restarted
-      when: postorius_conf is changed
-
     - name: Install the hyperkitty configuration file
       template: src=mailman-hyperkitty.cfg.j2 dest={{ mailman_conf_dir }}/mailman-hyperkitty.cfg owner=root group={{ mailman_user }} mode=0440
       when: mailman_use_hyperkitty_archiver | bool
@@ -56,6 +45,59 @@
 
   tags: [ 'mailman', 'postorius', 'hyperkitty', 'mailman_conf' ]
 
+- name: Manage the postorius and hyperkitty services
+  block:
+    - name: Create the postorius service rundir
+      file: dest={{ mailman_postorius_uwsgi_rundir }} state=directory owner={{ mailman_user }} group={{ mailman_user }} mode='0755'
+
+    - name: Create the tmpfile entry for the postorius run
+      template: src=rundir_mailmansuite.conf.j2 dest=/usr/lib/tmpfiles.d/mailmansuite.conf owner=root group=root mode=0644
+
+    - name: Install the UWSGI configuration
+      template: src=postorius_uwsgi.ini.j2 dest={{ mailman_postorius_uwsgi_config_file }} mode='0640'
+      notify: Restart mailmansuite
+
+    - name: Install the mailmansuite uwsgi startup unit
+      template: src=mailmansuite-uwsgi.service.systemd.j2 dest=/lib/systemd/system/mailmansuite-uwsgi.service owner=root group=root mode=0644
+      register: mailmansuite_unit_install
+
+    - name: Reload the systemd configuration
+      systemd: daemon_reload=yes
+      when: mailmansuite_unit_install is changed
+
+    - name: Ensure that the UWSGI postorius service is started and enabled
+      service: name={{ mailman_postorius_uwsgi_servicename }} state=started enabled=yes
+
+    - name: Restart the UWSGI service if needed
+      service: name={{ mailman_postorius_uwsgi_servicename }} state=restarted
+      when: postorius_conf is defined and postorius_conf is changed
+
+  tags: [ 'mailman', 'postorius', 'hyperkitty', 'mailman_conf', 'hyperkitty_async', 'mailmansuite_uwsgi' ]
+
+- name: Manage the hyperkitty async jobs
+  block:
+    - name: Install the hyperkitty qcluster startup unit
+      template: src=hyperkitty-qcluster.service.systemd.j2 dest=/lib/systemd/system/hyperkitty-qcluster.service owner=root group=root mode=0644
+      register: hyperkitty_unit_install
+
+    - name: Reload the systemd configuration
+      systemd: daemon_reload=yes
+      when: hyperkitty_unit_install is changed
+
+    - name: Ensure that the hyperkitty async jobs service is running and enabled
+      service: name=hyperkitty-qcluster state=started enabled=yes
+
+  when: mailman_hyperkitty_qcluster_standalone_service
+  tags: [ 'mailman', 'postorius', 'hyperkitty', 'mailman_conf', 'hyperkitty_async' ]
+
+- name: Manage the hyperkitty async jobs
+  block:
+    - name: Ensure that the hyperkitty async jobs service is stopped and disabled
+      service: name=hyperkitty-qcluster state=stopped enabled=no
+
+  when: not mailman_hyperkitty_qcluster_standalone_service
+  tags: [ 'mailman', 'postorius', 'hyperkitty', 'mailman_conf', 'hyperkitty_async' ]
+
 - name: Setup the postorius cron jobs
   block:
     - name: add a cron job that syncs the mailman core and postorius settings

templates/hyperkitty-qcluster.service.systemd.j2

@@ -0,0 +1,14 @@
+[Unit]
+Description=HyperKitty async tasks runner
+After=network.target remote-fs.target
+ConditionPathExists={{ mailman_postorius_dir }}/settings.py
+
+[Service]
+WorkingDirectory={{ mailman_postorius_dir }}
+User={{ mailman_user }}
+ExecStart={{ mailman_postorius_dir }}/manage.py qcluster
+Restart=always
+SyslogIdentifier=qcluster
+
+[Install]
+WantedBy=multi-user.target

templates/mailman-postorious-logrotate.j2

@@ -1,4 +1,4 @@
-{{ mailman_postorious_log_dir }}/*.log {
+{{ mailman_postorius_log_dir }}/*.log {
   copytruncate
   {{ mailman_logrotate_freq }}
   rotate {{ mailman_logrotate_retain }}

templates/mailman.service.systemd.j2

@@ -1,6 +1,6 @@
 [Unit]
 Description=Mailman 3 service
-After=network.target
+After=network.target remote-fs.target
 Documentation=https://mailman.readthedocs.io/
 ConditionPathExists={{ mailman_conf_dir }}/mailman.cfg
 

templates/mailmansuite-uwsgi.service.systemd.j2

@@ -0,0 +1,16 @@
+[Unit]
+Description=uWSGI Emperor Service for postorius and HyperKitty
+After=syslog.target
+
+[Service]
+ExecStart=/usr/sbin/uwsgi --ini {{ mailman_postorius_uwsgi_config_file }} --pidfile {{ mailman_postorius_uwsgi_rundir }}/uwsgi.pid --stats {{ mailman_postorius_uwsgi_rundir }}/stats.sock
+ExecReload=/bin/kill -HUP $MAINPID
+KillSignal=SIGINT
+Restart=always
+Type=notify
+StandardError=syslog
+NotifyAccess=all
+SyslogIdentifier=mailmansuite
+
+[Install]
+WantedBy=multi-user.target

templates/postorius_settings.py.j2

@@ -42,12 +42,12 @@ ADMINS = (
 {% endfor %}
 )
 
-SITE_ID = {{ mailman_postorious_site_id }}
+SITE_ID = {{ mailman_postorius_site_id }}
 
 # Hosts/domain names that are valid for this site; required if DEBUG is False
 # See https://docs.djangoproject.com/en/1.8/ref/settings/#allowed-hosts
 ALLOWED_HOSTS = [
-{% for h in mailman_postorious_allowed_hosts %}
+{% for h in mailman_postorius_allowed_hosts %}
     "{{ h }}",
 {% endfor %}
 ]
@@ -57,7 +57,7 @@ MAILMAN_REST_API_URL = 'http://localhost:8001'
 MAILMAN_REST_API_USER = '{{ mailman_api_user }}'
 MAILMAN_REST_API_PASS = '{{ mailman_vault_rest_api_pwd }}'
 MAILMAN_ARCHIVER_KEY = '{{ mailman_archiver_crypt_key }}'
-MAILMAN_ARCHIVER_FROM = ({{ mailman_postorious_hyperkitty_acl }})
+MAILMAN_ARCHIVER_FROM = ({{ mailman_postorius_hyperkitty_acl }})
 
 # Application definition
 
@@ -85,7 +85,7 @@ INSTALLED_APPS = (
     'allauth.account',
     'allauth.socialaccount'
 {% if mailman_use_social_account_providers %}
-{% for auth_provider in mailman_postorious_social_auth_providers %}
+{% for auth_provider in mailman_postorius_social_auth_providers %}
     '{{ auth_privider }}',
 {% endfor %}
 {% endif %}
@@ -142,15 +142,15 @@ DATABASES = {
         # Use 'sqlite3', 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
         'ENGINE': 'django.db.backends.postgresql_psycopg2',
         # DB name or path to database file if using sqlite3.
-        'NAME': '{{ mailman_postorious_db_name }}',
+        'NAME': '{{ mailman_postorius_db_name }}',
         # The following settings are not used with sqlite3:
-        'USER': '{{ mailman_postorious_db_user }}',
-        'PASSWORD': '{{ mailman_postorious_db_password }}',
+        'USER': '{{ mailman_postorius_db_user }}',
+        'PASSWORD': '{{ mailman_postorius_db_password }}',
         # HOST: empty for localhost through domain sockets or '127.0.0.1' for
         # localhost through TCP.
-        'HOST': '{{ mailman_postorious_db_host }}',
+        'HOST': '{{ mailman_postorius_db_host }}',
         # PORT: set to empty string for default.
-        'PORT': '{{ mailman_postorious_db_port }}',
+        'PORT': '{{ mailman_postorius_db_port }}',
     }
 }
 
@@ -415,7 +415,7 @@ LOGGING = {
             'level': 'INFO',
             #'class': 'logging.handlers.RotatingFileHandler',
             'class': 'logging.handlers.WatchedFileHandler',
-            'filename': os.path.join('..', '..', '..', '..', '{{ mailman_postorious_var_dir }}', '{{ mailman_postorious_log_base_dir }}', '{{ mailman_postorious_log_dest_dir }}', 'mailmansuite.log'),
+            'filename': os.path.join('..', '..', '..', '..', '{{ mailman_postorius_var_dir }}', '{{ mailman_postorius_log_base_dir }}', '{{ mailman_postorius_log_dest_dir }}', 'mailmansuite.log'),
             'formatter': 'verbose',
         },
         'console': {

templates/postorius_uwsgi.ini.j2

@@ -1,20 +1,19 @@
 [uwsgi]
 uid = {{ mailman_user }}
 gid = {{ mailman_user }}
-#pidfile = /run/uwsgi/uwsgi.pid
+#pidfile = {{ mailman_postorius_uwsgi_rundir }}/uwsgi.pid
 emperor = /etc/uwsgi.d
-#stats = /run/uwsgi/stats.sock
+#stats = {{ mailman_postorius_uwsgi_rundir }}/stats.sock
 chmod-socket = 660
 emperor-tyrant = true
 cap = setgid,setuid
-plugins = {{ mailman_postorious_uwsgi_plugins }}
-#plugins = systemd_logger,python36
+plugins = {{ mailman_postorius_uwsgi_plugins }}
 
 # Port on which uwsgi will be listening.
-uwsgi-socket = 127.0.0.1:{{ mailman_postorious_http_port }}
+uwsgi-socket = 127.0.0.1:{{ mailman_postorius_http_port }}
 
 # Move to the directory wher the django files are.
-chdir = {{ mailman_postorious_dir }}/
+chdir = {{ mailman_postorius_dir }}/
 
 # Use the wsgi file provided with the django project.
 wsgi-file = wsgi.py
@@ -24,20 +23,24 @@ master = true
 process = 2
 threads = 2
 
+{% if not mailman_hyperkitty_qcluster_standalone_service %}
 # Setup the django_q related worker processes.
 attach-daemon = ./manage.py qcluster
+{% endif %}
 
 # Setup the request log.
-req-logger = file:{{  mailman_postorious_log_dir }}/uwsgi.log
+req-logger = file:{{  mailman_postorius_log_dir }}/uwsgi.log
 
 # Log cron seperately.
-logger = cron file:{{  mailman_postorious_log_dir }}/uwsgi-cron.log
+logger = cron file:{{  mailman_postorius_log_dir }}/uwsgi-cron.log
 log-route = cron uwsgi-cron
 
+{% if not mailman_hyperkitty_qcluster_standalone_service %}
 # Log qcluster commands seperately.
-logger = qcluster file:{{  mailman_postorious_log_dir }}/uwsgi-qcluster.log
+logger = qcluster file:{{  mailman_postorius_log_dir }}/uwsgi-qcluster.log
 log-route = qcluster uwsgi-daemons
+{% endif %}
 
 # Last log and it logs the rest of the stuff.
-logger = file:{{  mailman_postorious_log_dir }}/uwsgi-error.log
+logger = file:{{  mailman_postorius_log_dir }}/uwsgi-error.log
 

templates/rundir_mailmansuite.conf.j2

@@ -0,0 +1 @@
+d {{ mailman_postorius_uwsgi_rundir }} 0775 {{ mailman_user }}  {{ mailman_user }}