Add support for the prometheus postgresql exporter.

README.md

@@ -32,6 +32,68 @@ psql_db_data:
   - { name: '{{ psql_db_name }}', encoding: 'UTF8', user: '{{ psql_db_user }}', pwd: '{{ psql_db_pwd }}', managedb: True, roles: 'NOCREATEDB,NOSUPERUSER', extensions: [ 'postgis', 'pgpool_regclass', 'pgpool_recovery' ], allowed_hosts: [ 'xxx.xxx.xxx.xxx/32', 'yyy.yyy.yyy.yyy/32' ], state=absent }
 ```
 
+### Prometheus exporter
+
+[postgres_exporter](https://github.com/prometheus-community/postgres_exporter)
+must run on the database host, so it is installed by this role instead of a
+separate one. It is off by default.
+
+``` yaml
+psql_prometheus_exporter_install: True
+psql_prometheus_exporter_version: "0.20.1"
+psql_prometheus_exporter_port: 9187
+# Scrape every database of the cluster and not only the one connected to
+psql_prometheus_exporter_auto_discover_dbs: True
+```
+
+The default connection is the unix socket with peer authentication: the
+exporter runs as the `postgres_exporter` system user, the database role has the
+same name, and there is no password anywhere. The role is granted `pg_monitor`,
+nothing else.
+
+To connect over TCP instead — for example when the exporter has to reach a
+cluster that only listens on an address — set:
+
+``` yaml
+psql_prometheus_exporter_use_socket: False
+psql_prometheus_exporter_db_host: 127.0.0.1
+psql_prometheus_exporter_db_pwd: '{{ a_vaulted_variable }}'
+```
+
+The DSN is written to `/etc/default/postgres_exporter` (`/etc/sysconfig` on EL)
+with mode 0640, and not into the systemd unit, which is world readable.
+
+On EL the exporter port is opened in firewalld when `firewalld_enabled` is
+true. On Debian/Ubuntu the port belongs to the `iptables` variable of the
+linux-firewall role, which is outside this role.
+
+#### pg_stat_statements
+
+``` yaml
+psql_prometheus_exporter_stat_statements: True
+# optional, applied after the restart
+psql_prometheus_exporter_stat_statements_parameters:
+  - { name: 'pg_stat_statements.track', value: 'all', set: 'true' }
+```
+
+It adds the library to `shared_preload_libraries`, **restarts the cluster**,
+creates the extension in the database the exporter connects to, and adds
+`--collector.stat_statements` to the exporter. Enable it before a server goes
+into production, or plan the restart.
+
+The current `shared_preload_libraries` is read and the library appended, so an
+extension that is already preloaded is not unloaded. For the same reason do
+**not** also set `shared_preload_libraries` in `psql_conf_custom_parameters`:
+the two would overwrite each other on every run.
+
+Turning the flag back to `False` stops the exporter collecting the metrics but
+deliberately does **not** remove the library, which would mean another restart.
+Remove it by hand if that is what you want.
+
+The exporter needs no extra privilege: `pg_monitor` already carries
+`pg_read_all_stats`, which is what lets a non superuser see the queries of every
+user instead of only its own.
+
 Dependencies
 ------------
 

defaults/main.yml

@@ -20,6 +20,11 @@ psql_db_size_w: 150000000
 psql_db_size_c: 170000000
 psql_max_connections: 1024
 psql_listen_on_ext_int: false
+# Only used when psql_listen_on_ext_int is true. '*' is every interface; a
+# comma separated list of addresses restricts the cluster to some of them, for
+# example 'localhost,192.168.0.5' when the server has a management interface
+# that must not carry database traffic
+psql_listen_addresses: '*'
 psql_use_alternate_data_dir: false
 # Deb/Ubuntu
 psql_data_root_dir: '/var/lib/postgresql/{{ psql_version }}'
@@ -188,5 +193,59 @@ pg_backup_use_auth: "no"
 pg_backup_pass_file: /root/.pgpass
 pg_backup_use_nagios: "yes"
 
+# Prometheus exporter: https://github.com/prometheus-community/postgres_exporter
+# It must run on the database host, so it is installed here and not by a
+# separate role.
+psql_prometheus_exporter_install: false
+psql_prometheus_exporter_version: "0.20.1"
+psql_prometheus_exporter_releases_url: 'https://github.com/prometheus-community/postgres_exporter/releases/download'
+psql_prometheus_exporter_download_url: '{{ psql_prometheus_exporter_releases_url }}/v{{ psql_prometheus_exporter_version }}/{{ psql_prometheus_exporter_file }}'
+psql_prometheus_exporter_port: 9187
+psql_prometheus_exporter_listen_address: '0.0.0.0'
+psql_prometheus_exporter_metrics_path: /metrics
+psql_prometheus_exporter_db_user: postgres_exporter
+# The database the exporter connects to. The cluster wide metrics do not depend
+# on it; per database metrics need psql_prometheus_exporter_auto_discover_dbs
+psql_prometheus_exporter_db_name: postgres
+# Default connection: unix socket with peer authentication. The exporter runs
+# as its own system user, whose name is the same as the database role, so
+# there is no password anywhere. Set to false to connect over TCP instead, and
+# then psql_prometheus_exporter_db_pwd becomes mandatory.
+psql_prometheus_exporter_use_socket: true
+psql_prometheus_exporter_db_socket_dir: /var/run/postgresql
+psql_prometheus_exporter_db_host: 127.0.0.1
+psql_prometheus_exporter_db_port: '{{ psql_db_port }}'
+psql_prometheus_exporter_db_sslmode: require
+psql_prometheus_exporter_db_pwd: ''
+# Scrape every database of the cluster, not only the one connected to. The
+# exporter role needs CONNECT on all of them (granted to PUBLIC by default)
+psql_prometheus_exporter_auto_discover_dbs: false
+psql_prometheus_exporter_exclude_dbs:
+  - template0
+  - template1
+# pg_stat_statements: slow queries, call counts and I/O per normalised
+# statement. Off by default because switching it on adds the library to
+# shared_preload_libraries, which needs a RESTART of the cluster: enable it
+# before a server goes into production, or plan the restart.
+# The exporter needs no extra privilege for it, pg_monitor already carries
+# pg_read_all_stats, which is what lets a non superuser see the queries of
+# every user instead of only its own.
+psql_prometheus_exporter_stat_statements: false
+psql_prometheus_exporter_stat_statements_library: pg_stat_statements
+# The database the extension is created in. It must be the one the exporter
+# connects to: the counters are cluster wide, but the view that reads them
+# exists only where the extension was created
+psql_prometheus_exporter_stat_statements_db: '{{ psql_prometheus_exporter_db_name }}'
+# Applied only after the restart, since the parameters do not exist until the
+# library is loaded. Empty means the PostgreSQL defaults (max 5000, track top).
+# Example: [{ name: 'pg_stat_statements.track', value: 'all', set: 'true' }]
+psql_prometheus_exporter_stat_statements_parameters: []
+# Appended verbatim to the command line, one entry per flag
+psql_prometheus_exporter_extra_opts: []
+# Managed only when the exporter connects over TCP: the socket case is covered
+# by the 'local' peer entry, which is always written
+psql_prometheus_exporter_manage_pg_hba: true
+
 # Used to configure firewalld
 postgresql_firewalld_zone: '{{ firewalld_default_zone }}'
+psql_prometheus_exporter_firewalld_zone: '{{ postgresql_firewalld_zone }}'

handlers/main.yml

@@ -32,3 +32,7 @@
 - name: Reload systemd
   ansible.builtin.systemd:
     daemon_reload: true
+- name: Restart postgres exporter
+  ansible.builtin.service:
+    name: postgres_exporter
+    state: restarted

tasks/main.yml

@@ -57,6 +57,11 @@
   when:
     - psql_postgresql_install
     - not postgresql_client_only
+- name: Prometheus exporter
+  ansible.builtin.import_tasks: postgresql-prometheus-exporter.yml
+  when:
+    - psql_postgresql_install
+    - not postgresql_client_only
 - name: Letsencrypt hook
   ansible.builtin.import_tasks: postgresql-letsencrypt-acmetool.yml
   when:

tasks/postgresql-config-deb.yml

@@ -95,7 +95,7 @@
       become_user: postgres
       community.postgresql.postgresql_set:
         name: 'listen_addresses'
-        value: "{% if psql_listen_on_ext_int %}*{% else %}default{% endif %}"
+        value: "{% if psql_listen_on_ext_int %}{{ psql_listen_addresses }}{% else %}default{% endif %}"
       notify: Restart postgresql
 
     - name: postgresql-config-deb | Custom configuration parameters

tasks/postgresql-config-el.yml

@@ -125,15 +125,7 @@
       become_user: postgres
       community.postgresql.postgresql_set:
         name: 'listen_addresses'
-        value: "{% if psql_listen_on_ext_int %}*{% else %}default{% endif %}"
-      notify: Restart postgresql
-
-    - name: postgresql-config-el | If postgresql is only accessed from localhost make it listen only on the localhost interface
-      become: true
-      become_user: postgres
-      community.postgresql.postgresql_set:
-        name: 'listen_addresses'
-        value: "{% if psql_listen_on_ext_int %}*{% else %}default{% endif %}"
+        value: "{% if psql_listen_on_ext_int %}{{ psql_listen_addresses }}{% else %}default{% endif %}"
       notify: Restart postgresql
 
     - name: postgresql-config-el | Flush flush_handlers Restart PostgreSQL

tasks/postgresql-prometheus-exporter.yml

@@ -0,0 +1,217 @@
+---
+- name: postgresql-prometheus-exporter | Database role used by the exporter
+  tags: ['postgresql', 'postgres', 'prometheus', 'postgres_exporter']
+  when: psql_prometheus_exporter_install
+  block:
+    - name: postgresql-prometheus-exporter | Create the database role {{ psql_prometheus_exporter_db_user }}
+      become: true
+      become_user: postgres
+      community.postgresql.postgresql_user:
+        name: '{{ psql_prometheus_exporter_db_user }}'
+        password: "{{ omit if psql_prometheus_exporter_use_socket else psql_prometheus_exporter_db_pwd }}"
+        role_attr_flags: 'LOGIN,NOSUPERUSER,NOCREATEDB,NOCREATEROLE'
+        state: present
+      no_log: "{{ not psql_prometheus_exporter_use_socket }}"
+
+    # pg_monitor carries pg_read_all_settings, pg_read_all_stats and
+    # pg_stat_scan_tables: everything the exporter reads, and nothing else.
+    - name: postgresql-prometheus-exporter | Grant pg_monitor to {{ psql_prometheus_exporter_db_user }}
+      become: true
+      become_user: postgres
+      community.postgresql.postgresql_membership:
+        group: pg_monitor
+        target_role: '{{ psql_prometheus_exporter_db_user }}'
+        state: present
+
+    # Peer authentication: the system user and the database role have the same
+    # name, so no map is needed. Always written, the socket is also the way to
+    # reach the cluster by hand.
+    - name: postgresql-prometheus-exporter | Local peer access for {{ psql_prometheus_exporter_db_user }}
+      community.postgresql.postgresql_pg_hba:
+        dest: '{{ psql_conf_dir }}/pg_hba.conf'
+        contype: local
+        users: '{{ psql_prometheus_exporter_db_user }}'
+        databases: all
+        method: peer
+        state: present
+        owner: root
+        group: postgres
+        mode: "0640"
+      when: psql_prometheus_exporter_use_socket
+      notify: Reload postgresql
+
+    - name: postgresql-prometheus-exporter | TCP access for {{ psql_prometheus_exporter_db_user }}
+      community.postgresql.postgresql_pg_hba:
+        dest: '{{ psql_conf_dir }}/pg_hba.conf'
+        contype: '{% if psql_force_ssl_client_connection or psql_enable_ssl %}hostssl{% else %}host{% endif %}'
+        users: '{{ psql_prometheus_exporter_db_user }}'
+        address: '{{ psql_prometheus_exporter_db_host }}/32'
+        databases: all
+        method: scram-sha-256
+        state: present
+        owner: root
+        group: postgres
+        mode: "0640"
+      when:
+        - not psql_prometheus_exporter_use_socket
+        - psql_prometheus_exporter_manage_pg_hba
+      notify: Reload postgresql
+
+    - name: postgresql-prometheus-exporter | Flush handlers
+      ansible.builtin.meta: flush_handlers
+
+- name: postgresql-prometheus-exporter | Manage pg_stat_statements
+  tags: ['postgresql', 'postgres', 'prometheus', 'postgres_exporter', 'pg_stat_statements']
+  when:
+    - psql_prometheus_exporter_install
+    - psql_prometheus_exporter_stat_statements
+  become: true
+  become_user: postgres
+  block:
+    # Read it instead of writing a fixed value: another extension may already be
+    # preloaded (auto_explain, pg_cron, ...) and overwriting the list would
+    # silently unload it.
+    - name: postgresql-prometheus-exporter | Read the current shared_preload_libraries
+      community.postgresql.postgresql_query:
+        db: postgres
+        query: 'SHOW shared_preload_libraries'
+      register: psql_current_preload_libraries
+      changed_when: false
+
+    - name: postgresql-prometheus-exporter | Preload the library {{ psql_prometheus_exporter_stat_statements_library }}
+      vars:
+        psql_preload_libraries: "{{ psql_current_preload_libraries.query_result[0]['shared_preload_libraries']
+                                    | split(',') | map('trim') | reject('equalto', '') | list }}"
+      community.postgresql.postgresql_set:
+        name: shared_preload_libraries
+        value: "{{ (psql_preload_libraries + [psql_prometheus_exporter_stat_statements_library]) | unique | join(',') }}"
+      when: psql_prometheus_exporter_stat_statements_library not in psql_preload_libraries
+      notify: Restart postgresql
+
+    # The library is only loaded at startup, and the parameters below do not
+    # exist until it is.
+    - name: postgresql-prometheus-exporter | Restart PostgreSQL so that the library is loaded
+      ansible.builtin.meta: flush_handlers
+
+    - name: postgresql-prometheus-exporter | Create the extension in {{ psql_prometheus_exporter_stat_statements_db }}
+      community.postgresql.postgresql_ext:
+        name: '{{ psql_prometheus_exporter_stat_statements_library }}'
+        db: '{{ psql_prometheus_exporter_stat_statements_db }}'
+        state: present
+
+    - name: postgresql-prometheus-exporter | Set the pg_stat_statements parameters
+      community.postgresql.postgresql_set:
+        name: '{{ item.name }}'
+        value: "{% if item.set | bool %}{{ item.value }}{% else %}default{% endif %}"
+      loop: '{{ psql_prometheus_exporter_stat_statements_parameters }}'
+      loop_control:
+        label: '{{ item.name }}'
+      notify: Restart postgresql
+
+    - name: postgresql-prometheus-exporter | Flush handlers
+      ansible.builtin.meta: flush_handlers
+
+- name: postgresql-prometheus-exporter | Install the exporter
+  tags: ['postgresql', 'postgres', 'prometheus', 'postgres_exporter']
+  when: psql_prometheus_exporter_install
+  block:
+    - name: postgresql-prometheus-exporter | Create the system user {{ psql_prometheus_exporter_user }}
+      ansible.builtin.user:
+        name: '{{ psql_prometheus_exporter_user }}'
+        home: '{{ psql_prometheus_exporter_home }}'
+        createhome: false
+        shell: /usr/sbin/nologin
+        system: true
+
+    - name: postgresql-prometheus-exporter | Create the exporter directories
+      ansible.builtin.file:
+        dest: '{{ item }}'
+        state: directory
+        owner: root
+        group: root
+        mode: "0755"
+      loop:
+        - '{{ psql_prometheus_exporter_home }}'
+        - '{{ psql_prometheus_exporter_dist_dir }}'
+
+    - name: postgresql-prometheus-exporter | Download postgres_exporter {{ psql_prometheus_exporter_version }}
+      ansible.builtin.get_url:
+        url: '{{ psql_prometheus_exporter_download_url }}'
+        dest: '/srv/{{ psql_prometheus_exporter_file }}'
+        owner: root
+        group: root
+        mode: "0644"
+
+    - name: postgresql-prometheus-exporter | Unarchive the postgres_exporter distribution
+      ansible.builtin.unarchive:
+        src: '/srv/{{ psql_prometheus_exporter_file }}'
+        dest: '{{ psql_prometheus_exporter_dist_dir }}'
+        remote_src: true
+        owner: root
+        group: root
+        creates: '{{ psql_prometheus_exporter_cmd }}'
+      notify: Restart postgres exporter
+
+    - name: postgresql-prometheus-exporter | Install the exporter environment file
+      ansible.builtin.template:
+        src: postgres_exporter-environment.j2
+        dest: '{{ psql_prometheus_exporter_env_file }}'
+        owner: root
+        group: '{{ psql_prometheus_exporter_user }}'
+        mode: "0640"
+      # The rendered file contains the database password when the exporter
+      # connects over TCP
+      diff: '{{ psql_prometheus_exporter_use_socket }}'
+      notify: Restart postgres exporter
+
+    - name: postgresql-prometheus-exporter | Install the exporter systemd unit
+      ansible.builtin.template:
+        src: postgres_exporter.service.j2
+        dest: /etc/systemd/system/postgres_exporter.service
+        owner: root
+        group: root
+        mode: "0644"
+      notify:
+        - Reload systemd
+        - Restart postgres exporter
+
+    - name: postgresql-prometheus-exporter | Flush handlers
+      ansible.builtin.meta: flush_handlers
+
+    - name: postgresql-prometheus-exporter | Ensure that postgres_exporter is started and enabled
+      ansible.builtin.service:
+        name: postgres_exporter
+        state: started
+        enabled: true
+
+- name: postgresql-prometheus-exporter | Remove the exporter
+  tags: ['postgresql', 'postgres', 'prometheus', 'postgres_exporter']
+  when: not psql_prometheus_exporter_install
+  block:
+    - name: postgresql-prometheus-exporter | Ensure that postgres_exporter is stopped and disabled
+      ansible.builtin.service:
+        name: postgres_exporter
+        state: stopped
+        enabled: false
+      failed_when: false
+
+    - name: postgresql-prometheus-exporter | Remove the exporter systemd unit
+      ansible.builtin.file:
+        dest: /etc/systemd/system/postgres_exporter.service
+        state: absent
+      notify: Reload systemd
+
+- name: postgresql-prometheus-exporter | Manage the exporter firewalld rules
+  when:
+    - psql_prometheus_exporter_install
+    - ansible_distribution_file_variety == "RedHat"
+    - firewalld_enabled is defined and firewalld_enabled | bool
+  tags: ['postgresql', 'postgres', 'prometheus', 'postgres_exporter', 'firewall', 'firewalld']
+  block:
+    - name: postgresql-prometheus-exporter | Open the exporter port {{ psql_prometheus_exporter_port }}
+      ansible.posix.firewalld:
+        port: '{{ psql_prometheus_exporter_port }}/tcp'
+        zone: '{{ psql_prometheus_exporter_firewalld_zone }}'
+        permanent: true
+        state: enabled
+        immediate: true

templates/postgres_exporter-environment.j2

@@ -0,0 +1,11 @@
+{{ ansible_managed | comment }}
+#
+# Environment of the postgres_exporter systemd unit. The DSN lives here and not
+# in the unit file because it can carry the database password.
+{% if psql_prometheus_exporter_use_socket %}
+# Unix socket: peer authentication, no password. lib/pq refuses any sslmode
+# other than 'disable' on a unix socket, and the traffic never leaves the host.
+DATA_SOURCE_NAME="user={{ psql_prometheus_exporter_db_user }} host={{ psql_prometheus_exporter_db_socket_dir }} port={{ psql_prometheus_exporter_db_port }} dbname={{ psql_prometheus_exporter_db_name }} sslmode=disable"
+{% else %}
+DATA_SOURCE_NAME="user={{ psql_prometheus_exporter_db_user }} password={{ psql_prometheus_exporter_db_pwd }} host={{ psql_prometheus_exporter_db_host }} port={{ psql_prometheus_exporter_db_port }} dbname={{ psql_prometheus_exporter_db_name }} sslmode={{ psql_prometheus_exporter_db_sslmode }}"
+{% endif %}

templates/postgres_exporter.service.j2

@@ -0,0 +1,47 @@
+{{ ansible_managed | comment }}
+
+[Unit]
+Description=postgres_exporter - Prometheus exporter for PostgreSQL
+Documentation=https://github.com/prometheus-community/postgres_exporter
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+Restart=on-failure
+RestartSec=5
+
+User={{ psql_prometheus_exporter_user }}
+Group={{ psql_prometheus_exporter_user }}
+
+EnvironmentFile={{ psql_prometheus_exporter_env_file }}
+
+ExecStart={{ psql_prometheus_exporter_cmd }} \
+  --web.listen-address={{ psql_prometheus_exporter_listen_address }}:{{ psql_prometheus_exporter_port }} \
+  --web.telemetry-path={{ psql_prometheus_exporter_metrics_path }} \
+{% if psql_prometheus_exporter_auto_discover_dbs %}
+  --auto-discover-databases \
+{% if psql_prometheus_exporter_exclude_dbs | length > 0 %}
+  --exclude-databases={{ psql_prometheus_exporter_exclude_dbs | join(',') }} \
+{% endif %}
+{% endif %}
+{% if psql_prometheus_exporter_stat_statements %}
+  --collector.stat_statements \
+{% endif %}
+{% for opt in psql_prometheus_exporter_extra_opts %}
+  {{ opt }} \
+{% endfor %}
+  --log.format=logfmt
+
+# ProtectSystem=full and not strict: connecting to the PostgreSQL unix socket
+# needs a writable /run
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectSystem=full
+ProtectHome=true
+ProtectKernelTunables=true
+ProtectControlGroups=true
+RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
+
+[Install]
+WantedBy=multi-user.target

vars/main.yml

@@ -1,2 +1,18 @@
 ---
-# vars file for ansible-role-template
+# vars file for ansible-role-template
+
+# postgres_exporter. Derived from psql_prometheus_exporter_version, which is a
+# default so that it can be overridden: the version is part of the paths, so a
+# version bump downloads, unpacks and restarts by itself.
+psql_prometheus_exporter_dir: 'postgres_exporter-{{ psql_prometheus_exporter_version }}.linux-amd64'
+psql_prometheus_exporter_file: '{{ psql_prometheus_exporter_dir }}.tar.gz'
+psql_prometheus_exporter_user: postgres_exporter
+psql_prometheus_exporter_home: /opt/postgres_exporter
+psql_prometheus_exporter_dist_dir: '{{ psql_prometheus_exporter_home }}/dist'
+psql_prometheus_exporter_cmd: '{{ psql_prometheus_exporter_dist_dir }}/{{ psql_prometheus_exporter_dir }}/postgres_exporter'
+# The DSN carries the password when the connection is over TCP, so it is kept
+# out of the systemd unit, which is world readable
+psql_prometheus_exporter_env_dir: >-
+  {% if ansible_distribution_file_variety == "Debian" %}{{ pg_backup_conf_dir }}
+  {%- elif ansible_distribution_file_variety == "RedHat" %}{{ pg_el_backup_conf_dir }}{% endif %}
+psql_prometheus_exporter_env_file: '{{ psql_prometheus_exporter_env_dir }}/postgres_exporter'