WPB-19318: Ensure high-availability of the postgress cluster

.github/workflows/offline.yml

@@ -1,15 +1,15 @@
 on:
   push:
     branches: [master, develop]
-    tags: [ v* ]
+    tags: [v*]
     paths-ignore:
-      - '*.md'
-      - '**/*.md'
+      - "*.md"
+      - "**/*.md"
   pull_request:
     branches: [master, develop]
     paths-ignore:
-      - '*.md'
-      - '**/*.md'
+      - "*.md"
+      - "**/*.md"
 jobs:
   offline:
     name: Prepare offline package
@@ -42,8 +42,8 @@ jobs:
       - name: Process the default profile build
         run: ./offline/default-build/build.sh
         env:
-          GPG_PRIVATE_KEY: '${{ secrets.GPG_PRIVATE_KEY }}'
-          DOCKER_LOGIN: '${{ secrets.DOCKER_LOGIN }}'
+          GPG_PRIVATE_KEY: "${{ secrets.GPG_PRIVATE_KEY }}"
+          DOCKER_LOGIN: "${{ secrets.DOCKER_LOGIN }}"
 
       - name: Copy default build assets tarball to S3 and clean up
         run: |
@@ -53,8 +53,8 @@ jobs:
           # removing everything except assets.tgz as it is not required anymore in the further builds
           find offline/default-build/output/ -mindepth 1 -maxdepth 1 ! -name 'assets.tgz' -exec rm -r {} +
         env:
-          AWS_ACCESS_KEY_ID: '${{ secrets.AWS_ACCESS_KEY_ID }}'
-          AWS_SECRET_ACCESS_KEY: '${{ secrets.AWS_SECRET_ACCESS_KEY }}'
+          AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
+          AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
           AWS_REGION: "eu-west-1"
 
       - name: Build and upload wire-server-deploy container
@@ -64,14 +64,14 @@ jobs:
             docker-archive:"$container_image" \
             "docker://quay.io/wire/wire-server-deploy:${{ steps.upload_name.outputs.UPLOAD_NAME }}"
         env:
-          DOCKER_LOGIN: '${{ secrets.DOCKER_LOGIN }}'
+          DOCKER_LOGIN: "${{ secrets.DOCKER_LOGIN }}"
 
       # demo profile build
       - name: Process the demo profile build
         run: ./offline/demo-build/build.sh
         env:
-          GPG_PRIVATE_KEY: '${{ secrets.GPG_PRIVATE_KEY }}'
-          DOCKER_LOGIN: '${{ secrets.DOCKER_LOGIN }}'
+          GPG_PRIVATE_KEY: "${{ secrets.GPG_PRIVATE_KEY }}"
+          DOCKER_LOGIN: "${{ secrets.DOCKER_LOGIN }}"
 
       - name: Copy demo build assets tarball to S3 and clean up
         run: |
@@ -81,16 +81,16 @@ jobs:
           # remove the assets from the build to optimize the space on the server
           rm -rf offline/demo-build/output/*
         env:
-          AWS_ACCESS_KEY_ID: '${{ secrets.AWS_ACCESS_KEY_ID }}'
-          AWS_SECRET_ACCESS_KEY: '${{ secrets.AWS_SECRET_ACCESS_KEY }}'
+          AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
+          AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
           AWS_REGION: "eu-west-1"
 
       # min profile build
       - name: Process the min profile build
         run: ./offline/min-build/build.sh
         env:
-          GPG_PRIVATE_KEY: '${{ secrets.GPG_PRIVATE_KEY }}'
-          DOCKER_LOGIN: '${{ secrets.DOCKER_LOGIN }}'
+          GPG_PRIVATE_KEY: "${{ secrets.GPG_PRIVATE_KEY }}"
+          DOCKER_LOGIN: "${{ secrets.DOCKER_LOGIN }}"
 
       - name: Copy min build assets tarball to S3
         run: |
@@ -100,8 +100,8 @@ jobs:
           # remove the archives from the build to optimize the space on the server
           rm -rf offline/min-build/output/*
         env:
-          AWS_ACCESS_KEY_ID: '${{ secrets.AWS_ACCESS_KEY_ID }}'
-          AWS_SECRET_ACCESS_KEY: '${{ secrets.AWS_SECRET_ACCESS_KEY }}'
+          AWS_ACCESS_KEY_ID: "${{ secrets.AWS_ACCESS_KEY_ID }}"
+          AWS_SECRET_ACCESS_KEY: "${{ secrets.AWS_SECRET_ACCESS_KEY }}"
           AWS_REGION: "eu-west-1"
 
       - name: Install terraform
@@ -114,7 +114,7 @@ jobs:
         run: |
           ./offline/cd.sh
         env:
-          HCLOUD_TOKEN: '${{ secrets.HCLOUD_TOKEN }}'
+          HCLOUD_TOKEN: "${{ secrets.HCLOUD_TOKEN }}"
 
       #- name: Clean up hetzner environment; just in case
       #  if: always()

.gitignore

@@ -18,7 +18,7 @@ values-init-done
 
 # Envrc local overrides
 .envrc.local
-
+.vscode
 # Nix-created result symlinks
 result
 result-*

ansible/inventory/offline/group_vars/postgresql/postgresql.yml

@@ -3,10 +3,44 @@ postgresql_version: 17
 postgresql_data_dir: /var/lib/postgresql/{{ postgresql_version }}/main
 postgresql_conf_dir: /etc/postgresql/{{ postgresql_version }}/main
 
-# Replication services configuration
-repsvc_user: repsvc
-repsvc_password: "securepassword"
-repsvc_database: repsvc_db
+# repmgr HA configuration
+repmgr_user: repmgr
+repmgr_password: "securepassword"
+repmgr_database: repmgr
+
+# Node configuration for repmgr
+repmgr_node_config:
+  postgresql1:  # Maps to postgresql_rw group
+    node_id: 1
+    priority: 150
+    role: primary
+  postgresql2:  # Maps to first postgresql_ro
+    node_id: 2
+    priority: 100
+    role: standby
+  postgresql3:  # Maps to second postgresql_ro
+    node_id: 3
+    priority: 50
+    role: standby
+
+# repmgr settings
+# repmgrd monitoring and reconnection configuration
+# Reference: https://repmgr.org/docs/current/repmgrd-basic-configuration.html
+#
+# monitor_interval_secs: Interval in seconds between monitoring checks
+#   - Default: 2 seconds
+#   - Controls how frequently repmgr monitors the primary server status
+#
+# reconnect_attempts: Maximum number of reconnection attempts
+#   - Default: 6 attempts
+#   - Number of times repmgr will attempt to reconnect to a failed primary
+#
+# reconnect_interval: Interval in seconds between reconnection attempts
+#   - Default: 10 seconds
+#   - Time to wait between each reconnection attempt
+monitor_interval_secs: 2
+reconnect_attempts: 6
+reconnect_interval: 5
 
 # Use local packages instead of repository
 postgresql_use_repository: false  # Set to true to use local packages from urls
@@ -35,3 +69,12 @@ postgresql_pkgs:
   - name: python3-psycopg2
     url: "{{ binaries_url }}/python3-psycopg2_2.9.10-1.pgdg22.04+1_amd64.deb"
     checksum: "sha256:cc2f749e3af292a67e012edeb4aa5d284f57f2d66a9a09fe5b81e5ffda73cab4"
+  - name: repmgr-common
+    url: "{{ binaries_url }}/repmgr-common_5.5.0+debpgdg-1.pgdg22.04+1_all.deb"
+    checksum: "sha256:34c660c66a9710fd4f20a66cc932741d3399dbba7e7ae4b67468b3e18f65f61c"
+  - name: repmgr
+    url: "{{ binaries_url }}/repmgr_5.5.0+debpgdg-1.pgdg22.04+1_all.deb"
+    checksum: "sha256:20c280811e758106335df1eb9954b61aa552823d3129f1e38c488fbd5efe0567"
+  - name: postgresql-17-repmgr
+    url: "{{ binaries_url }}/postgresql-17-repmgr_5.5.0+debpgdg-1.pgdg22.04+1_amd64.deb"
+    checksum: "sha256:520d6ed4d540a2bb9174ac8276f8cb686c0268c13cccb89b28a9cdbd12049df8"

ansible/postgresql-deploy.yml

@@ -1,3 +1,9 @@
+- name: Clean previous deployment state
+  import_playbook: postgresql-playbooks/clean_existing_setup.yml
+  tags:
+    - postgresql
+    - cleanup
+
 - name: Install PostgreSQL packages
   import_playbook: postgresql-playbooks/postgresql-install.yml
   tags:
@@ -27,3 +33,9 @@
   tags:
     - postgresql
     - wire-setup
+
+- name: Deploy cluster monitoring
+  import_playbook: postgresql-playbooks/postgresql-monitoring.yml
+  tags:
+    - postgresql
+    - monitoring

ansible/postgresql-playbooks/clean_existing_setup.yml

@@ -0,0 +1,173 @@
+- name: Clean previous deployment state
+  hosts: "{{ target_nodes | default('postgresql_rw,postgresql_ro') }}"
+  become: yes
+  tasks:
+    # ===== DETECT INSTALLATION TYPE =====
+    - name: Check if PostgreSQL is installed
+      stat:
+        path: "/usr/bin/psql"
+      register: postgresql_installed
+
+    - name: Check if PostgreSQL data directory exists
+      stat:
+        path: "/var/lib/postgresql/{{ postgresql_version }}/main/PG_VERSION"
+      register: postgresql_data_exists
+
+    - name: Check if repmgr configuration exists
+      stat:
+        path: "/etc/repmgr/{{ postgresql_version }}-main/repmgr.conf"
+      register: repmgr_config_exists
+
+    - name: Determine if this is a fresh installation
+      set_fact:
+        is_fresh_install: >-
+          {{
+            not postgresql_installed.stat.exists or
+            not postgresql_data_exists.stat.exists or
+            not repmgr_config_exists.stat.exists
+          }}
+
+    - name: Display installation type
+      debug:
+        msg: |
+          {{ inventory_hostname }}: {{ 'Fresh installation detected - skipping most cleanup tasks' if is_fresh_install else 'Existing deployment detected - performing full cleanup' }}
+
+    # ===== FRESH INSTALLATION TASKS (MINIMAL) =====
+    - name: Handle fresh installation
+      block:
+        - name: Ensure basic directories exist for fresh install
+          file:
+            path: "{{ item }}"
+            state: directory
+            owner: postgres
+            group: postgres
+            mode: "0755"
+          loop:
+            - "/etc/repmgr/{{ postgresql_version }}-main"
+            - "/opt/repmgr/scripts"
+            - "/var/log/postgresql"
+          when: postgresql_installed.stat.exists
+
+        - name: Skip cleanup message for fresh install
+          debug:
+            msg: "Fresh installation - cleanup tasks skipped"
+
+      when: is_fresh_install
+
+    # ===== EXISTING DEPLOYMENT CLEANUP =====
+    - name: Handle existing deployment cleanup
+      block:
+        - name: Check if PostgreSQL service exists
+          systemd:
+            name: "postgresql@{{ postgresql_version }}-main.service"
+          register: postgresql_service_exists
+          failed_when: false
+
+        - name: Check if repmgr database exists
+          ansible.builtin.shell: |
+            sudo -u postgres psql -t -A -c "SELECT COUNT(*) FROM pg_database WHERE datname = '{{ repmgr_database }}'" postgres 2>/dev/null || echo "0"
+          register: repmgr_db_exists
+          changed_when: false
+          failed_when: false
+          when:
+            - postgresql_installed.stat.exists
+            - postgresql_service_exists.status is defined
+            - postgresql_service_exists.status.LoadState != "not-found"
+
+        - name: Drop repmgr database completely (if exists)
+          ansible.builtin.shell: |
+            sudo -u postgres psql -c "DROP DATABASE IF EXISTS {{ repmgr_database }};" postgres 2>/dev/null || true
+          failed_when: false
+          when:
+            - postgresql_installed.stat.exists
+            - repmgr_db_exists is defined
+            - repmgr_db_exists.stdout | default('0') | trim != '0'
+
+        - name: Stop any existing split-brain monitoring timer
+          systemd:
+            name: detect-rogue-primary.timer
+            state: stopped
+          failed_when: false
+
+        - name: Stop any existing split-brain monitoring service
+          systemd:
+            name: detect-rogue-primary.service
+            state: stopped
+          failed_when: false
+
+        - name: Stop any existing repmgrd service
+          systemd:
+            name: "repmgrd@{{ postgresql_version }}-main.service"
+            state: stopped
+          failed_when: false
+
+        - name: Unmask PostgreSQL services from previous deployments
+          systemd:
+            name: "postgresql@{{ postgresql_version }}-main.service"
+            masked: no
+          failed_when: false
+
+        - name: Stop PostgreSQL service for clean state
+          systemd:
+            name: "postgresql@{{ postgresql_version }}-main.service"
+            state: stopped
+          failed_when: false
+
+        - name: Remove repmgr configuration files, scripts, and systemd units
+          file:
+            path: "{{ item }}"
+            state: absent
+          failed_when: false
+          loop:
+            - "/etc/repmgr/{{ postgresql_version }}-main/repmgr.conf"
+            - "/etc/repmgr/{{ postgresql_version }}"
+            - "/etc/repmgr/{{ postgresql_version }}-main"
+            - "/var/lib/postgresql/{{ postgresql_version }}/main/recovery.conf"
+            - "/var/lib/postgresql/{{ postgresql_version }}/main/standby.signal"
+            - "/opt/repmgr/scripts"
+            - "/usr/local/bin/repmgr"
+            - "/usr/local/bin/repmgrd"
+            - "/usr/local/bin/detect_rogue_primary.sh"
+            - "/etc/systemd/system/detect-rogue-primary.service"
+            - "/etc/systemd/system/detect-rogue-primary.timer"
+            - "/etc/systemd/system/[email protected]"
+            - "/etc/systemd/system/repmgrd@{{ postgresql_version }}-main.service"
+            - "/etc/systemd/system/repmgrd@{{ postgresql_version }}.service"
+            - "/etc/sudoers.d/postgres-postgresql-management"
+            - "/etc/sudoers.d/postgres-postgresql-service"
+
+        - name: Find rogue split-brain service files
+          find:
+            paths: /etc/systemd/system
+            patterns: "detect-rogue-primary.service*"
+          register: rogue_service_files
+
+        - name: Remove rogue split-brain service files
+          file:
+            path: "{{ item.path }}"
+            state: absent
+          loop: "{{ rogue_service_files.files }}"
+          when: rogue_service_files.matched > 0
+
+      when: not is_fresh_install
+
+    # ===== COMMON TASKS FOR ALL INSTALLATIONS =====
+    - name: Reload systemd daemon after cleanup
+      systemd:
+        daemon_reload: yes
+      failed_when: false
+
+    - name: Display cleanup status
+      debug:
+        msg: |
+          Cleanup completed for {{ inventory_hostname }}:
+          - Installation type: {{ 'Fresh' if is_fresh_install else 'Existing' }}
+          - PostgreSQL installed: {{ postgresql_installed.stat.exists }}
+          - PostgreSQL data exists: {{ postgresql_data_exists.stat.exists }}
+          - repmgr config exists: {{ repmgr_config_exists.stat.exists }}
+          {% if is_fresh_install %}
+          - Action taken: Minimal setup (directories created)
+          {% else %}
+          - Action taken: Full cleanup (services stopped, configs removed)
+          {% endif %}
+          - Ready for deployment: ✅