Add incremental backup and restore

docs/user/backup-restore.md

@@ -40,6 +40,7 @@ This creates a timestamped backup directory at `/var/backup/foreman-backup-YYYYM
 
 | Option | Description |
 |--------|-------------|
+| `--base-backup BASE_DIR` | Create an incremental backup based on a base backup. Only files changed since the base backup are included. The base backup directory must contain `.snar` snapshot files (generated by foremanctl backups). |
 | `--skip-pulp-content` | Skip backing up `/var/lib/pulp`. This is for debugging purposes or if you plan to copy `/var/lib/pulp` using other methods such as rsync or shared storage. **Warning:** You will not have a complete backup if you use this option. |
 | `--wait-for-tasks` | Wait for running Foreman and Pulp tasks to complete instead of failing immediately. The backup will poll until all tasks finish before proceeding. |
 
@@ -69,6 +70,38 @@ Allow in-progress tasks to complete before starting backup:
 foremanctl backup /var/backup --wait-for-tasks
 ```
 
+### Incremental Backup
+
+Create a full backup first:
+
+```bash
+foremanctl backup /var/backup
+# Creates: /var/backup/foreman-backup-20260629T120000/
+```
+
+Then create incremental backups referencing the base backup:
+
+```bash
+foremanctl backup /var/backup --base-backup /var/backup/foreman-backup-20260629T120000
+# Creates: /var/backup/foreman-backup-20260630T080000/ (incremental)
+```
+
+**How it works:**
+- The incremental backup contains only files changed since the previous backup
+- `.snar` snapshot files track which files were in the previous backup
+- Both `config.snar` (foremanctl state) and `pulp.snar` (Pulp content) are copied from the previous backup
+- Incremental backups are typically much smaller and faster than full backups
+
+**Typical backup strategy:**
+- Weekly full backup (Sunday)
+- Daily incremental backups (Monday-Saturday)
+- Each incremental references the previous day's backup
+
+**Important:**
+- You can chain incrementals: full -> inc1 -> inc2 -> inc3
+- All backups in the chain are required for restore
+- The previous backup must have been created with foremanctl (contains .snar files)
+
 ## Backup Contents
 
 ### Databases
@@ -269,6 +302,35 @@ foremanctl restore /var/backup/foreman-backup-20260617T104115 --validate
 
 This validates the backup and checks system requirements before proceeding.
 
+### Restore Incremental Backups
+
+To restore from an incremental backup, restore each backup in the chain sequentially, starting with the full backup that began the chain:
+
+```bash
+# Step 1: Restore the full (base) backup
+foremanctl restore /var/backup/foreman-backup-20260629T120000
+
+# Step 2: Restore the first incremental
+foremanctl restore /var/backup/foreman-backup-20260630T080000 --force
+
+# Step 3: Restore the second incremental
+foremanctl restore /var/backup/foreman-backup-20260701T080000 --force
+```
+
+**Important:**
+- Incremental backups contain only files changed since the previous backup
+- All backups in the chain must be restored in order (full -> inc1 -> inc2)
+- The restore command automatically detects incremental backups from metadata
+- The `--force` flag is required for subsequent restores since the system is already deployed
+
+**Automatic validation:**
+The restore command validates the backup chain:
+- Verifies the base backup directory exists
+- Confirms the base backup metadata matches the expected timestamp
+- Warns if the base backup has not been restored yet
+
+If any validation fails, a clear error message explains which backup is missing or incorrect.
+
 ## Prerequisites
 
 Before restoring, ensure:

src/playbooks/backup/metadata.obsah.yaml

@@ -1,6 +1,6 @@
 ---
 help: |
-  Create offline backup of Foreman databases and configuration
+  Create offline backup of databases and configuration
 
 variables:
   backup_dir:
@@ -23,3 +23,12 @@ variables:
     help: Wait for running tasks to complete instead of failing immediately
     action: store_true
     persist: false
+
+  backup_incremental:
+    parameter: --base-backup
+    help: |
+      Path to base backup directory for incremental backup.
+      Creates a differential backup containing only files changed since
+      the base backup. Requires .snar files from base backup.
+    type: AbsolutePath
+    persist: false

src/playbooks/health/metadata.obsah.yaml

@@ -1,6 +1,6 @@
 ---
 help: |
-  Check the health of your running Foreman server
+  Check the health of your running server
 variables:
   health_skip_check_foreman_tasks_param:
     help: |

src/playbooks/restore/metadata.obsah.yaml

@@ -3,11 +3,11 @@ help: |
   Restore from a backup
 
   Validates backup contents, extracts configuration files, restores databases,
-  restores Pulp content, and redeploys the system.
+  restores content, and redeploys the system.
 
 variables:
-  backup_dir:
-    parameter: backup_dir
+  restore_backup_dir:
+    parameter: restore_backup_dir
     help: Directory containing the backup files
     type: AbsolutePath
     persist: false

src/roles/backup/tasks/database_dumps.yaml

@@ -7,8 +7,8 @@
       --port={{ item.port }}
       --username={{ item.user }}
       --format=custom
-      --file={{ backup_dir_full }}/{{ item.database }}.dump
-      {{ item.database }}
+      --file="{{ backup_dir_full }}/{{ item.database }}.dump"
+      "{{ item.database }}"
   environment:
     PGPASSWORD: "{{ item.password }}"
   loop: "{{ backup_databases_config }}"

src/roles/backup/tasks/main.yaml

@@ -7,6 +7,35 @@
   ansible.builtin.set_fact:
     backup_dir_full: "{{ backup_dir }}/foreman-backup-{{ backup_timestamp }}"
 
+- name: Validate backup_incremental backup parameters
+  when: backup_incremental is defined and backup_incremental | length > 0
+  block:
+    - name: Check previous backup files exist
+      ansible.builtin.stat:
+        path: "{{ backup_incremental }}/{{ item }}"
+      register: backup_previous_files_check
+      loop:
+        - "metadata.yml"
+        - "config.snar"
+        - "pulp.snar"
+      failed_when: false
+
+    - name: Fail if previous backup is invalid
+      ansible.builtin.fail:
+        msg: |
+          Previous backup directory does not exist or is not a valid foremanctl backup: {{ backup_incremental }}
+          Missing files: {{ backup_previous_files_check.results | selectattr('stat.exists', 'equalto', false) | map(attribute='item') | list }}
+
+          The previous backup must contain metadata.yml and .snar files generated by foremanctl.
+          Create a new full backup first:
+            foremanctl backup {{ backup_dir }}
+      when: backup_previous_files_check.results | selectattr('stat.exists', 'equalto', false) | list | length > 0
+
+    - name: Load previous backup metadata
+      ansible.builtin.include_vars:
+        file: "{{ backup_incremental }}/metadata.yml"
+        name: backup_previous_metadata
+
 - name: Ensure backup directory exists
   ansible.builtin.file:
     path: "{{ backup_dir }}"
@@ -92,25 +121,53 @@
       ansible.builtin.include_tasks:
         file: database_dumps.yaml
 
-    - name: Archive foremanctl state on controller
-      community.general.archive:
-        path: "{{ obsah_state_path }}"
-        dest: "{{ obsah_state_path }}/foremanctl-state.tar.gz"
-        format: gz
+    - name: Copy pulp.snar from previous backup
+      ansible.builtin.copy:
+        src: "{{ backup_incremental }}/pulp.snar"
+        dest: "{{ backup_dir_full }}/pulp.snar"
+        remote_src: true
         mode: '0644'
+      when:
+        - backup_incremental is defined and backup_incremental | length > 0
+        - not skip_pulp_content | default(false)
+
+    - name: Fetch config.snar from previous backup to controller
+      ansible.builtin.fetch:
+        src: "{{ backup_incremental }}/config.snar"
+        dest: "{{ obsah_state_path | dirname }}/config.snar"
+        flat: true
+      when: backup_incremental is defined and backup_incremental | length > 0
+
+    - name: Backup foremanctl state directory  # noqa: command-instead-of-module
+      ansible.builtin.command:
+        cmd: >
+          tar -czf "{{ obsah_state_path | dirname }}/foremanctl-state.tar.gz"
+          --listed-incremental="{{ obsah_state_path | dirname }}/config.snar"
+          -C "{{ obsah_state_path | dirname }}"
+          "{{ obsah_state_path | basename }}"
       delegate_to: localhost
       become: false
+      changed_when: true
 
     - name: Copy foremanctl state archive to target
       ansible.builtin.copy:
-        src: "{{ obsah_state_path }}/foremanctl-state.tar.gz"
+        src: "{{ obsah_state_path | dirname }}/foremanctl-state.tar.gz"
         dest: "{{ backup_dir_full }}/foremanctl-state.tar.gz"
         mode: '0644'
 
-    - name: Clean up foremanctl state archive on controller
+    - name: Copy config.snar to backup directory
+      ansible.builtin.copy:
+        src: "{{ obsah_state_path | dirname }}/config.snar"
+        dest: "{{ backup_dir_full }}/config.snar"
+        mode: '0644'
+
+    - name: Clean up foremanctl state files on controller
       ansible.builtin.file:
-        path: "{{ obsah_state_path }}/foremanctl-state.tar.gz"
+        path: "{{ item }}"
         state: absent
+      loop:
+        - "{{ obsah_state_path | dirname }}/foremanctl-state.tar.gz"
+        - "{{ obsah_state_path | dirname }}/config.snar"
       delegate_to: localhost
       become: false
 

src/roles/backup/tasks/metadata.yaml

@@ -40,7 +40,10 @@
       os_version: "{{ ansible_facts['distribution'] }} {{ ansible_facts['distribution_version'] }}"
       foremanctl_version: "{{ ansible_facts.packages['foremanctl'][0].version | default('unknown') if 'foremanctl' in ansible_facts.packages else 'unknown' }}"
       type: offline
-      incremental: false
+      incremental:
+        is_incremental: "{{ (backup_incremental is defined and backup_incremental | length > 0) | bool }}"
+        base_backup_dir: "{{ backup_incremental | default('') }}"
+        base_backup_timestamp: "{{ (backup_previous_metadata | default({})).timestamp | default('') }}"
       timestamp: "{{ backup_timestamp }}"
       databases: "{{ backup_databases_to_backup }}"
       database_mapping: "{{ backup_databases_config | items2dict(key_name='name', value_name='database') }}"

src/roles/backup/tasks/pulp_content.yaml

@@ -5,8 +5,9 @@
     - name: Backup pulp content directory with encryption keys  # noqa: command-instead-of-module
       ansible.builtin.command:
         cmd: >
-          tar -czf {{ backup_dir_full }}/pulp-content.tar.gz
-          --directory={{ backup_pulp_storage_path }}
+          tar -czf "{{ backup_dir_full }}/pulp-content.tar.gz"
+          --listed-incremental="{{ backup_dir_full }}/pulp.snar"
+          --directory="{{ backup_pulp_storage_path }}"
           --exclude=media/exports
           --exclude=media/imports
           --exclude=media/sync_imports

src/roles/restore/tasks/main.yaml

@@ -6,26 +6,35 @@
 - name: Perform restore operations
   when: not validate | default(false)
   block:
-    - name: Prepare system for restore
-      ansible.builtin.include_tasks:
-        file: prepare_system.yaml
-
     - name: Restore foremanctl state
       ansible.builtin.include_tasks:
         file: restore_foremanctl_state.yaml
-      when: "'foremanctl_state' in restore_backup_metadata.backed_up_components | default(['foremanctl_state'])"
+
+    - name: Prepare system for restore
+      ansible.builtin.include_tasks:
+        file: prepare_system.yaml
 
     - name: Restore databases
       ansible.builtin.include_tasks:
         file: restore_databases.yaml
-      when:
-        - restore_database_mode == 'internal'
-        - "'databases' in restore_backup_metadata.backed_up_components | default(['databases'])"
 
     - name: Restore Pulp content
       ansible.builtin.include_tasks:
         file: restore_pulp_content.yaml
-      when: "'pulp_content' in restore_backup_metadata.backed_up_components | default([])"
+
+    - name: Record successful restore timestamp
+      ansible.builtin.copy:
+        content: "{{ restore_backup_metadata.timestamp }}"
+        dest: /var/lib/foremanctl/.last_restore_timestamp
+        mode: '0644'
+
+    - name: Display restore completion
+      ansible.builtin.debug:
+        msg: |
+          Restore completed successfully!
+          Backup: {{ restore_backup_dir }}
+          Type: {{ 'Incremental' if restore_is_incremental else 'Full' }}
+          Timestamp: {{ restore_backup_metadata.timestamp }}
 
   rescue:
     - name: Ensure services are stopped on failure

src/roles/restore/tasks/restore_databases.yaml

@@ -3,6 +3,7 @@
   containers.podman.podman_secret:
     name: postgresql-admin-password
     data: "{{ postgresql_admin_password }}"
+    force: true
 
 - name: Start PostgreSQL for restore
   ansible.builtin.systemd:
@@ -27,14 +28,13 @@
       dump_file: "{{ restore_backup_metadata.database_mapping[item.name] }}.dump"
       database: "{{ item.database }}"
       user: "{{ item.user }}"
-  loop: "{{ databases }}"
+  loop: "{{ databases | selectattr('name', 'in', restore_backup_metadata.databases) | list }}"
   loop_control:
     label: "{{ item.name }}"
-  when: item.name in restore_backup_metadata.database_mapping
 
 - name: Verify dump files exist
   ansible.builtin.stat:
-    path: "{{ backup_dir }}/{{ item.dump_file }}"
+    path: "{{ restore_backup_dir }}/{{ item.dump_file }}"
   register: restore_dump_files_check
   failed_when: not restore_dump_files_check.stat.exists
   loop: "{{ restore_databases_to_restore }}"
@@ -85,13 +85,14 @@
       pg_restore
       --host={{ database_host }}
       --port={{ database_port }}
-      --username={{ item.user }}
+      --username=postgres
       --dbname={{ item.database }}
-      {{ backup_dir }}/{{ item.dump_file }}
+      "{{ restore_backup_dir }}/{{ item.dump_file }}"
   environment:
     PGPASSWORD: "{{ postgresql_admin_password }}"
   loop: "{{ restore_databases_to_restore }}"
   loop_control:
-    label: "{{ item.dump_file }} → {{ item.database }}"
+    label: "{{ item.dump_file }} -> {{ item.database }}"
   changed_when: true
-  failed_when: false
+  register: restore_pg_restore_result
+  failed_when: restore_pg_restore_result.rc != 0 and 'WARNING' not in restore_pg_restore_result.stderr