From 0425517de01bb92871bc5e47c148860d9fc524e0 Mon Sep 17 00:00:00 2001 From: chyenne8 Date: Tue, 1 Sep 2026 08:46:37 -0400 Subject: [PATCH] Add incremental backup and restore --- docs/user/backup-restore.md | 62 ++++++++ src/playbooks/backup/metadata.obsah.yaml | 11 +- src/playbooks/health/metadata.obsah.yaml | 2 +- src/playbooks/restore/metadata.obsah.yaml | 6 +- src/roles/backup/tasks/database_dumps.yaml | 4 +- src/roles/backup/tasks/main.yaml | 73 ++++++++- src/roles/backup/tasks/metadata.yaml | 5 +- src/roles/backup/tasks/pulp_content.yaml | 5 +- src/roles/restore/tasks/main.yaml | 27 ++-- .../restore/tasks/restore_databases.yaml | 15 +- .../tasks/restore_foremanctl_state.yaml | 43 ++++- .../restore/tasks/restore_pulp_content.yaml | 36 ++++- src/roles/restore/tasks/validate.yaml | 148 ++++++++++++++++-- tests/backup_test.py | 2 +- 14 files changed, 386 insertions(+), 53 deletions(-) diff --git a/docs/user/backup-restore.md b/docs/user/backup-restore.md index 45a09b394..8cc40f469 100644 --- a/docs/user/backup-restore.md +++ b/docs/user/backup-restore.md @@ -40,6 +40,7 @@ This creates a timestamped backup directory at `/var/backup/foreman-backup-YYYYM | Option | Description | |--------|-------------| +| `--base-backup BASE_DIR` | Create an incremental backup based on a base backup. Only files changed since the base backup are included. The base backup directory must contain `.snar` snapshot files (generated by foremanctl backups). | | `--skip-pulp-content` | Skip backing up `/var/lib/pulp`. This is for debugging purposes or if you plan to copy `/var/lib/pulp` using other methods such as rsync or shared storage. **Warning:** You will not have a complete backup if you use this option. | | `--wait-for-tasks` | Wait for running Foreman and Pulp tasks to complete instead of failing immediately. The backup will poll until all tasks finish before proceeding. | @@ -69,6 +70,38 @@ Allow in-progress tasks to complete before starting backup: foremanctl backup /var/backup --wait-for-tasks ``` +### Incremental Backup + +Create a full backup first: + +```bash +foremanctl backup /var/backup +# Creates: /var/backup/foreman-backup-20260629T120000/ +``` + +Then create incremental backups referencing the base backup: + +```bash +foremanctl backup /var/backup --base-backup /var/backup/foreman-backup-20260629T120000 +# Creates: /var/backup/foreman-backup-20260630T080000/ (incremental) +``` + +**How it works:** +- The incremental backup contains only files changed since the previous backup +- `.snar` snapshot files track which files were in the previous backup +- Both `config.snar` (foremanctl state) and `pulp.snar` (Pulp content) are copied from the previous backup +- Incremental backups are typically much smaller and faster than full backups + +**Typical backup strategy:** +- Weekly full backup (Sunday) +- Daily incremental backups (Monday-Saturday) +- Each incremental references the previous day's backup + +**Important:** +- You can chain incrementals: full -> inc1 -> inc2 -> inc3 +- All backups in the chain are required for restore +- The previous backup must have been created with foremanctl (contains .snar files) + ## Backup Contents ### Databases @@ -269,6 +302,35 @@ foremanctl restore /var/backup/foreman-backup-20260617T104115 --validate This validates the backup and checks system requirements before proceeding. +### Restore Incremental Backups + +To restore from an incremental backup, restore each backup in the chain sequentially, starting with the full backup that began the chain: + +```bash +# Step 1: Restore the full (base) backup +foremanctl restore /var/backup/foreman-backup-20260629T120000 + +# Step 2: Restore the first incremental +foremanctl restore /var/backup/foreman-backup-20260630T080000 --force + +# Step 3: Restore the second incremental +foremanctl restore /var/backup/foreman-backup-20260701T080000 --force +``` + +**Important:** +- Incremental backups contain only files changed since the previous backup +- All backups in the chain must be restored in order (full -> inc1 -> inc2) +- The restore command automatically detects incremental backups from metadata +- The `--force` flag is required for subsequent restores since the system is already deployed + +**Automatic validation:** +The restore command validates the backup chain: +- Verifies the base backup directory exists +- Confirms the base backup metadata matches the expected timestamp +- Warns if the base backup has not been restored yet + +If any validation fails, a clear error message explains which backup is missing or incorrect. + ## Prerequisites Before restoring, ensure: diff --git a/src/playbooks/backup/metadata.obsah.yaml b/src/playbooks/backup/metadata.obsah.yaml index 775c22ab7..4717e2faf 100644 --- a/src/playbooks/backup/metadata.obsah.yaml +++ b/src/playbooks/backup/metadata.obsah.yaml @@ -1,6 +1,6 @@ --- help: | - Create offline backup of Foreman databases and configuration + Create offline backup of databases and configuration variables: backup_dir: @@ -23,3 +23,12 @@ variables: help: Wait for running tasks to complete instead of failing immediately action: store_true persist: false + + backup_incremental: + parameter: --base-backup + help: | + Path to base backup directory for incremental backup. + Creates a differential backup containing only files changed since + the base backup. Requires .snar files from base backup. + type: AbsolutePath + persist: false diff --git a/src/playbooks/health/metadata.obsah.yaml b/src/playbooks/health/metadata.obsah.yaml index 5733dc385..5cd929e97 100644 --- a/src/playbooks/health/metadata.obsah.yaml +++ b/src/playbooks/health/metadata.obsah.yaml @@ -1,6 +1,6 @@ --- help: | - Check the health of your running Foreman server + Check the health of your running server variables: health_skip_check_foreman_tasks_param: help: | diff --git a/src/playbooks/restore/metadata.obsah.yaml b/src/playbooks/restore/metadata.obsah.yaml index 54c126e19..25e8f72a6 100644 --- a/src/playbooks/restore/metadata.obsah.yaml +++ b/src/playbooks/restore/metadata.obsah.yaml @@ -3,11 +3,11 @@ help: | Restore from a backup Validates backup contents, extracts configuration files, restores databases, - restores Pulp content, and redeploys the system. + restores content, and redeploys the system. variables: - backup_dir: - parameter: backup_dir + restore_backup_dir: + parameter: restore_backup_dir help: Directory containing the backup files type: AbsolutePath persist: false diff --git a/src/roles/backup/tasks/database_dumps.yaml b/src/roles/backup/tasks/database_dumps.yaml index 207d509e1..466a5e595 100644 --- a/src/roles/backup/tasks/database_dumps.yaml +++ b/src/roles/backup/tasks/database_dumps.yaml @@ -7,8 +7,8 @@ --port={{ item.port }} --username={{ item.user }} --format=custom - --file={{ backup_dir_full }}/{{ item.database }}.dump - {{ item.database }} + --file="{{ backup_dir_full }}/{{ item.database }}.dump" + "{{ item.database }}" environment: PGPASSWORD: "{{ item.password }}" loop: "{{ backup_databases_config }}" diff --git a/src/roles/backup/tasks/main.yaml b/src/roles/backup/tasks/main.yaml index f73999b17..0a2e0699d 100644 --- a/src/roles/backup/tasks/main.yaml +++ b/src/roles/backup/tasks/main.yaml @@ -7,6 +7,35 @@ ansible.builtin.set_fact: backup_dir_full: "{{ backup_dir }}/foreman-backup-{{ backup_timestamp }}" +- name: Validate backup_incremental backup parameters + when: backup_incremental is defined and backup_incremental | length > 0 + block: + - name: Check previous backup files exist + ansible.builtin.stat: + path: "{{ backup_incremental }}/{{ item }}" + register: backup_previous_files_check + loop: + - "metadata.yml" + - "config.snar" + - "pulp.snar" + failed_when: false + + - name: Fail if previous backup is invalid + ansible.builtin.fail: + msg: | + Previous backup directory does not exist or is not a valid foremanctl backup: {{ backup_incremental }} + Missing files: {{ backup_previous_files_check.results | selectattr('stat.exists', 'equalto', false) | map(attribute='item') | list }} + + The previous backup must contain metadata.yml and .snar files generated by foremanctl. + Create a new full backup first: + foremanctl backup {{ backup_dir }} + when: backup_previous_files_check.results | selectattr('stat.exists', 'equalto', false) | list | length > 0 + + - name: Load previous backup metadata + ansible.builtin.include_vars: + file: "{{ backup_incremental }}/metadata.yml" + name: backup_previous_metadata + - name: Ensure backup directory exists ansible.builtin.file: path: "{{ backup_dir }}" @@ -92,25 +121,53 @@ ansible.builtin.include_tasks: file: database_dumps.yaml - - name: Archive foremanctl state on controller - community.general.archive: - path: "{{ obsah_state_path }}" - dest: "{{ obsah_state_path }}/foremanctl-state.tar.gz" - format: gz + - name: Copy pulp.snar from previous backup + ansible.builtin.copy: + src: "{{ backup_incremental }}/pulp.snar" + dest: "{{ backup_dir_full }}/pulp.snar" + remote_src: true mode: '0644' + when: + - backup_incremental is defined and backup_incremental | length > 0 + - not skip_pulp_content | default(false) + + - name: Fetch config.snar from previous backup to controller + ansible.builtin.fetch: + src: "{{ backup_incremental }}/config.snar" + dest: "{{ obsah_state_path | dirname }}/config.snar" + flat: true + when: backup_incremental is defined and backup_incremental | length > 0 + + - name: Backup foremanctl state directory # noqa: command-instead-of-module + ansible.builtin.command: + cmd: > + tar -czf "{{ obsah_state_path | dirname }}/foremanctl-state.tar.gz" + --listed-incremental="{{ obsah_state_path | dirname }}/config.snar" + -C "{{ obsah_state_path | dirname }}" + "{{ obsah_state_path | basename }}" delegate_to: localhost become: false + changed_when: true - name: Copy foremanctl state archive to target ansible.builtin.copy: - src: "{{ obsah_state_path }}/foremanctl-state.tar.gz" + src: "{{ obsah_state_path | dirname }}/foremanctl-state.tar.gz" dest: "{{ backup_dir_full }}/foremanctl-state.tar.gz" mode: '0644' - - name: Clean up foremanctl state archive on controller + - name: Copy config.snar to backup directory + ansible.builtin.copy: + src: "{{ obsah_state_path | dirname }}/config.snar" + dest: "{{ backup_dir_full }}/config.snar" + mode: '0644' + + - name: Clean up foremanctl state files on controller ansible.builtin.file: - path: "{{ obsah_state_path }}/foremanctl-state.tar.gz" + path: "{{ item }}" state: absent + loop: + - "{{ obsah_state_path | dirname }}/foremanctl-state.tar.gz" + - "{{ obsah_state_path | dirname }}/config.snar" delegate_to: localhost become: false diff --git a/src/roles/backup/tasks/metadata.yaml b/src/roles/backup/tasks/metadata.yaml index bff20540b..f9c79eb15 100644 --- a/src/roles/backup/tasks/metadata.yaml +++ b/src/roles/backup/tasks/metadata.yaml @@ -40,7 +40,10 @@ os_version: "{{ ansible_facts['distribution'] }} {{ ansible_facts['distribution_version'] }}" foremanctl_version: "{{ ansible_facts.packages['foremanctl'][0].version | default('unknown') if 'foremanctl' in ansible_facts.packages else 'unknown' }}" type: offline - incremental: false + incremental: + is_incremental: "{{ (backup_incremental is defined and backup_incremental | length > 0) | bool }}" + base_backup_dir: "{{ backup_incremental | default('') }}" + base_backup_timestamp: "{{ (backup_previous_metadata | default({})).timestamp | default('') }}" timestamp: "{{ backup_timestamp }}" databases: "{{ backup_databases_to_backup }}" database_mapping: "{{ backup_databases_config | items2dict(key_name='name', value_name='database') }}" diff --git a/src/roles/backup/tasks/pulp_content.yaml b/src/roles/backup/tasks/pulp_content.yaml index 7b691894e..bd625581d 100644 --- a/src/roles/backup/tasks/pulp_content.yaml +++ b/src/roles/backup/tasks/pulp_content.yaml @@ -5,8 +5,9 @@ - name: Backup pulp content directory with encryption keys # noqa: command-instead-of-module ansible.builtin.command: cmd: > - tar -czf {{ backup_dir_full }}/pulp-content.tar.gz - --directory={{ backup_pulp_storage_path }} + tar -czf "{{ backup_dir_full }}/pulp-content.tar.gz" + --listed-incremental="{{ backup_dir_full }}/pulp.snar" + --directory="{{ backup_pulp_storage_path }}" --exclude=media/exports --exclude=media/imports --exclude=media/sync_imports diff --git a/src/roles/restore/tasks/main.yaml b/src/roles/restore/tasks/main.yaml index 37a9e6d47..914ab3541 100644 --- a/src/roles/restore/tasks/main.yaml +++ b/src/roles/restore/tasks/main.yaml @@ -6,26 +6,35 @@ - name: Perform restore operations when: not validate | default(false) block: - - name: Prepare system for restore - ansible.builtin.include_tasks: - file: prepare_system.yaml - - name: Restore foremanctl state ansible.builtin.include_tasks: file: restore_foremanctl_state.yaml - when: "'foremanctl_state' in restore_backup_metadata.backed_up_components | default(['foremanctl_state'])" + + - name: Prepare system for restore + ansible.builtin.include_tasks: + file: prepare_system.yaml - name: Restore databases ansible.builtin.include_tasks: file: restore_databases.yaml - when: - - restore_database_mode == 'internal' - - "'databases' in restore_backup_metadata.backed_up_components | default(['databases'])" - name: Restore Pulp content ansible.builtin.include_tasks: file: restore_pulp_content.yaml - when: "'pulp_content' in restore_backup_metadata.backed_up_components | default([])" + + - name: Record successful restore timestamp + ansible.builtin.copy: + content: "{{ restore_backup_metadata.timestamp }}" + dest: /var/lib/foremanctl/.last_restore_timestamp + mode: '0644' + + - name: Display restore completion + ansible.builtin.debug: + msg: | + Restore completed successfully! + Backup: {{ restore_backup_dir }} + Type: {{ 'Incremental' if restore_is_incremental else 'Full' }} + Timestamp: {{ restore_backup_metadata.timestamp }} rescue: - name: Ensure services are stopped on failure diff --git a/src/roles/restore/tasks/restore_databases.yaml b/src/roles/restore/tasks/restore_databases.yaml index 20f88eba3..505a6ff81 100644 --- a/src/roles/restore/tasks/restore_databases.yaml +++ b/src/roles/restore/tasks/restore_databases.yaml @@ -3,6 +3,7 @@ containers.podman.podman_secret: name: postgresql-admin-password data: "{{ postgresql_admin_password }}" + force: true - name: Start PostgreSQL for restore ansible.builtin.systemd: @@ -27,14 +28,13 @@ dump_file: "{{ restore_backup_metadata.database_mapping[item.name] }}.dump" database: "{{ item.database }}" user: "{{ item.user }}" - loop: "{{ databases }}" + loop: "{{ databases | selectattr('name', 'in', restore_backup_metadata.databases) | list }}" loop_control: label: "{{ item.name }}" - when: item.name in restore_backup_metadata.database_mapping - name: Verify dump files exist ansible.builtin.stat: - path: "{{ backup_dir }}/{{ item.dump_file }}" + path: "{{ restore_backup_dir }}/{{ item.dump_file }}" register: restore_dump_files_check failed_when: not restore_dump_files_check.stat.exists loop: "{{ restore_databases_to_restore }}" @@ -85,13 +85,14 @@ pg_restore --host={{ database_host }} --port={{ database_port }} - --username={{ item.user }} + --username=postgres --dbname={{ item.database }} - {{ backup_dir }}/{{ item.dump_file }} + "{{ restore_backup_dir }}/{{ item.dump_file }}" environment: PGPASSWORD: "{{ postgresql_admin_password }}" loop: "{{ restore_databases_to_restore }}" loop_control: - label: "{{ item.dump_file }} → {{ item.database }}" + label: "{{ item.dump_file }} -> {{ item.database }}" changed_when: true - failed_when: false + register: restore_pg_restore_result + failed_when: restore_pg_restore_result.rc != 0 and 'WARNING' not in restore_pg_restore_result.stderr diff --git a/src/roles/restore/tasks/restore_foremanctl_state.yaml b/src/roles/restore/tasks/restore_foremanctl_state.yaml index 7de3d9024..ad8d3af73 100644 --- a/src/roles/restore/tasks/restore_foremanctl_state.yaml +++ b/src/roles/restore/tasks/restore_foremanctl_state.yaml @@ -1,11 +1,48 @@ --- -- name: Remove existing foremanctl state directory +- name: Check for config.snar file (incremental backup indicator) + ansible.builtin.stat: + path: "{{ restore_backup_dir }}/config.snar" + register: restore_config_snar_check + +- name: Remove existing foremanctl state directory for full restore ansible.builtin.file: path: "{{ obsah_state_path }}" state: absent + when: not restore_config_snar_check.stat.exists + +- name: Ensure state directory parent exists + ansible.builtin.file: + path: "{{ obsah_state_path | dirname }}" + state: directory + mode: '0755' -- name: Extract foremanctl state archive +- name: Extract foremanctl state archive (incremental with snar) # noqa: command-instead-of-module + ansible.builtin.command: + cmd: > + tar -xzf "{{ restore_backup_dir }}/foremanctl-state.tar.gz" + --listed-incremental=/dev/null + -C "{{ obsah_state_path | dirname }}" + when: restore_config_snar_check.stat.exists + changed_when: true + +- name: Extract foremanctl state archive (full backup) ansible.builtin.unarchive: - src: "{{ backup_dir }}/foremanctl-state.tar.gz" + src: "{{ restore_backup_dir }}/foremanctl-state.tar.gz" dest: "{{ obsah_state_path | dirname }}" remote_src: true + when: not restore_config_snar_check.stat.exists + +- name: Build list of expected state files + ansible.builtin.set_fact: + restore_expected_state_files: >- + {{ + ['parameters.yaml', 'oauth/foreman-oauth-consumer-key', 'oauth/foreman-oauth-consumer-secret', 'postgresql-admin-password'] + + (restore_backup_metadata.databases | map('regex_replace', '_', '-') | map('regex_replace', '$', '-db-password') | list) + }} + +- name: Verify critical files were restored + ansible.builtin.stat: + path: "{{ obsah_state_path }}/{{ item }}" + register: restore_state_files + failed_when: not restore_state_files.stat.exists + loop: "{{ restore_expected_state_files }}" diff --git a/src/roles/restore/tasks/restore_pulp_content.yaml b/src/roles/restore/tasks/restore_pulp_content.yaml index b0a3fa826..492ba56c1 100644 --- a/src/roles/restore/tasks/restore_pulp_content.yaml +++ b/src/roles/restore/tasks/restore_pulp_content.yaml @@ -1,7 +1,7 @@ --- - name: Check if pulp content archive exists ansible.builtin.stat: - path: "{{ backup_dir }}/pulp-content.tar.gz" + path: "{{ restore_backup_dir }}/pulp-content.tar.gz" register: restore_pulp_content_check - name: Restore pulp content and encryption keys @@ -13,16 +13,32 @@ state: directory mode: '0755' - - name: Remove existing pulp media directory + - name: Check for pulp.snar file (incremental backup indicator) + ansible.builtin.stat: + path: "{{ restore_backup_dir }}/pulp.snar" + register: restore_pulp_snar_check + + - name: Remove existing pulp media directory for full restore ansible.builtin.file: path: "{{ pulp_storage_path }}/media" state: absent + when: not restore_pulp_snar_check.stat.exists + + - name: Extract pulp content archive (incremental with snar) # noqa: command-instead-of-module + ansible.builtin.command: + cmd: > + tar -xzf "{{ restore_backup_dir }}/pulp-content.tar.gz" + --listed-incremental=/dev/null + -C "{{ pulp_storage_path }}" + when: restore_pulp_snar_check.stat.exists + changed_when: true - - name: Extract pulp content archive + - name: Extract pulp content archive (full backup) ansible.builtin.unarchive: - src: "{{ backup_dir }}/pulp-content.tar.gz" + src: "{{ restore_backup_dir }}/pulp-content.tar.gz" dest: "{{ pulp_storage_path }}" remote_src: true + when: not restore_pulp_snar_check.stat.exists - name: Verify Pulp encryption key was restored ansible.builtin.stat: @@ -35,3 +51,15 @@ path: "{{ pulp_storage_path }}/django_secret_key" register: restore_django_secret_key failed_when: not restore_django_secret_key.stat.exists + + - name: Count restored media files + ansible.builtin.find: + paths: "{{ pulp_storage_path }}/media" + file_type: file + recurse: true + register: restore_pulp_media_files + + - name: Get archive size + ansible.builtin.stat: + path: "{{ restore_backup_dir }}/pulp-content.tar.gz" + register: restore_pulp_content_size diff --git a/src/roles/restore/tasks/validate.yaml b/src/roles/restore/tasks/validate.yaml index 622365776..275dda87e 100644 --- a/src/roles/restore/tasks/validate.yaml +++ b/src/roles/restore/tasks/validate.yaml @@ -18,7 +18,7 @@ There is NO UNDO operation. If you are certain you want to proceed, re-run with --force: - foremanctl restore {{ backup_dir }} --force + foremanctl restore {{ restore_backup_dir }} --force when: - restore_existing_deployment.stat.exists - not force | default(false) @@ -26,33 +26,155 @@ - name: Check if backup directory exists ansible.builtin.stat: - path: "{{ backup_dir }}" + path: "{{ restore_backup_dir }}" register: restore_backup_dir_stat - name: Fail if backup directory does not exist ansible.builtin.fail: - msg: "Backup directory does not exist: {{ backup_dir }}" + msg: "Backup directory does not exist: {{ restore_backup_dir }}" when: not restore_backup_dir_stat.stat.exists - name: Check for metadata.yml ansible.builtin.stat: - path: "{{ backup_dir }}/metadata.yml" + path: "{{ restore_backup_dir }}/metadata.yml" register: restore_metadata_stat - name: Fail if metadata.yml is missing ansible.builtin.fail: - msg: "Backup metadata file not found: {{ backup_dir }}/metadata.yml" + msg: "Backup metadata file not found: {{ restore_backup_dir }}/metadata.yml" when: not restore_metadata_stat.stat.exists - name: Read backup metadata ansible.builtin.slurp: - path: "{{ backup_dir }}/metadata.yml" + path: "{{ restore_backup_dir }}/metadata.yml" register: restore_metadata_content - name: Parse backup metadata ansible.builtin.set_fact: restore_backup_metadata: "{{ restore_metadata_content['content'] | b64decode | from_yaml }}" +- name: Detect if backup is incremental + ansible.builtin.set_fact: + restore_is_incremental: "{{ restore_backup_metadata.incremental.is_incremental | bool }}" + restore_base_backup_dir: "{{ restore_backup_metadata.incremental.base_backup_dir }}" + restore_base_backup_timestamp: "{{ restore_backup_metadata.incremental.base_backup_timestamp }}" + +- name: Display backup type + ansible.builtin.debug: + msg: >- + Backup type: {{ 'Incremental' if restore_is_incremental else 'Full' }}{{ + ' (base: ' + restore_base_backup_dir + ')' if restore_is_incremental else '' }} + +- name: Validate incremental backup chain + when: restore_is_incremental + block: + - name: Fail if base backup directory not specified + ansible.builtin.fail: + msg: | + INCREMENTAL BACKUP CHAIN ERROR: + + This is an incremental backup but no base backup directory is specified in metadata. + Backup: {{ restore_backup_dir }} + + Incremental backups must reference a base (full) backup. + The metadata.yml file may be corrupted or incomplete. + when: restore_base_backup_dir | length == 0 + + - name: Check if base backup directory exists + ansible.builtin.stat: + path: "{{ restore_base_backup_dir }}" + register: restore_base_backup_stat + failed_when: false + + - name: Fail if base backup directory missing + ansible.builtin.fail: + msg: | + INCOMPLETE BACKUP CHAIN: + + This is an incremental backup that requires its base backup to be restored first. + + Incremental backup: {{ restore_backup_dir }} + Required base backup: {{ restore_base_backup_dir }} + + ERROR: Base backup directory does not exist! + + To restore an incremental backup, you must: + 1. First restore the full (base) backup: + foremanctl restore {{ restore_base_backup_dir }} + 2. Then restore this incremental backup: + foremanctl restore {{ restore_backup_dir }} + + All backups in the chain must be restored in chronological order. + when: not restore_base_backup_stat.stat.exists + + - name: Check for base backup metadata + ansible.builtin.stat: + path: "{{ restore_base_backup_dir }}/metadata.yml" + register: restore_base_metadata_stat + failed_when: false + + - name: Fail if base backup metadata missing + ansible.builtin.fail: + msg: | + INCOMPLETE BACKUP CHAIN: + + Base backup directory exists but metadata.yml is missing. + Base backup: {{ restore_base_backup_dir }} + + The base backup may be incomplete or corrupted. + Cannot validate backup chain integrity. + when: not restore_base_metadata_stat.stat.exists + + - name: Read base backup metadata + ansible.builtin.slurp: + path: "{{ restore_base_backup_dir }}/metadata.yml" + register: restore_base_metadata_content + + - name: Parse base backup metadata + ansible.builtin.set_fact: + restore_base_metadata: "{{ restore_base_metadata_content['content'] | b64decode | from_yaml }}" + + - name: Validate base backup timestamp matches + ansible.builtin.fail: + msg: | + BACKUP CHAIN MISMATCH: + + The incremental backup references a different base backup than provided. + + Incremental backup: {{ restore_backup_dir }} + Expected base timestamp: {{ restore_base_backup_timestamp }} + Actual base timestamp: {{ restore_base_metadata.timestamp }} + + This incremental backup is not based on the backup at {{ restore_base_backup_dir }}. + Verify you are using the correct base backup directory. + when: restore_base_backup_timestamp != restore_base_metadata.timestamp + + - name: Check if base backup has already been restored + ansible.builtin.stat: + path: /var/lib/foremanctl/.last_restore_timestamp + register: restore_last_restore_check + failed_when: false + + - name: Read last restore timestamp + ansible.builtin.slurp: + path: /var/lib/foremanctl/.last_restore_timestamp + register: restore_last_restore_content + when: restore_last_restore_check.stat.exists + failed_when: false + + - name: Warn if base backup not yet restored + ansible.builtin.debug: + msg: | + WARNING: Cannot verify if base backup has been restored. + + This is an incremental backup. Before proceeding, ensure you have already restored: + Base backup: {{ restore_base_backup_dir }} + + Restoring an incremental backup without its base will result in incomplete data! + when: > + not restore_last_restore_check.stat.exists or + (restore_last_restore_content['content'] | b64decode | trim) != restore_base_backup_timestamp + - name: Set database mode from backup metadata ansible.builtin.set_fact: restore_database_mode: "{{ restore_backup_metadata.database_mode | default('internal') }}" @@ -64,7 +186,7 @@ block: - name: Verify database dump files exist ansible.builtin.stat: - path: "{{ backup_dir }}/{{ restore_backup_metadata.database_mapping[item] }}.dump" + path: "{{ restore_backup_dir }}/{{ restore_backup_metadata.database_mapping[item] }}.dump" register: restore_dump_files_check failed_when: not restore_dump_files_check.stat.exists loop: "{{ restore_backup_metadata.databases }}" @@ -78,7 +200,7 @@ - name: Get total size of database dumps ansible.builtin.shell: - cmd: set -o pipefail && du -sB1 {{ backup_dir }}/*.dump 2>/dev/null | awk '{sum+=$1} END {print int(sum/1024/1024/1024)+1}' + cmd: set -o pipefail && du -sB1 '{{ restore_backup_dir }}'/*.dump 2>/dev/null | awk '{sum+=$1} END {print int(sum/1024/1024/1024)+1}' executable: /bin/bash register: restore_dumps_size changed_when: false @@ -96,7 +218,7 @@ - name: Verify foremanctl state archive exists ansible.builtin.stat: - path: "{{ backup_dir }}/foremanctl-state.tar.gz" + path: "{{ restore_backup_dir }}/foremanctl-state.tar.gz" register: restore_state_archive failed_when: not restore_state_archive.stat.exists when: "'foremanctl_state' in restore_backup_metadata.backed_up_components | default(['foremanctl_state'])" @@ -110,9 +232,8 @@ - name: Get Pulp content archive size ansible.builtin.stat: - path: "{{ backup_dir }}/pulp-content.tar.gz" + path: "{{ restore_backup_dir }}/pulp-content.tar.gz" register: restore_pulp_archive - when: restore_pulp_archive is defined - name: Fail if insufficient Pulp disk space ansible.builtin.fail: @@ -124,3 +245,8 @@ when: - restore_pulp_archive.stat.exists | default(false) - restore_pulp_space.stdout | int < (restore_pulp_archive.stat.size / 1024 / 1024 / 1024) | int + 1 + +- name: Check database mode matches backup + ansible.builtin.debug: + msg: "WARNING: Current database_mode ({{ database_mode }}) may not match backup configuration. Verify compatibility before proceeding." + when: restore_backup_metadata.database_mode is defined and restore_backup_metadata.database_mode != database_mode diff --git a/tests/backup_test.py b/tests/backup_test.py index ff76d15c8..7522084fb 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -242,7 +242,7 @@ def test_metadata_structure(backup_metadata, expected_databases): assert field in backup_metadata, f"Metadata should contain '{field}' field" assert backup_metadata['type'] == 'offline', "Backup type should be 'offline'" - assert backup_metadata['incremental'] is False, "Backup should not be incremental" + assert backup_metadata['incremental']['is_incremental'] is False, "Backup should not be incremental" assert backup_metadata['database_mode'] in ['internal', 'external'], "Database mode should be 'internal' or 'external'" # Verify expected databases are present