Main fdb backup

simplyblock_cli/cli-reference.yaml

@@ -2652,3 +2652,61 @@ commands:
             type: str
             required: false
             default: ""
+  - name: "db-backup"
+    help: "FDB Backup operations"
+    weight: 800
+    subcommands:
+      - name: create
+        help: "Creates an fdb backup"
+        arguments:
+          - name: "cluster_id"
+            help: "Cluster ID to create db backup for"
+            dest: cluster_id
+            type: str
+      - name: list
+        help: "Lists all fdb backups"
+        arguments:
+          - name: "cluster_id"
+            help: "Cluster ID to restore db backup to"
+            dest: cluster_id
+            type: str
+      - name: status
+        help: "get backup status"
+      - name: restore
+        help: "restore a backup"
+        arguments:
+          - name: "name"
+            help: "backup class name"
+            dest: name
+            type: str
+          - name: "cluster_id"
+            help: "Cluster ID to restore db backup to"
+            dest: cluster_id
+            type: str
+      - name: config
+        help: "Set backup configuration"
+        arguments:
+          - name: "cluster_id"
+            help: "Cluster ID to configure db backup for"
+            dest: cluster_id
+            type: str
+          - name: "--backup-path"
+            help: 'local backup path, defaults to /etc/foundationdb/backup'
+            dest: backup_path
+            type: str
+          - name: "--backup-frequency"
+            help: "backup frequency, can be 3h, 1d"
+            dest: backup_frequency
+            type: str
+          - name: "--s3-bucket"
+            help: 'AWS S3 bucket name'
+            dest: bucket_name
+            type: str
+          - name: "--s3-region"
+            help: 'AWS S3 region'
+            dest: region_name
+            type: str
+          - name: "--s3-credentials"
+            help: 'AWS S3 API key and secret, should be supplied like this: [API_KEY]:[API_SECRET]'
+            dest: backup_credentials
+            type: str

simplyblock_cli/cli.py

@@ -31,6 +31,7 @@ def __init__(self):
         self.init_snapshot()
         self.init_backup()
         self.init_qos()
+        self.init_db_backup()
         super().__init__()
 
     def init_storage_node(self):
@@ -1100,6 +1101,41 @@ def init_qos__delete(self, subparser):
         subcommand.add_argument('cluster_id', help='The cluster id.', type=str, default='')
 
 
+    def init_db_backup(self):
+        subparser = self.add_command('db-backup', 'FDB Backup operations')
+        self.init_db_backup__create(subparser)
+        self.init_db_backup__list(subparser)
+        self.init_db_backup__status(subparser)
+        self.init_db_backup__restore(subparser)
+        self.init_db_backup__config(subparser)
+
+
+    def init_db_backup__create(self, subparser):
+        subcommand = self.add_sub_command(subparser, 'create', 'Creates an fdb backup')
+        subcommand.add_argument('cluster_id', help='Cluster ID to create db backup for', type=str)
+
+    def init_db_backup__list(self, subparser):
+        subcommand = self.add_sub_command(subparser, 'list', 'Lists all fdb backups')
+        subcommand.add_argument('cluster_id', help='Cluster ID to restore db backup to', type=str)
+
+    def init_db_backup__status(self, subparser):
+        subcommand = self.add_sub_command(subparser, 'status', 'get backup status')
+
+    def init_db_backup__restore(self, subparser):
+        subcommand = self.add_sub_command(subparser, 'restore', 'restore a backup')
+        subcommand.add_argument('name', help='backup class name', type=str)
+        subcommand.add_argument('cluster_id', help='Cluster ID to restore db backup to', type=str)
+
+    def init_db_backup__config(self, subparser):
+        subcommand = self.add_sub_command(subparser, 'config', 'Set backup configuration')
+        subcommand.add_argument('cluster_id', help='Cluster ID to configure db backup for', type=str)
+        subcommand.add_argument('--backup-path', help='local backup path, defaults to /etc/foundationdb/backup', type=str, dest='backup_path')
+        subcommand.add_argument('--backup-frequency', help='backup frequency, can be 3h, 1d', type=str, dest='backup_frequency')
+        subcommand.add_argument('--s3-bucket', help='AWS S3 bucket name', type=str, dest='bucket_name')
+        subcommand.add_argument('--s3-region', help='AWS S3 region', type=str, dest='region_name')
+        subcommand.add_argument('--s3-credentials', help='AWS S3 API key and secret, should be supplied like this: [API_KEY]:[API_SECRET]', type=str, dest='backup_credentials')
+
+
     def run(self):
         args = self.parser.parse_args()
         if args.debug:
@@ -1550,6 +1586,21 @@ def run(self):
                 else:
                     self.parser.print_help()
 
+            elif args.command in ['db-backup']:
+                sub_command = args_dict['db-backup']
+                if sub_command in ['create']:
+                    ret = self.db_backup__create(sub_command, args)
+                elif sub_command in ['list']:
+                    ret = self.db_backup__list(sub_command, args)
+                elif sub_command in ['status']:
+                    ret = self.db_backup__status(sub_command, args)
+                elif sub_command in ['restore']:
+                    ret = self.db_backup__restore(sub_command, args)
+                elif sub_command in ['config']:
+                    ret = self.db_backup__config(sub_command, args)
+                else:
+                    self.parser.print_help()
+
             else:
                 self.parser.print_help()
 

simplyblock_cli/clibase.py

@@ -13,7 +13,7 @@
 from simplyblock_core import storage_node_ops as storage_ops
 from simplyblock_core import mgmt_node_ops as mgmt_ops
 from simplyblock_core.controllers import pool_controller, lvol_controller, snapshot_controller, device_controller, \
-    tasks_controller, qos_controller, migration_controller, backup_controller
+    tasks_controller, qos_controller, migration_controller, backup_controller, fdb_backup_controller
 from simplyblock_core.controllers import health_controller
 from simplyblock_core.models.pool import Pool
 from simplyblock_core.models.cluster import Cluster, HashicorpVaultSettings
@@ -1076,6 +1076,21 @@ def backup__source_switch(self, sub_command, args):
             print(f"Switched to external backup source: {target}")
         return True
 
+    def db_backup__create(self, sub_command, args):
+        return fdb_backup_controller.add_backup_task(args.cluster_id)
+
+    def db_backup__list(self, sub_command, args):
+        return fdb_backup_controller.list_backups(args.cluster_id)
+
+    def db_backup__status(self, sub_command, args):
+        return fdb_backup_controller.backup_status()
+
+    def db_backup__restore(self, sub_command, args):
+        return fdb_backup_controller.backup_restore(args.name, args.cluster_id)
+
+    def db_backup__config(self, sub_command, args):
+        return fdb_backup_controller.backup_configure(args.cluster_id, args.backup_path, args.backup_frequency, args.bucket_name, args.region_name, args.backup_credentials)
+
     def storage_node_list_devices(self, args):
         data = storage_ops.list_storage_devices(args.node_id)
 

simplyblock_core/cluster_ops.py

@@ -381,6 +381,7 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass,
     cluster.snode_api_port = snode_api_port
     cluster.container_image_prefix = container_image_prefix or ""
     cluster.hashicorp_vault_settings = hashicorp_vault_settings
+    cluster.backup_local_path = os.path.join(constants.KVD_DB_BACKUP_PATH, cluster.uuid)
 
     if nvmeof_tls_config:
         cluster.tls = True
@@ -596,6 +597,7 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn
     if backup_config:
         cluster.backup_config = backup_config
 
+    cluster.backup_local_path = os.path.join(constants.KVD_DB_BACKUP_PATH, cluster.uuid)
     cluster.status = Cluster.STATUS_UNREADY
     cluster.create_dt = str(datetime.datetime.now())
     cluster.write_to_db(db_controller.kv_store)

simplyblock_core/constants.py

@@ -26,6 +26,7 @@ def get_config_var(name, default=None):
 KVD_DB_VERSION = 730
 KVD_DB_FILE_PATH = os.getenv('FDB_CLUSTER_FILE', '/etc/foundationdb/fdb.cluster')
 KVD_DB_TIMEOUT_MS = 10000
+KVD_DB_BACKUP_PATH = "file:///etc/foundationdb/backup"
 SPK_DIR = '/home/ec2-user/spdk'
 LOG_LEVEL = logging.INFO
 LOG_WEB_LEVEL = logging.DEBUG

simplyblock_core/controllers/fdb_backup_controller.py

@@ -0,0 +1,207 @@
+# coding=utf-8
+import datetime
+import logging as lg
+import re
+import time
+import uuid
+
+import docker
+
+from simplyblock_core import utils, constants
+from simplyblock_core.db_controller import DBController
+from simplyblock_core.models.job_schedule import JobSchedule
+
+logger = lg.getLogger()
+db_controller = DBController()
+
+def __get_fdb_cont():
+    snode = db_controller.get_mgmt_nodes()[0]
+    if not snode:
+        return
+    node_docker = docker.DockerClient(base_url=f"tcp://{snode.docker_ip_port}", version="auto")
+    for container in node_docker.containers.list():
+        if container.name.startswith("app_fdb-backup-agent"): # type: ignore[union-attr]
+            return container
+
+def create_backup(cluster_id):
+    container = __get_fdb_cont()
+    if container:
+        cluster = db_controller.get_cluster_by_id(cluster_id)
+        backup_path = cluster.get_backup_path()
+        if cluster.backup_s3_bucket and cluster.backup_s3_cred:
+            folder = f"backup-{str(datetime.datetime.now())}"
+            folder = folder.replace(" ", "-")
+            folder = folder.replace(":", "-")
+            folder = folder.split(".")[0]
+            backup_path = f"blobstore://{cluster.backup_s3_cred}@s3.{cluster.backup_s3_region}.amazonaws.com/{folder}?bucket={cluster.backup_s3_bucket}&region={cluster.backup_s3_region}&sc=0"
+
+        res = container.exec_run(cmd=f"fdbbackup start -d {backup_path} -w")
+        cont = res.output.decode("utf-8")
+        logger.info(cont)
+        return True
+    return False
+
+def list_backups(cluster_id):
+    container = __get_fdb_cont()
+    data = []
+    if container:
+        cluster = db_controller.get_cluster_by_id(cluster_id)
+        backup_path = cluster.get_backup_path()
+        res = container.exec_run(cmd=f"fdbbackup list -b {backup_path}")
+        logger.info(f"backup list from : {backup_path}")
+        cont = res.output.decode("utf-8")
+        for line in cont.splitlines():
+            if not line or "backup-" not in line:
+                continue
+
+            name = line.split("/")[-1].strip()
+            name = name.split("?")[0]
+            size = 0
+            restorable = 0
+            date = ""
+            version = 0
+            res = container.exec_run(cmd=f"fdbbackup describe -d {cluster.get_backup_path(name)} --version-timestamps")
+            cont = res.output.decode("utf-8")
+            for line in cont.splitlines():
+                if line and line.startswith("SnapshotBytes"):
+                    size = line.split()[1].strip()
+                if line and line.startswith("Restorable"):
+                    restorable = line.split()[1].strip()
+                if line and line.startswith("Snapshot:"):
+                    for param in line.split():
+                        if param.startswith("startVersion"):
+                            version = param.split("=")[1].strip()
+                        elif param.startswith("(") and param.endswith(")") and not date:# 2025/12/28.10:10:20+0000
+                            try:
+                                date = datetime.datetime.strptime(param[1:-1], "%Y/%m/%d.%H:%M:%S+0000").strftime("%Y-%m-%d %H:%M:%S")
+                            except Exception:
+                                date = name.replace("backup-","")
+            if not date:
+                date = name.replace("backup-", "")
+
+            data.append({
+                "Name": name,
+                "Version": version,
+                "Size": utils.humanbytes(int(size)),
+                "Restorable": restorable,
+                "Date": date,
+            })
+
+        return utils.print_table(data)
+
+    return True
+
+
+
+def backup_status():
+    container = __get_fdb_cont()
+    if container:
+        res = container.exec_run(cmd="fdbbackup status")
+        cont = res.output.decode("utf-8")
+        logger.info(f"backup status: \n{cont.strip()}")
+        return True
+
+
+def backup_restore(backup_name, cluster_id):
+    container = __get_fdb_cont()
+    if container:
+        cluster = db_controller.get_cluster_by_id(cluster_id)
+        backup_path = cluster.get_backup_path(backup_name)
+        res = container.exec_run(cmd="fdbcli --exec \"writemode on; clearrange \\\"\\\" \\xff\"")
+        cont = res.output.decode("utf-8")
+        logger.info(cont.strip())
+        res = container.exec_run(cmd=f"fdbrestore start -r \"{backup_path}\" --dest-cluster-file {constants.KVD_DB_FILE_PATH}")
+        cont = res.output.decode("utf-8")
+        logger.info(cont.strip())
+        return True
+
+
+def parse_history_param(history_string):
+    if not history_string:
+        logger.error("Invalid history value")
+        return False
+    results = re.search(r'^(\d+[hmd])(\d+[hmd])?$', history_string.lower())
+    if not results:
+        logger.error(f"Error parsing history string: {history_string}")
+        return False
+    total_seconds = 0
+    for s in results.groups():
+        if not s:
+            continue
+        ind = s[-1]
+        v = int(s[:-1])
+        if ind == 'd':
+            total_seconds += v*60*60*24
+        if ind == 'h':
+            total_seconds += v*60*60
+        if ind == 'm':
+            total_seconds += v*60
+    return int(total_seconds)
+
+
+def backup_configure(cluster_id, backup_path, backup_frequency, bucket_name, region_name, backup_credentials):
+    cluster = db_controller.get_cluster_by_id(cluster_id)
+    if cluster:
+        if backup_frequency:
+            total_seconds = parse_history_param(backup_frequency)
+            cluster.backup_frequency_seconds = total_seconds
+        if backup_path:
+            if not backup_path.startswith("file://"):
+                backup_path = f"file://{backup_path}"
+            cluster.backup_local_path = backup_path
+            cluster.backup_s3_region = ""
+            cluster.backup_s3_bucket = ""
+            cluster.backup_s3_cred =  ""
+            cluster.write_to_db()
+            return True
+        else:
+            container = __get_fdb_cont()
+            if container:
+                backup_path = f"blobstore://{backup_credentials}@s3.{region_name}.amazonaws.com/?bucket={bucket_name}&region={region_name}&sc=0"
+                res = container.exec_run(cmd=f"fdbbackup list -b {backup_path}")
+                cont = res.output.decode("utf-8")
+                if res.exit_code == 0:
+                    logger.info(f"backup list from : {backup_path}")
+                    logger.info(cont)
+                    cluster.backup_s3_region = region_name if region_name else ""
+                    cluster.backup_s3_bucket = bucket_name if bucket_name else ""
+                    cluster.backup_s3_cred = backup_credentials if backup_credentials else ""
+                    cluster.write_to_db()
+                else:
+                    logger.error(f"Failed to list backup from s3: {backup_path}")
+                    logger.error(cont)
+                    return False
+        return True
+
+def add_backup_task(cluster_id):
+    try:
+        cluster = db_controller.get_cluster_by_id(cluster_id)
+    except Exception as e:
+        logger.error(f"Failed to get cluster {cluster_id}: {e}")
+        return False
+
+    tasks = get_backup_tasks(cluster_id)
+    for task in tasks:
+        if task.status != JobSchedule.STATUS_DONE:
+            logger.info(f"Backup task found: {tasks[0].uuid}")
+            return False
+
+    task_obj = JobSchedule()
+    task_obj.uuid = str(uuid.uuid4())
+    task_obj.cluster_id = cluster.get_id()
+    task_obj.date = int(time.time())
+    task_obj.max_retry = constants.TASK_EXEC_RETRY_COUNT
+    task_obj.status = JobSchedule.STATUS_NEW
+    task_obj.function_name = JobSchedule.FN_FDB_BACKUP
+    task_obj.write_to_db()
+    logger.info(f"Backup task created: {task_obj.uuid}")
+    return task_obj.uuid
+
+
+def get_backup_tasks(cluster_id):
+    backup_tasks = []
+    tasks = db_controller.get_job_tasks(cluster_id)
+    for task in tasks:
+        if task.function_name == JobSchedule.FN_FDB_BACKUP:
+            backup_tasks.append(task)
+    return backup_tasks