From 8ad8be641a80de8b719922266fe59487ce585085 Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Thu, 2 Jul 2026 23:23:25 +0300 Subject: [PATCH 1/8] Adds db-backup --- simplyblock_cli/cli-reference.yaml | 45 +++++ simplyblock_cli/cli.py | 48 +++++ simplyblock_cli/clibase.py | 15 ++ simplyblock_core/constants.py | 1 + .../controllers/fdb_backup_controller.py | 177 ++++++++++++++++++ .../controllers/tasks_controller.py | 34 ++++ simplyblock_core/models/cluster.py | 15 +- simplyblock_core/models/job_schedule.py | 1 + .../scripts/docker-compose-swarm.yml | 14 ++ simplyblock_core/services/cap_monitor.py | 16 +- .../services/tasks_runner_fdb_backup.py | 114 +++++++++++ 11 files changed, 477 insertions(+), 3 deletions(-) create mode 100644 simplyblock_core/controllers/fdb_backup_controller.py create mode 100644 simplyblock_core/services/tasks_runner_fdb_backup.py diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 5623175d4e..bbf9ca2cd4 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -2636,3 +2636,48 @@ commands: type: str required: false default: "" + - name: "db-backup" + help: "FDB Backup operations" + weight: 800 + subcommands: + - name: create + help: "Creates an fdb backup" + arguments: + - name: "cluster_id" + help: "Cluster ID to create db backup for" + dest: cluster_id + type: str + - name: list + help: "Lists all fdb backups" + - name: status + help: "get backup status" + - name: restore + help: "restore a backup" + arguments: + - name: "name" + help: "backup class name" + dest: name + type: str + - name: config + help: "Set backup configuration" + arguments: + - name: "--backup-path" + help: 'local backup path, defaults to /etc/foundationdb/backup' + dest: backup_path + type: str + - name: "--backup-frequency" + help: "backup frequency, can be 3h, 1d" + dest: backup_frequency + type: str + - name: "--s3-bucket" + help: 'AWS S3 bucket name' + dest: bucket_name + type: str + - name: "--s3-region" + help: 'AWS S3 region' + dest: region_name + type: str + - name: "--s3-credentials" + help: 'AWS S3 API key and secret, should be supplied like this: [API_KEY]:[API_SECRET]' + dest: backup_credentials + type: str \ No newline at end of file diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 6b3df0508a..140bfa6f54 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -31,6 +31,7 @@ def __init__(self): self.init_snapshot() self.init_backup() self.init_qos() + self.init_db_backup() super().__init__() def init_storage_node(self): @@ -1087,6 +1088,38 @@ def init_qos__delete(self, subparser): subcommand.add_argument('cluster_id', help='The cluster id.', type=str, default='') + def init_db_backup(self): + subparser = self.add_command('db-backup', 'FDB Backup operations') + self.init_db_backup__create(subparser) + self.init_db_backup__list(subparser) + self.init_db_backup__status(subparser) + self.init_db_backup__restore(subparser) + self.init_db_backup__config(subparser) + + + def init_db_backup__create(self, subparser): + subcommand = self.add_sub_command(subparser, 'create', 'Creates an fdb backup') + subcommand.add_argument('cluster_id', help='Cluster ID to create db backup for', type=str) + + def init_db_backup__list(self, subparser): + subcommand = self.add_sub_command(subparser, 'list', 'Lists all fdb backups') + + def init_db_backup__status(self, subparser): + subcommand = self.add_sub_command(subparser, 'status', 'get backup status') + + def init_db_backup__restore(self, subparser): + subcommand = self.add_sub_command(subparser, 'restore', 'restore a backup') + subcommand.add_argument('name', help='backup class name', type=str) + + def init_db_backup__config(self, subparser): + subcommand = self.add_sub_command(subparser, 'config', 'Set backup configuration') + subcommand.add_argument('--backup-path', help='local backup path, defaults to /etc/foundationdb/backup', type=str, dest='backup_path') + subcommand.add_argument('--backup-frequency', help='backup frequency, can be 3h, 1d', type=str, dest='backup_frequency') + subcommand.add_argument('--s3-bucket', help='AWS S3 bucket name', type=str, dest='bucket_name') + subcommand.add_argument('--s3-region', help='AWS S3 region', type=str, dest='region_name') + subcommand.add_argument('--s3-credentials', help='AWS S3 API key and secret, should be supplied like this: [API_KEY]:[API_SECRET]', type=str, dest='backup_credentials') + + def run(self): args = self.parser.parse_args() if args.debug: @@ -1533,6 +1566,21 @@ def run(self): else: self.parser.print_help() + elif args.command in ['db-backup']: + sub_command = args_dict['db-backup'] + if sub_command in ['create']: + ret = self.db_backup__create(sub_command, args) + elif sub_command in ['list']: + ret = self.db_backup__list(sub_command, args) + elif sub_command in ['status']: + ret = self.db_backup__status(sub_command, args) + elif sub_command in ['restore']: + ret = self.db_backup__restore(sub_command, args) + elif sub_command in ['config']: + ret = self.db_backup__config(sub_command, args) + else: + self.parser.print_help() + else: self.parser.print_help() diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 06c6285fe3..8418dabc55 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -1065,6 +1065,21 @@ def backup__source_switch(self, sub_command, args): print(f"Switched to external backup source: {target}") return True + def db_backup__create(self, sub_command, args): + return tasks_controller.add_backup_task(args.cluster_id) + + def db_backup__list(self, sub_command, args): + return backup_controller.list_backups() + + def db_backup__status(self, sub_command, args): + return backup_controller.backup_status() + + def db_backup__restore(self, sub_command, args): + return backup_controller.backup_restore(args.name) + + def db_backup__config(self, sub_command, args): + return backup_controller.backup_configure(args.backup_path, args.backup_frequency, args.bucket_name, args.region_name, args.backup_credentials) + def storage_node_list_devices(self, args): data = storage_ops.list_storage_devices(args.node_id) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index e6288e1c62..622ad775a6 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -26,6 +26,7 @@ def get_config_var(name, default=None): KVD_DB_VERSION = 730 KVD_DB_FILE_PATH = os.getenv('FDB_CLUSTER_FILE', '/etc/foundationdb/fdb.cluster') KVD_DB_TIMEOUT_MS = 10000 +KVD_DB_BACKUP_PATH = "file:///etc/foundationdb/backup" SPK_DIR = '/home/ec2-user/spdk' LOG_LEVEL = logging.INFO LOG_WEB_LEVEL = logging.DEBUG diff --git a/simplyblock_core/controllers/fdb_backup_controller.py b/simplyblock_core/controllers/fdb_backup_controller.py new file mode 100644 index 0000000000..470d39511a --- /dev/null +++ b/simplyblock_core/controllers/fdb_backup_controller.py @@ -0,0 +1,177 @@ +# coding=utf-8 +import datetime +import logging as lg +import re +import time +import uuid + +import docker + +from simplyblock_core import utils, constants +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.job_schedule import JobSchedule +from simplyblock_core.models.snapshot import SnapShot + +logger = lg.getLogger() +db_controller = DBController() +# backup_path = constants.KVD_DB_BACKUP_PATH +clusters = db_controller.get_clusters() +if clusters: + cluster = clusters[0] + + +def __get_fdb_cont(): + snode = db_controller.get_mgmt_nodes()[0] + if not snode: + return + node_docker = docker.DockerClient(base_url=f"tcp://{snode.docker_ip_port}", version="auto") + for container in node_docker.containers.list(): + if container.name.startswith("app_fdb-server"): # type: ignore[union-attr] + return container + +def create_backup(): + container = __get_fdb_cont() + if container: + backup_path = cluster.backup_local_path + if cluster.backup_s3_bucket and cluster.backup_s3_cred: + folder = f"backup-{str(datetime.datetime.now())}" + folder = folder.replace(" ", "-") + folder = folder.replace(":", "-") + folder = folder.split(".")[0] + backup_path = f"blobstore://{cluster.backup_s3_cred}@s3.{cluster.backup_s3_region}.amazonaws.com/{folder}?bucket={cluster.backup_s3_bucket}®ion={cluster.backup_s3_region}&sc=0" + + res = container.exec_run(cmd=f"fdbbackup start -d {backup_path} -w") + cont = res.output.decode("utf-8") + logger.info(cont) + return True + return False + +def list_backups(): + container = __get_fdb_cont() + data = [] + if container: + backup_path = cluster.get_backup_path() + res = container.exec_run(cmd=f"fdbbackup list -b {backup_path}") + logger.info(f"backup list from : {backup_path}") + cont = res.output.decode("utf-8") + for line in cont.splitlines(): + if not line or "backup-" not in line: + continue + + name = line.split("/")[-1].strip() + name = name.split("?")[0] + size = 0 + restorable = 0 + date = "" + version = 0 + res = container.exec_run(cmd=f"fdbbackup describe -d {cluster.get_backup_path(name)} --version-timestamps") + cont = res.output.decode("utf-8") + for line in cont.splitlines(): + if line and line.startswith("SnapshotBytes"): + size = line.split()[1].strip() + if line and line.startswith("Restorable"): + restorable = line.split()[1].strip() + if line and line.startswith("Snapshot:"): + for param in line.split(): + if param.startswith("startVersion"): + version = param.split("=")[1].strip() + elif param.startswith("(") and param.endswith(")") and not date:# 2025/12/28.10:10:20+0000 + try: + date = datetime.datetime.strptime(param[1:-1], "%Y/%m/%d.%H:%M:%S+0000").strftime("%Y-%m-%d %H:%M:%S") + except Exception: + date = name.replace("backup-","") + if not date: + date = name.replace("backup-", "") + + data.append({ + "Name": name, + "Version": version, + "Size": utils.humanbytes(int(size)), + "Restorable": restorable, + "Date": date, + }) + + return utils.print_table(data) + + return True + + + +def backup_status(): + container = __get_fdb_cont() + if container: + res = container.exec_run(cmd="fdbbackup status") + cont = res.output.decode("utf-8") + logger.info(f"backup status: \n{cont.strip()}") + return True + + +def backup_restore(backup_name): + container = __get_fdb_cont() + if container: + backup_path = cluster.get_backup_path(backup_name) + res = container.exec_run(cmd="fdbcli --exec \"writemode on; clearrange \\\"\\\" \\xff\"") + cont = res.output.decode("utf-8") + logger.info(cont.strip()) + res = container.exec_run(cmd=f"fdbrestore start -r \"{backup_path}\" --dest-cluster-file {constants.KVD_DB_FILE_PATH}") + cont = res.output.decode("utf-8") + logger.info(cont.strip()) + return True + + +def parse_history_param(history_string): + if not history_string: + logger.error("Invalid history value") + return False + results = re.search(r'^(\d+[hmd])(\d+[hmd])?$', history_string.lower()) + if not results: + logger.error(f"Error parsing history string: {history_string}") + return False + total_seconds = 0 + for s in results.groups(): + if not s: + continue + ind = s[-1] + v = int(s[:-1]) + if ind == 'd': + total_seconds += v*60*60*24 + if ind == 'h': + total_seconds += v*60*60 + if ind == 'm': + total_seconds += v*60 + return int(total_seconds) + + +def backup_configure(backup_path, backup_frequency, bucket_name, region_name, backup_credentials): + clusters = db_controller.get_clusters() + if clusters: + if backup_path: + if not backup_path.startswith("file://"): + backup_path = f"file://{backup_path}" + clusters[0].backup_local_path = backup_path + if backup_frequency: + total_seconds = parse_history_param(backup_frequency) + clusters[0].backup_frequency_seconds = total_seconds + clusters[0].backup_s3_region = region_name if region_name else "" + clusters[0].backup_s3_bucket = bucket_name if bucket_name else "" + clusters[0].backup_s3_cred = backup_credentials if backup_credentials else "" + clusters[0].write_to_db() + return True + + +def create_snapshot_backup(snapshot: SnapShot): + container = __get_fdb_cont() + if container: + backup_path = cluster.backup_local_path + if cluster.backup_s3_bucket and cluster.backup_s3_cred: + folder = f"backup-{str(datetime.datetime.now())}" + folder = folder.replace(" ", "-") + folder = folder.replace(":", "-") + folder = folder.split(".")[0] + backup_path = f"blobstore://{cluster.backup_s3_cred}@s3.{cluster.backup_s3_region}.amazonaws.com/{folder}?bucket={cluster.backup_s3_bucket}®ion={cluster.backup_s3_region}&sc=0" + + res = container.exec_run(cmd=f"fdbbackup start -d {backup_path} -w") + cont = res.output.decode("utf-8") + logger.info(cont) + return True + return False diff --git a/simplyblock_core/controllers/tasks_controller.py b/simplyblock_core/controllers/tasks_controller.py index ece14cccb2..8768393852 100644 --- a/simplyblock_core/controllers/tasks_controller.py +++ b/simplyblock_core/controllers/tasks_controller.py @@ -471,6 +471,40 @@ def get_active_node_mig_task(cluster_id, node_id, distr_name=None): return False +def add_backup_task(cluster_id): + try: + cluster = db.get_cluster_by_id(cluster_id) + except Exception as e: + logger.error(f"Failed to get cluster {cluster_id}: {e}") + return False + + tasks = get_backup_tasks(cluster_id) + for task in tasks: + if task.status != JobSchedule.STATUS_DONE: + logger.info(f"Backup task found: {tasks[0].uuid}") + return False + + task_obj = JobSchedule() + task_obj.uuid = str(uuid.uuid4()) + task_obj.cluster_id = cluster.get_id() + task_obj.date = int(time.time()) + task_obj.max_retry = constants.TASK_EXEC_RETRY_COUNT + task_obj.status = JobSchedule.STATUS_NEW + task_obj.function_name = JobSchedule.FN_FDB_BACKUP + task_obj.write_to_db() + logger.info(f"Backup task created: {task_obj.uuid}") + return task_obj.uuid + + +def get_backup_tasks(cluster_id): + backup_tasks = [] + tasks = db.get_job_tasks(cluster_id) + for task in tasks: + if task.function_name == JobSchedule.FN_FDB_BACKUP: + backup_tasks.append(task) + return backup_tasks + + def add_device_failed_mig_task(device_id): device = db.get_storage_device_by_id(device_id) for node in db.get_storage_nodes_by_cluster_id(device.cluster_id): diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index f06a4997c6..38746282ae 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -1,9 +1,10 @@ # coding=utf-8 - +import os.path from typing import List, Optional from pydantic import SecretStr +from simplyblock_core import constants from simplyblock_core.models.base_model import BaseModel @@ -155,6 +156,11 @@ class Cluster(BaseModel): # ``simplyblock_core.controllers.cluster_expansion.planner``. See the feature plan # ``single_node_expansion_plan.md`` for the schema. expand_state: dict = {} + backup_local_path: str = constants.KVD_DB_BACKUP_PATH + backup_frequency_seconds: int = 3*60*60 + backup_s3_bucket: str = "" + backup_s3_region: str = "" + backup_s3_cred: str = "" def get_status_code(self): if self.status in self.STATUS_CODE_MAP: @@ -171,6 +177,13 @@ def is_qos_set(self) -> bool: return True return False + def get_backup_path(self, path=""): + if self.backup_s3_bucket and self.backup_s3_cred: + backup_path = f"blobstore://{self.backup_s3_cred}@s3.{self.backup_s3_region}.amazonaws.com/{path}?bucket={self.backup_s3_bucket}" \ + + f"®ion={self.backup_s3_region}&sc=0" + else: + backup_path = os.path.join(self.backup_local_path, path) + return backup_path class ClusterAddNodeLock(BaseModel): diff --git a/simplyblock_core/models/job_schedule.py b/simplyblock_core/models/job_schedule.py index 859f5fcfb1..d563dfbf79 100644 --- a/simplyblock_core/models/job_schedule.py +++ b/simplyblock_core/models/job_schedule.py @@ -33,6 +33,7 @@ class JobSchedule(BaseModel): # lvol delta to the target, flip ANA so the client fails over. Used for # migration commit and fail-back (fresh or recovered source). FN_REPLICATION_FINAL = "replication_final" + FN_FDB_BACKUP = "fdb_backup" canceled: bool = False cluster_id: str = "" diff --git a/simplyblock_core/scripts/docker-compose-swarm.yml b/simplyblock_core/scripts/docker-compose-swarm.yml index e7b6255680..ad574de017 100644 --- a/simplyblock_core/scripts/docker-compose-swarm.yml +++ b/simplyblock_core/scripts/docker-compose-swarm.yml @@ -596,6 +596,20 @@ services: environment: SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL" + BackupService: + <<: *service-base + image: $SIMPLYBLOCK_DOCKER_IMAGE + command: "python3 simplyblock_core/services/tasks_runner_fdb_backup.py" + deploy: + placement: + constraints: [node.role == manager] + volumes: + - "/etc/foundationdb:/etc/foundationdb" + networks: + - hostnet + environment: + SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL" + networks: monitoring-net: external: true diff --git a/simplyblock_core/services/cap_monitor.py b/simplyblock_core/services/cap_monitor.py index f9ae66a444..176275a53f 100644 --- a/simplyblock_core/services/cap_monitor.py +++ b/simplyblock_core/services/cap_monitor.py @@ -4,9 +4,9 @@ from datetime import datetime, timezone from simplyblock_core import db_controller, constants, cluster_ops, utils -from simplyblock_core.controllers import cluster_events +from simplyblock_core.controllers import cluster_events, tasks_controller from simplyblock_core.models.cluster import Cluster - +from simplyblock_core.models.job_schedule import JobSchedule logger = utils.get_logger(__name__) @@ -14,6 +14,17 @@ db = db_controller.DBController() last_event: dict[str, dict] = {} + +def create_fdb_backup_if_needed(cluster): + last_backup_task = None + tasks = tasks_controller.get_backup_tasks(cluster.get_id()) + if tasks: + last_backup_task = tasks[0] + if last_backup_task and last_backup_task.status == JobSchedule.STATUS_DONE: + if last_backup_task.date + cluster.backup_frequency_seconds < time.time(): + tasks_controller.add_backup_task(cluster.get_id()) + + logger.info("Starting capacity monitoring service...") while True: try: @@ -24,6 +35,7 @@ continue clusters = db.get_clusters() for cl in clusters: + create_fdb_backup_if_needed(cl) logger.info(f"Checking cluster: {cl.get_id()}") records = db.get_cluster_capacity(cl, 1) if not records: diff --git a/simplyblock_core/services/tasks_runner_fdb_backup.py b/simplyblock_core/services/tasks_runner_fdb_backup.py new file mode 100644 index 0000000000..6806682e3a --- /dev/null +++ b/simplyblock_core/services/tasks_runner_fdb_backup.py @@ -0,0 +1,114 @@ +# coding=utf-8 +import time + + +from simplyblock_core import db_controller, utils, constants +from simplyblock_core.controllers import backup_controller, tasks_controller +from simplyblock_core.models.job_schedule import JobSchedule +from simplyblock_core.models.cluster import Cluster +from simplyblock_core.models.storage_node import StorageNode + +logger = utils.get_logger(__name__) + +# get DB controller +db = db_controller.DBController() + + +def process_fdb_backup_task(task): + task = db.get_task_by_id(task.uuid) + if task.canceled: + task.function_result = "canceled" + task.status = JobSchedule.STATUS_DONE + task.write_to_db(db.kv_store) + return + + if task.retry >= task.max_retry: + task.function_result = "max retry reached, stopping task" + task.status = JobSchedule.STATUS_DONE + task.write_to_db(db.kv_store) + return + + if task.status != JobSchedule.STATUS_RUNNING: + task.status = JobSchedule.STATUS_RUNNING + task.write_to_db(db.kv_store) + + ret = backup_controller.create_backup() + if ret: + task.function_result = "Backup created" + task.status = JobSchedule.STATUS_DONE + task.write_to_db(db.kv_store) + + +def process_snap_backup_task(task): + + task = db.get_task_by_id(task.uuid) + + if task.canceled: + task.function_result = "canceled" + task.status = JobSchedule.STATUS_DONE + task.write_to_db(db.kv_store) + return + + node = db.get_storage_node_by_id(task.node_id) + + if not node: + task.function_result = "node not found" + task.status = JobSchedule.STATUS_DONE + task.write_to_db(db.kv_store) + return + + if node.status not in [StorageNode.STATUS_DOWN, StorageNode.STATUS_ONLINE]: + msg = f"Node is {node.status}, retry task" + logger.info(msg) + task.function_result = msg + task.status = JobSchedule.STATUS_SUSPENDED + task.write_to_db(db.kv_store) + return + + if task.status != JobSchedule.STATUS_RUNNING: + task.status = JobSchedule.STATUS_RUNNING + task.write_to_db(db.kv_store) + + cluster = db.get_cluster_by_id(task.cluster_id) + snapshot_id = task.function_params["snapshot_id"] + snapshot = db.get_snapshot_by_id(snapshot_id) + logger.info(f"backing up: {snapshot_id}") + if cluster.backup_s3_bucket and cluster.backup_s3_cred: + folder = f"backup-{snapshot_id}" + folder = folder.replace(" ", "-") + folder = folder.replace(":", "-") + folder = folder.split(".")[0] + backup_path = f"blobstore://{cluster.backup_s3_cred}@s3.{cluster.backup_s3_region}.amazonaws.com/{folder}?bucket={cluster.backup_s3_bucket}®ion={cluster.backup_s3_region}&sc=0" + ret, err = node.rpc_client().bdev_lvol_s3_backup(backup_path, snapshot.snap_bdev) + if not ret: + logger.error( + f"Failed to backup snapshot: {snapshot_id}, error: {err}") + + task.function_result = f"snap backup error" + else: + task.function_result = f"snap backup done" + + task.status = JobSchedule.STATUS_DONE + task.write_to_db(db.kv_store) + + +logger.info("Starting Tasks runner fdb backup...") + +while True: + clusters = db.get_clusters() + if not clusters: + logger.error("No clusters found!") + else: + for cl in clusters: + if cl.status == Cluster.STATUS_IN_ACTIVATION: + continue + + tasks = db.get_job_tasks(cl.get_id()) + for task in tasks: + if task.status != JobSchedule.STATUS_DONE: + if task.function_name == JobSchedule.FN_FDB_BACKUP: + process_fdb_backup_task(task) + elif task.function_name == JobSchedule.FN_SNAPSHOT_BACKUP: + process_snap_backup_task(task) + + time.sleep(constants.TASK_EXEC_INTERVAL_SEC) From 123ceadb775bd4c995de7fc449bb8a528d83399c Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Wed, 15 Jul 2026 03:30:13 +0300 Subject: [PATCH 2/8] Refactor: consolidate backup task logic under `fdb_backup_controller` and remove redundant methods from `tasks_controller`. --- simplyblock_cli/clibase.py | 12 ++-- .../controllers/fdb_backup_controller.py | 33 +++++++++++ .../controllers/tasks_controller.py | 32 ---------- simplyblock_core/env_var | 2 +- simplyblock_core/services/cap_monitor.py | 7 ++- .../services/tasks_runner_fdb_backup.py | 59 +------------------ 6 files changed, 46 insertions(+), 99 deletions(-) diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 6920be95f6..2a6b2265e5 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -13,7 +13,7 @@ from simplyblock_core import storage_node_ops as storage_ops from simplyblock_core import mgmt_node_ops as mgmt_ops from simplyblock_core.controllers import pool_controller, lvol_controller, snapshot_controller, device_controller, \ - tasks_controller, qos_controller, migration_controller, backup_controller + tasks_controller, qos_controller, migration_controller, backup_controller, fdb_backup_controller from simplyblock_core.controllers import health_controller from simplyblock_core.models.pool import Pool from simplyblock_core.models.cluster import Cluster, HashicorpVaultSettings @@ -1074,19 +1074,19 @@ def backup__source_switch(self, sub_command, args): return True def db_backup__create(self, sub_command, args): - return tasks_controller.add_backup_task(args.cluster_id) + return fdb_backup_controller.add_backup_task(args.cluster_id) def db_backup__list(self, sub_command, args): - return backup_controller.list_backups() + return fdb_backup_controller.list_backups() def db_backup__status(self, sub_command, args): - return backup_controller.backup_status() + return fdb_backup_controller.backup_status() def db_backup__restore(self, sub_command, args): - return backup_controller.backup_restore(args.name) + return fdb_backup_controller.backup_restore(args.name) def db_backup__config(self, sub_command, args): - return backup_controller.backup_configure(args.backup_path, args.backup_frequency, args.bucket_name, args.region_name, args.backup_credentials) + return fdb_backup_controller.backup_configure(args.backup_path, args.backup_frequency, args.bucket_name, args.region_name, args.backup_credentials) def storage_node_list_devices(self, args): data = storage_ops.list_storage_devices(args.node_id) diff --git a/simplyblock_core/controllers/fdb_backup_controller.py b/simplyblock_core/controllers/fdb_backup_controller.py index 470d39511a..6789026640 100644 --- a/simplyblock_core/controllers/fdb_backup_controller.py +++ b/simplyblock_core/controllers/fdb_backup_controller.py @@ -175,3 +175,36 @@ def create_snapshot_backup(snapshot: SnapShot): logger.info(cont) return True return False + +def add_backup_task(cluster_id): + try: + cluster = db_controller.get_cluster_by_id(cluster_id) + except Exception as e: + logger.error(f"Failed to get cluster {cluster_id}: {e}") + return False + + tasks = get_backup_tasks(cluster_id) + for task in tasks: + if task.status != JobSchedule.STATUS_DONE: + logger.info(f"Backup task found: {tasks[0].uuid}") + return False + + task_obj = JobSchedule() + task_obj.uuid = str(uuid.uuid4()) + task_obj.cluster_id = cluster.get_id() + task_obj.date = int(time.time()) + task_obj.max_retry = constants.TASK_EXEC_RETRY_COUNT + task_obj.status = JobSchedule.STATUS_NEW + task_obj.function_name = JobSchedule.FN_FDB_BACKUP + task_obj.write_to_db() + logger.info(f"Backup task created: {task_obj.uuid}") + return task_obj.uuid + + +def get_backup_tasks(cluster_id): + backup_tasks = [] + tasks = db_controller.get_job_tasks(cluster_id) + for task in tasks: + if task.function_name == JobSchedule.FN_FDB_BACKUP: + backup_tasks.append(task) + return backup_tasks diff --git a/simplyblock_core/controllers/tasks_controller.py b/simplyblock_core/controllers/tasks_controller.py index 06a7bdeda6..2e6d1d2681 100644 --- a/simplyblock_core/controllers/tasks_controller.py +++ b/simplyblock_core/controllers/tasks_controller.py @@ -548,38 +548,6 @@ def get_active_node_mig_task(cluster_id, node_id, distr_name=None): return False -def add_backup_task(cluster_id): - try: - cluster = db.get_cluster_by_id(cluster_id) - except Exception as e: - logger.error(f"Failed to get cluster {cluster_id}: {e}") - return False - - tasks = get_backup_tasks(cluster_id) - for task in tasks: - if task.status != JobSchedule.STATUS_DONE: - logger.info(f"Backup task found: {tasks[0].uuid}") - return False - - task_obj = JobSchedule() - task_obj.uuid = str(uuid.uuid4()) - task_obj.cluster_id = cluster.get_id() - task_obj.date = int(time.time()) - task_obj.max_retry = constants.TASK_EXEC_RETRY_COUNT - task_obj.status = JobSchedule.STATUS_NEW - task_obj.function_name = JobSchedule.FN_FDB_BACKUP - task_obj.write_to_db() - logger.info(f"Backup task created: {task_obj.uuid}") - return task_obj.uuid - - -def get_backup_tasks(cluster_id): - backup_tasks = [] - tasks = db.get_job_tasks(cluster_id) - for task in tasks: - if task.function_name == JobSchedule.FN_FDB_BACKUP: - backup_tasks.append(task) - return backup_tasks def add_device_failed_mig_task(device_id): diff --git a/simplyblock_core/env_var b/simplyblock_core/env_var index 5abdecd440..ae25be9791 100644 --- a/simplyblock_core/env_var +++ b/simplyblock_core/env_var @@ -1,5 +1,5 @@ SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev SIMPLY_BLOCK_VERSION=19.2.34 -SIMPLY_BLOCK_DOCKER_IMAGE=public.ecr.aws/simply-block/simplyblock:main +SIMPLY_BLOCK_DOCKER_IMAGE=public.ecr.aws/simply-block/simplyblock:main-fdb-backup SIMPLY_BLOCK_SPDK_ULTRA_IMAGE=public.ecr.aws/simply-block/ultra:main-latest diff --git a/simplyblock_core/services/cap_monitor.py b/simplyblock_core/services/cap_monitor.py index 385c03cfcb..9e041e9309 100644 --- a/simplyblock_core/services/cap_monitor.py +++ b/simplyblock_core/services/cap_monitor.py @@ -4,7 +4,7 @@ from datetime import datetime, timezone from simplyblock_core import db_controller, constants, cluster_ops, utils -from simplyblock_core.controllers import cluster_events, tasks_controller +from simplyblock_core.controllers import cluster_events, fdb_backup_controller from simplyblock_core.models.cluster import Cluster from simplyblock_core.models.job_schedule import JobSchedule @@ -16,12 +16,13 @@ def create_fdb_backup_if_needed(cluster): last_backup_task = None - tasks = tasks_controller.get_backup_tasks(cluster.get_id()) + tasks = fdb_backup_controller.get_backup_tasks(cluster.get_id()) if tasks: last_backup_task = tasks[0] if last_backup_task and last_backup_task.status == JobSchedule.STATUS_DONE: if last_backup_task.date + cluster.backup_frequency_seconds < time.time(): - tasks_controller.add_backup_task(cluster.get_id()) + fdb_backup_controller.add_backup_task(cluster.get_id()) + def main(): logger.info("Starting capacity monitoring service...") diff --git a/simplyblock_core/services/tasks_runner_fdb_backup.py b/simplyblock_core/services/tasks_runner_fdb_backup.py index 6806682e3a..d872af191a 100644 --- a/simplyblock_core/services/tasks_runner_fdb_backup.py +++ b/simplyblock_core/services/tasks_runner_fdb_backup.py @@ -3,10 +3,9 @@ from simplyblock_core import db_controller, utils, constants -from simplyblock_core.controllers import backup_controller, tasks_controller +from simplyblock_core.controllers import fdb_backup_controller from simplyblock_core.models.job_schedule import JobSchedule from simplyblock_core.models.cluster import Cluster -from simplyblock_core.models.storage_node import StorageNode logger = utils.get_logger(__name__) @@ -32,65 +31,13 @@ def process_fdb_backup_task(task): task.status = JobSchedule.STATUS_RUNNING task.write_to_db(db.kv_store) - ret = backup_controller.create_backup() + ret = fdb_backup_controller.create_backup() if ret: task.function_result = "Backup created" task.status = JobSchedule.STATUS_DONE task.write_to_db(db.kv_store) -def process_snap_backup_task(task): - - task = db.get_task_by_id(task.uuid) - - if task.canceled: - task.function_result = "canceled" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return - - node = db.get_storage_node_by_id(task.node_id) - - if not node: - task.function_result = "node not found" - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - return - - if node.status not in [StorageNode.STATUS_DOWN, StorageNode.STATUS_ONLINE]: - msg = f"Node is {node.status}, retry task" - logger.info(msg) - task.function_result = msg - task.status = JobSchedule.STATUS_SUSPENDED - task.write_to_db(db.kv_store) - return - - if task.status != JobSchedule.STATUS_RUNNING: - task.status = JobSchedule.STATUS_RUNNING - task.write_to_db(db.kv_store) - - cluster = db.get_cluster_by_id(task.cluster_id) - snapshot_id = task.function_params["snapshot_id"] - snapshot = db.get_snapshot_by_id(snapshot_id) - logger.info(f"backing up: {snapshot_id}") - if cluster.backup_s3_bucket and cluster.backup_s3_cred: - folder = f"backup-{snapshot_id}" - folder = folder.replace(" ", "-") - folder = folder.replace(":", "-") - folder = folder.split(".")[0] - backup_path = f"blobstore://{cluster.backup_s3_cred}@s3.{cluster.backup_s3_region}.amazonaws.com/{folder}?bucket={cluster.backup_s3_bucket}®ion={cluster.backup_s3_region}&sc=0" - ret, err = node.rpc_client().bdev_lvol_s3_backup(backup_path, snapshot.snap_bdev) - if not ret: - logger.error( - f"Failed to backup snapshot: {snapshot_id}, error: {err}") - - task.function_result = f"snap backup error" - else: - task.function_result = f"snap backup done" - - task.status = JobSchedule.STATUS_DONE - task.write_to_db(db.kv_store) - logger.info("Starting Tasks runner fdb backup...") @@ -108,7 +55,5 @@ def process_snap_backup_task(task): if task.status != JobSchedule.STATUS_DONE: if task.function_name == JobSchedule.FN_FDB_BACKUP: process_fdb_backup_task(task) - elif task.function_name == JobSchedule.FN_SNAPSHOT_BACKUP: - process_snap_backup_task(task) time.sleep(constants.TASK_EXEC_INTERVAL_SEC) From 76c537652a34d18f0d507a805abc26dd1cf7010f Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Wed, 15 Jul 2026 03:43:55 +0300 Subject: [PATCH 3/8] Add `cluster_id` support for database backup and restore commands --- simplyblock_cli/cli-reference.yaml | 4 +++ simplyblock_cli/cli.py | 1 + simplyblock_cli/clibase.py | 2 +- .../controllers/fdb_backup_controller.py | 33 ++++--------------- .../services/tasks_runner_fdb_backup.py | 2 +- 5 files changed, 13 insertions(+), 29 deletions(-) diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 1bd168b7ab..2f7e1a5269 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -2667,6 +2667,10 @@ commands: help: "backup class name" dest: name type: str + - name: "cluster_id" + help: "Cluster ID to restore db backup to" + dest: cluster_id + type: str - name: config help: "Set backup configuration" arguments: diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 0c8bd52644..911d3e75b0 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -1118,6 +1118,7 @@ def init_db_backup__status(self, subparser): def init_db_backup__restore(self, subparser): subcommand = self.add_sub_command(subparser, 'restore', 'restore a backup') subcommand.add_argument('name', help='backup class name', type=str) + subcommand.add_argument('cluster_id', help='Cluster ID to restore db backup to', type=str) def init_db_backup__config(self, subparser): subcommand = self.add_sub_command(subparser, 'config', 'Set backup configuration') diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 2a6b2265e5..0629d9c0b9 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -1083,7 +1083,7 @@ def db_backup__status(self, sub_command, args): return fdb_backup_controller.backup_status() def db_backup__restore(self, sub_command, args): - return fdb_backup_controller.backup_restore(args.name) + return fdb_backup_controller.backup_restore(args.name, args.cluster_id) def db_backup__config(self, sub_command, args): return fdb_backup_controller.backup_configure(args.backup_path, args.backup_frequency, args.bucket_name, args.region_name, args.backup_credentials) diff --git a/simplyblock_core/controllers/fdb_backup_controller.py b/simplyblock_core/controllers/fdb_backup_controller.py index 6789026640..45e12d1a7c 100644 --- a/simplyblock_core/controllers/fdb_backup_controller.py +++ b/simplyblock_core/controllers/fdb_backup_controller.py @@ -10,15 +10,9 @@ from simplyblock_core import utils, constants from simplyblock_core.db_controller import DBController from simplyblock_core.models.job_schedule import JobSchedule -from simplyblock_core.models.snapshot import SnapShot logger = lg.getLogger() db_controller = DBController() -# backup_path = constants.KVD_DB_BACKUP_PATH -clusters = db_controller.get_clusters() -if clusters: - cluster = clusters[0] - def __get_fdb_cont(): snode = db_controller.get_mgmt_nodes()[0] @@ -29,9 +23,10 @@ def __get_fdb_cont(): if container.name.startswith("app_fdb-server"): # type: ignore[union-attr] return container -def create_backup(): +def create_backup(cluster_id): container = __get_fdb_cont() if container: + cluster = db_controller.get_cluster_by_id(cluster_id) backup_path = cluster.backup_local_path if cluster.backup_s3_bucket and cluster.backup_s3_cred: folder = f"backup-{str(datetime.datetime.now())}" @@ -46,10 +41,11 @@ def create_backup(): return True return False -def list_backups(): +def list_backups(cluster_id): container = __get_fdb_cont() data = [] if container: + cluster = db_controller.get_cluster_by_id(cluster_id) backup_path = cluster.get_backup_path() res = container.exec_run(cmd=f"fdbbackup list -b {backup_path}") logger.info(f"backup list from : {backup_path}") @@ -106,9 +102,10 @@ def backup_status(): return True -def backup_restore(backup_name): +def backup_restore(backup_name, cluster_id): container = __get_fdb_cont() if container: + cluster = db_controller.get_cluster_by_id(cluster_id) backup_path = cluster.get_backup_path(backup_name) res = container.exec_run(cmd="fdbcli --exec \"writemode on; clearrange \\\"\\\" \\xff\"") cont = res.output.decode("utf-8") @@ -158,24 +155,6 @@ def backup_configure(backup_path, backup_frequency, bucket_name, region_name, ba clusters[0].write_to_db() return True - -def create_snapshot_backup(snapshot: SnapShot): - container = __get_fdb_cont() - if container: - backup_path = cluster.backup_local_path - if cluster.backup_s3_bucket and cluster.backup_s3_cred: - folder = f"backup-{str(datetime.datetime.now())}" - folder = folder.replace(" ", "-") - folder = folder.replace(":", "-") - folder = folder.split(".")[0] - backup_path = f"blobstore://{cluster.backup_s3_cred}@s3.{cluster.backup_s3_region}.amazonaws.com/{folder}?bucket={cluster.backup_s3_bucket}®ion={cluster.backup_s3_region}&sc=0" - - res = container.exec_run(cmd=f"fdbbackup start -d {backup_path} -w") - cont = res.output.decode("utf-8") - logger.info(cont) - return True - return False - def add_backup_task(cluster_id): try: cluster = db_controller.get_cluster_by_id(cluster_id) diff --git a/simplyblock_core/services/tasks_runner_fdb_backup.py b/simplyblock_core/services/tasks_runner_fdb_backup.py index d872af191a..8fb738d42d 100644 --- a/simplyblock_core/services/tasks_runner_fdb_backup.py +++ b/simplyblock_core/services/tasks_runner_fdb_backup.py @@ -31,7 +31,7 @@ def process_fdb_backup_task(task): task.status = JobSchedule.STATUS_RUNNING task.write_to_db(db.kv_store) - ret = fdb_backup_controller.create_backup() + ret = fdb_backup_controller.create_backup(task.cluster_id) if ret: task.function_result = "Backup created" task.status = JobSchedule.STATUS_DONE From fe2524ec4089977960acfb89cd170b9c57f208de Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Wed, 15 Jul 2026 03:57:15 +0300 Subject: [PATCH 4/8] Add `cluster_id` support for database backup and restore commands --- simplyblock_cli/cli-reference.yaml | 5 +++++ simplyblock_cli/cli.py | 1 + simplyblock_cli/clibase.py | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 2f7e1a5269..80f8bff0ae 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -2658,6 +2658,11 @@ commands: type: str - name: list help: "Lists all fdb backups" + arguments: + - name: "cluster_id" + help: "Cluster ID to restore db backup to" + dest: cluster_id + type: str - name: status help: "get backup status" - name: restore diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 911d3e75b0..a46186940a 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -1111,6 +1111,7 @@ def init_db_backup__create(self, subparser): def init_db_backup__list(self, subparser): subcommand = self.add_sub_command(subparser, 'list', 'Lists all fdb backups') + subcommand.add_argument('cluster_id', help='Cluster ID to restore db backup to', type=str) def init_db_backup__status(self, subparser): subcommand = self.add_sub_command(subparser, 'status', 'get backup status') diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 0629d9c0b9..bd8db3cf87 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -1077,7 +1077,7 @@ def db_backup__create(self, sub_command, args): return fdb_backup_controller.add_backup_task(args.cluster_id) def db_backup__list(self, sub_command, args): - return fdb_backup_controller.list_backups() + return fdb_backup_controller.list_backups(args.cluster_id) def db_backup__status(self, sub_command, args): return fdb_backup_controller.backup_status() From 2870a936b3dc02c0bac5dd0af5fd2a8da5f1fb4e Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Wed, 15 Jul 2026 06:28:33 +0300 Subject: [PATCH 5/8] Add `cluster_id` support for DB backup configuration and improve backup path handling --- simplyblock_cli/cli-reference.yaml | 4 +++ simplyblock_cli/cli.py | 1 + simplyblock_cli/clibase.py | 2 +- simplyblock_core/cluster_ops.py | 2 ++ .../controllers/fdb_backup_controller.py | 32 +++++++++++++------ simplyblock_core/models/cluster.py | 4 ++- simplyblock_core/services/cap_monitor.py | 8 +++-- 7 files changed, 38 insertions(+), 15 deletions(-) diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index 80f8bff0ae..f7521d89ec 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -2679,6 +2679,10 @@ commands: - name: config help: "Set backup configuration" arguments: + - name: "cluster_id" + help: "Cluster ID to configure db backup for" + dest: cluster_id + type: str - name: "--backup-path" help: 'local backup path, defaults to /etc/foundationdb/backup' dest: backup_path diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index a46186940a..91d3dc9160 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -1123,6 +1123,7 @@ def init_db_backup__restore(self, subparser): def init_db_backup__config(self, subparser): subcommand = self.add_sub_command(subparser, 'config', 'Set backup configuration') + subcommand.add_argument('cluster_id', help='Cluster ID to configure db backup for', type=str) subcommand.add_argument('--backup-path', help='local backup path, defaults to /etc/foundationdb/backup', type=str, dest='backup_path') subcommand.add_argument('--backup-frequency', help='backup frequency, can be 3h, 1d', type=str, dest='backup_frequency') subcommand.add_argument('--s3-bucket', help='AWS S3 bucket name', type=str, dest='bucket_name') diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index bd8db3cf87..44a11fe770 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -1086,7 +1086,7 @@ def db_backup__restore(self, sub_command, args): return fdb_backup_controller.backup_restore(args.name, args.cluster_id) def db_backup__config(self, sub_command, args): - return fdb_backup_controller.backup_configure(args.backup_path, args.backup_frequency, args.bucket_name, args.region_name, args.backup_credentials) + return fdb_backup_controller.backup_configure(args.cluster_id, args.backup_path, args.backup_frequency, args.bucket_name, args.region_name, args.backup_credentials) def storage_node_list_devices(self, args): data = storage_ops.list_storage_devices(args.node_id) diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index fbbc959688..91af2d219d 100755 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -378,6 +378,7 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, cluster.snode_api_port = snode_api_port cluster.container_image_prefix = container_image_prefix or "" cluster.hashicorp_vault_settings = hashicorp_vault_settings + cluster.backup_local_path = os.path.join(constants.KVD_DB_BACKUP_PATH, cluster.uuid) if nvmeof_tls_config: cluster.tls = True @@ -593,6 +594,7 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn if backup_config: cluster.backup_config = backup_config + cluster.backup_local_path = os.path.join(constants.KVD_DB_BACKUP_PATH, cluster.uuid) cluster.status = Cluster.STATUS_UNREADY cluster.create_dt = str(datetime.datetime.now()) cluster.write_to_db(db_controller.kv_store) diff --git a/simplyblock_core/controllers/fdb_backup_controller.py b/simplyblock_core/controllers/fdb_backup_controller.py index 45e12d1a7c..9c7bf2841f 100644 --- a/simplyblock_core/controllers/fdb_backup_controller.py +++ b/simplyblock_core/controllers/fdb_backup_controller.py @@ -27,7 +27,7 @@ def create_backup(cluster_id): container = __get_fdb_cont() if container: cluster = db_controller.get_cluster_by_id(cluster_id) - backup_path = cluster.backup_local_path + backup_path = cluster.get_backup_path() if cluster.backup_s3_bucket and cluster.backup_s3_cred: folder = f"backup-{str(datetime.datetime.now())}" folder = folder.replace(" ", "-") @@ -139,20 +139,32 @@ def parse_history_param(history_string): return int(total_seconds) -def backup_configure(backup_path, backup_frequency, bucket_name, region_name, backup_credentials): - clusters = db_controller.get_clusters() - if clusters: +def backup_configure(cluster_id, backup_path, backup_frequency, bucket_name, region_name, backup_credentials): + cluster = db_controller.get_cluster_by_id(cluster_id) + if cluster: if backup_path: if not backup_path.startswith("file://"): backup_path = f"file://{backup_path}" - clusters[0].backup_local_path = backup_path + cluster.backup_local_path = backup_path if backup_frequency: total_seconds = parse_history_param(backup_frequency) - clusters[0].backup_frequency_seconds = total_seconds - clusters[0].backup_s3_region = region_name if region_name else "" - clusters[0].backup_s3_bucket = bucket_name if bucket_name else "" - clusters[0].backup_s3_cred = backup_credentials if backup_credentials else "" - clusters[0].write_to_db() + cluster.backup_frequency_seconds = total_seconds + container = __get_fdb_cont() + if container: + backup_path = f"blobstore://{backup_credentials}@s3.{region_name}.amazonaws.com/?bucket={bucket_name}®ion={region_name}&sc=0" + res = container.exec_run(cmd=f"fdbbackup list -b {backup_path}") + cont = res.output.decode("utf-8") + if res.exit_code == 0: + logger.info(f"backup list from : {backup_path}") + logger.info(cont) + cluster.backup_s3_region = region_name if region_name else "" + cluster.backup_s3_bucket = bucket_name if bucket_name else "" + cluster.backup_s3_cred = backup_credentials if backup_credentials else "" + cluster.write_to_db() + else: + logger.error(f"Failed to list backup from s3: {backup_path}") + logger.error(cont) + return False return True def add_backup_task(cluster_id): diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index 9165201b66..72017e4c62 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -190,8 +190,10 @@ def get_backup_path(self, path=""): if self.backup_s3_bucket and self.backup_s3_cred: backup_path = f"blobstore://{self.backup_s3_cred}@s3.{self.backup_s3_region}.amazonaws.com/{path}?bucket={self.backup_s3_bucket}" \ + f"®ion={self.backup_s3_region}&sc=0" - else: + elif self.backup_local_path: backup_path = os.path.join(self.backup_local_path, path) + else: + backup_path = os.path.join(constants.KVD_DB_BACKUP_PATH, self.uuid, path) return backup_path diff --git a/simplyblock_core/services/cap_monitor.py b/simplyblock_core/services/cap_monitor.py index 9e041e9309..13238048f9 100644 --- a/simplyblock_core/services/cap_monitor.py +++ b/simplyblock_core/services/cap_monitor.py @@ -15,10 +15,12 @@ last_event: dict[str, dict] = {} def create_fdb_backup_if_needed(cluster): - last_backup_task = None tasks = fdb_backup_controller.get_backup_tasks(cluster.get_id()) - if tasks: - last_backup_task = tasks[0] + if not tasks: + fdb_backup_controller.add_backup_task(cluster.get_id()) + return + + last_backup_task = tasks[0] if last_backup_task and last_backup_task.status == JobSchedule.STATUS_DONE: if last_backup_task.date + cluster.backup_frequency_seconds < time.time(): fdb_backup_controller.add_backup_task(cluster.get_id()) From 7689c905d5f873118da7b1f31de323c11de73968 Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Wed, 15 Jul 2026 06:37:45 +0300 Subject: [PATCH 6/8] Add `cluster_id` support for DB backup configuration and improve backup path handling --- .../controllers/fdb_backup_controller.py | 44 +++++++++++-------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/simplyblock_core/controllers/fdb_backup_controller.py b/simplyblock_core/controllers/fdb_backup_controller.py index 9c7bf2841f..76fb34f232 100644 --- a/simplyblock_core/controllers/fdb_backup_controller.py +++ b/simplyblock_core/controllers/fdb_backup_controller.py @@ -142,29 +142,35 @@ def parse_history_param(history_string): def backup_configure(cluster_id, backup_path, backup_frequency, bucket_name, region_name, backup_credentials): cluster = db_controller.get_cluster_by_id(cluster_id) if cluster: + if backup_frequency: + total_seconds = parse_history_param(backup_frequency) + cluster.backup_frequency_seconds = total_seconds if backup_path: if not backup_path.startswith("file://"): backup_path = f"file://{backup_path}" cluster.backup_local_path = backup_path - if backup_frequency: - total_seconds = parse_history_param(backup_frequency) - cluster.backup_frequency_seconds = total_seconds - container = __get_fdb_cont() - if container: - backup_path = f"blobstore://{backup_credentials}@s3.{region_name}.amazonaws.com/?bucket={bucket_name}®ion={region_name}&sc=0" - res = container.exec_run(cmd=f"fdbbackup list -b {backup_path}") - cont = res.output.decode("utf-8") - if res.exit_code == 0: - logger.info(f"backup list from : {backup_path}") - logger.info(cont) - cluster.backup_s3_region = region_name if region_name else "" - cluster.backup_s3_bucket = bucket_name if bucket_name else "" - cluster.backup_s3_cred = backup_credentials if backup_credentials else "" - cluster.write_to_db() - else: - logger.error(f"Failed to list backup from s3: {backup_path}") - logger.error(cont) - return False + cluster.backup_s3_region = "" + cluster.backup_s3_bucket = "" + cluster.backup_s3_cred = "" + cluster.write_to_db() + return True + else: + container = __get_fdb_cont() + if container: + backup_path = f"blobstore://{backup_credentials}@s3.{region_name}.amazonaws.com/?bucket={bucket_name}®ion={region_name}&sc=0" + res = container.exec_run(cmd=f"fdbbackup list -b {backup_path}") + cont = res.output.decode("utf-8") + if res.exit_code == 0: + logger.info(f"backup list from : {backup_path}") + logger.info(cont) + cluster.backup_s3_region = region_name if region_name else "" + cluster.backup_s3_bucket = bucket_name if bucket_name else "" + cluster.backup_s3_cred = backup_credentials if backup_credentials else "" + cluster.write_to_db() + else: + logger.error(f"Failed to list backup from s3: {backup_path}") + logger.error(cont) + return False return True def add_backup_task(cluster_id): From c21c42c9618ca85e6ab065bb8ee4e77ad82352c2 Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Wed, 15 Jul 2026 06:55:15 +0300 Subject: [PATCH 7/8] Reverse task list order in `cap_monitor` to use the most recent task first --- simplyblock_core/services/cap_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplyblock_core/services/cap_monitor.py b/simplyblock_core/services/cap_monitor.py index 13238048f9..32a1a27687 100644 --- a/simplyblock_core/services/cap_monitor.py +++ b/simplyblock_core/services/cap_monitor.py @@ -19,7 +19,7 @@ def create_fdb_backup_if_needed(cluster): if not tasks: fdb_backup_controller.add_backup_task(cluster.get_id()) return - + tasks.reverse() last_backup_task = tasks[0] if last_backup_task and last_backup_task.status == JobSchedule.STATUS_DONE: if last_backup_task.date + cluster.backup_frequency_seconds < time.time(): From 889c0a3f30b29bb44f6bd8d2ad8b8d1523605e74 Mon Sep 17 00:00:00 2001 From: hamdykhader Date: Wed, 15 Jul 2026 07:31:24 +0300 Subject: [PATCH 8/8] Update container name match in `fdb_backup_controller` to `app_fdb-backup-agent` --- simplyblock_core/controllers/fdb_backup_controller.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/simplyblock_core/controllers/fdb_backup_controller.py b/simplyblock_core/controllers/fdb_backup_controller.py index 76fb34f232..6fdfc09552 100644 --- a/simplyblock_core/controllers/fdb_backup_controller.py +++ b/simplyblock_core/controllers/fdb_backup_controller.py @@ -20,7 +20,7 @@ def __get_fdb_cont(): return node_docker = docker.DockerClient(base_url=f"tcp://{snode.docker_ip_port}", version="auto") for container in node_docker.containers.list(): - if container.name.startswith("app_fdb-server"): # type: ignore[union-attr] + if container.name.startswith("app_fdb-backup-agent"): # type: ignore[union-attr] return container def create_backup(cluster_id):