diff --git a/simplyblock_cli/cli-reference.yaml b/simplyblock_cli/cli-reference.yaml index b17152c82..67c51aa57 100644 --- a/simplyblock_cli/cli-reference.yaml +++ b/simplyblock_cli/cli-reference.yaml @@ -2652,3 +2652,61 @@ commands: type: str required: false default: "" + - name: "db-backup" + help: "FDB Backup operations" + weight: 800 + subcommands: + - name: create + help: "Creates an fdb backup" + arguments: + - name: "cluster_id" + help: "Cluster ID to create db backup for" + dest: cluster_id + type: str + - name: list + help: "Lists all fdb backups" + arguments: + - name: "cluster_id" + help: "Cluster ID to restore db backup to" + dest: cluster_id + type: str + - name: status + help: "get backup status" + - name: restore + help: "restore a backup" + arguments: + - name: "name" + help: "backup class name" + dest: name + type: str + - name: "cluster_id" + help: "Cluster ID to restore db backup to" + dest: cluster_id + type: str + - name: config + help: "Set backup configuration" + arguments: + - name: "cluster_id" + help: "Cluster ID to configure db backup for" + dest: cluster_id + type: str + - name: "--backup-path" + help: 'local backup path, defaults to /etc/foundationdb/backup' + dest: backup_path + type: str + - name: "--backup-frequency" + help: "backup frequency, can be 3h, 1d" + dest: backup_frequency + type: str + - name: "--s3-bucket" + help: 'AWS S3 bucket name' + dest: bucket_name + type: str + - name: "--s3-region" + help: 'AWS S3 region' + dest: region_name + type: str + - name: "--s3-credentials" + help: 'AWS S3 API key and secret, should be supplied like this: [API_KEY]:[API_SECRET]' + dest: backup_credentials + type: str \ No newline at end of file diff --git a/simplyblock_cli/cli.py b/simplyblock_cli/cli.py index 851de9d83..6aea3322a 100755 --- a/simplyblock_cli/cli.py +++ b/simplyblock_cli/cli.py @@ -31,6 +31,7 @@ def __init__(self): self.init_snapshot() self.init_backup() self.init_qos() + self.init_db_backup() super().__init__() def init_storage_node(self): @@ -1100,6 +1101,41 @@ def init_qos__delete(self, subparser): subcommand.add_argument('cluster_id', help='The cluster id.', type=str, default='') + def init_db_backup(self): + subparser = self.add_command('db-backup', 'FDB Backup operations') + self.init_db_backup__create(subparser) + self.init_db_backup__list(subparser) + self.init_db_backup__status(subparser) + self.init_db_backup__restore(subparser) + self.init_db_backup__config(subparser) + + + def init_db_backup__create(self, subparser): + subcommand = self.add_sub_command(subparser, 'create', 'Creates an fdb backup') + subcommand.add_argument('cluster_id', help='Cluster ID to create db backup for', type=str) + + def init_db_backup__list(self, subparser): + subcommand = self.add_sub_command(subparser, 'list', 'Lists all fdb backups') + subcommand.add_argument('cluster_id', help='Cluster ID to restore db backup to', type=str) + + def init_db_backup__status(self, subparser): + subcommand = self.add_sub_command(subparser, 'status', 'get backup status') + + def init_db_backup__restore(self, subparser): + subcommand = self.add_sub_command(subparser, 'restore', 'restore a backup') + subcommand.add_argument('name', help='backup class name', type=str) + subcommand.add_argument('cluster_id', help='Cluster ID to restore db backup to', type=str) + + def init_db_backup__config(self, subparser): + subcommand = self.add_sub_command(subparser, 'config', 'Set backup configuration') + subcommand.add_argument('cluster_id', help='Cluster ID to configure db backup for', type=str) + subcommand.add_argument('--backup-path', help='local backup path, defaults to /etc/foundationdb/backup', type=str, dest='backup_path') + subcommand.add_argument('--backup-frequency', help='backup frequency, can be 3h, 1d', type=str, dest='backup_frequency') + subcommand.add_argument('--s3-bucket', help='AWS S3 bucket name', type=str, dest='bucket_name') + subcommand.add_argument('--s3-region', help='AWS S3 region', type=str, dest='region_name') + subcommand.add_argument('--s3-credentials', help='AWS S3 API key and secret, should be supplied like this: [API_KEY]:[API_SECRET]', type=str, dest='backup_credentials') + + def run(self): args = self.parser.parse_args() if args.debug: @@ -1550,6 +1586,21 @@ def run(self): else: self.parser.print_help() + elif args.command in ['db-backup']: + sub_command = args_dict['db-backup'] + if sub_command in ['create']: + ret = self.db_backup__create(sub_command, args) + elif sub_command in ['list']: + ret = self.db_backup__list(sub_command, args) + elif sub_command in ['status']: + ret = self.db_backup__status(sub_command, args) + elif sub_command in ['restore']: + ret = self.db_backup__restore(sub_command, args) + elif sub_command in ['config']: + ret = self.db_backup__config(sub_command, args) + else: + self.parser.print_help() + else: self.parser.print_help() diff --git a/simplyblock_cli/clibase.py b/simplyblock_cli/clibase.py index 9668e07ff..c76eebd98 100755 --- a/simplyblock_cli/clibase.py +++ b/simplyblock_cli/clibase.py @@ -13,7 +13,7 @@ from simplyblock_core import storage_node_ops as storage_ops from simplyblock_core import mgmt_node_ops as mgmt_ops from simplyblock_core.controllers import pool_controller, lvol_controller, snapshot_controller, device_controller, \ - tasks_controller, qos_controller, migration_controller, backup_controller + tasks_controller, qos_controller, migration_controller, backup_controller, fdb_backup_controller from simplyblock_core.controllers import health_controller from simplyblock_core.models.pool import Pool from simplyblock_core.models.cluster import Cluster, HashicorpVaultSettings @@ -1076,6 +1076,21 @@ def backup__source_switch(self, sub_command, args): print(f"Switched to external backup source: {target}") return True + def db_backup__create(self, sub_command, args): + return fdb_backup_controller.add_backup_task(args.cluster_id) + + def db_backup__list(self, sub_command, args): + return fdb_backup_controller.list_backups(args.cluster_id) + + def db_backup__status(self, sub_command, args): + return fdb_backup_controller.backup_status() + + def db_backup__restore(self, sub_command, args): + return fdb_backup_controller.backup_restore(args.name, args.cluster_id) + + def db_backup__config(self, sub_command, args): + return fdb_backup_controller.backup_configure(args.cluster_id, args.backup_path, args.backup_frequency, args.bucket_name, args.region_name, args.backup_credentials) + def storage_node_list_devices(self, args): data = storage_ops.list_storage_devices(args.node_id) diff --git a/simplyblock_core/cluster_ops.py b/simplyblock_core/cluster_ops.py index bf7803a85..38f9c2d2d 100755 --- a/simplyblock_core/cluster_ops.py +++ b/simplyblock_core/cluster_ops.py @@ -381,6 +381,7 @@ def create_cluster(blk_size, page_size_in_blocks, cli_pass, cluster.snode_api_port = snode_api_port cluster.container_image_prefix = container_image_prefix or "" cluster.hashicorp_vault_settings = hashicorp_vault_settings + cluster.backup_local_path = os.path.join(constants.KVD_DB_BACKUP_PATH, cluster.uuid) if nvmeof_tls_config: cluster.tls = True @@ -596,6 +597,7 @@ def add_cluster(blk_size, page_size_in_blocks, cap_warn, cap_crit, prov_cap_warn if backup_config: cluster.backup_config = backup_config + cluster.backup_local_path = os.path.join(constants.KVD_DB_BACKUP_PATH, cluster.uuid) cluster.status = Cluster.STATUS_UNREADY cluster.create_dt = str(datetime.datetime.now()) cluster.write_to_db(db_controller.kv_store) diff --git a/simplyblock_core/constants.py b/simplyblock_core/constants.py index de09667aa..9344ab4b9 100644 --- a/simplyblock_core/constants.py +++ b/simplyblock_core/constants.py @@ -26,6 +26,7 @@ def get_config_var(name, default=None): KVD_DB_VERSION = 730 KVD_DB_FILE_PATH = os.getenv('FDB_CLUSTER_FILE', '/etc/foundationdb/fdb.cluster') KVD_DB_TIMEOUT_MS = 10000 +KVD_DB_BACKUP_PATH = "file:///etc/foundationdb/backup" SPK_DIR = '/home/ec2-user/spdk' LOG_LEVEL = logging.INFO LOG_WEB_LEVEL = logging.DEBUG diff --git a/simplyblock_core/controllers/fdb_backup_controller.py b/simplyblock_core/controllers/fdb_backup_controller.py new file mode 100644 index 000000000..6fdfc0955 --- /dev/null +++ b/simplyblock_core/controllers/fdb_backup_controller.py @@ -0,0 +1,207 @@ +# coding=utf-8 +import datetime +import logging as lg +import re +import time +import uuid + +import docker + +from simplyblock_core import utils, constants +from simplyblock_core.db_controller import DBController +from simplyblock_core.models.job_schedule import JobSchedule + +logger = lg.getLogger() +db_controller = DBController() + +def __get_fdb_cont(): + snode = db_controller.get_mgmt_nodes()[0] + if not snode: + return + node_docker = docker.DockerClient(base_url=f"tcp://{snode.docker_ip_port}", version="auto") + for container in node_docker.containers.list(): + if container.name.startswith("app_fdb-backup-agent"): # type: ignore[union-attr] + return container + +def create_backup(cluster_id): + container = __get_fdb_cont() + if container: + cluster = db_controller.get_cluster_by_id(cluster_id) + backup_path = cluster.get_backup_path() + if cluster.backup_s3_bucket and cluster.backup_s3_cred: + folder = f"backup-{str(datetime.datetime.now())}" + folder = folder.replace(" ", "-") + folder = folder.replace(":", "-") + folder = folder.split(".")[0] + backup_path = f"blobstore://{cluster.backup_s3_cred}@s3.{cluster.backup_s3_region}.amazonaws.com/{folder}?bucket={cluster.backup_s3_bucket}®ion={cluster.backup_s3_region}&sc=0" + + res = container.exec_run(cmd=f"fdbbackup start -d {backup_path} -w") + cont = res.output.decode("utf-8") + logger.info(cont) + return True + return False + +def list_backups(cluster_id): + container = __get_fdb_cont() + data = [] + if container: + cluster = db_controller.get_cluster_by_id(cluster_id) + backup_path = cluster.get_backup_path() + res = container.exec_run(cmd=f"fdbbackup list -b {backup_path}") + logger.info(f"backup list from : {backup_path}") + cont = res.output.decode("utf-8") + for line in cont.splitlines(): + if not line or "backup-" not in line: + continue + + name = line.split("/")[-1].strip() + name = name.split("?")[0] + size = 0 + restorable = 0 + date = "" + version = 0 + res = container.exec_run(cmd=f"fdbbackup describe -d {cluster.get_backup_path(name)} --version-timestamps") + cont = res.output.decode("utf-8") + for line in cont.splitlines(): + if line and line.startswith("SnapshotBytes"): + size = line.split()[1].strip() + if line and line.startswith("Restorable"): + restorable = line.split()[1].strip() + if line and line.startswith("Snapshot:"): + for param in line.split(): + if param.startswith("startVersion"): + version = param.split("=")[1].strip() + elif param.startswith("(") and param.endswith(")") and not date:# 2025/12/28.10:10:20+0000 + try: + date = datetime.datetime.strptime(param[1:-1], "%Y/%m/%d.%H:%M:%S+0000").strftime("%Y-%m-%d %H:%M:%S") + except Exception: + date = name.replace("backup-","") + if not date: + date = name.replace("backup-", "") + + data.append({ + "Name": name, + "Version": version, + "Size": utils.humanbytes(int(size)), + "Restorable": restorable, + "Date": date, + }) + + return utils.print_table(data) + + return True + + + +def backup_status(): + container = __get_fdb_cont() + if container: + res = container.exec_run(cmd="fdbbackup status") + cont = res.output.decode("utf-8") + logger.info(f"backup status: \n{cont.strip()}") + return True + + +def backup_restore(backup_name, cluster_id): + container = __get_fdb_cont() + if container: + cluster = db_controller.get_cluster_by_id(cluster_id) + backup_path = cluster.get_backup_path(backup_name) + res = container.exec_run(cmd="fdbcli --exec \"writemode on; clearrange \\\"\\\" \\xff\"") + cont = res.output.decode("utf-8") + logger.info(cont.strip()) + res = container.exec_run(cmd=f"fdbrestore start -r \"{backup_path}\" --dest-cluster-file {constants.KVD_DB_FILE_PATH}") + cont = res.output.decode("utf-8") + logger.info(cont.strip()) + return True + + +def parse_history_param(history_string): + if not history_string: + logger.error("Invalid history value") + return False + results = re.search(r'^(\d+[hmd])(\d+[hmd])?$', history_string.lower()) + if not results: + logger.error(f"Error parsing history string: {history_string}") + return False + total_seconds = 0 + for s in results.groups(): + if not s: + continue + ind = s[-1] + v = int(s[:-1]) + if ind == 'd': + total_seconds += v*60*60*24 + if ind == 'h': + total_seconds += v*60*60 + if ind == 'm': + total_seconds += v*60 + return int(total_seconds) + + +def backup_configure(cluster_id, backup_path, backup_frequency, bucket_name, region_name, backup_credentials): + cluster = db_controller.get_cluster_by_id(cluster_id) + if cluster: + if backup_frequency: + total_seconds = parse_history_param(backup_frequency) + cluster.backup_frequency_seconds = total_seconds + if backup_path: + if not backup_path.startswith("file://"): + backup_path = f"file://{backup_path}" + cluster.backup_local_path = backup_path + cluster.backup_s3_region = "" + cluster.backup_s3_bucket = "" + cluster.backup_s3_cred = "" + cluster.write_to_db() + return True + else: + container = __get_fdb_cont() + if container: + backup_path = f"blobstore://{backup_credentials}@s3.{region_name}.amazonaws.com/?bucket={bucket_name}®ion={region_name}&sc=0" + res = container.exec_run(cmd=f"fdbbackup list -b {backup_path}") + cont = res.output.decode("utf-8") + if res.exit_code == 0: + logger.info(f"backup list from : {backup_path}") + logger.info(cont) + cluster.backup_s3_region = region_name if region_name else "" + cluster.backup_s3_bucket = bucket_name if bucket_name else "" + cluster.backup_s3_cred = backup_credentials if backup_credentials else "" + cluster.write_to_db() + else: + logger.error(f"Failed to list backup from s3: {backup_path}") + logger.error(cont) + return False + return True + +def add_backup_task(cluster_id): + try: + cluster = db_controller.get_cluster_by_id(cluster_id) + except Exception as e: + logger.error(f"Failed to get cluster {cluster_id}: {e}") + return False + + tasks = get_backup_tasks(cluster_id) + for task in tasks: + if task.status != JobSchedule.STATUS_DONE: + logger.info(f"Backup task found: {tasks[0].uuid}") + return False + + task_obj = JobSchedule() + task_obj.uuid = str(uuid.uuid4()) + task_obj.cluster_id = cluster.get_id() + task_obj.date = int(time.time()) + task_obj.max_retry = constants.TASK_EXEC_RETRY_COUNT + task_obj.status = JobSchedule.STATUS_NEW + task_obj.function_name = JobSchedule.FN_FDB_BACKUP + task_obj.write_to_db() + logger.info(f"Backup task created: {task_obj.uuid}") + return task_obj.uuid + + +def get_backup_tasks(cluster_id): + backup_tasks = [] + tasks = db_controller.get_job_tasks(cluster_id) + for task in tasks: + if task.function_name == JobSchedule.FN_FDB_BACKUP: + backup_tasks.append(task) + return backup_tasks diff --git a/simplyblock_core/controllers/tasks_controller.py b/simplyblock_core/controllers/tasks_controller.py index e903dbb95..2e6d1d268 100644 --- a/simplyblock_core/controllers/tasks_controller.py +++ b/simplyblock_core/controllers/tasks_controller.py @@ -548,6 +548,8 @@ def get_active_node_mig_task(cluster_id, node_id, distr_name=None): return False + + def add_device_failed_mig_task(device_id): device = db.get_storage_device_by_id(device_id) for node in db.get_storage_nodes_by_cluster_id(device.cluster_id): diff --git a/simplyblock_core/env_var b/simplyblock_core/env_var index 5abdecd44..ae25be979 100644 --- a/simplyblock_core/env_var +++ b/simplyblock_core/env_var @@ -1,5 +1,5 @@ SIMPLY_BLOCK_COMMAND_NAME=sbcli-dev SIMPLY_BLOCK_VERSION=19.2.34 -SIMPLY_BLOCK_DOCKER_IMAGE=public.ecr.aws/simply-block/simplyblock:main +SIMPLY_BLOCK_DOCKER_IMAGE=public.ecr.aws/simply-block/simplyblock:main-fdb-backup SIMPLY_BLOCK_SPDK_ULTRA_IMAGE=public.ecr.aws/simply-block/ultra:main-latest diff --git a/simplyblock_core/models/cluster.py b/simplyblock_core/models/cluster.py index 6ee003215..72017e4c6 100644 --- a/simplyblock_core/models/cluster.py +++ b/simplyblock_core/models/cluster.py @@ -1,9 +1,10 @@ # coding=utf-8 - +import os.path from typing import List, Optional from pydantic import SecretStr +from simplyblock_core import constants from simplyblock_core.models.base_model import BaseModel @@ -164,6 +165,11 @@ class Cluster(BaseModel): # ``simplyblock_core.controllers.cluster_expansion.planner``. See the feature plan # ``single_node_expansion_plan.md`` for the schema. expand_state: dict = {} + backup_local_path: str = constants.KVD_DB_BACKUP_PATH + backup_frequency_seconds: int = 3*60*60 + backup_s3_bucket: str = "" + backup_s3_region: str = "" + backup_s3_cred: str = "" def get_status_code(self): if self.status in self.STATUS_CODE_MAP: @@ -180,6 +186,15 @@ def is_qos_set(self) -> bool: return True return False + def get_backup_path(self, path=""): + if self.backup_s3_bucket and self.backup_s3_cred: + backup_path = f"blobstore://{self.backup_s3_cred}@s3.{self.backup_s3_region}.amazonaws.com/{path}?bucket={self.backup_s3_bucket}" \ + + f"®ion={self.backup_s3_region}&sc=0" + elif self.backup_local_path: + backup_path = os.path.join(self.backup_local_path, path) + else: + backup_path = os.path.join(constants.KVD_DB_BACKUP_PATH, self.uuid, path) + return backup_path class ClusterAddNodeLock(BaseModel): diff --git a/simplyblock_core/models/job_schedule.py b/simplyblock_core/models/job_schedule.py index 00564f4fc..88476b450 100644 --- a/simplyblock_core/models/job_schedule.py +++ b/simplyblock_core/models/job_schedule.py @@ -36,6 +36,7 @@ class JobSchedule(BaseModel): # lvol delta to the target, flip ANA so the client fails over. Used for # migration commit and fail-back (fresh or recovered source). FN_REPLICATION_FINAL = "replication_final" + FN_FDB_BACKUP = "fdb_backup" canceled: bool = False cluster_id: str = "" diff --git a/simplyblock_core/scripts/docker-compose-swarm.yml b/simplyblock_core/scripts/docker-compose-swarm.yml index e7b625568..ad574de01 100644 --- a/simplyblock_core/scripts/docker-compose-swarm.yml +++ b/simplyblock_core/scripts/docker-compose-swarm.yml @@ -596,6 +596,20 @@ services: environment: SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL" + BackupService: + <<: *service-base + image: $SIMPLYBLOCK_DOCKER_IMAGE + command: "python3 simplyblock_core/services/tasks_runner_fdb_backup.py" + deploy: + placement: + constraints: [node.role == manager] + volumes: + - "/etc/foundationdb:/etc/foundationdb" + networks: + - hostnet + environment: + SIMPLYBLOCK_LOG_LEVEL: "$LOG_LEVEL" + networks: monitoring-net: external: true diff --git a/simplyblock_core/services/cap_monitor.py b/simplyblock_core/services/cap_monitor.py index 0655d4ddf..32a1a2768 100644 --- a/simplyblock_core/services/cap_monitor.py +++ b/simplyblock_core/services/cap_monitor.py @@ -4,9 +4,9 @@ from datetime import datetime, timezone from simplyblock_core import db_controller, constants, cluster_ops, utils -from simplyblock_core.controllers import cluster_events +from simplyblock_core.controllers import cluster_events, fdb_backup_controller from simplyblock_core.models.cluster import Cluster - +from simplyblock_core.models.job_schedule import JobSchedule logger = utils.get_logger(__name__) @@ -14,6 +14,17 @@ db = db_controller.DBController() last_event: dict[str, dict] = {} +def create_fdb_backup_if_needed(cluster): + tasks = fdb_backup_controller.get_backup_tasks(cluster.get_id()) + if not tasks: + fdb_backup_controller.add_backup_task(cluster.get_id()) + return + tasks.reverse() + last_backup_task = tasks[0] + if last_backup_task and last_backup_task.status == JobSchedule.STATUS_DONE: + if last_backup_task.date + cluster.backup_frequency_seconds < time.time(): + fdb_backup_controller.add_backup_task(cluster.get_id()) + def main(): logger.info("Starting capacity monitoring service...") @@ -26,6 +37,7 @@ def main(): continue clusters = db.get_clusters() for cl in clusters: + create_fdb_backup_if_needed(cl) logger.info(f"Checking cluster: {cl.get_id()}") records = db.get_cluster_capacity(cl, 1) if not records: diff --git a/simplyblock_core/services/tasks_runner_fdb_backup.py b/simplyblock_core/services/tasks_runner_fdb_backup.py new file mode 100644 index 000000000..8fb738d42 --- /dev/null +++ b/simplyblock_core/services/tasks_runner_fdb_backup.py @@ -0,0 +1,59 @@ +# coding=utf-8 +import time + + +from simplyblock_core import db_controller, utils, constants +from simplyblock_core.controllers import fdb_backup_controller +from simplyblock_core.models.job_schedule import JobSchedule +from simplyblock_core.models.cluster import Cluster + +logger = utils.get_logger(__name__) + +# get DB controller +db = db_controller.DBController() + + +def process_fdb_backup_task(task): + task = db.get_task_by_id(task.uuid) + if task.canceled: + task.function_result = "canceled" + task.status = JobSchedule.STATUS_DONE + task.write_to_db(db.kv_store) + return + + if task.retry >= task.max_retry: + task.function_result = "max retry reached, stopping task" + task.status = JobSchedule.STATUS_DONE + task.write_to_db(db.kv_store) + return + + if task.status != JobSchedule.STATUS_RUNNING: + task.status = JobSchedule.STATUS_RUNNING + task.write_to_db(db.kv_store) + + ret = fdb_backup_controller.create_backup(task.cluster_id) + if ret: + task.function_result = "Backup created" + task.status = JobSchedule.STATUS_DONE + task.write_to_db(db.kv_store) + + + +logger.info("Starting Tasks runner fdb backup...") + +while True: + clusters = db.get_clusters() + if not clusters: + logger.error("No clusters found!") + else: + for cl in clusters: + if cl.status == Cluster.STATUS_IN_ACTIVATION: + continue + + tasks = db.get_job_tasks(cl.get_id()) + for task in tasks: + if task.status != JobSchedule.STATUS_DONE: + if task.function_name == JobSchedule.FN_FDB_BACKUP: + process_fdb_backup_task(task) + + time.sleep(constants.TASK_EXEC_INTERVAL_SEC)