diff --git a/.dockerignore b/.dockerignore index 4a4f22f..3267cca 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,8 +9,11 @@ PyBackUpper.code-workspace .test-logs .test-target .test-source +.test-appconfig source target logs secrets -test_docker_compose.yaml \ No newline at end of file +test_docker_compose.yaml +src/log_dev.conf +src/__pycache__ \ No newline at end of file diff --git a/.gitignore b/.gitignore index d10432f..dcfb475 100644 --- a/.gitignore +++ b/.gitignore @@ -130,9 +130,10 @@ dmypy.json .vscode *.code-workspace -.test-source -.test-target -.test-logs +test-source/* +test-target/* +test-logs/* +test-appconfig/* test.cmd test_docker_compose.yaml # ignore all files in source directory @@ -144,4 +145,8 @@ target/* # ignore all files in logs directory logs/* -secrets/* \ No newline at end of file +secrets/* + +*.png.bkp +*.png.dtmp +.sync-exclude.lst \ No newline at end of file diff --git a/PyBackUpper Schema.png b/PyBackUpper Schema.png new file mode 100644 index 0000000..3dace8b Binary files /dev/null and b/PyBackUpper Schema.png differ diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a3a6d58 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,16 @@ +[project] +name = "pybackupper" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "APScheduler == 3.10.1", + "boto3 == 1.26.158", + "botocore == 1.29.165", + "Flask == 2.3.2", + "flask_wtf == 1.2.1", + "psutil == 5.9.6", + "Requests == 2.32.3", + "tzlocal == 5.2", +] diff --git a/src/backup.py b/src/backup.py new file mode 100644 index 0000000..1b397e5 --- /dev/null +++ b/src/backup.py @@ -0,0 +1,691 @@ +"""Backup class for pybackupper.""" + +import logging +import logging.config +import inspect +import shutil +from os import walk, remove, cpu_count +from os.path import exists, join, normpath, getsize +from zipfile import ZipFile, ZIP_BZIP2 +from threading import Lock +from concurrent.futures import ThreadPoolExecutor +from hashlib import md5, sha256, sha512, sha1 +from tools import size_to_human_readable + +class Backup(dict): + """Backup class for pybackupper.""" + def __init__(self, + name:str, + dest_path:str, + ignored:str = None, + logger:logging.Logger=None) -> None: + """Initializes Backup object. + + Args: + name (str): Backup name. + dest_path (str): Destination path of the backup. + ignored (str): Ignored patterns of the backup. + logger (logging.Logger, optional): Logger for the class. Defaults to None. + """ + self.logger = logger + self.name = name + self.dest_path = dest_path + self.ignored = ignored + super().__init__(self.__dict__()) + + try: + size = self.get_raw_size() + self.completed = True if size > 0 else False + except FileNotFoundError: + self.completed = False + + self.compressed = True if exists(f"{join(self.dest_path, self.name)}.zip") else False + super().update(self.__dict__()) + self.logger.debug(f"Backup {self.name} initialized.\n{self}") + + def __str__(self) -> str: + """Returns string representation of the backup. + + Returns: + str: String representation of the backup. + """ + size = size_to_human_readable(self.get_size()) + + return f"Backup {self.name}:\n" \ + f" Destination path: {self.dest_path}\n" \ + f" Ignored: {self.ignored}\n" \ + f" Size: {size}\n" \ + f" Completed: {self.completed}\n" \ + f" Compressed: {self.compressed}\n" + + def __dict__(self) -> dict: + """Returns dictionary representation of the backup. + + Returns: + dict: Dictionary representation of the backup. + """ + return { + "name": self.name, + "size": size_to_human_readable(self.get_size()), + "ignored": self.ignored, + "completed": self.completed, + "compressed": self.compressed, + "raw_hash": self.calculate_raw_hash(method="sha256") if self.completed else None, + "compressed_hash": self.calculate_compressed_hash(method="sha256") if self.compressed else None, + } + + @property + def logger(self) -> logging.Logger: + """Returns logger for the class. + + Returns: + logging.Logger: Logger for the class. + """ + return self._logger + + @logger.setter + def logger(self, logger:logging.Logger) -> None: + """Sets logger for the class. + + Args: + logger (logging.Logger): Logger for the class. + """ + if logger is None: + logging.config.fileConfig("log_dev.conf") + self._logger = logging.getLogger('pybackupper_logger') + else: + self._logger = logger + + @property + def name(self) -> str: + """Returns name of the backup. + + Returns: + str: Name of the backup. + """ + return self._name + + @name.setter + def name(self, name:str) -> None: + """Sets name of the backup. + + Args: + name (str): Name of the backup. + + Raises: + ValueError: Name of the backup is not valid. + PermissionError: Change of `name` property is not allowed for Backup. + """ + if name is None or name == "": + self.logger.error(f"Backup {name} is not valid.") + raise ValueError(f"Backup {name} is not valid.") + + try: + if self._name != "": + self.logger.error("Cannot change name of the backup.") + raise PermissionError("Cannot change name of the backup.") + except AttributeError: + self.logger.debug(f"Setting name of the backup to {name}.") + self._name = name + + @property + def dest_path(self) -> str: + """Returns destination path of the backup. + + Returns: + str: Destination path of the backup. + """ + return self._dest_path + + @dest_path.setter + def dest_path(self, dest_path:str) -> None: + """Sets destination path of the backup. + + Args: + dest_path (str): Destination path of the backup. + + Raises: + ValueError: Destination path of the backup is not valid. + FileNotFoundError: Destination path of the backup does not exist. + PermissionError: Change of `dest_path` property is not allowed for Backup. + """ + if dest_path is None or dest_path == "": + self.logger.error(f"Backup {dest_path} is not valid.") + raise ValueError(f"Backup {dest_path} is not valid.") + + if not exists(dest_path): + self.logger.error(f"Backup {dest_path} does not exist.") + raise FileNotFoundError(f"Backup {dest_path} does not exist.") + + try: + if self._dest_path != "": + self.logger.error("Cannot change destination path of the backup.") + raise PermissionError("Cannot change destination path of the backup.") + except AttributeError: + self.logger.debug(f"Setting destination path of the backup to {dest_path}.") + self._dest_path = dest_path + + backup_path = join(dest_path, self.name) + + try: + if exists(backup_path) and self.get_raw_size() > 0: + self.logger.debug(f"Backup {backup_path} already exists. "\ + "Marking it as completed.") + self.completed = True + except FileNotFoundError: + pass + + + @property + def completed(self) -> bool: + """Returns True if backup is completed, False otherwise. + + Returns: + bool: True if backup is completed, False otherwise. + """ + try: + return self._completed + except AttributeError: + return False + + @completed.setter + def completed(self, completed:bool) -> None: + """Sets completed property of the backup. + + Args: + completed (bool): True if backup is completed, False otherwise. + + Raises: + PermissionError: Change of `completed` property is not allowed for Backup. + """ + caller_class = inspect.currentframe().f_back.f_locals.get("self").__class__.__name__ + + if caller_class == self.__class__.__name__: + self.logger.debug(f"Setting completed property of the backup to {completed}.") + self._completed = completed + super().__init__(self.__dict__()) + else: + self.logger.error(f"Change of `completed` property is not allowed for {caller_class}.") + raise PermissionError( + f"Change of `completed` property is not allowed for {caller_class}.") + + @property + def ignored(self) -> str: + """Returns ignored patterns of the backup. + + Returns: + str: Ignored patterns of the backup. + """ + try: + return self._ignored + except AttributeError: + return "*.sock, *.pid, *.lock" + + @ignored.setter + def ignored(self, ignored:str) -> None: + """Sets ignored files of the backup. + + Args: + ignored (str): Ignored files of the backup. + + Raises: + ValueError: Ignored files of the backup is not valid. + PermissionError: Change of `ignored` property is not allowed for Backup. + """ + + if ignored is None or ignored == "": + ignored = "*.sock, *.pid, *.lock" + + # check if ignored will match pattern "*.ext1, *.ext2, *.ext3, ..." + if not all([pattern.startswith("*.") for pattern in ignored.split(", ")]): + self.logger.error(f"Backup {ignored} is not valid.") + raise ValueError(f"Backup {ignored} is not valid.") + + try: + if self._ignored != "": + self.logger.error("Cannot change ignored files of the backup.") + raise PermissionError("Cannot change ignored files of the backup.") + except AttributeError: + self.logger.debug(f"Setting ignored files of the backup to {ignored}.") + super().__init__(self.__dict__()) + self._ignored = ignored + + @property + def compressed(self) -> bool: + """Returns True if backup is compressed, False otherwise. + + Returns: + bool: True if backup is compressed, False otherwise. + """ + try: + return self._compressed + except AttributeError: + return False + + @compressed.setter + def compressed(self, compressed:bool) -> None: + """Sets compressed property of the backup. + + Args: + compressed (bool): Compressed property of the backup. + + Raises: + PermissionError: Change of `compressed` property is not allowed for Backup. + """ + caller_class = inspect.currentframe().f_back.f_locals.get("self").__class__.__name__ + + if caller_class == self.__class__.__name__: + self.logger.debug(f"Setting compressed property of the backup to {compressed}.") + self._compressed = compressed + super().__init__(self.__dict__()) + else: + self.logger.error(f"Change of `compressed` property is not allowed for {caller_class}.") + raise PermissionError( + f"Change of `compressed` property is not allowed for {caller_class}.") + + + def get_raw_size(self) -> int: + """Returns raw size of the backup. + + Returns: + int: Raw size of the backup. + """ + backup_path = normpath(join(self.dest_path, self.name)) + self.logger.debug(f"Getting raw size of the backup {self.name}.") + + if not exists(backup_path): + self.logger.debug(f"Backup {backup_path} does not exist.") + return 0 + + size = sum(getsize(join(root, file)) + for root, _, files in walk(backup_path) for file in files) + self.logger.debug(f"Raw size of the backup {self.name} is {size_to_human_readable(size)}.") + return size + + def get_compressed_size(self) -> int: + """Returns compressed size of the backup. + + Raises: + FileNotFoundError: Backup does not exist. + + Returns: + int: Compressed size of the backup. + """ + backup_path = join(self.dest_path, self.name) + self.logger.debug(f"Getting compressed size of the backup {self.name}.") + + if not exists(f"{backup_path}.zip"): + self.logger.error(f"Backup {backup_path}.zip does not exist.") + raise FileNotFoundError(f"Backup {backup_path}.zip does not exist.") + + size = getsize(f"{backup_path}.zip") + self.logger.debug( + f"Compressed size of the backup {self.name} is {size_to_human_readable(size)}.") + return size + + def get_size(self) -> int: + """Returns size of the backup. + + Returns: + int: Size of the backup. + """ + try: + raw_size = self.get_raw_size() if self.completed else 0 + except FileNotFoundError: + raw_size = 0 + + try: + compressed_size = self.get_compressed_size() if self.compressed else 0 + except FileNotFoundError: + compressed_size = 0 + + size = raw_size + compressed_size + + self.logger.debug( + f"Size of the backup {self.name} is {size}. "\ + f"Human readable: {size_to_human_readable(size)}.") + return size + + + def create_raw_backup(self, src_path:str) -> None: + """Creates raw backup of the `src_path` to the `dest_path`. + + Args: + src_path (str): Source path of the backup. + + Raises: + FileExistsError: Backup is already completed. + FileNotFoundError: Source path of the backup does not exist. + shutil.Error: Backup failed. + """ + if self.completed: + self.logger.error(f"Backup {self.name} is already completed.") + raise FileExistsError(f"Backup {self.name} is already completed.") + + if not exists(src_path): + self.logger.error(f"Backup {src_path} does not exist.") + raise FileNotFoundError(f"Backup {src_path} does not exist.") + + ignored_extensions = self.ignored.split(", ") + + backup_path = join(self.dest_path, self.name) + + try: + self.logger.debug(f"Creating raw backup of {src_path} to {self.dest_path}.") + shutil.copytree(src_path, + backup_path, + symlinks=True, + ignore_dangling_symlinks=True, + ignore=shutil.ignore_patterns(*ignored_extensions)) + except shutil.Error as e: + self.logger.exception(f"Backup {self.name} failed. Exception: {e}.") + raise e + + self.completed = True + self.logger.debug(f"Backup {self.name} completed.") + + def _add_to_zip(self, lock: Lock, handle: ZipFile, file_paths_batch: list) -> None: + """Adds files to the zip file. + + Args: + lock (Lock): Lock for the zip file. + handle (ZipFile): Zip file handle. + file_paths_batch (list): List of file paths to add to the zip file. + """ + backup_path = normpath(join(self.dest_path, self.name)) + + with lock: + for file_path in file_paths_batch: + handle.write(file_path, + normpath(file_path).replace(backup_path, "").lstrip("\\").lstrip("/")) + + def compress_raw_backup(self) -> None: + """Compresses raw backup to the zip file. + + Raises: + FileNotFoundError: Backup is not completed. + FileNotFoundError: Zip file was not created. + """ + + if not self.completed: + self.logger.error(f"Backup {self.name} is not completed.") + raise FileNotFoundError(f"Backup {self.name} is not completed.") + + if self.compressed: + self.logger.info(f"Backup {self.name} is already compressed. Nothing to do :).") + return + + self.logger.debug(f"Compressing raw backup {self.name}.") + backup_path = join(self.dest_path, self.name) + + file_paths = [] + + for root, _, files in walk(backup_path): + for file in files: + file_paths.append(normpath(join(root, file))) + + lock = Lock() + + n_workers = cpu_count() * 2 + + self.logger.debug(f"Using {n_workers} workers to compress the backup.") + + chunk_size = len(file_paths) // n_workers + if chunk_size == 0: + chunk_size = 1 + + with ZipFile(f"{backup_path}.zip", 'w', compression=ZIP_BZIP2) as handle: + with ThreadPoolExecutor(max_workers=n_workers) as executor: + for i in range(0, len(file_paths), chunk_size): + + file_paths_batch = file_paths[i:i+chunk_size] + + _ = executor.submit(self._add_to_zip, lock, handle, file_paths_batch) + + if not exists(f"{backup_path}.zip"): + self.logger.error(f"Zip file {backup_path}.zip was not created.") + raise FileNotFoundError(f"Zip file {backup_path}.zip was not created.") + + self.compressed = True + self.logger.debug(f"Backup {self.name} compressed.") + + def delete_raw_backup(self) -> None: + """Deletes raw backup. + """ + self.logger.debug(f"Deleting raw backup {self.name}.") + backup_path = join(self.dest_path, self.name) + + shutil.rmtree(backup_path, ignore_errors=True) + + self.completed = False + self.logger.debug(f"Backup {self.name} deleted.") + + def delete_compressed_backup(self) -> None: + """Deletes compressed backup. + """ + self.logger.debug(f"Deleting compressed backup {self.name}.") + backup_path = join(self.dest_path, self.name) + + try: + remove(f"{backup_path}.zip") + except FileNotFoundError: + pass + + self.compressed = False + self.logger.debug(f"Backup {self.name} deleted.") + + def delete_backup(self) -> None: + """Deletes backup. + """ + self.logger.debug(f"Deleting backup {self.name}.") + + self.delete_raw_backup() + self.delete_compressed_backup() + + self.completed = False + self.compressed = False + self.logger.debug(f"Backup {self.name} deleted.") + + def restore_backup_from_raw(self, restore_path:str) -> None: + """Restores backup from raw. + + Args: + restore_path (str): Destination path of the backup. + + Raises: + FileExistsError: Backup is not completed. + FileNotFoundError: Backup does not exist. + shutil.Error: Backup failed. + """ + backup_path = join(self.dest_path, self.name) + + if not self.completed or not exists(backup_path): + self.logger.error(f"Backup {self.name} is not completed.") + raise FileExistsError(f"Backup {self.name} is not completed.") + + try: + self.logger.debug(f"Restoring backup {self.name} from raw to {restore_path}.") + shutil.copytree(backup_path, + restore_path, + symlinks=True, + dirs_exist_ok=True, + ignore_dangling_symlinks=True) + except shutil.Error as e: + self.logger.exception(f"Backup {self.name} failed. Exception: {e}.") + raise e + + self.completed = True + self.logger.debug(f"Backup {self.name} completed.") + + def unpack_compressed(self) -> None: + """Unpacks compressed backup. + + Raises: + FileNotFoundError: Zip file was not created. + """ + backup_path = join(self.dest_path, self.name) + if not self.compressed or not exists(f"{backup_path}.zip"): + self.logger.error(f"Backup {self.name} is not compressed.") + raise FileNotFoundError(f"Backup {self.name} is not compressed.") + + self.logger.debug(f"Unpacking compressed backup {self.name}.") + backup_path = join(self.dest_path, self.name) + + with ZipFile(f"{backup_path}.zip", 'r') as handle: + handle.extractall(backup_path) + + self.completed = True + + self.logger.debug(f"Backup {self.name} unpacked.") + + def calculate_raw_hash(self, method:str) -> str: + """Calculates hash of the raw backup. + + Returns: + str: hash of the raw backup. + Raises: + FileNotFoundError: Backup is not completed. + ValueError: Method is not supported. + """ + methods = { + "md5": md5, + "sha1": sha1, + "sha256": sha256, + "sha512": sha512 + } + + if method not in methods: + self.logger.error(f"Method {method} is not supported.") + raise ValueError(f"Method {method} is not supported. "\ + "Supported methods: md5, sha1, sha256, sha512.") + + backup_path = join(self.dest_path, self.name) + + if not exists(backup_path): + self.logger.error(f"Backup {self.name} is not completed.") + raise FileNotFoundError(f"Backup {self.name} is not completed.") + + if not self.completed: + self.logger.warning(f"Backup {self.name} is not completed. "\ + "Calculating hash of the incomplete backup.") + + dir_hash = methods[method]() + + for root, _, files in walk(backup_path): + for file in files: + with open(join(root, file), "rb") as handle: + dir_hash.update(handle.read()) + + dir_hash = dir_hash.hexdigest() + + self.logger.debug(f"{method} hash of the raw backup {self.name} is {dir_hash}.") + return dir_hash + + def calculate_compressed_hash(self, method) -> str: + """Calculates hash of the compressed backup. + + Returns: + str: hash of the compressed backup. + Raises: + FileNotFoundError: Backup is not completed. + ValueError: Method is not supported. + """ + + methods = { + "md5": md5, + "sha1": sha1, + "sha256": sha256, + "sha512": sha512 + } + + if method not in methods: + self.logger.error(f"Method {method} is not supported.") + raise ValueError(f"Method {method} is not supported. "\ + "Supported methods: md5, sha1, sha256, sha512.") + + backup_path = join(self.dest_path, self.name) + + if not exists(f"{backup_path}.zip"): + self.logger.error(f"Backup {self.name} is not completed.") + raise FileNotFoundError(f"Backup {self.name} is not completed.") + + zip_hash = methods[method]() + + with open(f"{backup_path}.zip", "rb") as handle: + zip_hash.update(handle.read()) + + zip_hash = zip_hash.hexdigest() + + self.logger.debug(f"{method} hash of the compressed backup {self.name} is {zip_hash}.") + return zip_hash + + def restore_backup(self, restore_path:str) -> bool: + """Restores backup. + + Args: + restore_path: Path to restore the backup. + + Raises: + FileNotFoundError: Restore path does not exist. + FileNotFoundError: Backup is not available. + ValueError: Restore path is not valid. + + Returns: + bool: True if backup was restored successfully, False if errors occurred. + """ + if restore_path is None or restore_path == "": + self.logger.error(f"Restore path {restore_path} is not valid.") + raise ValueError(f"Restore path {restore_path} is not valid.") + + if not exists(restore_path): + self.logger.error(f"Restore path {restore_path} does not exist.") + raise FileNotFoundError(f"Restore path {restore_path} does not exist.") + + if self.completed: + self.logger.info(f"Restoring backup {self.name} from raw to {restore_path}.") + self.restore_backup_from_raw(restore_path) + + elif self.compressed: + self.logger.info(f"Restoring backup {self.name} from compressed to {restore_path}.") + self.unpack_compressed() + self.restore_backup_from_raw(restore_path) + + else: + self.logger.error(f"Backup {self.name} is not available.") + raise FileNotFoundError(f"Backup {self.name} is not available.") + + backup_hash = self.calculate_raw_hash(method="sha256") + restore_hash = sha256() + + for root, _, files in walk(restore_path): + for file in files: + with open(join(root, file), "rb") as handle: + restore_hash.update(handle.read()) + + restore_hash = restore_hash.hexdigest() + + if backup_hash == restore_hash: + self.logger.info(f"Backup {self.name} restored to {restore_path} successfully.") + return True + + self.logger.warning( + f"Backup {self.name} restored to {restore_path}, but hashes are different. "\ + f"Backup hash: {backup_hash}, restore hash: {restore_hash}.") + return False + + def calculate_compression_ratio(self) -> float: + """Calculates compression ratio of the backup. + + Returns: + float: Compression ratio of the backup. + """ + try: + raw_size = self.get_raw_size() + compressed_size = self.get_compressed_size() + + ratio = raw_size / compressed_size + except FileNotFoundError: + ratio = 0.0 + + self.logger.debug(f"Compression ratio of the backup {self.name} is {ratio}.") + return ratio diff --git a/src/backup_manager.py b/src/backup_manager.py new file mode 100644 index 0000000..61671fd --- /dev/null +++ b/src/backup_manager.py @@ -0,0 +1,1163 @@ +"""BackupManager class""" + +import logging +import logging.config +from logging.handlers import TimedRotatingFileHandler +from singleton import Singleton +from os.path import exists, normpath, getsize, join, isfile, isdir +from os import makedirs, walk, listdir, remove +from datetime import datetime +from psutil import disk_usage +from json import dump, load +from pprint import pformat +from shutil import Error as shutilError +from shutil import rmtree +from concurrent.futures import ThreadPoolExecutor +from re import fullmatch +from botocore.exceptions import ClientError as botocoreClientError +from filecmp import dircmp +from backup import Backup +from s3_handler import S3Handler +from telegram_handler import TelegramHandler +from tools import * + +class BackupManager(metaclass=Singleton): + """BackupManager class""" + def __init__(self, + src_path:str, + dest_path:str, + raw_to_keep:int, + compressed_to_keep:int, + s3_to_keep:int = 0, + ignored:str=None, + s3_handler=None, + telegram_handler=None, + logger:logging.Logger=None) -> None: + """Initialize the BackupManager class. + + Args: + src_path (str): Source path. + dest_path (str): Destination path. + raw_to_keep (int): How many raw backups to keep. + compressed_to_keep (int): How many compressed backups to keep. + s3_to_keep (int, optional): How many S3 backups to keep. Defaults to 0. + ignored (str, optional): Ignored paths. Defaults to None. + s3_handler (_type_, optional): S3 handler. Defaults to None. + telegram_handler (_type_, optional): Telegram handler. Defaults to None. + logger (logging.Logger, optional): Logger. Defaults to None. + """ + self.logger = logger + self.pending_backup = False + self.src_path = src_path + self.dest_path = dest_path + self.ignored = ignored + self.raw_to_keep = raw_to_keep + self.compressed_to_keep = compressed_to_keep + self.s3_to_keep = s3_to_keep + + self.s3_handler = s3_handler + self.telegram_handler = telegram_handler + + self.backups = { + "local": [], + "s3": [], + } + + try: + self.load_backup_info() + except FileNotFoundError: + print(listdir(self.dest_path)) + self.logger.warning(f"Backup info not found. Creating it.") + self.restore_backup_info() + self.logger.info("BackupManager initialized.") + + def __str__(self) -> str: + """Get the string representation of the object. + + Returns: + str: String representation of the object. + """ + local_size = 0 + s3_size = 0 + + for backup in self.backups["local"]: + local_size += backup.get_size() + + s3_size = self.s3_handler.get_bucket_size() if not self.s3_handler is None else 0 + + return f"BackupManager:\n" \ + f" src_path: {self.src_path}\n" \ + f" dest_path: {self.dest_path}\n" \ + f" ignored: {self.ignored}\n" \ + f" raw_to_keep: {self.raw_to_keep}\n" \ + f" compressed_to_keep: {self.compressed_to_keep}\n" \ + f" s3_handler: {True if self.s3_handler else False}\n" \ + f" telegram_handler: {True if self.telegram_handler else False}\n" \ + f" backups:\n" \ + f" local:\n" \ + f" count: {len(self.backups['local'])}\n" \ + f" size: {size_to_human_readable(local_size)}\n" \ + f" s3:\n" \ + f" count: {len(self.backups['s3'])}\n" \ + f" size: {size_to_human_readable(s3_size)}\n" \ + f" last:\n" \ + f" {self.backups['local'][-1].name if len(self.backups['local']) > 0 else None}" + + def __dict__(self) -> dict: + """Get the dictionary representation of the object. + + Returns: + dict: Dictionary representation of the object. + """ + return { + "src_path": self.src_path, + "dest_path": self.dest_path, + "ignored": self.ignored, + "raw_to_keep": self.raw_to_keep, + "compressed_to_keep": self.compressed_to_keep, + "local_size": size_to_human_readable(sum([backup.get_size() for backup in self.backups["local"]])), + "s3_size": size_to_human_readable(self.s3_handler.get_bucket_size() if not self.s3_handler is None else 0), + "s3_to_keep": self.s3_to_keep, + "backups": self.backups, + } + + @property + def logger(self) -> logging.Logger: + """Get the logger. + + Returns: + logging.Logger: Logger. + """ + return self._logger + + @logger.setter + def logger(self, logger:logging.Logger) -> None: + """Set the logger. + + Args: + logger (logging.Logger): Logger. + """ + if logger is None: + logging.config.fileConfig("log_dev.conf") + self._logger = logging.getLogger('pybackupper_logger') + else: + self._logger = logger + + @property + def src_path(self) -> str: + """Get the source path. + + Returns: + str: Source path. + """ + return self._src_path + + @src_path.setter + def src_path(self, src_path:str) -> None: + """ + + Args: + src_path (str): Source path. + + Raises: + ValueError: Source path cannot be None or empty. + FileNotFoundError: Source path does not exist. + """ + if src_path is None or src_path == "": + self.logger.error("src_path cannot be None or empty.") + raise ValueError("src_path cannot be None or empty.") + + src_path = normpath(src_path) + if not exists(src_path): + self.logger.error(f"src_path {src_path} does not exist.") + raise FileNotFoundError(f"src_path {src_path} does not exist.") + + self._src_path = src_path + + @property + def dest_path(self) -> str: + """Get the destination path. + + Returns: + str: Destination path. + """ + return self._dest_path + + @dest_path.setter + def dest_path(self, dest_path:str) -> None: + """ + + Args: + dest_path (str): Destination path. + + Raises: + ValueError: Destination path cannot be None or empty. + OSError: Destination path cannot be created. + """ + if dest_path is None or dest_path == "": + self.logger.error("dest_path cannot be None or empty.") + raise ValueError("dest_path cannot be None or empty.") + + dest_path = normpath(dest_path) + if not exists(dest_path): + self.logger.warning(f"dest_path {dest_path} does not exist. Creating it.") + try: + makedirs(dest_path) + except OSError as e: + self.logger.error(f"dest_path {dest_path} cannot be created. {e}") + raise OSError(f"dest_path {dest_path} cannot be created. {e}") + + self._dest_path = dest_path + + @property + def ignored(self) -> str: + """Get the ignored paths. + + Returns: + str: Ignored paths. + """ + return self._ignored + + @ignored.setter + def ignored(self, ignored:str) -> None: + """ + + Args: + ignored (str): Ignored paths. + Raises: + TypeError: ignored must be a string or None. + """ + if type(ignored) is str or ignored is None: + self._ignored = ignored + else: + self.logger.error("ignored must be a string or None.") + raise TypeError("ignored must be a string or None.") + + @property + def raw_to_keep(self) -> int: + """Get the number of raw backups to keep. + + Returns: + int: Number of raw backups to keep. + """ + return self._raw_to_keep + + @raw_to_keep.setter + def raw_to_keep(self, raw_to_keep:int) -> None: + """ + + Args: + raw_to_keep (int): Number of raw backups to keep. + + Raises: + TypeError: raw_to_keep must be an integer. + ValueError: raw_to_keep must be greater or equal to 0. + """ + if not type(raw_to_keep) is int: + self.logger.error("raw_to_keep must be an integer.") + raise TypeError("raw_to_keep must be an integer.") + + if raw_to_keep < 0: + self.logger.error("raw_to_keep must be greater or equal to 0.") + raise ValueError("raw_to_keep must be greater or equal to 0.") + + self._raw_to_keep = raw_to_keep + + @property + def compressed_to_keep(self) -> int: + """Get the number of compressed backups to keep. + + Returns: + int: Number of compressed backups to keep. + """ + return self._compressed_to_keep + + @compressed_to_keep.setter + def compressed_to_keep(self, compressed_to_keep:int) -> None: + """ + + Args: + compressed_to_keep (int): Number of compressed backups to keep. + + Raises: + TypeError: compressed_to_keep must be an integer. + ValueError: compressed_to_keep must be greater or equal to 0. + """ + if not type(compressed_to_keep) is int: + self.logger.error("compressed_to_keep must be an integer.") + raise TypeError("compressed_to_keep must be an integer.") + + if compressed_to_keep < 0: + self.logger.error("compressed_to_keep must be greater or equal to 0.") + raise ValueError("compressed_to_keep must be greater or equal to 0.") + + self._compressed_to_keep = compressed_to_keep + + @property + def s3_to_keep(self) -> int: + """Get the number of S3 backups to keep. + + Returns: + int: Number of S3 backups to keep. + """ + return self._s3_to_keep + + @s3_to_keep.setter + def s3_to_keep(self, s3_to_keep:int) -> None: + """ + + Args: + s3_to_keep (int): Number of S3 backups to keep. + + Raises: + TypeError: s3_to_keep must be an integer. + ValueError: s3_to_keep must be greater or equal to 0. + """ + if not type(s3_to_keep) is int: + self.logger.error("s3_to_keep must be an integer.") + raise TypeError("s3_to_keep must be an integer.") + + if s3_to_keep < 0: + self.logger.error("s3_to_keep must be greater or equal to 0.") + raise ValueError("s3_to_keep must be greater or equal to 0.") + + self._s3_to_keep = s3_to_keep + + @property + def s3_handler(self): + """Get the S3 handler. + + Returns: + S3Handler: S3 handler. + """ + return self._s3_handler + + @s3_handler.setter + def s3_handler(self, s3_handler) -> None: + """Set the S3 handler. + + Args: + s3_handler (S3Handler): S3 handler. + + Raises: + TypeError: s3_handler must be a S3Handler or None. + """ + if s3_handler is None: + self._s3_handler = None + return + + if not type(s3_handler) is S3Handler: + self.logger.error("s3_handler must be a S3Handler or None.") + raise TypeError("s3_handler must be a S3Handler or None.") + + if not s3_handler.test_connection(): + self.logger.error("S3 connection test failed. S3 handler will be set to None.") + self._s3_handler = None + else: + self._s3_handler = s3_handler + + @property + def telegram_handler(self): + """Get the Telegram handler. + + Returns: + TelegramHandler: Telegram handler. + """ + return self._telegram_handler + + @telegram_handler.setter + def telegram_handler(self, telegram_handler) -> None: + """Set the Telegram handler. + + Args: + telegram_handler (TelegramHandler): Telegram handler. + + Raises: + TypeError: telegram_handler must be a TelegramHandler or None. + """ + if telegram_handler is None: + self._telegram_handler = None + return + + if not type(telegram_handler) is TelegramHandler: + self.logger.error("telegram_handler must be a TelegramHandler or None.") + raise TypeError("telegram_handler must be a TelegramHandler or None.") + + if not telegram_handler.test_connection(): + self.logger.error("Telegram connection test failed. Telegram handler will be set to None.") + self._telegram_handler = None + else: + self._telegram_handler = telegram_handler + + @property + def pending_backup(self) -> bool: + """Get the pending backup flag. + + Returns: + bool: Pending backup flag. + """ + return self._pending_backup + + @pending_backup.setter + def pending_backup(self, pending_backup:bool) -> None: + """Set the pending backup flag. + + Args: + pending_backup (bool): Pending backup flag. + """ + if not type(pending_backup) is bool: + self.logger.error("pending_backup must be a boolean.") + raise TypeError("pending_backup must be a boolean.") + + self._pending_backup = pending_backup + + def generate_backup_name(self) -> str: + """Generate a backup name. + + Returns: + str: Backup name. + """ + return timestamp_to_file_name(datetime.now().timestamp()) + + def get_src_size(self) -> int: + """Get the size of the source. + + Returns: + int: Size of the source. + """ + self.logger.debug(f"Getting size of {self.src_path}...") + size = 0 + + for root, _, files in walk(self.src_path): + for file in files: + size += getsize(join(root, file)) + + self.logger.debug(f"Size of {self.src_path} is {size_to_human_readable(size)}.") + return size + + def get_dest_space(self): + """Get the size of the destination. + + Returns: + int: Size of the destination. + """ + self.logger.debug(f"Getting available space of {self.dest_path}...") + + try: + disk = disk_usage(self.dest_path) + total = disk.total + space = disk.free + + if total == 0: + self.logger.error(f"Error getting available space of {self.dest_path}.") + return 0 + + if space < total * 0.1: + self.logger.warning(f"Available space of {self.dest_path} is less than 10% of total space.") + if self.telegram_handler: + self.telegram_handler.send_message( + f"Available space of {self.dest_path} is less than 10% of total space.\n" + f"Space: {size_to_human_readable(space)} / {size_to_human_readable(total)}") + except OSError: + space = 0 + + self.logger.debug(f"Available space of {self.dest_path} is {size_to_human_readable(space)}.") + return space + + def check_available_space(self) -> bool: + """Check if there is enough space to create a backup. + + Returns: + bool: True if there is enough space, False otherwise. + """ + src_size = self.get_src_size() + dest_space = self.get_dest_space() + + if src_size > dest_space: + self.logger.error( + f"Not enough space to create a backup." + f" Source size is {size_to_human_readable(src_size)} " + f"and destination space is {size_to_human_readable(dest_space)}.") + return False + + if len(self.backups["local"]) > 0: + try: + compression_ratio = self.backups["local"][-1].calculate_compression_ratio() + if compression_ratio == 0: + compression_ratio = 1 + except FileNotFoundError: + compression_ratio = 1 + + return True if ((src_size + (src_size / compression_ratio)) * 1.05) < dest_space else False + + elif self.compressed_to_keep > 0: + return True if (src_size * 2 * 1.05) < dest_space else False + + return True if (src_size * 1.05) < dest_space else False + + def save_backup_info(self, dest_path:str=None) -> None: + """Save the backup info to a file. + + Args: + dest_path (str): Path to save the backup info. + + Raises: + OSError: Path cannot be created. + botocoreClientError: Error uploading backup info to S3. + """ + if dest_path is None or dest_path == "": + dest_path = self.dest_path + + self.logger.debug(f"Saving backup info to {dest_path}...") + dest_path = normpath(dest_path) + + if not exists(dest_path): + self.logger.warning(f"Path {dest_path} does not exist. Creating it.") + try: + makedirs(dest_path) + except OSError as e: + self.logger.error(f"Path {dest_path} cannot be created. {e}") + raise OSError(f"Path {dest_path} cannot be created. {e}") + + path = normpath(join(dest_path, "backup_info.json")) + with open(path, "w") as file: + dump(self.__dict__(), file, indent=4) + self.logger.debug(f"Backup info saved locally to {dest_path}.") + + if self.s3_handler: + try: + self.s3_handler.upload_file(path, "backup_info.json") + except botocoreClientError as e: + self.logger.exception(f"Error uploading backup info to S3. {e}", exc_info=True) + raise botocoreClientError(f"Error uploading backup info to S3. {e}") + + self.logger.debug(f"Backup info saved to S3.") + + def load_backup_info(self, src_path:str=None, ignore_hash_mismatch:bool=True) -> None: + """Load the backup info from a file. + + Args: + src_path (str): Path to load the backup info. + ignore_hash_mismatch (bool): Ignore hash mismatch and load backup anyway. + + Raises: + FileNotFoundError: File does not exist. + """ + if src_path is None or src_path == "": + src_path = self.dest_path + + src_path = normpath(src_path) + path = normpath(join(src_path, "backup_info.json")) + self.logger.debug(f"Loading backup info from {path}...") + + if not exists(path): + self.logger.error(f"File {path} does not exist.") + raise FileNotFoundError(f"File {path} does not exist.") + + with open(path, "r") as file: + backup_info = load(file) + + for backup in backup_info["backups"]["local"]: + try: + tmp_backup = Backup(backup["name"], self.dest_path, self.ignored, self.logger) + except FileNotFoundError: + self.logger.error(f"Backup {backup['name']} not found.") + continue + + if tmp_backup.completed and tmp_backup.compressed: + if (backup["raw_hash"] != tmp_backup.calculate_raw_hash(method="sha256")) or \ + (backup["compressed_hash"] != tmp_backup.calculate_compressed_hash(method="sha256")): + if ignore_hash_mismatch: + self.logger.warning(f"Hash mismatch for backup {backup['name']}. Loading backup anyway.") + else: + self.logger.error(f"Hash mismatch for backup {backup['name']}. Skipping backup.") + continue + + elif tmp_backup.compressed: + if (backup["compressed_hash"] != tmp_backup.calculate_compressed_hash(method="sha256")): + if ignore_hash_mismatch: + self.logger.warning(f"Hash mismatch for backup {backup['name']}. Loading backup anyway.") + else: + self.logger.error(f"Hash mismatch for backup {backup['name']}. Skipping backup.") + continue + + self.backups["local"].append(tmp_backup) + + for backup in backup_info["backups"]["s3"]: + if backup in self.backups["local"]: + self.backups["s3"].append(backup) + else: + try: + self.backups["s3"].append(Backup(backup["name"], self.dest_path, self.ignored, self.logger)) + except FileNotFoundError: + self.logger.error(f"Backup {backup} not found.") + continue + + self.logger.debug(f"Backup info loaded from {src_path}.") + + def restore_backup_info(self, src_path:str=None) -> None: + """Restore the backup info file. + + Args: + src_path (str): Path to restore the backup info. + """ + pattern = r"\b\d{4}_\d{2}_\d{2}_\d{2}_\d{2}_\d{2}(?:\.zip)?\b" + + if src_path is None or src_path == "": + src_path = self.dest_path + + self.logger.debug(f"Restoring backup info from {src_path}...") + src_path = normpath(src_path) + + backups_list = set() + for item in listdir(src_path): + if fullmatch(pattern, item): + if item.endswith(".zip"): + backups_list.add(item.split(".")[0]) + else: + backups_list.add(item) + + if len(backups_list) > 0: + with ThreadPoolExecutor(len(backups_list)) as executor: + for backup in backups_list: + executor.submit(lambda: + self.backups["local"].append( + Backup( backup, + self.dest_path, + self.ignored, + self.logger))) + + self.backups["local"].sort(key=lambda x: x.name, reverse=False) + self.backups["s3"].sort(key=lambda x: x.name, reverse=False) + if len(self.backups["local"]) > 0: + self.logger.debug(f"Restored backups\n{pformat(backups_list, sort_dicts=False, compact=True, indent=2)}\nfrom {src_path}") + self.save_backup_info() + if self.telegram_handler: + self.telegram_handler.send_message(f"Restored backups\n{pformat(backups_list, sort_dicts=False, compact=True, indent=2)}\nfrom {src_path}") + return + + self.logger.warning(f"No backups found in {src_path}.") + if self.telegram_handler: + self.telegram_handler.send_message(f"No backups found in {src_path}.") + + # TODO: Add method to verify backup_info.json integrity + + def create_backup(self) -> bool: + """Create a backup. + + Returns: + bool: True if the backup has been created, False otherwise. + """ + name = self.generate_backup_name() + self.logger.info(f"Creating backup {name}...") + + if not self.check_available_space(): + self.logger.error("Not enough space to create a backup.") + return False + + backup = Backup(name, self.dest_path, self.ignored, self.logger) + + try: + backup.create_raw_backup(self.src_path) + except FileExistsError: + self.logger.error(f"Backup {name} already exists.") + return False + except FileNotFoundError: + self.logger.error(f"Source path {self.src_path} does not exist.") + return False + except shutilError as e: + self.logger.error(f"Error creating backup {name}. {e}") + return False + try: + raw_hash = backup.calculate_raw_hash(method="sha256") + except FileNotFoundError: + self.logger.error(f"Error calculating raw hash of {backup.name}.") + return False + + self.logger.debug(f"Raw hash of {backup.name} is {raw_hash}.") + + if self.compressed_to_keep > 0: + try: + backup.compress_raw_backup() + compressed_hash = backup.calculate_compressed_hash(method="sha256") + except FileNotFoundError: + self.logger.error(f"Error compressing backup {backup.name}.") + return False + + self.logger.debug(f"Compressed hash of {backup.name} is {compressed_hash}.") + self.logger.debug(f"Compression ratio of {backup.name} is {backup.calculate_compression_ratio():.2f}.") + + self.backups["local"].append(backup) + + self.logger.info(f"Backup {backup.name} created.") + self.logger.debug(f"Backup {backup.name} size is {size_to_human_readable(backup.get_size())}.") + return True + + def upload_backup_to_s3(self, backup_name:str) -> bool: + """Upload a backup to S3. + + Args: + backup_name (str): Name of the backup to upload. + + Returns: + bool: True if the backup has been uploaded, False otherwise. + """ + if self.s3_handler is None: + self.logger.error("S3 handler is not set.") + return False + + self.logger.debug(f"Uploading backup {backup_name} to S3...") + index = self.get_backup_index_by_name(backup_name) + + if index == -1: + self.logger.error(f"Backup {backup_name} not found.") + return False + + if not self.backups["local"][index].completed or \ + not self.backups["local"][index].compressed: + self.logger.error(f"Backup {backup_name} is not completed or compressed.") + return False + + try: + self.s3_handler.upload_file(self.backups["local"][index].dest_path + "/" + + backup_name + ".zip", backup_name + ".zip") + except FileNotFoundError: + self.logger.error(f"Backup {backup_name} not found.") + return False + except botocoreClientError as e: + self.logger.exception(f"Error uploading backup {backup_name} to S3. {e}", exc_info=True) + return False + + self.backups["s3"].append(self.backups["local"][index]) + self.logger.debug(f"Backup {backup_name} uploaded to S3.") + return True + + def download_backup_from_s3(self, backup_name:str) -> bool: + """Download a backup from S3. + + Args: + backup_name (str): Name of the backup to download. + + Returns: + bool: True if the backup has been downloaded, False otherwise. + """ + if self.pending_backup: + self.logger.error("Backup task is running, need to wait for it to finish.") + return False + self.pending_backup = True + if self.s3_handler is None: + self.logger.error("S3 handler is not set.") + return False + + self.logger.debug(f"Downloading backup {backup_name} from S3...") + + for backup in self.backups["local"]: + if backup.name == backup_name and backup.compressed: + self.logger.debug(f"Backup {backup_name} already exists.") + self.pending_backup = False + return True + + try: + self.s3_handler.download_file(backup_name + ".zip", + self.dest_path + "/" + backup_name + ".zip") + except botocoreClientError as e: + self.logger.exception(f"Error downloading backup {backup_name} from S3. {e}", exc_info=True) + self.pending_backup = False + return False + + if not exists(self.dest_path + "/" + backup_name + ".zip"): + self.logger.error(f"Downloaded backup {backup_name} not found.") + self.pending_backup = False + return False + + self.logger.debug(f"Backup {backup_name} downloaded from S3.") + backup = Backup(backup_name, self.dest_path, self.ignored, self.logger) + self.backups["local"].append(backup) + self.backups["s3"][self.get_backup_index_by_name(backup_name, from_s3=True)].update(backup) + self.logger.debug(f"Backup {backup_name} added to local backups.") + try: + self.save_backup_info() + except Exception as e: + self.logger.exception(f"Error saving backup info. {e}", exc_info=True) + self.pending_backup = False + return True + + def get_backup_index_by_name(self, backup_name:str, from_s3:bool=False) -> int: + """Get the index of a backup by its name. + + Args: + backup_name (str): Name of the backup. + + Returns: + int: Index of the backup. + """ + if from_s3 and self.s3_handler is None: + self.logger.error("S3 handler is not set.") + return -1 + + for index, backup in enumerate(self.backups["local"] if not from_s3 else self.backups["s3"]): + if backup.name == backup_name: + return index + return -1 + + def delete_raw_backup(self, backup_name:str) -> bool: + """Delete a raw backup. + + Args: + backup_name (str): Name of the backup to delete. + + Returns: + bool: True if the backup has been deleted, False otherwise. + """ + self.logger.debug(f"Deleting raw backup {backup_name}...") + index = self.get_backup_index_by_name(backup_name) + + if index == -1: + self.logger.error(f"Backup {backup_name} not found.") + return False + + try: + backup = self.backups["local"][index] + self.backups["local"][index].delete_raw_backup() + if not self.backups["local"][index].completed and \ + not self.backups["local"][index].compressed: + _ = self.backups["local"].pop(index) + if self.s3_handler: + index_s3 = self.get_backup_index_by_name(backup.name, from_s3=True) + self.backups["s3"][index_s3].update(backup) + except FileNotFoundError: + self.logger.error(f"Backup {backup_name} not found.") + return False + except botocoreClientError as e: + self.logger.exception(f"Error updating backup {backup_name} in S3. {e}", exc_info=True) + pass + + self.logger.debug(f"Raw backup {backup_name} deleted.") + return True + + def delete_compressed_backup(self, backup_name:str) -> bool: + """Delete a compressed backup. + + Args: + backup_name (str): Name of the backup to delete. + + Returns: + bool: True if the backup has been deleted, False otherwise. + """ + self.logger.debug(f"Deleting compressed backup {backup_name}...") + index = self.get_backup_index_by_name(backup_name) + + if index == -1: + self.logger.error(f"Backup {backup_name} not found.") + return False + + try: + backup = self.backups["local"][index] + self.backups["local"][index].delete_compressed_backup() + if not self.backups["local"][index].completed and \ + not self.backups["local"][index].compressed: + _ = self.backups["local"].pop(index) + if self.s3_handler: + index_s3 = self.get_backup_index_by_name(backup.name, from_s3=True) + self.backups["s3"][index_s3].update(backup) + except FileNotFoundError: + self.logger.error(f"Backup {backup_name} not found.") + return False + except botocoreClientError as e: + self.logger.exception(f"Error updating backup {backup_name} in S3. {e}", exc_info=True) + pass + + self.logger.debug(f"Compressed backup {backup_name} deleted.") + return True + + def delete_s3_backup(self, backup_name:str) -> bool: + """Delete a S3 backup. + + Args: + backup_name (str): Name of the backup to delete. + + Returns: + bool: True if the backup has been deleted, False otherwise. + """ + if self.s3_handler is None: + self.logger.error("S3 handler is not set.") + return False + + self.logger.debug(f"Deleting S3 backup {backup_name}...") + index = self.get_backup_index_by_name(backup_name, from_s3=True) + + if index == -1: + self.logger.error(f"Backup {backup_name} not found.") + return False + + try: + self.s3_handler.delete_file(backup_name + ".zip") + _ = self.backups["s3"].pop(index) + except botocoreClientError as e: + self.logger.exception(f"Error deleting backup {backup_name} from S3. {e}", exc_info=True) + return False + + self.logger.debug(f"S3 backup {backup_name} deleted.") + return True + + def delete_backup(self, backup_name:str) -> bool: + """Delete a backup. + + Args: + backup_name (str): Name of the backup to delete. + + Returns: + bool: True if the backup has been deleted, False otherwise. + """ + if self.pending_backup: + self.logger.error("Backup task is running, need to wait for it to finish.") + return False + self.pending_backup = True + self.logger.debug(f"Deleting backup {backup_name}...") + index = self.get_backup_index_by_name(backup_name) + + if self.s3_handler: + try: + self.s3_handler.delete_file(backup_name + ".zip") + _ = self.backups["s3"].pop(self.get_backup_index_by_name(backup_name, from_s3=True)) + except botocoreClientError as e: + self.logger.exception(f"Error deleting backup {backup_name} from S3. {e}", exc_info=True) + + if index == -1: + self.logger.error(f"Backup {backup_name} not found.") + else: + try: + self.backups["local"][index].delete_backup() + except FileNotFoundError: + pass + _ = self.backups["local"].pop(index) + + self.logger.debug(f"Backup {backup_name} deleted.") + try: + self.save_backup_info() + except Exception as e: + self.logger.exception(f"Error saving backup info. {e}", exc_info=True) + self.pending_backup = False + return True + + def delete_old_backups(self) -> None: + """Delete old backups.""" + self.logger.debug(f"Deleting old backups...") + backups_to_delete = { + "raw": [], + "compressed": [], + "s3": [], + } + if len(self.backups["local"]) > self.raw_to_keep: + for backup in self.backups["local"][0:len(self.backups["local"]) - self.raw_to_keep]: + if backup.completed: + backups_to_delete["raw"].append(backup.name) + + if len(self.backups["local"]) > self.compressed_to_keep: + for backup in self.backups["local"][0:len(self.backups["local"]) - self.compressed_to_keep]: + if backup.compressed: + backups_to_delete["compressed"].append(backup.name) + + if len(self.backups["s3"]) > self.s3_to_keep: + for backup in self.backups["s3"][0:len(self.backups["s3"]) - self.s3_to_keep]: + backups_to_delete["s3"].append(backup.name) + + self.logger.info(f"Deleting backups\n{pformat(backups_to_delete, sort_dicts=False, compact=True, indent=2)}\n...") + + for backup in backups_to_delete["raw"]: + self.delete_raw_backup(backup) + for backup in backups_to_delete["compressed"]: + self.delete_compressed_backup(backup) + for backup in backups_to_delete["s3"]: + self.delete_s3_backup(backup) + + self.logger.debug(f"Old backups deleted.") + + def clear_dest_path(self) -> None: + self.logger.debug(f"Clearing {self.dest_path}...") + for item in listdir(self.dest_path): + item_path = join(self.dest_path, item) + if isfile(item_path): + remove(item_path) + elif isdir(item_path): + rmtree(item_path) + self.logger.debug(f"{self.dest_path} cleared.") + + def unzip_backup(self, backup_name:str) -> bool: + """Unzip a backup. + + Args: + backup_name (str): Name of the backup to unzip. + + Returns: + bool: True if the backup has been unzipped, False otherwise. + """ + if self.pending_backup: + self.logger.error("Backup task is running, need to wait for it to finish.") + return False + self.pending_backup = True + self.logger.debug(f"Unzipping backup {backup_name}...") + index = self.get_backup_index_by_name(backup_name) + + if index == -1: + self.logger.error(f"Backup {backup_name} not found.") + return False + + try: + self.backups["local"][index].unpack_compressed() + except FileNotFoundError: + self.logger.error(f"Backup {backup_name} not found.") + return False + + if not exists(join(self.dest_path, backup_name)): + self.logger.error(f"Unzipped backup {backup_name} not found.") + return False + + self.logger.debug(f"Backup {backup_name} unzipped.") + + if self.s3_handler: + self.backups["s3"][self.get_backup_index_by_name(backup_name, from_s3=True)].update(self.backups["local"][index]) + + self.pending_backup = False + return True + + def restore_backup(self, backup_name:str, restore_path:str) -> bool: + """Restore a backup. + + Args: + backup_name (str): Name of the backup to restore. + restore_path (str): Path to restore the backup. + + Returns: + bool: True if the backup has been restored, False otherwise. + """ + if self.pending_backup: + self.logger.error("Backup task is running, need to wait for it to finish.") + return False + self.pending_backup = True + self.logger.debug(f"Restoring backup {backup_name}...") + index = self.get_backup_index_by_name(backup_name) + + if index == -1: + self.logger.error(f"Backup {backup_name} not found.") + return False + + try: + self.backups["local"][index].restore_backup_from_raw(restore_path) + except FileNotFoundError as e: + self.logger.exception(f"Error restoring backup {backup_name}. {e}", exc_info=True) + return False + except shutilError as e: + self.logger.exception(f"Error restoring backup {backup_name}. {e}", exc_info=True) + return False + + if dircmp(self.src_path, join(self.dest_path, backup_name)).diff_files != []: + self.logger.error(f"Error restoring backup {backup_name}.") + return False + + self.logger.debug(f"Backup {backup_name} restored.") + + if self.s3_handler: + self.backups["s3"][self.get_backup_index_by_name(backup_name, from_s3=True)].update(self.backups["local"][index]) + + self.pending_backup = False + return True + + def run_backup(self, callback=None) -> str: + """Run a backup. + + Returns: + str: Backup name. + """ + if self.pending_backup: + self.logger.error("A backup is already running.") + return None + self.pending_backup = True + start_time = datetime.now().timestamp() + self.logger.info(f"Running backup. Start time: {timestamp_to_human_readable(start_time)}.") + + if self.create_backup(): + end_time = datetime.now().timestamp() + + if self.s3_handler: + s3_result = self.upload_backup_to_s3(self.backups["local"][-1].name) + upload_end_time = datetime.now().timestamp() + + if callback: + callback(True, "Backup completed.") + + self.delete_old_backups() + + self.logger.info(f"Backup completed. End time: {timestamp_to_human_readable(end_time)}.") + self.logger.info(f"Backup duration: {time_diff_to_human_readable(round(end_time - start_time))}.") + self.logger.info(f"""Backup info:\n{pformat(self.backups["local"][-1].__dict__(), sort_dicts=False, indent=2, compact=True)}.""") + if self.s3_handler and s3_result: + self.logger.info(f"Backup uploaded to S3. Upload duration: {time_diff_to_human_readable(round(upload_end_time - end_time))}.") + elif self.s3_handler and not s3_result: + self.logger.error(f"Backup upload to S3 failed.") + + if self.telegram_handler: + telegram_message = \ + f"*Backup completed*\\.\n" \ + f"""Start time: *{timestamp_to_human_readable(start_time).replace("-", "\\-")}*\\.\n""" \ + f"End time: *{timestamp_to_human_readable(end_time).replace("-", "\\-")}*\\.\n" \ + f"Backup duration: *{time_diff_to_human_readable(round(end_time - start_time))}*\\.\n" \ + f"Backup info:\n```json\n{pformat(self.backups['local'][-1].__dict__(), sort_dicts=False, indent=2, compact=True)}```\n" + + if self.s3_handler: + if s3_result: + telegram_message += \ + f"Backup uploaded to S3: *{True if self.s3_handler and s3_result else False}*\\.\n" \ + f"Upload duration: *{time_diff_to_human_readable(round(upload_end_time - end_time)) if s3_result else 0}*\\.\n" + + try: + s3_size = self.s3_handler.get_bucket_size() + telegram_message += \ + f"S3 size: *{size_to_human_readable(s3_size).replace(".", "\\.")}*\\.\n" + except botocoreClientError: + self.logger.error(f"Error getting S3 size.") + telegram_message += \ + f"*Error getting S3 size\\.*\n" + + else: + telegram_message += \ + f"*Backup uploaded to S3 failed\\.*\n" + + self.telegram_handler.send_message( + telegram_message, + markdown=True) + else: + self.logger.error(f"Backup failed.") + + if self.telegram_handler: + for index, handler in enumerate(self.logger.handlers): + if type(handler) is TimedRotatingFileHandler: + break + + path = normpath(self.logger.handlers[index].baseFilename) + self.telegram_handler.send_file(path, caption=f"Backup failed at {timestamp_to_human_readable(datetime.now().timestamp())}.") + + try: + self.save_backup_info() + except OSError: + self.logger.error(f"Backup info cannot be saved.") + self.logger.error(f"Printing backup info:\n{pformat(self.__dict__(), sort_dicts=False, compact=True, indent=2)}") + + if self.telegram_handler: + self.telegram_handler.send_message( + f"*Backup info cannot be saved\\.*\n" \ + f"Printing backup info:\n```json\n{pformat(self.__dict__(), sort_dicts=False, compact=True, indent=2)}```\n", + markdown=True) + except botocoreClientError: + self.logger.error(f"Backup info cannot be uploaded to S3.") + self.logger.error(f"Printing backup info:\n{pformat(self.__dict__(), sort_dicts=False, compact=True, indent=2)}") + + if self.telegram_handler: + self.telegram_handler.send_message( + f"*Backup info cannot be uploaded to S3\\.*\n" \ + f"Printing backup info:\n```json\n{pformat(self.__dict__(), sort_dicts=False, compact=True, indent=2)}```\n", + markdown=True) + + self.pending_backup = False + return self.backups["local"][-1].name + diff --git a/src/backups_manager.py b/src/backups_manager.py deleted file mode 100644 index 282c062..0000000 --- a/src/backups_manager.py +++ /dev/null @@ -1,924 +0,0 @@ -"""Module to manage backups - -Raises: - FileNotFoundError: Exception raised if source_path does not exist - FileNotFoundError: Exception raised if target_path does not exist - -Returns: - _type_: BackupManager object -""" -import logging -import logging.config -import json -import os -import shutil -from datetime import datetime -from time import perf_counter -import math -from multiprocessing import cpu_count - -class BackupManager(): - def __init__(self, - logger=None, - source_path: str = "/source", - target_path: str = "/target", - s3handler=None, - backup_info_file: str = None, - is_compression_enabled: bool = True, - archive_format: str = "tar.gz", - raw_backup_keep: int = 1, - compressed_backup_keep: int = 7, - s3_raw_keep: int = 1, - s3_compressed_keep: int = 3, - ignored_extensions: list = None, - puid: int = None, - pgid: int = None,): - """BackupManager class constructor - - Args: - logger (_type_, optional): Logger if available. Defaults to None. - source_path (str, optional): Custom absolute path from which backups will be created. Defaults to "/source". - target_path (str, optional): Custom absolute path where backups will be saved. Defaults to "/target". - s3handler (_type_, optional): Handle for S3Handler object. Defaults to None. - backup_info_file (str, optional): Custom backup_info.json file path. Defaults to None. - raw_backup_keep (int, optional): Number of raw backups to keep. Defaults to 1. - compressed_backup_keep (int, optional): Number of compressed backups to keep. Defaults to 7. - s3_raw_keep (int, optional): Number of raw backups to keep in S3. Defaults to 1. - s3_compressed_keep (int, optional): Number of compressed backups to keep in S3. Defaults to 3. - - Raises: - FileNotFoundError: Exception raised if source_path does not exist - FileNotFoundError: Exception raised if target_path does not exist - """ - if logger is None: - logging.config.fileConfig("log.conf") - self.logger = logging.getLogger('pybackupper_logger') - else: - self.logger = logger - - if not os.path.exists(source_path): - logger.error(f"Source path {source_path} does not exist") - raise FileNotFoundError(f"Source path {source_path} does not exist") - - self.source_path = source_path - - if not os.path.exists(target_path): - logger.error(f"Target path {target_path} does not exist") - raise FileNotFoundError(f"Target path {target_path} does not exist") - - self.target_path = target_path - - self.is_compression_enabled = is_compression_enabled - self.archive_format = archive_format - - if self.is_compression_enabled and self.map_archive_format(self.archive_format) is None: - self.logger.error(f"Archive format {self.archive_format} not supported") - raise ValueError(f"Archive format {self.archive_format} not supported") - - self.raw_backup_keep = raw_backup_keep - self.compressed_backup_keep = compressed_backup_keep - self.s3_raw_keep = s3_raw_keep - self.s3_compressed_keep = s3_compressed_keep - self.ignored_extensions = ignored_extensions - - self.s3handler = s3handler - - self.backups = self.load_backup_info_from_file(backup_info_file) - self.verify_backup_info() - - def save_backup_info_to_file(self, file_path: str=None, backup_info: dict = None) -> bool: - """Function to save the backup_info dictionary to a file - - Args: - file_path (str, optional): Absolute path to save the file. Defaults to None. - backup_info (dict, optional): Use custom backup_info instead of the generated one. Defaults to None. - - Returns: - bool: True if the file was saved successfully, False otherwise - """ - if file_path is None: - file_path = os.path.join(self.target_path, "backup_info.json") - - if backup_info is None: - backup_info = self.backups - - try: - with open(file_path, 'w') as file: - json.dump(backup_info, file, indent=4) - self.logger.debug(f"Backup info saved in file {file_path}") - except Exception as e: - self.logger.error(e, exc_info=True) - return False - return True - - def load_backup_info_from_file(self, file_path: str = None) -> dict: - """Function to load the backup_info dictionary from a file - Args: - file_path (str, optional): Custom absolute path to load the file from. Defaults to None. - - Returns: - dict: backup_info dictionary - """ - if file_path is None: - file_path = os.path.join(self.target_path, "backup_info.json") - - try: - with open(file_path, 'r') as file: - backup_info = json.load(file) - self.logger.debug(f"Backup info loaded from file {file_path}") - except Exception as e: - self.logger.warning(e) - self.logger.warning(f"Backup info file {file_path} not found, creating new one") - default_backup_info = { - "local_raw": [], - "local_compressed": [], - "s3_raw": [], - "s3_compressed": [] - } - self.save_backup_info_to_file(backup_info=default_backup_info) - return default_backup_info - return backup_info - - def verify_backup_info(self, target_path:str = None) -> bool: - """Function to verify that the backup info is correct and fix it if it is not - - Args: - target_path (str, optional): Custom absolute path were backups are saved. Defaults to None. - - Returns: - bool: True if the backup info is correct or was fixed, False otherwise - """ - if target_path is None: - target_path = self.target_path - - if not os.path.exists(target_path): - self.logger.error(f"Target path {target_path} does not exist") - return False - - directories = [f for f in os.listdir(target_path) if os.path.isdir(os.path.join(target_path, f))] - - for directory in directories: - if directory not in self.backups["local_raw"]: - self.logger.warning(f"Backup {directory} not listed in backup info") - self.backups["local_raw"].append(directory) - self.save_backup_info_to_file() - - for backup in self.backups["local_raw"]: - if backup not in directories: - self.logger.warning(f"Backup {backup} listed in backup info but not found in target path") - self.backups["local_raw"].remove(backup) - self.save_backup_info_to_file() - - if self.is_compression_enabled: - archives = [f for f in os.listdir(target_path) if os.path.isfile(os.path.join(target_path, f))] - try: - archives.remove("backup_info.json") - except ValueError: - pass - - for archive in archives: - if archive not in self.backups["local_compressed"]: - self.logger.warning(f"Backup {archive} not listed in backup info") - self.backups["local_compressed"].append(archive) - self.save_backup_info_to_file() - - for backup in self.backups["local_compressed"]: - if backup not in archives: - self.logger.warning(f"Backup {backup} listed in backup info but not found in target path") - self.backups["local_compressed"].remove(backup) - self.save_backup_info_to_file() - - if self.s3handler is not None: - s3_directories = self.s3handler.list_directories() - - for directory in s3_directories: - if directory not in self.backups["s3_raw"]: - self.logger.warning(f"Backup {directory} not listed in backup info") - self.backups["s3_raw"].append(directory) - self.save_backup_info_to_file() - - for backup in self.backups["s3_raw"]: - if backup not in s3_directories: - self.logger.warning(f"Backup {backup} listed in backup info but not found in s3") - self.backups["s3_raw"].remove(backup) - self.save_backup_info_to_file() - - if self.is_compression_enabled: - s3_archives = self.s3handler.list_files() - - for archive in s3_archives: - if archive not in self.backups["s3_compressed"]: - self.logger.warning(f"Backup {archive} not listed in backup info") - self.backups["s3_compressed"].append(archive) - self.save_backup_info_to_file() - - for backup in self.backups["s3_compressed"]: - if backup not in s3_archives: - self.logger.warning(f"Backup {backup} listed in backup info but not found in s3") - self.backups["s3_compressed"].remove(backup) - self.save_backup_info_to_file() - - return True - - - def create_raw_backup(self, backup_name: str = None, backup_path: str = None, source_path: str = None) -> bool: - """Function to create a raw backup - - Args: - backup_name (str, optional): Custom name for backup. Defaults to None. - backup_path (str, optional): Custom absolute path to save the backup. Defaults to None. - source_path (str, optional): Custom absolute path for backup source. Defaults to None. - - Returns: - bool: True if the backup was created, False otherwise - """ - if backup_name is None: - backup_name = datetime.today().strftime("%Y_%m_%d_%H_%M_%S") - - if backup_path is None: - backup_path = os.path.join(self.target_path, backup_name) - - if source_path is None: - source_path = self.source_path - - if not os.path.exists(source_path): - self.logger.error(f"Source path {source_path} does not exist") - return False - - if os.path.exists(backup_path): - logging.warning(f"Backup {backup_path} already exists") - if backup_name not in self.backups["local_raw"]: - self.backups["local_raw"].append(backup_name) - self.save_backup_info_to_file() - return True - - try: - shutil.copytree(source_path, backup_path, symlinks=True, ignore_dangling_symlinks=True, ignore=shutil.ignore_patterns(*self.ignored_extensions)) - self.logger.debug(f"Backup {backup_name} created") - except Exception as e: - self.logger.error(e, exc_info=True) - return False - - for root, dirs, files in os.walk(backup_path): - for file in files: - try: - source_file = os.path.join(root.replace(backup_path, self.source_path), file) - source_file_stat = os.stat(source_file) - target_file = os.path.join(root, file) - self.logger.debug(f"Copying stats from {source_file} to {target_file}") - shutil.copymode(source_file, target_file) - self.logger.debug(f"Chowning {target_file} to {source_file_stat.st_uid}:{source_file_stat.st_gid}") - shutil.chown(target_file, user=source_file_stat.st_uid, group=source_file_stat.st_gid) - except FileNotFoundError: - self.logger.error(f"File {source_file} not found") - except Exception as e: - self.logger.error(e, exc_info=True) - return False - - for dir in dirs: - try: - source_dir = os.path.join(root.replace(backup_path, self.source_path), dir) - source_dir_stat = os.stat(source_dir) - target_dir = os.path.join(root, dir) - self.logger.debug(f"Copying stats from {source_dir} to {target_dir}") - shutil.copymode(source_dir, target_dir) - self.logger.debug(f"Chowning {target_dir} to {source_dir_stat.st_uid}:{source_dir_stat.st_gid}") - shutil.chown(target_dir, user=source_dir_stat.st_uid, group=source_dir_stat.st_gid) - except FileNotFoundError: - self.logger.error(f"Directory {source_dir} not found") - except Exception as e: - self.logger.error(e, exc_info=True) - return False - - if backup_name not in self.backups["local_raw"]: - self.backups["local_raw"].append(backup_name) - self.save_backup_info_to_file() - - return True - - def map_archive_format(self, archive_format: str, reverse:bool= False) -> str: - """Function to map archive formatsf - - Args: - archive_format (str): Archive format to map - reverse (bool, optional): Map key to value accordingly. Defaults to False. - - Returns: - str: Mapped archive format - """ - if archive_format == "" or archive_format is None: - self.logger.error(f"Empty archive format") - return None - - mapping = { - "tar": "tar", - "tar.gz": "gztar", - "tar.bz2": "bztar", - "tar.xz": "xztar", - "zip": "zip" - } - - for k, v in mapping.items(): - if reverse: - if v == archive_format: - return k - if k == archive_format: - return v - return None - - def compress_backup(self, backup_path: str = None, archive_format: str = None) -> bool: - """Function to compress a backup - - Args: - backup_path (str, optional): Custom absolute path to read the backup from. Defaults to None. - archive_format (str, optional): Custom arichve format to compress to. Defaults to "tar.gz". - - Returns: - bool: True if the backup was compressed, False otherwise - """ - if backup_path is None: - if len(self.backups["local_raw"]) == 0: - self.logger.error(f"No local raw backups found") - return False - backup_path = os.path.join(self.target_path, self.backups["local_raw"][-1]) - - if not os.path.exists(backup_path): - self.logger.error(f"Backup path {backup_path} does not exist") - return False - - if archive_format is None: - archive_format = self.archive_format - if self.map_archive_format(archive_format) is None: - self.logger.error(f"Archive format {archive_format} not supported") - return False - - archive_path = backup_path + "." + archive_format - - if os.path.exists(archive_path): - logging.warning(f"Archive {archive_path} already exists") - if os.path.basename(archive_path) not in self.backups["local_compressed"]: - self.backups["local_compressed"].append(os.path.basename(archive_path)) - self.save_backup_info_to_file() - return True - - archive_name = os.path.basename(archive_path) - - try: - shutil_archive_format = self.map_archive_format(archive_format) - self.logger.debug(f"Compressing backup {backup_path} to {archive_path}") - # shutil.make_archive(backup_path, shutil_archive_format, backup_path) - - match archive_format: - case "tar": - command = f"tar -cf {archive_path} {backup_path}" - case "tar.gz": - command = f"""tar --use-compress-program="pigz -9 -N" -cf {archive_path} {backup_path}""" - case "tar.bz2": - command = f"tar -cjf {archive_path} {backup_path}" - case "tar.xz": - command = f"""tar --use-compress-program="pixz -9" -cf {archive_path} {backup_path}""" - case "zip": - command = f"""tar --use-compress-program="pigz -9 -N --zip" -cf {archive_path} {backup_path}""" - - os.system(command) - - self.logger.debug(f"Backup {backup_path} compressed to {archive_name}") - except Exception as e: - self.logger.error(e, exc_info=True) - return False - - self.backups["local_compressed"].append(os.path.basename(archive_name)) - self.save_backup_info_to_file() - - return True - - def send_raw_backup_to_s3(self, backup_path: str = None) -> bool: - """Function to send a raw backup to S3 - - Args: - backup_path (str, optional): Custom absolute path to read the backup from. Defaults to None. - - Returns: - bool: True if the backup was sent to S3, False otherwise - """ - if self.s3handler is None: - self.logger.error(f"No S3 handler found") - return False - - if self.s3_raw_keep == 0: - self.logger.error(f"S3 raw backups disabled") - return True - - if backup_path is None: - if len(self.backups["local_raw"]) == 0: - self.logger.error(f"No local raw backups found") - return False - backup_path = os.path.join(self.target_path, self.backups["local_raw"][-1]) - - if not os.path.exists(backup_path): - logging.error(f"Backup {backup_path} does not exist") - return False - - backup_name = os.path.basename(backup_path) - - if backup_name in self.backups["s3_raw"]: - logging.warning(f"Backup {backup_path} already exists in S3") - return True - - self.s3handler.upload_directory(backup_path, backup_name) - self.logger.debug(f"Backup {backup_path} sent to S3") - self.backups["s3_raw"].append(backup_name) - self.save_backup_info_to_file() - - return True - - def send_archive_to_s3(self, archive_path: str = None) -> bool: - """Function to send an archive to S3 - - Args: - archive_path (str, optional): Custom absolute path to read the backup archive from. Defaults to None. - - Returns: - bool: True if the archive was sent to S3, False otherwise - """ - if self.s3handler is None: - self.logger.error(f"No S3 handler found") - return False - - if self.s3_compressed_keep == 0: - self.logger.error(f"S3 compressed backups disabled") - return True - - if archive_path is None: - if len(self.backups["local_compressed"]) == 0: - self.logger.error(f"No local compressed backups found") - return False - archive_path = os.path.join(self.target_path, self.backups["local_compressed"][-1]) - - if not os.path.exists(archive_path): - logging.error(f"Archive {archive_path} does not exist") - return False - - archive_name = os.path.basename(archive_path) - - if archive_name in self.backups["s3_compressed"]: - logging.warning(f"Archive {archive_path} already exists in S3") - return True - - self.s3handler.upload_file(archive_path, archive_name) - self.logger.debug(f"Archive {archive_path} sent to S3") - self.backups["s3_compressed"].append(archive_name) - self.save_backup_info_to_file() - - return True - - def delete_raw_backup(self, backup_name: str = None, backup_path: str = None) -> bool: - """Function to delete a raw backup - - Args: - backup_name (str, optional): Custom backup name to be deleted. Defaults to None. - backup_path (str, optional): Custom absolute path were the backups are saved. Defaults to None. - - Returns: - bool: True if the backup was deleted, False otherwise - """ - if backup_name is None: - if len(self.backups["local_raw"]) == 0: - self.logger.error(f"No local raw backups found") - return False - backup_name = self.backups["local_raw"][0] - - if backup_path is None: - backup_path = os.path.join(self.target_path, backup_name) - - if not os.path.exists(backup_path): - logging.error(f"Backup {backup_path} does not exist") - return False - - try: - shutil.rmtree(backup_path) - self.logger.debug(f"Backup {backup_path} deleted") - except Exception as e: - self.logger.error(e, exc_info=True) - return False - - self.backups["local_raw"].remove(backup_name) - self.save_backup_info_to_file() - - return True - - def delete_compressed_backup(self, backup_name: str = None, backup_path: str = None) -> bool: - """Function to delete a compressed backup - - Args: - backup_name (str, optional): Custom archive name to be deleted. Defaults to None. - backup_path (str, optional): Custom absolute path were archives are saved. Defaults to None. - - Returns: - bool: True if the archive was deleted, False otherwise - """ - if backup_name is None: - if len(self.backups["local_compressed"]) == 0: - self.logger.error(f"No local compressed backups found") - return False - backup_name = self.backups["local_compressed"][0] - - if backup_path is None: - backup_path = os.path.join(self.target_path, backup_name) - - if not os.path.exists(backup_path): - logging.error(f"Backup {backup_path} does not exist") - return False - - try: - os.remove(backup_path) - self.logger.debug(f"Backup {backup_path} deleted") - except Exception as e: - self.logger.error(e, exc_info=True) - return False - - self.backups["local_compressed"].remove(backup_name) - self.save_backup_info_to_file() - - return True - - def delete_s3_raw_backup(self, backup_name: str = None) -> bool: - """Function to delete a raw backup from S3 - - Args: - backup_name (str, optional): Custom backup name to be deleted. Defaults to None. - - Returns: - bool: True if the backup was deleted, False otherwise - """ - if backup_name is None: - if len(self.backups["s3_raw"]) == 0: - self.logger.error(f"No S3 raw backups found") - return False - backup_name = self.backups["s3_raw"][0] - - try: - self.s3handler.delete_directory(backup_name) - self.logger.debug(f"S3 backup {backup_name} deleted") - except Exception as e: - self.logger.error(e, exc_info=True) - return False - - self.backups["s3_raw"].remove(backup_name) - self.save_backup_info_to_file() - - return True - - def delete_s3_compressed_backup(self, backup_name: str = None) -> bool: - """Function to delete a compressed backup from S3 - - Args: - backup_name (str, optional): Custom archive name to be deleted. Defaults to None. - - Returns: - bool: True if the archive was deleted, False otherwise - """ - if backup_name is None: - if len(self.backups["s3_compressed"]) == 0: - self.logger.error(f"No S3 compressed backups found") - return False - backup_name = self.backups["s3_compressed"][0] - - try: - self.s3handler.delete_file(backup_name) - self.logger.debug(f"S3 backup {backup_name} deleted") - except Exception as e: - self.logger.error(e, exc_info=True) - return False - - self.backups["s3_compressed"].remove(backup_name) - self.save_backup_info_to_file() - - return True - - def delete_old_backups_from(self, backup_type: str, max_backups: int) -> bool: - """Function to delete old backups - - Args: - backup_type (str): Type of backup to delete (raw, compressed, s3_raw, s3_compressed) - max_backups (int): Maximum number of backups to keep - - Returns: - bool: True if the old backups were deleted, False otherwise - """ - if backup_type not in ["local_raw", "local_compressed", "s3_raw", "s3_compressed"]: - self.logger.error(f"Invalid backup type {backup_type}") - return False - - if max_backups < 0: - self.logger.error(f"Invalid max backups {max_backups}") - return False - - backups = self.backups[backup_type] - - if len(backups) <= max_backups: - self.logger.debug(f"No old backups to delete") - return True - - backups_to_delete = len(backups) - max_backups - self.logger.debug(f"Deleting {backups_to_delete} old backups from {backup_type}") - - for _ in range(backups_to_delete): - match backup_type: - case "local_raw": - self.delete_raw_backup() - case "local_compressed": - self.delete_compressed_backup() - case "s3_raw": - self.delete_s3_raw_backup() - case "s3_compressed": - self.delete_s3_compressed_backup() - - return True - - def delete_old_backups(self) -> bool: - """Function to delete old backups - - Returns: - bool: True if the old backups were deleted, False otherwise - """ - if not self.delete_old_backups_from("local_raw", self.raw_backup_keep): - return False - - if not self.delete_old_backups_from("local_compressed", self.compressed_backup_keep): - return False - - if self.s3handler is not None: - if not self.delete_old_backups_from("s3_raw", self.s3_raw_keep): - return False - - if not self.delete_old_backups_from("s3_compressed", self.s3_compressed_keep): - return False - - return True - - def convert_to_human_readable(self, size: int) -> str: - """Function to convert a size in bytes to human readable format - - Args: - size (int): Size in bytes - - Returns: - str: Size in human readable format - """ - if size == 0: - return "0B" - - power = int(math.log(size, 1024)) - units = ["B", "KB", "MB", "GB", "TB", "PB"] - converted_size = round(size / 1024**power, 2) - - return f"{converted_size}{units[power]}" - - def convert_time_to_human_readable(self, time: float) -> str: - """Function to convert time in seconds to human readable format - - Args: - time (float): Time in seconds - - Returns: - str: Time in human readable format - """ - if time == 0: - return "0s" - - power = int(math.log(time, 60)) - units = ["s", "m", "h"] - if power < 0: - power = 0 - if power > len(units) - 1: - power = len(units) - 1 - converted_time = round(time / 60**power, 2) - - return f"{converted_time}{units[power]}" - - def get_backup_dir_size(self, backup_dir: str=None) -> int: - """Function to get the size of a backup directory - - Args: - backup_dir (str, optional): Custom backup directory to get the size from. Defaults to None. - - Raises: - FileNotFoundError: Exception raised if the backup directory does not exist - - Returns: - int: Size of the backup directory in bytes - """ - if backup_dir is None: - backup_dir = self.target_path - - if not os.path.exists(backup_dir): - self.logger.error(f"Backup directory {backup_dir} does not exist") - raise FileNotFoundError(f"Backup directory {backup_dir} does not exist") - - backup_dir_size = 0 - - backup_dir_size = shutil.disk_usage(backup_dir).used - - # for root, _, files in os.walk(backup_dir): - # for file in files: - # try: - # backup_dir_size += os.path.getsize(os.path.join(root, file)) - # except FileNotFoundError as e: - # self.logger.error(f"File {os.path.join(root, file)} not found") - # except Exception as e: - # self.logger.error(e, exc_info=True) - # raise e - - self.logger.debug(f"Backup directory {backup_dir} size: {self.convert_to_human_readable(backup_dir_size)}") - - return backup_dir_size - - def get_last_backup_size(self) -> dict: - """Function to get the size of the last backup - - Returns: - dict: Dictionary with the size of the last backup in raw and compressed format - """ - backup_size = {} - backup_size["raw"] = self.convert_to_human_readable(self.get_backup_dir_size(os.path.join(self.target_path, self.backups["local_raw"][-1]))) - - if self.backups["local_compressed"][-1].startswith(self.backups["local_raw"][-1]): - backup_size["compressed"] = self.convert_to_human_readable(os.path.getsize(os.path.join(self.target_path, self.backups["local_compressed"][-1]))) - - return backup_size - - def get_backups_size(self) -> dict: - """Function to get the size of the backup directory and the S3 bucket - - Returns: - dict: Dictionary with the size of the backup directory and the S3 bucket - """ - backup_size = {} - - backup_size["local"] = self.convert_to_human_readable(self.get_backup_dir_size()) - - if self.s3handler is not None: - backup_size["s3"] = self.convert_to_human_readable(self.s3handler.get_bucket_size()) - - return backup_size - - def get_backup_size(self, backup:str) -> int: - """Function to get the size of a backup - - Args: - backup (str): Name of the backup - - Returns: - int: Size of the backup in bytes - """ - if backup in self.backups["local_raw"]: - return self.get_backup_dir_size(os.path.join(self.target_path, backup)) - elif backup in self.backups["local_compressed"]: - return os.path.getsize(os.path.join(self.target_path, backup)) - - return 0 - - def get_backup_dir_free_space(self, backup_dir: str=None) -> int: - """Function to get the free space in the backup directory - - Args: - backup_dir (str, optional): Path to the backup directory. Defaults to None. - - Raises: - FileNotFoundError: Exception raised if the backup directory does not exist - - Returns: - int: Free space in the backup directory - """ - if backup_dir is None: - backup_dir = self.target_path - - if not os.path.exists(backup_dir): - self.logger.error(f"Backup directory {backup_dir} does not exist") - raise FileNotFoundError(f"Backup directory {backup_dir} does not exist") - - backup_dir_free_space = shutil.disk_usage(backup_dir).free - - self.logger.debug(f"Backup directory free space: {self.convert_to_human_readable(backup_dir_free_space)}") - - return backup_dir_free_space - - def get_source_dir_size(self, source_path:str=None) -> int: - """Function to get the size of the source directory - - Args: - source_path (str, optional): Path to the source directory. Defaults to None. - - Raises: - FileNotFoundError: Exception raised if the source directory does not exist - - Returns: - int: Size of the source directory in bytes - """ - if source_path is None: - source_path = self.source_path - - if not os.path.exists(source_path): - self.logger.error(f"Source directory {source_path} does not exist") - raise FileNotFoundError(f"Source directory {source_path} does not exist") - - source_dir_size = 0 - for root, _, files in os.walk(source_path): - for file in files: - source_dir_size += os.path.getsize(os.path.join(root, file)) - - self.logger.debug(f"Source directory: {source_path} size: {self.convert_to_human_readable(source_dir_size)}") - - return source_dir_size - - def check_if_enough_free_space(self, with_archive:bool=True) -> bool: - """Function to check if there is enough free space in the backup directory - - Args: - with_archive (bool, optional): If True, the size of the archive will be checked. Defaults to True. - - Returns: - bool: True if there is enough free space, False otherwise - """ - source_dir_size = self.get_source_dir_size() - target_dir_free_space = self.get_backup_dir_free_space() - - if with_archive: - return source_dir_size * 2 < target_dir_free_space - return source_dir_size < target_dir_free_space - - def get_backup_info(self) -> dict: - """Function to get information about the backup - - Returns: - dict: Dictionary with information about the backup - """ - backup_info = {} - backup_info["backup_dir_free_space"] = self.convert_to_human_readable(self.get_backup_dir_free_space()) - backup_info["last_backup"] = self.backups["local_raw"][-1] - backup_info["last_backup_size"] = self.get_last_backup_size() - backup_info["backup_size"] = self.get_backups_size() - - if self.s3handler is not None: - backup_info["s3_bucket_size"] = self.convert_to_human_readable(self.s3handler.get_bucket_size()) - - try: - if self.s3handler.check_directory_exists(self.backups["s3_raw"][-1]): - backup_info["last_s3_backup"] = self.backups["s3_raw"][-1] - except Exception: - try: - if self.s3handler.check_file_exists(self.backups["s3_compressed"][-1]): - backup_info["last_s3_backup"] = self.backups["s3_compressed"][-1] - except Exception: - pass - finally: - if backup_info.get("last_s3_backup", None) is not None: - backup_info["last_s3_backup_size"] = "ERROR" - - - return backup_info - - def perform_backup(self)-> str: - """Function to perform a backup - - Returns: - str: If the backup was successful, returns string with statistics. If the backup failed, returns None - - Raises: - Exception: Exception raised if any of the backup steps fails - """ - self.logger.info(f"Performing backup. Starting at {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") - backup_start_time = perf_counter() - if not self.create_raw_backup(): - raise Exception("Failed to create raw backup") - - compress_start_time = perf_counter() - if self.is_compression_enabled: - if not self.compress_backup(): - raise Exception("Failed to compress backup") - compress_end_time = perf_counter() - - upload_start_time = perf_counter() - if self.s3handler is not None: - if not self.send_raw_backup_to_s3(): - raise Exception("Failed to send raw backup to S3") - if self.is_compression_enabled: - if not self.send_archive_to_s3(): - raise Exception("Failed to send archive to S3") - upload_end_time = perf_counter() - - if not self.delete_old_backups(): - raise Exception("Failed to delete old backups") - backup_end_time = perf_counter() - - response = f"\ -Backup performed in {self.convert_time_to_human_readable(backup_end_time - backup_start_time)}, \ -raw backup took {self.convert_time_to_human_readable(compress_start_time - backup_start_time)}, \ -compression took {self.convert_time_to_human_readable(compress_end_time - compress_start_time)}, \ -upload to S3 took {self.convert_time_to_human_readable(upload_end_time - upload_start_time)}, \ -deletion took {self.convert_time_to_human_readable(backup_end_time - upload_end_time)}" - - self.logger.info(response) - - return response \ No newline at end of file diff --git a/src/log_dev.conf b/src/log_dev.conf new file mode 100644 index 0000000..0e502fd --- /dev/null +++ b/src/log_dev.conf @@ -0,0 +1,36 @@ +[loggers] +keys=root,pybackupper + +[handlers] +keys=consoleHandler,fileHandler + +[formatters] +keys=consoleFormatter,fileFormater + +[logger_root] +level=DEBUG +handlers=consoleHandler + +[logger_pybackupper] +level=DEBUG +handlers=consoleHandler,fileHandler +qualname=pybackupper_logger +propagate=0 + +[handler_consoleHandler] +class=StreamHandler +level=INFO +formatter=consoleFormatter +args=(sys.stdout, ) + +[handler_fileHandler] +class=handlers.TimedRotatingFileHandler +level=DEBUG +formatter=fileFormater +args=('../test-logs/log.log', "D", 7, 10) + +[formatter_consoleFormatter] +format=%(levelname)10s()s - %(module)20s() - %(funcName)30s() - %(message)s + +[formatter_fileFormater] +format=%(asctime)s - %(levelname)10s()s - %(module)20s() - %(funcName)30s() - %(message)s \ No newline at end of file diff --git a/src/main.py b/src/main.py index 5d516cf..42710aa 100644 --- a/src/main.py +++ b/src/main.py @@ -1,394 +1,59 @@ -"""PyBackUpper main module. - -Raises: - ValueError: Exception raised when required environment variable has invalid value. - KeyError: Exception raised required environment variable is not set. -""" - -import logging -import logging.config -import os -from s3_handler import S3Handler -from telegram_handler import TelegramHandler -from backups_manager import BackupManager -from pprint import pformat -from flask import Flask, render_template -import threading -from apscheduler.schedulers.blocking import BlockingScheduler -from apscheduler.triggers.cron import CronTrigger -from datetime import datetime class PyBackUpper(): - """PyBackUpper class. - """ - - DAY_NAMES = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'] - - def __init__(self, logger:logging.Logger=None): - """PyBackUpper constructor. + def __init__(self, config:dict) -> None: + """Initialize the PyBackUpper class. Args: - logger (logging.Logger, optional): Logger to use. Defaults to None. + config (dict): Configuration dictionary. """ - if logger is None: - logging.config.fileConfig("log.conf") - self.logger = logging.getLogger('pybackupper_logger') - else: - self.logger = logger + self.config = config + logging.config.fileConfig("log_dev.conf") + self.logger = logging.getLogger('pybackupper_logger') + + self.telegram_handler = TelegramHandler( + token=config["telegram"]["token"], + chat_id=config["telegram"]["chat_id"], + logger=self.logger) if "telegram" in config else None + + self.s3_handler = S3Handler( + bucket_name=config["s3"]["bucket"], + access_key=config["s3"]["access_key"], + secret_key=config["s3"]["secret_key"], + acl=config["s3"]["acl"] if "acl" in config["s3"] else None, + region=config["s3"]["region"] if "region" in config["s3"] else None, + url=config["s3"]["url"] if "url" in config["s3"] else None, + logger=self.logger) if "s3" in config else None + + self.backup_manager = BackupManager( + src_path=config["src_path"], + dest_path=config["dest_path"], + raw_to_keep=config["raw_to_keep"], + compressed_to_keep=config["compressed_to_keep"], + s3_to_keep=config["s3_to_keep"], + ignored=config["ignored"], + s3_handler=self.s3_handler, + telegram_handler=self.telegram_handler, + ) + + self.server = Server(self.backup_manager, logger=self.logger) + self.logger.info("PyBackUpper initialized.") - self.config = {} - self.read_env() - - if self.config["S3_BUCKET"] is not None and self.config["S3_ACCESS_KEY_ID"] is not None and self.config["S3_SECRET_ACCESS_KEY"] is not None: - self.s3_handler = S3Handler( - self.config["S3_BUCKET"], - self.config["S3_ACCESS_KEY_ID"], - self.config["S3_SECRET_ACCESS_KEY"], - self.config["S3_ACL"] if self.config["S3_ACL"] is not None else 'public-read', - self.config["S3_REGION_NAME"] if self.config["S3_REGION_NAME"] is not None else 'us-east-1', - self.config["S3_ENDPOINT_URL"] if self.config["S3_ENDPOINT_URL"] is not None else 'https://s3.amazonaws.com', - logger=self.logger) - if not self.s3_handler.test_connection(): - self.logger.error("S3 connection test failed. S3 upload will not be available.") - self.s3_handler = None - else: - self.logger.info("S3 connection test successful.") - - else: - self.logger.warning("S3_BUCKET, S3_ACCESS_KEY and S3_SECRET_KEY not set. S3 upload will not be available.") - self.s3_handler = None - - if self.config["TELEGRAM_TOKEN"] is not None and self.config["TELEGRAM_CHAT_ID"] is not None: - self.telegram_handler = TelegramHandler(self.config["TELEGRAM_TOKEN"], self.config["TELEGRAM_CHAT_ID"], logger=self.logger) - if not self.telegram_handler.test_connection(): - self.logger.error("Telegram connection test failed. Telegram notifications will not be available.") - self.telegram_handler = None - else: - self.logger.info("Telegram connection test successful.") - else: - self.logger.warning("TELEGRAM_TOKEN and TELEGRAM_CHAT_ID not set. Telegram notifications will not be available.") - - self.backups_manager = BackupManager( - logger=self.logger, - s3handler = self.s3_handler, - is_compression_enabled = self.config["COMPRESSION_ENABLED"], - archive_format = self.config["ARCHIVE_FORMAT"], - raw_backup_keep = self.config["LOCAL_RAW_BACKUPS_KEEP"], - compressed_backup_keep = self.config["LOCAL_COMPRESSED_BACKUPS_KEEP"], - s3_raw_keep = self.config["S3_RAW_BACKUPS_KEEP"], - s3_compressed_keep = self.config["S3_COMPRESSED_BACKUPS_KEEP"], - ignored_extensions = self.config["IGNORED_EXTENSIONS"] - ) - -# if self.telegram_handler: -# self.telegram_handler.send_message(f"\ -# PyBackUpper initialized.\n\n\ -# Config:\n\ -# `{self.print_config()}`") - - def read_env(self): - """Reads environment variables and stores them in self.config. - Raises: - ValueError: Exception raised when required environment variable has invalid value. - KeyError: Exception raised required environment variable is not set. - """ - self.logger.info("Reading environment variables.") - - try: - self.config["HOSTNAME"] = os.environ['HOSTNAME'].strip().replace('"', '') - except KeyError as e: - raise KeyError("HOSTNAME not set.") from e - - try: - self.config["PUID"] = int(os.environ['PUID'].strip().replace('"', '')) - if self.config["PUID"] < 0 or self.config["PUID"] > 65535: - self.logger.error("Value of PUID must be between 0 and 65535, not %s", self.config["PUID"]) - raise ValueError("Value of PUID must be between 0 and 65535") - except KeyError as e: - raise KeyError("PUID not set.") from e - - try: - self.config["PGID"] = int(os.environ['PGID'].strip().replace('"', '')) - if self.config["PGID"] < 0 or self.config["PGID"] > 65535: - self.logger.error("Value of PGID must be between 0 and 65535, not %s", self.config["PGID"]) - raise ValueError("Value of PGID must be between 0 and 65535") - except KeyError as e: - raise KeyError("PGID not set.") from e - - try: - self.config["DAYS_TO_RUN"] = [int(x) for x in os.environ['DAYS_TO_RUN'].strip().replace('"', '').split(',')] - for day in self.config["DAYS_TO_RUN"]: - if day < 0 or day > 6: - self.logger.error("Value of DAYS_TO_RUN must be between 0 and 6, not %s", day) - raise ValueError("Value of DAYS_TO_RUN must be between 0 and 6") - - if len(self.config["DAYS_TO_RUN"]) != len(set(self.config["DAYS_TO_RUN"])): - self.logger.error("DAYS_TO_RUN contains duplicates.") - raise ValueError("DAYS_TO_RUN contains duplicates") - - except KeyError as e: - raise KeyError("DAYS_TO_RUN not set.") from e - - try: - self.config["HOUR"] = int(os.environ['HOUR'].strip().replace('"', '')) - if self.config["HOUR"] < 0 or self.config["HOUR"] > 23: - self.logger.error("Value of HOUR must be between 0 and 23, not %s", self.config["HOUR"]) - raise ValueError("Value of HOUR must be between 0 and 23") - except KeyError as e: - raise KeyError("HOUR not set.") from e - - try: - self.config["MINUTE"] = int(os.environ['MINUTE'].strip().replace('"', '')) - if self.config["MINUTE"] < 0 or self.config["MINUTE"] > 59: - self.logger.error("Value of MINUTE must be between 0 and 59, not %s", self.config["MINUTE"]) - raise ValueError("Value of MINUTE must be between 0 and 59") - except KeyError as e: - raise KeyError("MINUTE not set.") from e - - try: - if os.environ['COMPRESSION_ENABLED'].strip().replace('"', '').lower() == "true": - self.config["COMPRESSION_ENABLED"] = True - elif os.environ['COMPRESSION_ENABLED'].strip().replace('"', '').lower() == "false": - self.config["COMPRESSION_ENABLED"] = False - else: - self.logger.error("COMPRESSION_ENABLED must be either true or false.") - raise ValueError("COMPRESSION_ENABLED must be either true or false") - except KeyError: - self.logger.warning("COMPRESSION_ENABLED not set. Defaulting to true.") - self.config["COMPRESSION_ENABLED"] = True - - if self.config["COMPRESSION_ENABLED"]: - try: - self.config["ARCHIVE_FORMAT"] = os.environ['ARCHIVE_FORMAT'].strip().replace('"', '') - if self.config["ARCHIVE_FORMAT"] not in ["tar", "tar.gz", "tar.bz2", "tar.xz", "zip"]: - self.logger.error("ARCHIVE_FORMAT must be one of tar, tar.gz, tar.bz2, tar.xz, zip, not %s", self.config["ARCHIVE_FORMAT"]) - raise ValueError("ARCHIVE_FORMAT must be one of tar, tar.gz, tar.bz2, tar.xz, zip") - except KeyError: - self.logger.warning("ARCHIVE_FORMAT not set. Defaulting to tar.gz.") - self.config["ARCHIVE_FORMAT"] = "tar.gz" - else: - self.config["ARCHIVE_FORMAT"] = None - - try: - self.config["LOCAL_RAW_BACKUPS_KEEP"] = int(os.environ['LOCAL_RAW_BACKUPS_KEEP'].strip().replace('"', '')) - if self.config["LOCAL_RAW_BACKUPS_KEEP"] < 0: - self.logger.error("Value of LOCAL_RAW_BACKUPS_KEEP must be at least 0, not %s", self.config["LOCAL_RAW_BACKUPS_KEEP"]) - raise ValueError("Value of LOCAL_RAW_BACKUPS_KEEP must be at least 0") - except KeyError: - self.logger.warning("LOCAL_RAW_BACKUPS_KEEP not set. Defaulting to 1.") - self.config["LOCAL_RAW_BACKUPS_KEEP"] = 1 - - if self.config["COMPRESSION_ENABLED"]: - try: - self.config["LOCAL_COMPRESSED_BACKUPS_KEEP"] = int(os.environ['LOCAL_COMPRESSED_BACKUPS_KEEP'].strip().replace('"', '')) - if self.config["LOCAL_COMPRESSED_BACKUPS_KEEP"] < 0: - self.logger.error("Value of LOCAL_COMPRESSED_BACKUPS_KEEP must be at least 0, not %s", self.config["LOCAL_COMPRESSED_BACKUPS_KEEP"]) - raise ValueError("Value of LOCAL_COMPRESSED_BACKUPS_KEEP must be at least 0") - except KeyError: - self.logger.warning("LOCAL_COMPRESSED_BACKUPS_KEEP not set. Defaulting to 1.") - self.config["LOCAL_COMPRESSED_BACKUPS_KEEP"] = 1 - else: - self.config["LOCAL_COMPRESSED_BACKUPS_KEEP"] = 0 - - try: - self.config["S3_RAW_BACKUPS_KEEP"] = int(os.environ['S3_RAW_BACKUPS_KEEP'].strip().replace('"', '')) - if self.config["S3_RAW_BACKUPS_KEEP"] < 0: - self.logger.error("Value of S3_RAW_BACKUPS_KEEP must be at least 0, not %s", self.config["S3_RAW_BACKUPS_KEEP"]) - raise ValueError("Value of S3_RAW_BACKUPS_KEEP must be at least 0") - except KeyError: - self.logger.warning("S3_RAW_BACKUPS_KEEP not set. Defaulting to 0.") - self.config["S3_RAW_BACKUPS_KEEP"] = 0 - - if self.config["COMPRESSION_ENABLED"]: - try: - self.config["S3_COMPRESSED_BACKUPS_KEEP"] = int(os.environ['S3_COMPRESSED_BACKUPS_KEEP'].strip().replace('"', '')) - if self.config["S3_COMPRESSED_BACKUPS_KEEP"] < 0: - self.logger.error("Value of S3_COMPRESSED_BACKUPS_KEEP must be at least 0, not %s", self.config["S3_COMPRESSED_BACKUPS_KEEP"]) - raise ValueError("Value of S3_COMPRESSED_BACKUPS_KEEP must be at least 0") - except KeyError: - self.logger.warning("S3_COMPRESSED_BACKUPS_KEEP not set. Defaulting to 0.") - self.config["S3_COMPRESSED_BACKUPS_KEEP"] = 0 - else: - self.config["S3_COMPRESSED_BACKUPS_KEEP"] = 0 - - try: - self.config["S3_BUCKET"] = os.environ['S3_BUCKET'].strip().replace('"', '') - except KeyError as e: - self.logger.warning("S3_BUCKET not set.") - self.config["S3_BUCKET"] = None - - try: - self.config["S3_ENDPOINT_URL"] = os.environ['S3_ENDPOINT_URL'].strip().replace('"', '') - except KeyError: - self.logger.warning("S3_ENDPOINT_URL not set. Defaulting to None.") - self.config["S3_ENDPOINT_URL"] = None - - - try: - with open(os.environ['S3_ACCESS_KEY_ID_FILE'].strip().replace('"', ''), 'r') as f: - self.config["S3_ACCESS_KEY_ID"] = f.read().strip() - except (FileNotFoundError, KeyError): - try: - self.config["S3_ACCESS_KEY_ID"] = os.environ['S3_ACCESS_KEY_ID'].strip().replace('"', '') - except KeyError: - self.logger.warning("S3_ACCESS_KEY_ID not set. Defaulting to None.") - self.config["S3_ACCESS_KEY_ID"] = None - - try: - with open(os.environ['S3_SECRET_ACCESS_KEY_FILE'].strip().replace('"', ''), 'r') as f: - self.config["S3_SECRET_ACCESS_KEY"] = f.read().strip() - except (FileNotFoundError, KeyError) as e: - print(e) - try: - self.config["S3_SECRET_ACCESS_KEY"] = os.environ['S3_SECRET_ACCESS_KEY'].strip().replace('"', '') - except KeyError: - self.logger.warning("S3_SECRET_ACCESS_KEY not set. Defaulting to None.") - self.config["S3_SECRET_ACCESS_KEY"] = None - - try: - self.config["S3_REGION_NAME"] = os.environ['S3_REGION_NAME'].strip().replace('"', '') - except KeyError: - self.logger.warning("S3_REGION_NAME not set. Defaulting to None.") - self.config["S3_REGION_NAME"] = None - - try: - self.config["S3_ACL"] = os.environ['S3_ACL'].strip().replace('"', '') - except KeyError: - self.logger.warning("S3_ACL not set. Defaulting to None.") - self.config["S3_ACL"] = None - - try: - self.config["IGNORED_EXTENSIONS"] = os.environ['IGNORED_EXTENSIONS'].strip().replace('"', '') - if self.config["IGNORED_EXTENSIONS"] == "": - self.config["IGNORED_EXTENSIONS"] = [] - else: - self.config["IGNORED_EXTENSIONS"] = self.config["IGNORED_EXTENSIONS"].split(",") - except KeyError: - self.config["IGNORED_EXTENSIONS"] = [] - - try: - with open(os.environ['TELEGRAM_TOKEN_FILE'].strip().replace('"', ''), 'r') as f: - self.config["TELEGRAM_TOKEN"] = f.read().strip() - except (FileNotFoundError, KeyError): - try: - self.config["TELEGRAM_TOKEN"] = os.environ['TELEGRAM_TOKEN'].strip().replace('"', '') - except KeyError: - self.logger.warning("TELEGRAM_TOKEN not set. Telegram notifications disabled.") - self.config["TELEGRAM_TOKEN"] = None - - try: - with open(os.environ['TELEGRAM_CHAT_ID_FILE'].strip().replace('"', ''), 'r') as f: - self.config["TELEGRAM_CHAT_ID"] = f.read().strip() - except (FileNotFoundError, KeyError): - try: - self.config["TELEGRAM_CHAT_ID"] = os.environ['TELEGRAM_CHAT_ID'].strip().replace('"', '') - except KeyError: - self.logger.warning("TELEGRAM_CHAT_ID not set. Telegram notifications disabled.") - self.config["TELEGRAM_CHAT_ID"] = None - - def print_config(self) -> str: - config = self.config.copy() - - if config["S3_ACCESS_KEY_ID"] is not None: - config["S3_ACCESS_KEY_ID"] = "********" - - if config["S3_SECRET_ACCESS_KEY"] is not None: - config["S3_SECRET_ACCESS_KEY"] = "********" - - if config["TELEGRAM_TOKEN"] is not None: - config["TELEGRAM_TOKEN"] = "********" - - if config["TELEGRAM_CHAT_ID"] is not None: - config["TELEGRAM_CHAT_ID"] = "********" - - return pformat(config, sort_dicts=False) - - def create_backup(self): - try: - response = pybackupper.backups_manager.perform_backup() - if response is not None and response != "": - pybackupper.logger.info("Backup completed successfully.") - pybackupper.telegram_handler.send_backup_info(pybackupper.config["HOSTNAME"], response, pybackupper.backups_manager.get_backup_info()) - else: - pybackupper.logger.error("Backup failed.") - pybackupper.telegram_handler.send_message(pybackupper.config["HOSTNAME"] + ": Backup failed.") - except Exception as e: - pybackupper.telegram_handler.send_message(pybackupper.config["HOSTNAME"] + ": Error occured while creating a backup.") - pybackupper.logger.exception("Error occured while creating a backup.") +def main(): - def run(self): - days_string = ','.join([self.DAY_NAMES[day] for day in self.config["DAYS_TO_RUN"]]) - - sched = BlockingScheduler() - cron_trigger = CronTrigger( - day_of_week=days_string, - hour=self.config["HOUR"], - minute=self.config["MINUTE"]) - sched.add_job(self.create_backup, - trigger=cron_trigger, - id='backup', - name='Create backup', - replace_existing=True) - - self.display_webpage(cron_trigger) - self.logger.info("Next backup will be created on " + cron_trigger.get_next_fire_time(datetime.now(), datetime.now()).strftime("%d/%m/%Y %H:%M:%S")) - if self.telegram_handler is not None: - self.telegram_handler.send_message(self.config["HOSTNAME"] + ": PyBackUpper started. Next backup will be created on " + cron_trigger.get_next_fire_time(datetime.now(), datetime.now()).strftime("%d/%m/%Y %H:%M:%S")) - sched.start() + with open("../test-appconfig/config.json", "r") as file: + config = json_load(file) - def display_webpage(self, cron_trigger: CronTrigger): - self.logger.info("Starting web server.") - - app = Flask(__name__, template_folder="templates") - - def backup_info_formatter() -> dict: - formatted_backup = dict() - info = self.backups_manager.get_backup_info() - formatted_backup["last_backup"] = info["last_backup"] - formatted_backup["local_size"] = info["backup_size"]["local"] - formatted_backup["s3_size"] = info["backup_size"]["s3"] if info["backup_size"]["s3"] is not None else "N/A" - formatted_backup["free_space"] = info["backup_dir_free_space"] - - backups = [] - all_backups = set() - for content in self.backups_manager.backups.values(): - for item in content: - all_backups.add(item) - - for backup in sorted(all_backups, reverse=True, key=lambda x:x.split(".")[0]): - item = dict() - item["name"] = backup - item['size'] = self.backups_manager.convert_to_human_readable(self.backups_manager.get_backup_size(backup)) - item["local"] = True if backup in self.backups_manager.backups["local_raw"] or backup in self.backups_manager.backups["local_compressed"] else False - item["s3"] = True if backup in self.backups_manager.backups["s3_raw"] or backup in self.backups_manager.backups["s3_compressed"] else False - - backups.append(item) - - formatted_backup["backups"] = backups - - return formatted_backup - - @app.route("/") - def index(): - next_run = cron_trigger.get_next_fire_time(datetime.now(), datetime.now()) - backup_info = backup_info_formatter() - - try: - last_backup = datetime.strptime(backup_info["last_backup"], "%Y_%m_%d_%H_%M_%S").strftime("%Y_%m_%d %H:%M:%S") - except ValueError: - last_backup = backup_info["last_backup"] - return render_template("index.html", - hostname=self.config["HOSTNAME"], - next_backup=next_run.strftime("%Y_%m_%d %H:%M:%S"), - last_backup=last_backup, - local_size=backup_info["local_size"], - s3_size=backup_info["s3_size"], - free_space=backup_info["free_space"], - backups=backup_info["backups"]) - server_thread = threading.Thread(target=app.run, kwargs={"host": "0.0.0.0", "port": 5000, "debug": False, "use_reloader": False, "threaded": True}) - server_thread.start() - + pybackupper = PyBackUpper(config) + pybackupper.server.run() + if __name__ == "__main__": - pybackupper = PyBackUpper() - pybackupper.run() + import logging + import logging.config + from json import load as json_load + from backup_manager import BackupManager + from s3_handler import S3Handler + from telegram_handler import TelegramHandler + from server import Server + + main() diff --git a/src/main_old.py b/src/main_old.py new file mode 100644 index 0000000..5d516cf --- /dev/null +++ b/src/main_old.py @@ -0,0 +1,394 @@ +"""PyBackUpper main module. + +Raises: + ValueError: Exception raised when required environment variable has invalid value. + KeyError: Exception raised required environment variable is not set. +""" + +import logging +import logging.config +import os +from s3_handler import S3Handler +from telegram_handler import TelegramHandler +from backups_manager import BackupManager +from pprint import pformat +from flask import Flask, render_template +import threading +from apscheduler.schedulers.blocking import BlockingScheduler +from apscheduler.triggers.cron import CronTrigger +from datetime import datetime + +class PyBackUpper(): + """PyBackUpper class. + """ + + DAY_NAMES = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'] + + def __init__(self, logger:logging.Logger=None): + """PyBackUpper constructor. + + Args: + logger (logging.Logger, optional): Logger to use. Defaults to None. + """ + if logger is None: + logging.config.fileConfig("log.conf") + self.logger = logging.getLogger('pybackupper_logger') + else: + self.logger = logger + self.logger.info("PyBackUpper initialized.") + self.config = {} + self.read_env() + + if self.config["S3_BUCKET"] is not None and self.config["S3_ACCESS_KEY_ID"] is not None and self.config["S3_SECRET_ACCESS_KEY"] is not None: + self.s3_handler = S3Handler( + self.config["S3_BUCKET"], + self.config["S3_ACCESS_KEY_ID"], + self.config["S3_SECRET_ACCESS_KEY"], + self.config["S3_ACL"] if self.config["S3_ACL"] is not None else 'public-read', + self.config["S3_REGION_NAME"] if self.config["S3_REGION_NAME"] is not None else 'us-east-1', + self.config["S3_ENDPOINT_URL"] if self.config["S3_ENDPOINT_URL"] is not None else 'https://s3.amazonaws.com', + logger=self.logger) + if not self.s3_handler.test_connection(): + self.logger.error("S3 connection test failed. S3 upload will not be available.") + self.s3_handler = None + else: + self.logger.info("S3 connection test successful.") + + else: + self.logger.warning("S3_BUCKET, S3_ACCESS_KEY and S3_SECRET_KEY not set. S3 upload will not be available.") + self.s3_handler = None + + if self.config["TELEGRAM_TOKEN"] is not None and self.config["TELEGRAM_CHAT_ID"] is not None: + self.telegram_handler = TelegramHandler(self.config["TELEGRAM_TOKEN"], self.config["TELEGRAM_CHAT_ID"], logger=self.logger) + if not self.telegram_handler.test_connection(): + self.logger.error("Telegram connection test failed. Telegram notifications will not be available.") + self.telegram_handler = None + else: + self.logger.info("Telegram connection test successful.") + else: + self.logger.warning("TELEGRAM_TOKEN and TELEGRAM_CHAT_ID not set. Telegram notifications will not be available.") + + self.backups_manager = BackupManager( + logger=self.logger, + s3handler = self.s3_handler, + is_compression_enabled = self.config["COMPRESSION_ENABLED"], + archive_format = self.config["ARCHIVE_FORMAT"], + raw_backup_keep = self.config["LOCAL_RAW_BACKUPS_KEEP"], + compressed_backup_keep = self.config["LOCAL_COMPRESSED_BACKUPS_KEEP"], + s3_raw_keep = self.config["S3_RAW_BACKUPS_KEEP"], + s3_compressed_keep = self.config["S3_COMPRESSED_BACKUPS_KEEP"], + ignored_extensions = self.config["IGNORED_EXTENSIONS"] + ) + +# if self.telegram_handler: +# self.telegram_handler.send_message(f"\ +# PyBackUpper initialized.\n\n\ +# Config:\n\ +# `{self.print_config()}`") + + def read_env(self): + """Reads environment variables and stores them in self.config. + + Raises: + ValueError: Exception raised when required environment variable has invalid value. + KeyError: Exception raised required environment variable is not set. + """ + self.logger.info("Reading environment variables.") + + try: + self.config["HOSTNAME"] = os.environ['HOSTNAME'].strip().replace('"', '') + except KeyError as e: + raise KeyError("HOSTNAME not set.") from e + + try: + self.config["PUID"] = int(os.environ['PUID'].strip().replace('"', '')) + if self.config["PUID"] < 0 or self.config["PUID"] > 65535: + self.logger.error("Value of PUID must be between 0 and 65535, not %s", self.config["PUID"]) + raise ValueError("Value of PUID must be between 0 and 65535") + except KeyError as e: + raise KeyError("PUID not set.") from e + + try: + self.config["PGID"] = int(os.environ['PGID'].strip().replace('"', '')) + if self.config["PGID"] < 0 or self.config["PGID"] > 65535: + self.logger.error("Value of PGID must be between 0 and 65535, not %s", self.config["PGID"]) + raise ValueError("Value of PGID must be between 0 and 65535") + except KeyError as e: + raise KeyError("PGID not set.") from e + + try: + self.config["DAYS_TO_RUN"] = [int(x) for x in os.environ['DAYS_TO_RUN'].strip().replace('"', '').split(',')] + for day in self.config["DAYS_TO_RUN"]: + if day < 0 or day > 6: + self.logger.error("Value of DAYS_TO_RUN must be between 0 and 6, not %s", day) + raise ValueError("Value of DAYS_TO_RUN must be between 0 and 6") + + if len(self.config["DAYS_TO_RUN"]) != len(set(self.config["DAYS_TO_RUN"])): + self.logger.error("DAYS_TO_RUN contains duplicates.") + raise ValueError("DAYS_TO_RUN contains duplicates") + + except KeyError as e: + raise KeyError("DAYS_TO_RUN not set.") from e + + try: + self.config["HOUR"] = int(os.environ['HOUR'].strip().replace('"', '')) + if self.config["HOUR"] < 0 or self.config["HOUR"] > 23: + self.logger.error("Value of HOUR must be between 0 and 23, not %s", self.config["HOUR"]) + raise ValueError("Value of HOUR must be between 0 and 23") + except KeyError as e: + raise KeyError("HOUR not set.") from e + + try: + self.config["MINUTE"] = int(os.environ['MINUTE'].strip().replace('"', '')) + if self.config["MINUTE"] < 0 or self.config["MINUTE"] > 59: + self.logger.error("Value of MINUTE must be between 0 and 59, not %s", self.config["MINUTE"]) + raise ValueError("Value of MINUTE must be between 0 and 59") + except KeyError as e: + raise KeyError("MINUTE not set.") from e + + try: + if os.environ['COMPRESSION_ENABLED'].strip().replace('"', '').lower() == "true": + self.config["COMPRESSION_ENABLED"] = True + elif os.environ['COMPRESSION_ENABLED'].strip().replace('"', '').lower() == "false": + self.config["COMPRESSION_ENABLED"] = False + else: + self.logger.error("COMPRESSION_ENABLED must be either true or false.") + raise ValueError("COMPRESSION_ENABLED must be either true or false") + except KeyError: + self.logger.warning("COMPRESSION_ENABLED not set. Defaulting to true.") + self.config["COMPRESSION_ENABLED"] = True + + if self.config["COMPRESSION_ENABLED"]: + try: + self.config["ARCHIVE_FORMAT"] = os.environ['ARCHIVE_FORMAT'].strip().replace('"', '') + if self.config["ARCHIVE_FORMAT"] not in ["tar", "tar.gz", "tar.bz2", "tar.xz", "zip"]: + self.logger.error("ARCHIVE_FORMAT must be one of tar, tar.gz, tar.bz2, tar.xz, zip, not %s", self.config["ARCHIVE_FORMAT"]) + raise ValueError("ARCHIVE_FORMAT must be one of tar, tar.gz, tar.bz2, tar.xz, zip") + except KeyError: + self.logger.warning("ARCHIVE_FORMAT not set. Defaulting to tar.gz.") + self.config["ARCHIVE_FORMAT"] = "tar.gz" + else: + self.config["ARCHIVE_FORMAT"] = None + + try: + self.config["LOCAL_RAW_BACKUPS_KEEP"] = int(os.environ['LOCAL_RAW_BACKUPS_KEEP'].strip().replace('"', '')) + if self.config["LOCAL_RAW_BACKUPS_KEEP"] < 0: + self.logger.error("Value of LOCAL_RAW_BACKUPS_KEEP must be at least 0, not %s", self.config["LOCAL_RAW_BACKUPS_KEEP"]) + raise ValueError("Value of LOCAL_RAW_BACKUPS_KEEP must be at least 0") + except KeyError: + self.logger.warning("LOCAL_RAW_BACKUPS_KEEP not set. Defaulting to 1.") + self.config["LOCAL_RAW_BACKUPS_KEEP"] = 1 + + if self.config["COMPRESSION_ENABLED"]: + try: + self.config["LOCAL_COMPRESSED_BACKUPS_KEEP"] = int(os.environ['LOCAL_COMPRESSED_BACKUPS_KEEP'].strip().replace('"', '')) + if self.config["LOCAL_COMPRESSED_BACKUPS_KEEP"] < 0: + self.logger.error("Value of LOCAL_COMPRESSED_BACKUPS_KEEP must be at least 0, not %s", self.config["LOCAL_COMPRESSED_BACKUPS_KEEP"]) + raise ValueError("Value of LOCAL_COMPRESSED_BACKUPS_KEEP must be at least 0") + except KeyError: + self.logger.warning("LOCAL_COMPRESSED_BACKUPS_KEEP not set. Defaulting to 1.") + self.config["LOCAL_COMPRESSED_BACKUPS_KEEP"] = 1 + else: + self.config["LOCAL_COMPRESSED_BACKUPS_KEEP"] = 0 + + try: + self.config["S3_RAW_BACKUPS_KEEP"] = int(os.environ['S3_RAW_BACKUPS_KEEP'].strip().replace('"', '')) + if self.config["S3_RAW_BACKUPS_KEEP"] < 0: + self.logger.error("Value of S3_RAW_BACKUPS_KEEP must be at least 0, not %s", self.config["S3_RAW_BACKUPS_KEEP"]) + raise ValueError("Value of S3_RAW_BACKUPS_KEEP must be at least 0") + except KeyError: + self.logger.warning("S3_RAW_BACKUPS_KEEP not set. Defaulting to 0.") + self.config["S3_RAW_BACKUPS_KEEP"] = 0 + + if self.config["COMPRESSION_ENABLED"]: + try: + self.config["S3_COMPRESSED_BACKUPS_KEEP"] = int(os.environ['S3_COMPRESSED_BACKUPS_KEEP'].strip().replace('"', '')) + if self.config["S3_COMPRESSED_BACKUPS_KEEP"] < 0: + self.logger.error("Value of S3_COMPRESSED_BACKUPS_KEEP must be at least 0, not %s", self.config["S3_COMPRESSED_BACKUPS_KEEP"]) + raise ValueError("Value of S3_COMPRESSED_BACKUPS_KEEP must be at least 0") + except KeyError: + self.logger.warning("S3_COMPRESSED_BACKUPS_KEEP not set. Defaulting to 0.") + self.config["S3_COMPRESSED_BACKUPS_KEEP"] = 0 + else: + self.config["S3_COMPRESSED_BACKUPS_KEEP"] = 0 + + try: + self.config["S3_BUCKET"] = os.environ['S3_BUCKET'].strip().replace('"', '') + except KeyError as e: + self.logger.warning("S3_BUCKET not set.") + self.config["S3_BUCKET"] = None + + try: + self.config["S3_ENDPOINT_URL"] = os.environ['S3_ENDPOINT_URL'].strip().replace('"', '') + except KeyError: + self.logger.warning("S3_ENDPOINT_URL not set. Defaulting to None.") + self.config["S3_ENDPOINT_URL"] = None + + + try: + with open(os.environ['S3_ACCESS_KEY_ID_FILE'].strip().replace('"', ''), 'r') as f: + self.config["S3_ACCESS_KEY_ID"] = f.read().strip() + except (FileNotFoundError, KeyError): + try: + self.config["S3_ACCESS_KEY_ID"] = os.environ['S3_ACCESS_KEY_ID'].strip().replace('"', '') + except KeyError: + self.logger.warning("S3_ACCESS_KEY_ID not set. Defaulting to None.") + self.config["S3_ACCESS_KEY_ID"] = None + + try: + with open(os.environ['S3_SECRET_ACCESS_KEY_FILE'].strip().replace('"', ''), 'r') as f: + self.config["S3_SECRET_ACCESS_KEY"] = f.read().strip() + except (FileNotFoundError, KeyError) as e: + print(e) + try: + self.config["S3_SECRET_ACCESS_KEY"] = os.environ['S3_SECRET_ACCESS_KEY'].strip().replace('"', '') + except KeyError: + self.logger.warning("S3_SECRET_ACCESS_KEY not set. Defaulting to None.") + self.config["S3_SECRET_ACCESS_KEY"] = None + + try: + self.config["S3_REGION_NAME"] = os.environ['S3_REGION_NAME'].strip().replace('"', '') + except KeyError: + self.logger.warning("S3_REGION_NAME not set. Defaulting to None.") + self.config["S3_REGION_NAME"] = None + + try: + self.config["S3_ACL"] = os.environ['S3_ACL'].strip().replace('"', '') + except KeyError: + self.logger.warning("S3_ACL not set. Defaulting to None.") + self.config["S3_ACL"] = None + + try: + self.config["IGNORED_EXTENSIONS"] = os.environ['IGNORED_EXTENSIONS'].strip().replace('"', '') + if self.config["IGNORED_EXTENSIONS"] == "": + self.config["IGNORED_EXTENSIONS"] = [] + else: + self.config["IGNORED_EXTENSIONS"] = self.config["IGNORED_EXTENSIONS"].split(",") + except KeyError: + self.config["IGNORED_EXTENSIONS"] = [] + + try: + with open(os.environ['TELEGRAM_TOKEN_FILE'].strip().replace('"', ''), 'r') as f: + self.config["TELEGRAM_TOKEN"] = f.read().strip() + except (FileNotFoundError, KeyError): + try: + self.config["TELEGRAM_TOKEN"] = os.environ['TELEGRAM_TOKEN'].strip().replace('"', '') + except KeyError: + self.logger.warning("TELEGRAM_TOKEN not set. Telegram notifications disabled.") + self.config["TELEGRAM_TOKEN"] = None + + try: + with open(os.environ['TELEGRAM_CHAT_ID_FILE'].strip().replace('"', ''), 'r') as f: + self.config["TELEGRAM_CHAT_ID"] = f.read().strip() + except (FileNotFoundError, KeyError): + try: + self.config["TELEGRAM_CHAT_ID"] = os.environ['TELEGRAM_CHAT_ID'].strip().replace('"', '') + except KeyError: + self.logger.warning("TELEGRAM_CHAT_ID not set. Telegram notifications disabled.") + self.config["TELEGRAM_CHAT_ID"] = None + + def print_config(self) -> str: + config = self.config.copy() + + if config["S3_ACCESS_KEY_ID"] is not None: + config["S3_ACCESS_KEY_ID"] = "********" + + if config["S3_SECRET_ACCESS_KEY"] is not None: + config["S3_SECRET_ACCESS_KEY"] = "********" + + if config["TELEGRAM_TOKEN"] is not None: + config["TELEGRAM_TOKEN"] = "********" + + if config["TELEGRAM_CHAT_ID"] is not None: + config["TELEGRAM_CHAT_ID"] = "********" + + return pformat(config, sort_dicts=False) + + def create_backup(self): + try: + response = pybackupper.backups_manager.perform_backup() + if response is not None and response != "": + pybackupper.logger.info("Backup completed successfully.") + pybackupper.telegram_handler.send_backup_info(pybackupper.config["HOSTNAME"], response, pybackupper.backups_manager.get_backup_info()) + else: + pybackupper.logger.error("Backup failed.") + pybackupper.telegram_handler.send_message(pybackupper.config["HOSTNAME"] + ": Backup failed.") + except Exception as e: + pybackupper.telegram_handler.send_message(pybackupper.config["HOSTNAME"] + ": Error occured while creating a backup.") + pybackupper.logger.exception("Error occured while creating a backup.") + + def run(self): + days_string = ','.join([self.DAY_NAMES[day] for day in self.config["DAYS_TO_RUN"]]) + + sched = BlockingScheduler() + cron_trigger = CronTrigger( + day_of_week=days_string, + hour=self.config["HOUR"], + minute=self.config["MINUTE"]) + sched.add_job(self.create_backup, + trigger=cron_trigger, + id='backup', + name='Create backup', + replace_existing=True) + + self.display_webpage(cron_trigger) + self.logger.info("Next backup will be created on " + cron_trigger.get_next_fire_time(datetime.now(), datetime.now()).strftime("%d/%m/%Y %H:%M:%S")) + if self.telegram_handler is not None: + self.telegram_handler.send_message(self.config["HOSTNAME"] + ": PyBackUpper started. Next backup will be created on " + cron_trigger.get_next_fire_time(datetime.now(), datetime.now()).strftime("%d/%m/%Y %H:%M:%S")) + sched.start() + + def display_webpage(self, cron_trigger: CronTrigger): + self.logger.info("Starting web server.") + + app = Flask(__name__, template_folder="templates") + + def backup_info_formatter() -> dict: + formatted_backup = dict() + info = self.backups_manager.get_backup_info() + formatted_backup["last_backup"] = info["last_backup"] + formatted_backup["local_size"] = info["backup_size"]["local"] + formatted_backup["s3_size"] = info["backup_size"]["s3"] if info["backup_size"]["s3"] is not None else "N/A" + formatted_backup["free_space"] = info["backup_dir_free_space"] + + backups = [] + all_backups = set() + for content in self.backups_manager.backups.values(): + for item in content: + all_backups.add(item) + + for backup in sorted(all_backups, reverse=True, key=lambda x:x.split(".")[0]): + item = dict() + item["name"] = backup + item['size'] = self.backups_manager.convert_to_human_readable(self.backups_manager.get_backup_size(backup)) + item["local"] = True if backup in self.backups_manager.backups["local_raw"] or backup in self.backups_manager.backups["local_compressed"] else False + item["s3"] = True if backup in self.backups_manager.backups["s3_raw"] or backup in self.backups_manager.backups["s3_compressed"] else False + + backups.append(item) + + formatted_backup["backups"] = backups + + return formatted_backup + + @app.route("/") + def index(): + next_run = cron_trigger.get_next_fire_time(datetime.now(), datetime.now()) + backup_info = backup_info_formatter() + + try: + last_backup = datetime.strptime(backup_info["last_backup"], "%Y_%m_%d_%H_%M_%S").strftime("%Y_%m_%d %H:%M:%S") + except ValueError: + last_backup = backup_info["last_backup"] + return render_template("index.html", + hostname=self.config["HOSTNAME"], + next_backup=next_run.strftime("%Y_%m_%d %H:%M:%S"), + last_backup=last_backup, + local_size=backup_info["local_size"], + s3_size=backup_info["s3_size"], + free_space=backup_info["free_space"], + backups=backup_info["backups"]) + server_thread = threading.Thread(target=app.run, kwargs={"host": "0.0.0.0", "port": 5000, "debug": False, "use_reloader": False, "threaded": True}) + server_thread.start() + +if __name__ == "__main__": + pybackupper = PyBackUpper() + pybackupper.run() diff --git a/src/requirements.txt b/src/requirements.txt index f7c94e5..7254618 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -1,4 +1,8 @@ APScheduler==3.10.1 boto3==1.26.158 +botocore==1.29.165 Flask==2.3.2 -Requests==2.31.0 +flask_wtf==1.2.1 +psutil==5.9.6 +Requests==2.32.3 +tzlocal==5.2 diff --git a/src/s3_handler.py b/src/s3_handler.py index 6ba3ae9..9b7fdeb 100644 --- a/src/s3_handler.py +++ b/src/s3_handler.py @@ -1,99 +1,336 @@ -import boto3 +"""S3Handler class.""" import logging import logging.config -import os -import concurrent.futures +from os import walk, cpu_count, makedirs +from os.path import basename, exists, join, normpath, dirname +from concurrent.futures import ThreadPoolExecutor +from time import sleep +import boto3 +from botocore.exceptions import ClientError +from tools import size_to_human_readable class S3Handler: - def __init__(self, bucket_name, access_key, secret_key, acl='public-read', region='us-east-1', url='https://s3.amazonaws.com', logger: logging.Logger = None): - self.client = boto3.client( + """S3Handler class.""" + def __init__(self, + bucket_name:str, + access_key:str, + secret_key:str, + acl:str='public-read', + region:str='us-east-1', + url:str='https://s3.amazonaws.com', + logger:logging.Logger = None): + """Initialize the S3Handler class. + + Args: + bucket_name (str): Bucket name. + access_key (str): Access key. + secret_key (str): Secret key. + acl (str, optional): ACL. Defaults to 'public-read'. + region (str, optional): Region. Defaults to 'us-east-1'. + url (_type_, optional): URL. Defaults to 'https://s3.amazonaws.com'. + logger (logging.Logger, optional): Logger. Defaults to None. + + Raises: + ConnectionError: If the connection to the bucket fails. + """ + + self.logger = logger + self.bucket_name = bucket_name + self.acl = acl + + self.bucket = boto3.resource( 's3', aws_access_key_id=access_key, aws_secret_access_key=secret_key, region_name=region, endpoint_url=url - ) - self.bucket_name = bucket_name - self.acl = acl - + ).Bucket(self.bucket_name) + + if not self.test_connection(): + raise ConnectionError(f"Could not connect to bucket {self.bucket_name}") + + self.logger.debug(f"Connected to bucket {self.bucket_name}") + + @property + def bucket_name(self) -> str: + """Get the bucket name. + + Returns: + str: The bucket name. + """ + return self._bucket_name + + @bucket_name.setter + def bucket_name(self, bucket_name:str) -> None: + """Set the bucket name. + + Args: + bucket_name (str): The bucket name. + + Raises: + ValueError: If the bucket name is None or empty. + TypeError: If the bucket name is not a string. + """ + if bucket_name is None: + raise ValueError("bucket_name cannot be None") + + if not isinstance(bucket_name, str): + raise TypeError("bucket_name must be a string") + + if bucket_name == "": + raise ValueError("bucket_name cannot be empty") + + self._bucket_name = bucket_name + + @property + def acl(self) -> str: + """Get the acl. + + Returns: + str: The acl. + """ + return self._acl + + @acl.setter + def acl(self, acl:str) -> None: + """Set the acl. + + Args: + acl (str): The acl. + + Raises: + ValueError: If the acl is None or empty. + TypeError: If the acl is not a string. + """ + if acl is None: + acl = "private" + + if not isinstance(acl, str): + raise TypeError("acl must be a string") + + if acl == "": + raise ValueError("acl cannot be empty") + + self._acl = acl + + @property + def logger(self) -> logging.Logger: + """Get the logger. + + Returns: + logging.Logger: The logger. + """ + return self._logger + + @logger.setter + def logger(self, logger:logging.Logger) -> None: + """Set the logger. + + Args: + logger (logging.Logger): The logger. + + Raises: + ValueError: If the logger is None. + TypeError: If the logger is not a logging.Logger. + """ if logger is None: - logging.config.fileConfig("log.conf") - self.logger = logging.getLogger('pybackupper_logger') + logging.config.fileConfig("log_dev.conf") + self._logger = logging.getLogger('pybackupper_logger') else: - self.logger = logger - - def upload_file(self, file_name, object_name=None): - if object_name is None: - object_name = os.path.basename(file_name) - - self.logger.debug(f"Uploading file {file_name} to {object_name}") + if not isinstance(logger, logging.Logger): + raise TypeError("logger must be a logging.Logger") + self._logger = logger + + def test_connection(self) -> bool: + """Test the connection to the bucket. + + Returns: + bool: True if the connection is successful, False otherwise. + """ + self.logger.debug(f"Testing connection to bucket {self.bucket_name}") try: - _ = self.client.upload_file(file_name, self.bucket_name, object_name, ExtraArgs={'ACL': self.acl}) - self.logger.debug(f"File {file_name} uploaded successfully") - except Exception as e: - self.logger.error(e, exc_info=True) + _ = self.bucket.meta.client.head_bucket(Bucket=self.bucket_name) + except ClientError as e: + self.logger.exception(e, exc_info=True) return False return True - - def upload_directory(self, directory_path, object_name=None): + + def upload_file(self, file_path:str, object_name:str=None): + """Upload a file to the bucket. + + Args: + file_path (str): The file path. + object_name (str, optional): The object name. Defaults to None. + + Raises: + FileNotFoundError: If the file does not exist. + error: botocore.exceptions: If the upload fails. + """ + + if not exists(file_path): + self.logger.error(f"File {file_path} does not exist") + raise FileNotFoundError(f"File {file_path} does not exist") + if object_name is None: - object_name = os.path.basename(directory_path) - + object_name = basename(file_path) + object_name = object_name.replace('\\', '/') + + self.logger.debug(f"Uploading file {file_path} to {object_name}") + try: + _ = self.bucket.upload_file(file_path, object_name, ExtraArgs={'ACL': self.acl}) + self.logger.debug(f"File {file_path} uploaded successfully") + except ClientError as error: + if error.response['Error']['Code'] == 'LimitExceededException': + self.logger.warn( + 'API call limit exceeded; backing off and retrying in 5 seconds...') + sleep(5) + self.upload_file(file_path, object_name) + else: + self.logger.exception(error, exc_info=True) + raise error + + def upload_directory(self, directory_path:str, object_name:str=None): + """Upload a directory to the bucket. + + Args: + directory_path (str): The directory path. + object_name (str, optional): The object name. Defaults to None. + + Raises: + FileNotFoundError: If the directory does not exist. + error: botocore.exceptions: If the upload fails. + """ + if not exists(directory_path): + self.logger.error(f"Directory {directory_path} does not exist") + raise FileNotFoundError(f"Directory {directory_path} does not exist") + + if object_name is None: + object_name = basename(directory_path) + self.logger.debug(f"Uploading directory {directory_path} to {object_name}") + files_to_upload = [] + for path, _, files in walk(directory_path): + dest_path = path.replace(directory_path, "") + for file in files: + files_to_upload.append((join(path, file), + normpath(object_name + '/' + + dest_path + '/' + file))) + + n_workers = cpu_count() * 2 + self.logger.debug(f"Uploading {len(files_to_upload)} files with {n_workers} workers") + try: - with concurrent.futures.ThreadPoolExecutor(max_workers=2*os.cpu_count()) as executor: - for path, _, files in os.walk(directory_path): - dest_path = path.replace(directory_path, "") - for file in files: - s3file = os.path.normpath(object_name + '/' + dest_path + '/' + file) - local_file = os.path.join(path, file) - self.logger.debug(f"upload : {local_file} to target: {s3file}") - executor.submit(self.upload_file, local_file, s3file) - except Exception as e: - self.logger.error(e, exc_info=True) - raise e - - def delete_file(self, file_name): + with ThreadPoolExecutor(max_workers=n_workers) as executor: + for file_path, file_name in files_to_upload: + executor.submit(self.upload_file, file_path, file_name) + except ClientError as error: + self.logger.exception(error, exc_info=True) + raise error + + def delete_file(self, file_name:str) -> None: + """Delete a file from the bucket. + + Args: + file_name (str): The file name. + + Raises: + ValueError: If the file name is None or empty. + TypeError: If the file name is not a string. + e: botocore.exceptions: If the delete fails. + """ + if file_name is None: + self.logger.error("file_name cannot be None") + raise ValueError("file_name cannot be None") + + if not isinstance(file_name, str): + self.logger.error("file_name must be a string") + raise TypeError("file_name must be a string") + + if file_name == "": + self.logger.error("file_name cannot be empty") + raise ValueError("file_name cannot be empty") + try: - _ = self.client.delete_object(Bucket=self.bucket_name, Key=file_name) + _ = self.bucket.delete_objects(Delete={'Objects': [{'Key': file_name}]}) self.logger.debug(f"File {file_name} deleted successfully") - except Exception as e: - self.logger.error(e, exc_info=True) - return False - return True - - def delete_directory(self, directory_path): + except ClientError as e: + self.logger.exception(e, exc_info=True) + raise e + + def delete_directory(self, directory_path:str) -> None: + """Delete a directory from the bucket. + + Args: + directory_path (str): The directory path. + + Raises: + ValueError: If the directory path is None or empty. + TypeError: If the directory path is not a string. + e: botocore.exceptions: If the delete fails. + """ + if directory_path is None: + self.logger.error("directory_path cannot be None") + raise ValueError("directory_path cannot be None") + + if not isinstance(directory_path, str): + self.logger.error("directory_path must be a string") + raise TypeError("directory_path must be a string") + + if directory_path == "": + self.logger.error("directory_path cannot be empty") + raise ValueError("directory_path cannot be empty") + + if directory_path[-1] != '/': + self.logger.debug(f"Adding '/' to directory_path {directory_path}") + directory_path += '/' + try: - response = self.client.list_objects_v2(Bucket=self.bucket_name, Prefix=directory_path) - with concurrent.futures.ThreadPoolExecutor(max_workers=2*os.cpu_count()) as executor: - for content in response['Contents']: - if content['Key'].find('/') != -1: - self.logger.debug(f"Deleting file {content['Key']}") - executor.submit(self.delete_file, content['Key']) - except Exception as e: - self.logger.error(e, exc_info=True) - return False - return True - - def list_buckets(self): + _ = self.bucket.objects.filter(Prefix=directory_path).delete() + except ClientError as e: + self.logger.exception(e, exc_info=True) + raise e + + def list_buckets(self) -> list: + """List all the buckets. + + Returns: + list: The list of buckets. + + Raises: + e: botocore.exceptions: If the list fails. + """ + self.logger.debug("Listing buckets") try: - response = self.client.list_buckets() - print(response) - for bucket in response['Buckets']: - self.logger.debug(f"Bucket: {bucket['Name']}") - except Exception as e: - self.logger.error(e, exc_info=True) - return False - return True - - def list_files(self, prefix=None) -> list: + return [bucket["Name"] for bucket in self.bucket.meta.client.list_buckets()['Buckets']] + except ClientError as e: + self.logger.exception(e, exc_info=True) + raise e + + def list_files(self, prefix:str=None) -> list: + """List all the files in the bucket. + + Args: + prefix (str, optional): The prefix. Defaults to None. + + Returns: + list: The list of files. + + Raises: + e: botocore.exceptions: If the list fails. + """ + self.logger.debug(f"Listing files in bucket {self.bucket_name}") files = [] + if prefix is not None and prefix[-1] != '/': + self.logger.debug(f"Adding '/' to prefix {prefix}") + prefix += '/' try: if prefix is None: - response = self.client.list_objects_v2(Bucket=self.bucket_name) + response = self.bucket.meta.client.list_objects_v2( + Bucket=self.bucket_name) else: - response = self.client.list_objects_v2(Bucket=self.bucket_name, Prefix=prefix) + response = self.bucket.meta.client.list_objects_v2( + Bucket=self.bucket_name, Prefix=prefix) for content in response['Contents']: if prefix is not None: content['Key'] = content['Key'].replace(prefix, '') @@ -101,111 +338,339 @@ def list_files(self, prefix=None) -> list: files.append(content['Key']) except KeyError: return [] - except Exception as e: - self.logger.error(e, exc_info=True) - return [] + except ClientError as e: + self.logger.exception(e, exc_info=True) + raise e return files - - def list_directories(self, prefix=None) -> list: + + def list_directories(self, prefix:str=None) -> list: + """List all the directories in the bucket. + + Args: + prefix (str, optional): The prefix. Defaults to None. + + Returns: + list: The list of directories. + + Raises: + e: botocore.exceptions: If the list fails. + """ + self.logger.debug(f"Listing directories in bucket {self.bucket_name}") directories = [] + if prefix is not None and prefix[-1] != '/': + self.logger.debug(f"Adding '/' to prefix {prefix}") + prefix += '/' try: if prefix is None: - response = self.client.list_objects_v2(Bucket=self.bucket_name, Delimiter='/') + response = self.bucket.meta.client.list_objects_v2( + Bucket=self.bucket_name, Delimiter='/') else: - response = self.client.list_objects_v2(Bucket=self.bucket_name, Prefix=prefix, Delimiter='/') + response = self.bucket.meta.client.list_objects_v2( + Bucket=self.bucket_name, Prefix=prefix, Delimiter='/') for content in response.get('CommonPrefixes', []): if prefix is not None: content['Prefix'] = content['Prefix'].replace(prefix, '') directories.append(content['Prefix'].replace('/', '')) except KeyError: return [] - except Exception as e: - self.logger.error(e, exc_info=True) - return [] + except ClientError as e: + self.logger.exception(e, exc_info=True) + raise e return directories - - def list_tree(self, prefix=None) -> list: - tree = [] - try: - if prefix is None: - response = self.client.list_objects_v2(Bucket=self.bucket_name) - else: - response = self.client.list_objects_v2(Bucket=self.bucket_name, Prefix=prefix) - for content in response['Contents']: - if prefix is not None: - content['Key'] = content['Key'].replace(prefix, '') - tree.append(content['Key']) - except Exception as e: - self.logger.error(e, exc_info=True) - return [] - return tree - - def download_file(self, file_path, object_name=None) -> bool: + + def download_file(self, object_name:str, save_path:str) -> None: + """Download a file from the bucket. + + Args: + object_name (str): The object name. + save_path (str): The save path. + + Raises: + ValueError: If the object name or save path is None or empty. + TypeError: If the object name or save path is not a string. + error: botocore.exceptions: If the download fails. + """ if object_name is None: - object_name = os.path.basename(file_path) - + self.logger.error("object_name cannot be None") + raise ValueError("object_name cannot be None") + + if not isinstance(object_name, str): + self.logger.error("object_name must be a string") + raise TypeError("object_name must be a string") + + if object_name == "": + self.logger.error("object_name cannot be empty") + raise ValueError("object_name cannot be empty") + + if save_path is None: + self.logger.error("save_path cannot be None") + raise ValueError("save_path cannot be None") + + if not isinstance(save_path, str): + self.logger.error("save_path must be a string") + raise TypeError("save_path must be a string") + + if save_path == "": + self.logger.error("save_path cannot be empty") + raise ValueError("save_path cannot be empty") + + save_path = normpath(save_path) + + if not exists(dirname(save_path)): + self.logger.debug(f"Creating directory {dirname(save_path)}") + makedirs(dirname(save_path)) + + self.logger.debug(f"Downloading file {object_name} to {save_path}") try: - _ = self.client.download_file(self.bucket_name, object_name, file_path) - self.logger.debug(f"File {file_path} downloaded successfully") - except Exception as e: - self.logger.error(e, exc_info=True) - return False - return True - - def download_directory(self, directory_path, object_name=None) -> bool: + with open(save_path, 'wb') as f: + self.bucket.download_fileobj(object_name, f) + except ClientError as error: + self.logger.exception(error, exc_info=True) + raise error + + self.logger.debug(f"File {object_name} downloaded successfully") + + def download_directory(self, object_name:str, save_path:str) -> None: + """Download a directory from the bucket. + + Args: + object_name (str): The object name. + save_path (str): The save path. + + Raises: + ValueError: If the object name or save path is None or empty. + TypeError: If the object name or save path is not a string. + error: botocore.exceptions: If the download fails. + """ if object_name is None: - object_name = os.path.basename(directory_path) - - try: - response = self.client.list_objects_v2(Bucket=self.bucket_name, Prefix=object_name) - for content in response['Contents']: - path = os.path.join(directory_path, os.path.dirname(content['Key'])) - if not os.path.exists(path): - os.makedirs(path) - self.download_file(os.path.join(path, os.path.basename(content['Key'])), content['Key']) - except Exception as e: - self.logger.error(e, exc_info=True) - return False - return True + self.logger.error("object_name cannot be None") + raise ValueError("object_name cannot be None") + + if not isinstance(object_name, str): + self.logger.error("object_name must be a string") + raise TypeError("object_name must be a string") + + if object_name == "": + self.logger.error("object_name cannot be empty") + raise ValueError("object_name cannot be empty") + + if save_path is None: + self.logger.error("save_path cannot be None") + raise ValueError("save_path cannot be None") + + if not isinstance(save_path, str): + self.logger.error("save_path must be a string") + raise TypeError("save_path must be a string") + + if save_path == "": + self.logger.error("save_path cannot be empty") + raise ValueError("save_path cannot be empty") + + save_path = normpath(save_path) - def get_bucket_size(self): - # TODO: Fix this, now it is only getting the size of the files in the root of the bucket and not the size of the bucket + if not exists(save_path): + self.logger.debug(f"Creating directory {save_path}") + makedirs(save_path) + + self.logger.debug(f"Downloading directory {object_name} to {save_path}") + + n_workers = cpu_count() * 2 + self.logger.debug(f"Downloading with {n_workers} workers") + + for file in self.list_files(object_name): + try: + self.download_file(object_name + '/' + file, join(save_path, file)) + except ClientError as error: + self.logger.exception(error, exc_info=True) + raise error + + with ThreadPoolExecutor(max_workers=n_workers) as executor: + for directory in self.list_directories(object_name): + try: + executor.submit( + self.download_directory, + object_name + '/' + directory, + join(save_path, directory)) + except ClientError as error: + self.logger.exception(error, exc_info=True) + raise error + + self.logger.debug(f"Directory {object_name} downloaded successfully") + + def get_bucket_size(self) -> int: + """Get the bucket size. + + Returns: + int: The bucket size. + + Raises: + e: botocore.exceptions: If the size cannot be calculated. + """ + self.logger.debug(f"Calculating size of bucket {self.bucket_name}") try: - # Get size of whole bucket - response = self.client.list_objects_v2(Bucket=self.bucket_name) - size = sum([content['Size'] for content in response['Contents']]) - except KeyError: - return 0 - except Exception as e: - self.logger.error(e, exc_info=True) + total_size = 0 + for key in self.bucket.objects.all(): + total_size += key.size + except ClientError as e: + self.logger.exception(e, exc_info=True) raise e - return size - - def check_file_exists(self, file_name): + self.logger.debug( + f"Size of bucket {self.bucket_name} is {size_to_human_readable(total_size)}") + return total_size + + def get_object_size(self, object_name:str) -> int: + """Get the object size. + + Args: + object_name (str): The object name. + + Returns: + int: The object size. + + Raises: + ValueError: If the object name is None or empty. + TypeError: If the object name is not a string. + e: botocore.exceptions: If the size cannot be calculated. + """ + if object_name is None: + self.logger.error("object_name cannot be None") + raise ValueError("object_name cannot be None") + + if not isinstance(object_name, str): + self.logger.error("object_name must be a string") + raise TypeError("object_name must be a string") + + if object_name == "": + self.logger.error("object_name cannot be empty") + raise ValueError("object_name cannot be empty") + + self.logger.debug(f"Calculating size of object {object_name}") try: - _ = self.client.head_object(Bucket=self.bucket_name, Key=file_name) - except Exception as e: - self.logger.error(e, exc_info=True) - return False - return True - - def check_directory_exists(self, directory_path): + total_size = 0 + for key in self.bucket.objects.all(): + if key.key.find(object_name) != -1: + total_size += key.size + except ClientError as e: + self.logger.exception(e, exc_info=True) + raise e + self.logger.debug(f"Size of object {object_name} is {size_to_human_readable(total_size)}") + return total_size + + def check_object_exists(self, object_name:str) -> bool: + """Check if an object exists. + + Args: + object_name (str): The object name. + + Returns: + bool: True if the object exists, False otherwise. + + Raises: + ValueError: If the object name is None or empty. + TypeError: If the object name is not a string. + e: botocore.exceptions: If the check fails. + """ + if object_name is None: + self.logger.error("object_name cannot be None") + raise ValueError("object_name cannot be None") + + if not isinstance(object_name, str): + self.logger.error("object_name must be a string") + raise TypeError("object_name must be a string") + + if object_name == "": + self.logger.error("object_name cannot be empty") + raise ValueError("object_name cannot be empty") + try: - response = self.client.list_objects_v2(Bucket=self.bucket_name, Prefix=directory_path) - for content in response['Contents']: - if content['Key'].find('/') != -1: + for key in self.bucket.objects.all(): + if key.key.find(object_name) != -1: return True - except KeyError: - return False - except Exception as e: - self.logger.error(e, exc_info=True) - return False + except ClientError as e: + self.logger.exception(e, exc_info=True) + raise e return False - - def test_connection(self) -> bool: + + def get_object_path(self, object_name:str) -> str: + """Get the object path. + + Args: + object_name (str): The object name. + + Returns: + str: The object path. + + Raises: + ValueError: If the object name is None or empty. + TypeError: If the object name is not a string. + e: botocore.exceptions: If the check fails. + """ + if object_name is None: + self.logger.error("object_name cannot be None") + raise ValueError("object_name cannot be None") + + if not isinstance(object_name, str): + self.logger.error("object_name must be a string") + raise TypeError("object_name must be a string") + + if object_name == "": + self.logger.error("object_name cannot be empty") + raise ValueError("object_name cannot be empty") + try: - _ = self.client.head_bucket(Bucket=self.bucket_name) - except Exception as e: - self.logger.error(e, exc_info=True) - return False - return True \ No newline at end of file + for key in self.bucket.objects.all(): + if key.key.find(object_name) != -1: + return key.key + except ClientError as e: + self.logger.exception(e, exc_info=True) + raise e + return None + + def clear_bucket(self) -> None: + """Clear the bucket. + + Raises: + e: botocore.exceptions: If the clear fails. + """ + self.logger.debug(f"Clearing bucket {self.bucket_name}") + try: + _ = self.bucket.objects.all().delete() + except ClientError as e: + self.logger.exception(e, exc_info=True) + raise e + + def create_download_link(self, object_name:str, expiration:int=3600) -> str: + """Create a download link for an object. + + Args: + object_name (str): The object name. + expiration (int, optional): The expiration time in seconds. Defaults to 3600. + + Returns: + str: The download link. + + Raises: + ValueError: If the object name is None or empty. + TypeError: If the object name is not a string. + e: botocore.exceptions: If the link cannot be created. + """ + if object_name is None: + self.logger.error("object_name cannot be None") + raise ValueError("object_name cannot be None") + + if not isinstance(object_name, str): + self.logger.error("object_name must be a string") + raise TypeError("object_name must be a string") + + if object_name == "": + self.logger.error("object_name cannot be empty") + raise ValueError("object_name cannot be empty") + + try: + return self.bucket.meta.client.generate_presigned_url( + 'get_object', + Params={'Bucket': self.bucket_name, 'Key': object_name}, + ExpiresIn=expiration) + except ClientError as e: + self.logger.exception(e, exc_info=True) + raise e \ No newline at end of file diff --git a/src/scheduler.py b/src/scheduler.py new file mode 100644 index 0000000..a211173 --- /dev/null +++ b/src/scheduler.py @@ -0,0 +1,222 @@ +""""This module contains the Scheduler class.""" + +import logging +import logging.config +from re import fullmatch +from datetime import datetime +from apscheduler.schedulers.blocking import BlockingScheduler +from apscheduler.triggers.cron import CronTrigger +from backup_manager import BackupManager +from tools import timestamp_to_human_readable + +class Scheduler(BlockingScheduler): + """The Scheduler class.""" + def __init__( + self, + backupper:BackupManager, + trigger:CronTrigger, + logger:logging.Logger=None) -> None: + """Initialize the Scheduler class. + + Args: + backupper (BackupManager): BackupManager instance. + trigger (CronTrigger): CronTrigger instance. + logger (logging.Logger, optional): Logger. Defaults to None. + """ + super().__init__() + self.logger = logger + self.backupper = backupper + self.trigger = trigger + self.sched_job = self.add_job( + self.backupper.run_backup, + trigger=self.trigger, + id="backup_job_" + "_".join(str(x) for x in self.trigger.fields), + name=f"Backup job with cron: {self.trigger.fields} and timezone: {self.trigger.timezone}") + self.logger.info( + f"Scheduler configured with cron: {self.trigger.fields} and timezone: {self.trigger.timezone}") + + def __del__(self) -> None: + """Destructor. + """ + self.shutdown() + self.logger.info("Scheduler stopped") + + def __str__(self) -> str: + """String representation. + + Returns: + str: String representation. + """ + return f"""Scheduler with cron: {self.trigger.fields} \ + and timezone: {self.trigger.timezone}. \ + Next run: {timestamp_to_human_readable( + self.trigger.get_next_fire_time( + datetime.now(), + datetime.now()).timestamp())}""" + + def __dict__(self) -> dict: + """Dictionary representation. + + Returns: + dict: Dictionary representation. + """ + return { + "id": self.sched_job.id, + "cron": ", ".join(f"{x.name}: {str(x)}" for x in self.trigger.fields), + "timezone": self.trigger.timezone, + "next_run": timestamp_to_human_readable(self.trigger.get_next_fire_time(datetime.now(), datetime.now()).timestamp()) + } + + @property + def logger(self) -> logging.Logger: + """The logger property. + + Returns: + logging.Logger: The logger instance + """ + return self._logger + + @logger.setter + def logger(self, logger:logging.Logger) -> None: + """Set the logger. + + Args: + logger (logging.Logger): Logger. + """ + if logger is None: + logging.config.fileConfig("log_dev.conf") + self._logger = logging.getLogger('pybackupper_logger') + else: + self._logger = logger + + @property + def backupper(self) -> BackupManager: + """The backupper property. + + Returns: + BackupManager: The backupper instance + """ + return self._backupper + + @backupper.setter + def backupper(self, backupper:BackupManager) -> None: + """Set the backupper. + + Raises: + ValueError: If the backupper is invalid. + + Args: + backupper (BackupManager): Backupper. + """ + if backupper is None or not isinstance(backupper, BackupManager): + self.logger.error("Invalid backupper") + raise ValueError("Invalid backupper") + self._backupper = backupper + + @staticmethod + def to_CronTrigger( + minute:str="0", + hour:str="0", + day_of_week:str="*", + day_of_month:str="*", + month:str="*", + timezone:str="UTC") -> CronTrigger: + """Convert the given parameters to a cron expression. + + Args: + minute (str, optional): Minute. Defaults to "0". + hour (str, optional): Hour. Defaults to "0". + day_of_week (str, optional): Day of week. Defaults to "*". + For example: "0,1,2,3,4,5,6" or "MON,TUE,WED,THU,FRI,SAT,SUN". + day_of_month (str, optional): Day of month. Defaults to "*". + month (str, optional): Month. Defaults to "*". + timezone (str, optional): Timezone. Defaults to "UTC". + + Raises: + ValueError: If one of the parameters is invalid. + + Returns: + CronTrigger: The CronTrigger instance. + """ + + if minute is None or not isinstance(minute, str): + raise ValueError("Invalid minute") + if minute != "*": + try: + i_minute = int(minute) + except ValueError as e: + raise ValueError("Invalid minute") from e + + if i_minute < 0 or i_minute > 59: + raise ValueError("Invalid minute") + + if hour is None or not isinstance(hour, str): + raise ValueError("Invalid hour") + if hour != "*": + try: + i_hour = int(hour) + except ValueError as e: + raise ValueError("Invalid hour") from e + + if i_hour < 0 or i_hour > 23: + raise ValueError("Invalid hour") + + if day_of_week is None or not isinstance(day_of_week, str): + raise ValueError("Invalid day of week") + + if day_of_month is None or not isinstance(day_of_month, str): + raise ValueError("Invalid day of month") + if day_of_month != "*": + try: + i_day_of_month = int(day_of_month) + except ValueError as e: + raise ValueError("Invalid day of month") from e + + if i_day_of_month < 0 or i_day_of_month > 31: + raise ValueError("Invalid day of month") + + if month is None or not isinstance(month, str): + raise ValueError("Invalid month") + if month != "*": + try: + i_month = int(month) + except ValueError as e: + raise ValueError("Invalid month") from e + + if i_month < 0 or i_month > 12: + raise ValueError("Invalid month") + + if timezone is None or not isinstance(timezone, str): + raise ValueError("Invalid timezone") + if not fullmatch(r'^[\w\/\-]+$', timezone): + raise ValueError("Invalid timezone") + + if day_of_week == "*": + return CronTrigger( + minute=minute, + hour=hour, + day=day_of_month, + month=month, + timezone=timezone) + + day_of_week = day_of_week.lower().split(",") + + for i, day in enumerate(day_of_week): + if day not in ["mon", "tue", "wed", "thu", "fri", "sat", "sun"]: + try: + i_day = int(day) + except ValueError as e: + raise ValueError("Invalid days of week") from e + + if i_day < 0 or i_day > 6: + raise ValueError("Invalid days of week") + + day_of_week[i] = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"][i_day] + + return CronTrigger( + minute=minute, + hour=hour, + day=day_of_month, + day_of_week=",".join(day_of_week), + month=month, + timezone=timezone) diff --git a/src/server.py b/src/server.py new file mode 100644 index 0000000..06f1ff4 --- /dev/null +++ b/src/server.py @@ -0,0 +1,446 @@ +"""Module to represent the server""" + +import logging +import logging.config +from secrets import token_hex +from threading import Thread +from flask_wtf.csrf import CSRFProtect +from flask.logging import default_handler +from flask import Flask, render_template, redirect, url_for, session, request, send_file +from tzlocal import get_localzone +from backup_manager import BackupManager +from scheduler import Scheduler +from tools import size_to_human_readable + +class Message(dict): + """Class to represent a message to be displayed on the website + """ + def __init__(self, message: str, level: str) -> None: + """Constructor for Message class + + Args: + message (str): Message to be displayed + level (str): Level of the message, can be one of the following: + primary, secondary, success, danger, warning, info, light, dark + """ + self.message = message if isinstance(message, str) else str(message) + self.level = level if level in ["primary", + "secondary", + "success", + "danger", + "warning", + "info", + "light", + "dark"] else "info" + + def __str__(self) -> str: + """String representation of the Message object + + Returns: + str: String representation of the Message object + """ + return f"{self.level}: {self.message}" + + def __dict__(self) -> dict: + """Dictionary representation of the Message object + + Returns: + dict: Dictionary representation of the Message object + """ + return { + "message": self.message, + "level": self.level + } + + def to_dict(self) -> dict: + """Dictionary representation of the Message object + + Returns: + dict: Dictionary representation of the Message object + """ + return dict(self) + +class Server(Flask): + """Class to represent the server + + Args: + Flask (Flask): Flask object + """ + def __init__(self, backupper: BackupManager, logger: logging.Logger=None) -> None: + """Constructor for Server class + + Args: + backupper (BackupManager): BackupManager object + logger (logging.Logger, optional): Logger object. Defaults to None. + """ + super().__init__(__name__) + self.backupper = backupper + self.logger = logger + self.schedulers_list = [] + + super().logger.removeHandler(default_handler) + super().logger.addHandler(x for x in self.logger.handlers) + self.add_url_rule('/', view_func=self.index) + self.add_url_rule('/backup_info', view_func=self.backup_info, methods=['GET']) + self.add_url_rule('/info/', view_func=self.single_backup_info, methods=['GET']) + self.add_url_rule('/backup_now', view_func=self.backup_now, methods=['POST']) + self.add_url_rule('/download/', view_func=self.download_backup, methods=['GET']) + + self.add_url_rule('/restore_backup', view_func=self.restore_backup, methods=['POST']) + self.add_url_rule('/unzip_backup', view_func=self.unzip_backup, methods=['POST']) + self.add_url_rule('/download_backup', view_func=self.download_from_s3, methods=['POST']) + self.add_url_rule('/delete_backup', view_func=self.delete_backup, methods=['POST']) + + self.add_url_rule('/schedulers', view_func=self.schedulers) + self.add_url_rule('/add_scheduler', view_func=self.add_scheduler, methods=['POST']) + self.add_url_rule('/delete_scheduler', view_func=self.delete_scheduler, methods=['POST']) + + self.add_url_rule('/logs', view_func=self.logs) + + self.config["SERVER_NAME"] = "127.0.0.1:5000" + self.config["SECRET_KEY"] = token_hex(16) + self.csrf = CSRFProtect(self) + + @property + def logger(self) -> logging.Logger: + """Logger object + + Returns: + logging.Logger: Logger object + """ + return self._logger + + @logger.setter + def logger(self, logger: logging.Logger) -> None: + """Logger object setter + + Args: + logger (logging.Logger): Logger object + """ + if logger is None: + logging.config.fileConfig("log_dev.conf") + self._logger = logging.getLogger('pybackupper_logger') + else: + self._logger = logger + + def index(self) -> str: + """Index page + + Returns: + str: HTML page + """ + backups_dict = self.backupper.__dict__() + local_size = backups_dict["local_size"] if "local_size" in backups_dict else 0 + s3_size = backups_dict["s3_size"] if "s3_size" in backups_dict else 0 + + backups = [] + for backup in backups_dict["backups"]["local"]: + backups.append({ + "name": backup["name"], + "size": backup["size"], + "raw": backup["completed"], + "zip": backup["compressed"], + "s3": backup in backups_dict["backups"]["s3"], + }) + + for backup in backups_dict["backups"]["s3"]: + if backup not in backups_dict["backups"]["local"]: + backups.append({ + "name": backup["name"], + "size": size_to_human_readable( + self.backupper.s3_handler.get_object_size( + backup["name"])), + "raw": False, + "zip": False, + "s3": True, + }) + + backups.sort(key=lambda x: x["name"], reverse=True) + + return render_template('home.html', + pending_backup=self.backupper.pending_backup, + message=session.pop("message", None), + backups=backups, + local_size=local_size, + s3_size=s3_size) + + def backup_info(self) -> dict: + """Backup info + + Returns: + dict: Backup info + """ + return self.backupper.__dict__() + + def single_backup_info(self, name: str) -> dict: + """Single backup info action button + + Args: + name (str): Backup name + + Returns: + dict: Single backup info + """ + try: + for backup in self.backupper.backups["local"]: + if backup["name"] == name: + return backup.__dict__() + except FileNotFoundError: + session["message"] = Message(f"Backup with name {name} not found", "danger").to_dict() + return redirect(url_for('index')) + + try: + for backup in self.backupper.backups["s3"]: + if backup["name"] == name: + info = backup.__dict__() + try: + info["size"] = size_to_human_readable( + self.backupper.s3_handler.get_object_size( + name + ".zip")) + except Exception: + info["size"] = "Unknown" + return info + except Exception: + session["message"] = Message(f"Backup with name {name} not found", "danger").to_dict() + return redirect(url_for('index')) + + session["message"] = Message(f"Backup with name {name} not found", "danger").to_dict() + return redirect(url_for('index')) + + def download_backup(self, name: str)-> str: + """Download backup action button + + Args: + name (str): Backup name + + Returns: + str: HTML page + """ + try: + for backup in self.backupper.backups["local"]: + if backup["name"] == name: + return send_file( + f"""{self.backupper.dest_path}/{backup["name"]}.zip""", + as_attachment=True) + except FileNotFoundError: + pass + + try: + for backup in self.backupper.backups["s3"]: + if backup["name"] == name: + return redirect(self.backupper.s3_handler.create_download_link(name+".zip")) + except Exception: + pass + + session["message"] = Message( + f"Backup with name {name} not found", "danger").to_dict() + return redirect(url_for('index')) + + def restore_backup(self) -> str: + """Restore backup action button + + Returns: + str: HTML page + """ + name = request.form.get("backup_name", None) + file_path = request.form.get("file_path", None) + + if name is None or file_path is None: + session["message"] = Message( + "No backup name or file path provided", "danger").to_dict() + self.logger.error("No backup name or file path provided") + return redirect(url_for('index')) + + if not self.backupper.pending_backup: + Thread( + target=self.backupper.restore_backup, + args=(name, file_path)).start() + session["message"] = Message( + f"Backup {name} restore requested", "info").to_dict() + return redirect(url_for('index')) + + session["message"] = Message( + "Backup task is running, need to wait", "warning").to_dict() + return redirect(url_for('index')) + + def unzip_backup(self) -> str: + """Unzip backup action button + + Returns: + str: HTML page + """ + name = request.form.get("name", None) + if name is None: + session["message"] = Message("No backup name provided", "danger").to_dict() + self.logger.error("No backup name provided") + return redirect(url_for('index')) + + if not self.backupper.pending_backup: + Thread(target=self.backupper.unzip_backup, args=(name,)).start() + session["message"] = Message(f"Backup {name} unzip requested", "info").to_dict() + return redirect(url_for('index')) + + session["message"] = Message("Backup task is running, need to wait", "warning").to_dict() + return redirect(url_for('index')) + + def download_from_s3(self) -> str: + """Download backup from s3 action button + + Returns: + str: HTML page + """ + name = request.form.get("name", None) + if name is None: + session["message"] = Message("No backup name provided", "danger").to_dict() + self.logger.error("No backup name provided") + return redirect(url_for('index')) + + if not self.backupper.pending_backup: + Thread(target=self.backupper.download_backup_from_s3, args=(name,)).start() + session["message"] = Message(f"Backup {name} download requested", "info").to_dict() + return redirect(url_for('index')) + + session["message"] = Message("Backup task is running, need to wait", "warning").to_dict() + return redirect(url_for('index')) + + def delete_backup(self) -> str: + """Delete backup action button + + Returns: + str: HTML page + """ + name = request.form.get("name", None) + if name is None: + session["message"] = Message("No backup name provided", "danger").to_dict() + self.logger.error("No backup name provided") + return redirect(url_for('index')) + + if not self.backupper.pending_backup: + if self.backupper.delete_backup(name): + session["message"] = Message(f"Backup {name} deleted", "success").to_dict() + self.logger.info(f"Backup {name} deleted") + return redirect(url_for('index')) + + session["message"] = Message(f"Error while deleting backup {name}", "danger").to_dict() + return redirect(url_for('index')) + + session["message"] = Message("Backup task is running, need to wait", "warning").to_dict() + return redirect(url_for('index')) + + def backup_now(self) -> str: + """Backup now action button + + Returns: + str: HTML page + """ + if not self.backupper.pending_backup: + self.logger.info("Backup requested") + message = Message("Backup requested", "info") + session["message"] = message.to_dict() + Thread(target=self.backupper.run_backup, args=(self.backup_callback,)).start() + else: + self.logger.info("Backup already running") + message = Message("Backup already running", "warning") + session["message"] = message.to_dict() + + with self.app_context(): + return redirect(url_for('index')) + + def backup_callback(self, success: bool, message: str) -> str: + """Backup callback + + Args: + success (bool): If the backup was successful + message (str): Message to be displayed + + Returns: + str: HTML page + """ + message = Message(message, "success" if success else "danger") + with self.app_context(): + return redirect(url_for('index')) + + def schedulers(self) -> str: + """Schedulers page + + Returns: + str: HTML page + """ + tmp_schedulers = [scheduler.__dict__() for scheduler in self.schedulers_list] + return render_template( + 'schedulers.html', + schedulers=tmp_schedulers, + pending_backup=self.backupper.pending_backup, + message=session.pop("message", None)) + + def add_scheduler(self) -> str: + """Add scheduler action button + + Returns: + str: HTML page + """ + form = {} + for key in request.form: + form[key] = request.form[key] + + try: + cron = Scheduler.to_CronTrigger( + form["minute1"], + form["hour1"], + form["dow1"], + form["day1"], + form["month1"], + timezone=str(get_localzone())) + except ValueError as e: + self.logger.exception("Failed to create cron trigger", exc_info=e) + session["message"] = Message("Failed to create cron trigger", "danger").to_dict() + return redirect(url_for('schedulers')) + + scheduler = Scheduler(self.backupper, trigger=cron) + self.schedulers_list.append(scheduler) + Thread(target=scheduler.start).start() + return redirect(url_for('schedulers')) + + def delete_scheduler(self) -> str: + """Delete scheduler action button + + Returns: + str: HTML page + """ + shed_id = request.form.get("sched_id", None) + if shed_id is None: + session["message"] = Message("No scheduler id provided", "danger").to_dict() + self.logger.error("No scheduler id provided") + return redirect(url_for('schedulers')) + for scheduler in self.schedulers_list: + if scheduler.sched_job.id == shed_id: + session["message"] = Message( + f"Scheduler with id {shed_id} found and stopped", "success").to_dict() + self.logger.info(f"Scheduler with id {shed_id} found and stopped") + scheduler.shutdown() + self.schedulers_list.remove(scheduler) + break + else: + session["message"] = Message( + f"Scheduler with id {shed_id} not found", "danger").to_dict() + self.logger.error(f"Scheduler with id {shed_id} not found") + return redirect(url_for('schedulers')) + + def logs(self) -> str: + """Logs page + + Returns: + str: HTML page + """ + for handler in self.logger.handlers: + if isinstance(handler, logging.FileHandler): + log_file = handler.baseFilename + break + else: + log = "No log file found" + return render_template('logs.html', log=log) + with open(log_file, "r", encoding="utf8") as f: + log = f.read() + if log == "": + log = "No logs yet" + return render_template('logs.html', log=log) diff --git a/src/singleton.py b/src/singleton.py new file mode 100644 index 0000000..8a09d95 --- /dev/null +++ b/src/singleton.py @@ -0,0 +1,6 @@ +class Singleton(type): + _instances = {} + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs) + return cls._instances[cls] \ No newline at end of file diff --git a/src/static/style.css b/src/static/style.css index 7304f96..33e6206 100644 --- a/src/static/style.css +++ b/src/static/style.css @@ -1,35 +1,4 @@ -table, th, td { - border: 1px solid black; - border-collapse: collapse; - text-align: center; - width: 100%; - } - - th, td { - padding: 15px; - } - -tr:nth-child(odd) { background-color:#eee; } -tr:nth-child(even) { background-color:#fff; } - - button { - padding: 15px 20px; - margin: 8px 0; - border: 1px solid #ccc; - border-radius: 4px; - box-sizing: border-box; - cursor: pointer; - } - -.inline { - display: inline; +.actions-column { + width: 1%; + white-space: nowrap; } - -.align-right { - /* make element input align right */ - float: right; -} - -button:hover{ - background: #383; -} \ No newline at end of file diff --git a/src/telegram_handler.py b/src/telegram_handler.py index e917245..0b92595 100644 --- a/src/telegram_handler.py +++ b/src/telegram_handler.py @@ -1,48 +1,137 @@ -"""Module for handling Telegram bot commands. -""" +"""TelegramHandler class.""" import logging import logging.config +from os.path import exists, isfile import requests -import os -from pprint import pformat +from singleton import Singleton -class TelegramHandler(): - """Class for handling Telegram bot commands. - """ - def __init__(self, token:str, chat_id:str, logger:logging.Logger=None): - """_summary_ +class TelegramHandler(metaclass=Singleton): + """TelegramHandler class.""" + def __init__(self, token:str, chat_id:str, logger:logging.Logger=None) -> None: + """Initializes TelegramHandler class. Args: token (str): Telegram bot token. chat_id (str): Telegram chat id. - logger (logging.Logger, optional): Logger to use. Defaults to None. + logger (logging.Logger, optional): Logger. Defaults to None. + """ + self.logger = logger + self.token = token + self.chat_id = chat_id + if not self.test_connection(): + self.logger.error("TelegramHandler initialization failed.") + raise ConnectionError("TelegramHandler initialization failed.") + self.logger.debug("TelegramHandler initialized.") + + @property + def token(self) -> str: + """Telegram bot token. + + Returns: + str: Telegram bot token. + """ + return self._token + + @token.setter + def token(self, token:str) -> None: + """Sets Telegram bot token. + + Args: + token (str): Telegram bot token. Raises: - ValueError: Exception raised when required argument has invalid value. + ValueError: Empty token. """ - if logger is None: - logging.config.fileConfig("log.conf") - self.logger = logging.getLogger('pybackupper_logger') - else: - self.logger = logger - if token is None or token == "": - self.logger.error("Telegram token is not set.") - raise ValueError("Telegram token is not set.") - + self.logger.error(f"Telegram {token=} is not valid.") + raise ValueError(f"Telegram {token=} is not valid.") + self._token = token + + @property + def chat_id(self) -> str: + """Telegram chat id. + + Returns: + str: Telegram chat id. + """ + return self._chat_id + + @chat_id.setter + def chat_id(self, chat_id:str) -> None: + """Sets Telegram chat id. + + Args: + chat_id (str): Telegram chat id. + + Raises: + ValueError: Empty chat id. + """ if chat_id is None or chat_id == "": - self.logger.error("Telegram chat_id is not set.") - raise ValueError("Telegram chat_id is not set.") - - self.token = token - self.chat_id = chat_id - self.logger.info("TelegramHandler initialized.") - - def send_message(self, message:str): + self.logger.error(f"Telegram {chat_id=} is not valid.") + raise ValueError(f"Telegram {chat_id=} is not valid.") + self._chat_id = chat_id + + @property + def logger(self) -> logging.Logger: + """Logger. + + Returns: + logging.Logger: Logger. + """ + return self._logger + + @logger.setter + def logger(self, logger:logging.Logger) -> None: + """Sets logger. + + Args: + logger (logging.Logger): Logger. + """ + if logger is None: + logging.config.fileConfig("log_dev.conf") + self._logger = logging.getLogger('pybackupper_logger') + else: + self._logger = logger + + def test_connection(self) -> bool: + """Tests connection to Telegram bot. + + Returns: + bool: True if connection is successful, False otherwise. + """ + url = f"https://api.telegram.org/bot{self.token}/getMe" + try: + response = requests.get(url, timeout=10) + + if response.status_code != 200: + self.logger.error( + f"Telegram connection test failed. Status code: {response.status_code}.") + return False + + if not response.json()['ok']: + self.logger.error( + f"Telegram connection test failed. Status code: {response.status_code}. "\ + f"Response: {response.json()}.") + return False + + self.logger.debug("Telegram connection test successful.") + return True + except Exception as e: + self.logger.error(f"Telegram connection test failed. Exception: {e}.") + return False + + def send_message(self, + message:str, + silent:bool=False, + markdown:bool=False, + html:bool=False) -> None: """Sends message to Telegram chat. Args: message (str): Message to send. + silent (bool, optional): Whether to send message silently. Defaults to False. + markdown (bool, optional): Whether to parse message as markdown. Defaults to False. + html (bool, optional): Whether to parse message as html. Defaults to False. Raises: ValueError: Empty message. @@ -52,33 +141,50 @@ def send_message(self, message:str): if message is None or message == "": self.logger.error("Message is empty.") raise ValueError("Message is empty.") - + url = f"https://api.telegram.org/bot{self.token}/sendMessage" data = { "chat_id": self.chat_id, "text": message, - "parse_mode": "markdown" + "disable_notification": silent, } - + + if markdown: + data["parse_mode"] = "MarkdownV2" + + if html: + data["parse_mode"] = "HTML" + + if markdown and html: + self.logger.error("Message can't be parsed as markdown and html at the same time.") + raise ValueError("Message can't be parsed as markdown and html at the same time.") + try: - response = requests.post(url, data=data) - if response.status_code != 200: - self.logger.error(f"Failed to send message to Telegram chat. Status code: {response.status_code}. Response: {response.text}") - raise Exception(f"Failed to send message to Telegram chat. Status code: {response.status_code}") + response = requests.post(url, data=data, timeout=10) + if response.status_code != 200 or not response.json()['ok']: + self.logger.error( + f"Failed to send message to Telegram chat. "\ + f"Status code: {response.status_code}. Response: {response.json()}.") + raise ConnectionError( + f"Failed to send message to Telegram chat. "\ + f"Status code: {response.status_code}. Response: {response.json()}.") + self.logger.debug("Message sent to Telegram chat.") except Exception as e: - self.logger.error(e, exc_info=True) - self.logger.error("Failed to send message to Telegram chat.") + self.logger.exception(e, exc_info=True) + self.logger.exception("Failed to send message to Telegram chat.") raise e - - def send_file(self, file_path:str): + + def send_file(self, file_path:str, caption:str=None, silent:bool=False) -> None: """Sends file to Telegram chat. Args: file_path (str): Path to file to send. + caption (str, optional): Caption for file. Defaults to None. + silent (bool, optional): Whether to send message silently. Defaults to False. Raises: - ValueError: File path is empty. + ValueError: Empty file path. FileNotFoundError: File does not exist. Exception: Failed to send file to Telegram chat. e: Exception raised when failed to send file to Telegram chat. @@ -86,91 +192,41 @@ def send_file(self, file_path:str): if file_path is None or file_path == "": self.logger.error("File path is empty.") raise ValueError("File path is empty.") - - if not os.path.exists(file_path): - self.logger.error(f"File {file_path} does not exist.") - raise FileNotFoundError(f"File {file_path} does not exist.") - + + if not exists(file_path): + self.logger.error(f"File {file_path=} does not exist.") + raise FileNotFoundError(f"File {file_path=} does not exist.") + + if not isfile(file_path): + self.logger.error(f"File {file_path=} is not a file.") + raise FileNotFoundError(f"File {file_path=} is not a file.") + + if caption is not None and caption == "": + self.logger.error("Caption is provided, but is empty.") + raise ValueError("Caption is provided, but is empty.") + url = f"https://api.telegram.org/bot{self.token}/sendDocument" data = { "chat_id": self.chat_id, + "disable_notification": silent, } - files = { - "document": open(file_path, "rb"), - } - - try: - response = requests.post(url, data=data, files=files) - if response.status_code != 200: - self.logger.error(f"Failed to send file to Telegram chat. Status code: {response.status_code}. Response: {response.text}") - raise Exception(f"Failed to send file to Telegram chat. Status code: {response.status_code}") - self.logger.debug("File sent to Telegram chat.") - except Exception as e: - self.logger.error(e, exc_info=True) - self.logger.error("Failed to send file to Telegram chat.") - raise e - - def send_backup_info(self, hostname:str, response:str, backup_info:dict): - """Sends backup info to Telegram chat. - Args: - hostname (str): Hostname. - backup_info (dict): Backup info. + if caption is not None: + data["caption"] = caption - Raises: - ValueError: Hostname or backup info is empty. - Exception: Failed to send message to Telegram chat. - e: Exception raised when failed to send message to Telegram chat. - """ - - if hostname is None or hostname == "": - self.logger.error("Hostname is empty.") - raise ValueError("Hostname is empty.") - - if response is None or response == "": - self.logger.error("Response is empty.") - raise ValueError("Response is empty.") - - if backup_info is None or backup_info == {}: - self.logger.error("Backup info is empty.") - raise ValueError("Backup info is empty.") - - url = f"https://api.telegram.org/bot{self.token}/sendMessage" - data = { - "chat_id": self.chat_id, - "text": f"""*PyBackUpper*\n*Hostname: {hostname}*\n\nOutput: {response}\n\nBackup info:\n`{pformat(backup_info)}`""", - "parse_mode": "markdown", - } - try: - response = requests.post(url, data=data) - if response.status_code != 200: - self.logger.error(f"Failed to send message to Telegram chat. Status code: {response.status_code}. Response: {response.text}") - raise Exception(f"Failed to send message to Telegram chat. Status code: {response.status_code}") - self.logger.debug("Message sent to Telegram chat.") - except Exception as e: - self.logger.error(e, exc_info=True) - self.logger.error("Failed to send message to Telegram chat.") - raise e - - def test_connection(self) -> bool: - """Tests connection to Telegram chat. + with open(file_path, "rb") as file: + response = requests.post(url, data=data, files={"document": file}, timeout=10) + if response.status_code != 200 or not response.json()['ok']: + self.logger.error( + "Failed to send file to Telegram chat. "\ + f"Status code: {response.status_code}. Response: {response.json()}.") + raise ConnectionError( + "Failed to send file to Telegram chat. "\ + f"Status code: {response.status_code}. Response: {response.json()}.") - Returns: - bool: True if connection is successful. - """ - url = f"https://api.telegram.org/bot{self.token}/getMe" - - try: - response = requests.post(url) - if response.status_code != 200: - self.logger.error(f"Failed to test connection to Telegram chat. Status code: {response.status_code}. Response: {response.text}") - return False - self.logger.debug("Connection to Telegram chat successful.") - return True + self.logger.debug("File sent to Telegram chat.") except Exception as e: - self.logger.error(e, exc_info=True) - self.logger.error("Failed to test connection to Telegram chat.") - return False - - + self.logger.exception(e, exc_info=True) + self.logger.exception("Failed to send file to Telegram chat.") + raise e diff --git a/src/templates/base.html b/src/templates/base.html new file mode 100644 index 0000000..6e7dadb --- /dev/null +++ b/src/templates/base.html @@ -0,0 +1,57 @@ + + + + PyBackUpper - {% block title %}{% endblock %} + {% block styles %} + + {% endblock %} + + +{% block body %} + +
+
+
+ {% if message %} +

{{ message.message }}

+ {% endif %} + {% block content %} + {% endblock %} +
+
+
+{% endblock %} + + +Backups + {% if backups | length == 0 %} +

No backups found.

+ {% else %} +

Found {{ backups|length }} backups.

+

Local size: {{ local_size }}, S3 size: {{ s3_size }}

+ + + + + + + + + + + + + {% for backup in backups %} + + + + {% if backup.raw %} + + {% else %} + + {% endif %} + {% if backup.zip %} + + {% else %} + + {% endif %} + {% if backup.s3 %} + + {% else %} + + {% endif %} + + + {% endfor %} + +
NameSizeRAWZIPS3Actions
{{ backup.name }}{{ backup.size }}✅❌✅❌✅❌ + {% if backup.raw %} + + {% endif %} + {% if backup.zip and not backup.raw %} +
+ + + +
+ {% endif %} + {% if backup.s3 and not backup.zip %} +
+ + + +
+ {% endif %} +
+ + + +
+ Info + Download +
+ {% endif %} + + + +{% endblock %} \ No newline at end of file diff --git a/src/templates/index.html b/src/templates/index.html deleted file mode 100644 index 13ed02e..0000000 --- a/src/templates/index.html +++ /dev/null @@ -1,47 +0,0 @@ - - - - - - PyBackUpper - {% block styles %} - - {% endblock %} - - - -{% block body %} -

PyBackUpper

- - - -

Hostname: {{hostname}}

- -

Last backup: {{last_backup}}

-

Next backup: {{next_backup}}

- -

Local size: {{local_size}}

-

Free space: {{free_space}}

-

S3 size: {{s3_size}}

- - - - - - - - - {% for backup in backups %} - - - - - - - {% endfor %} -
BackupSizeLocalS3
{{ backup.name }}{{ backup.size }}{% if backup.local == True %}✅{% else %}❌{% endif %}{% if backup.s3 == True %}✅{% else %}❌{% endif %}
-{% endblock %} - - - - \ No newline at end of file diff --git a/src/templates/logs.html b/src/templates/logs.html new file mode 100644 index 0000000..fd140ac --- /dev/null +++ b/src/templates/logs.html @@ -0,0 +1,27 @@ +{% extends "base.html" %} + +{% block title %}Logs{% endblock %} + +{% block content %} +
+ 
+{{ log }}
+
+
+ + +{% endblock %} \ No newline at end of file diff --git a/src/templates/schedulers.html b/src/templates/schedulers.html new file mode 100644 index 0000000..b973002 --- /dev/null +++ b/src/templates/schedulers.html @@ -0,0 +1,127 @@ +{% extends "base.html" %} + +{% block title %}Schedulers{% endblock %} + +{% block content %} + + + +
+

Schedulers

+ +
+ + {% if schedulers %} +

Found {{ schedulers|length }} schedulers.

+ + + + + + + + + + + {% for scheduler in schedulers %} + + + + + + {% endfor %} + +
IDCronNext runAction
{{ scheduler.id }}{{ scheduler.cron }}{{ scheduler.next_run }} +
+ + + +
+
+ {% else %} +

No schedulers found.

+ {% endif %} + + +{% endblock %} \ No newline at end of file diff --git a/src/tools.py b/src/tools.py new file mode 100644 index 0000000..0c6a529 --- /dev/null +++ b/src/tools.py @@ -0,0 +1,112 @@ +from datetime import datetime +from time import perf_counter +from os.path import exists, normpath, getsize, join, isfile, isdir +from json import load as json_load + +def size_to_human_readable(size: int) -> str: + """Converts the size in bytes to a human readable format. + + Args: + size (int): Size in bytes. + + Returns: + str: Human readable size. + """ + if isinstance(size, str): + try: + size = int(size) + except ValueError: + return f"{size}B" + power = 2 ** 10 + n = 0 + power_labels = {0: '', 1: 'k', 2: 'M', 3: 'G', 4: 'T'} + while size > power: + size /= power + n += 1 + return f"{size:.2f}{power_labels[n]}B" + +def timestamp_to_human_readable(timestamp: int) -> str: + """Converts the timestamp to a human readable format. + + Args: + timestamp (int): Timestamp. + + Returns: + str: Human readable timestamp. + """ + return datetime.fromtimestamp(timestamp).strftime("%Y-%m-%d %H:%M:%S") + +def timestamp_to_file_name(timestamp: int) -> str: + """Converts the timestamp to a file name. + + Args: + timestamp (int): Timestamp. + + Returns: + str: File name. + """ + return datetime.fromtimestamp(timestamp).strftime("%Y_%m_%d_%H_%M_%S") + +def time_diff_to_human_readable(time_diff: int) -> str: + """Converts the time difference to a human readable format. + + Args: + time_diff (int): Time difference. + + Returns: + str: Human readable time difference. + """ + if time_diff < 60: + return f"{time_diff}s" + elif time_diff < 3600: + return f"{time_diff // 60}m {time_diff % 60}s" + elif time_diff < 86400: + return f"{time_diff // 3600}h {(time_diff % 3600) // 60}m {time_diff % 60}s" + else: + return f"{time_diff // 86400}d {(time_diff % 86400) // 3600}h {(time_diff % 3600) // 60}m {time_diff % 60}s" + + +def timeit(func): + """Decorator to measure the execution time of a function. + + Args: + func (function): Function to measure. + + Returns: + function: Decorated function. + """ + def wrapper(*args, **kwargs): + start = perf_counter() + result = func(*args, **kwargs) + end = perf_counter() + print(f"Time elapsed: {end - start:.2f}s") + return result + return wrapper + +def read_config_from_file(file_path: str) -> dict: + """Function to read PyBackupper config from json file + + Args: + filePath (str): json config file + + Returns: + dict: parsed config + """ + + if file_path is None or file_path == "": + raise ValueError("file_path cannot be None or empty.") + + file_path = normpath(file_path) + + if not exists(file_path) or not isfile(file_path): + raise FileNotFoundError(f"src_path {file_path} does not exist.") + + if getsize(file_path) == 0: + raise ValueError("File is empty.") + + with open(file_path, "r") as file: + try: + return json_load(file) + except ValueError: + raise ValueError("File is not in valid JSON format") + diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..8d67ee9 --- /dev/null +++ b/uv.lock @@ -0,0 +1,387 @@ +version = 1 +requires-python = ">=3.12" + +[[package]] +name = "apscheduler" +version = "3.10.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytz" }, + { name = "setuptools" }, + { name = "six" }, + { name = "tzlocal" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ea/ed/f1ad88e88208c24db80dcaae7a5a339bb283956984f8fa59933d2806413a/APScheduler-3.10.1.tar.gz", hash = "sha256:0293937d8f6051a0f493359440c1a1b93e882c57daf0197afeff0e727777b96e", size = 100376 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/08/952d9570f4897dc2b30166fca5afd3a2cd19b3d408abdb470978484e8a09/APScheduler-3.10.1-py3-none-any.whl", hash = "sha256:e813ad5ada7aff36fb08cdda746b520531eaac7757832abc204868ba78e0c8f6", size = 59238 }, +] + +[[package]] +name = "blinker" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458 }, +] + +[[package]] +name = "boto3" +version = "1.26.158" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a7/26/9318f38e41e4ff257d4400118e46faf42d4a1a6d3f296a57c6248fc0e0e1/boto3-1.26.158.tar.gz", hash = "sha256:7f88d9403f81e6f3fc770c424f7089b15eb0553b168b1d2f979fa0d12b663b42", size = 103589 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/05/48d42bd5ea19e0d979f348a744199452306333e4fc46e9eae17ab04b917e/boto3-1.26.158-py3-none-any.whl", hash = "sha256:0be407c2e941b422634766c0d754132ad4d33b5d0f84d9a30426b713b31ce3ab", size = 135906 }, +] + +[[package]] +name = "botocore" +version = "1.29.165" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/f6/d35a27c73dc1053abdfe8524d1e488073fccb51e43c88da61b8fe29522e3/botocore-1.29.165.tar.gz", hash = "sha256:988b948be685006b43c4bbd8f5c0cb93e77c66deb70561994e0c5b31b5a67210", size = 11180165 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/20/e7a9a8e6746872afcc4e3ad5ab503702c38813b3a532df27cce95c98b8cb/botocore-1.29.165-py3-none-any.whl", hash = "sha256:6f35d59e230095aed7cd747604fe248fa384bebb7d09549077892f936a8ca3df", size = 10975299 }, +] + +[[package]] +name = "certifi" +version = "2025.1.31" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/ab/c9f1e32b7b1bf505bf26f0ef697775960db7932abeb7b516de930ba2705f/certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651", size = 167577 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/fc/bce832fd4fd99766c04d1ee0eead6b0ec6486fb100ae5e74c1d91292b982/certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe", size = 166393 }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/16/b0/572805e227f01586461c80e0fd25d65a2115599cc9dad142fee4b747c357/charset_normalizer-3.4.1.tar.gz", hash = "sha256:44251f18cd68a75b56585dd00dae26183e102cd5e0f9f1466e6df5da2ed64ea3", size = 123188 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0a/9a/dd1e1cdceb841925b7798369a09279bd1cf183cef0f9ddf15a3a6502ee45/charset_normalizer-3.4.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:73d94b58ec7fecbc7366247d3b0b10a21681004153238750bb67bd9012414545", size = 196105 }, + { url = "https://files.pythonhosted.org/packages/d3/8c/90bfabf8c4809ecb648f39794cf2a84ff2e7d2a6cf159fe68d9a26160467/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dad3e487649f498dd991eeb901125411559b22e8d7ab25d3aeb1af367df5efd7", size = 140404 }, + { url = "https://files.pythonhosted.org/packages/ad/8f/e410d57c721945ea3b4f1a04b74f70ce8fa800d393d72899f0a40526401f/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c30197aa96e8eed02200a83fba2657b4c3acd0f0aa4bdc9f6c1af8e8962e0757", size = 150423 }, + { url = "https://files.pythonhosted.org/packages/f0/b8/e6825e25deb691ff98cf5c9072ee0605dc2acfca98af70c2d1b1bc75190d/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2369eea1ee4a7610a860d88f268eb39b95cb588acd7235e02fd5a5601773d4fa", size = 143184 }, + { url = "https://files.pythonhosted.org/packages/3e/a2/513f6cbe752421f16d969e32f3583762bfd583848b763913ddab8d9bfd4f/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc2722592d8998c870fa4e290c2eec2c1569b87fe58618e67d38b4665dfa680d", size = 145268 }, + { url = "https://files.pythonhosted.org/packages/74/94/8a5277664f27c3c438546f3eb53b33f5b19568eb7424736bdc440a88a31f/charset_normalizer-3.4.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffc9202a29ab3920fa812879e95a9e78b2465fd10be7fcbd042899695d75e616", size = 147601 }, + { url = "https://files.pythonhosted.org/packages/7c/5f/6d352c51ee763623a98e31194823518e09bfa48be2a7e8383cf691bbb3d0/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:804a4d582ba6e5b747c625bf1255e6b1507465494a40a2130978bda7b932c90b", size = 141098 }, + { url = "https://files.pythonhosted.org/packages/78/d4/f5704cb629ba5ab16d1d3d741396aec6dc3ca2b67757c45b0599bb010478/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:0f55e69f030f7163dffe9fd0752b32f070566451afe180f99dbeeb81f511ad8d", size = 149520 }, + { url = "https://files.pythonhosted.org/packages/c5/96/64120b1d02b81785f222b976c0fb79a35875457fa9bb40827678e54d1bc8/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:c4c3e6da02df6fa1410a7680bd3f63d4f710232d3139089536310d027950696a", size = 152852 }, + { url = "https://files.pythonhosted.org/packages/84/c9/98e3732278a99f47d487fd3468bc60b882920cef29d1fa6ca460a1fdf4e6/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:5df196eb874dae23dcfb968c83d4f8fdccb333330fe1fc278ac5ceeb101003a9", size = 150488 }, + { url = "https://files.pythonhosted.org/packages/13/0e/9c8d4cb99c98c1007cc11eda969ebfe837bbbd0acdb4736d228ccaabcd22/charset_normalizer-3.4.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e358e64305fe12299a08e08978f51fc21fac060dcfcddd95453eabe5b93ed0e1", size = 146192 }, + { url = "https://files.pythonhosted.org/packages/b2/21/2b6b5b860781a0b49427309cb8670785aa543fb2178de875b87b9cc97746/charset_normalizer-3.4.1-cp312-cp312-win32.whl", hash = "sha256:9b23ca7ef998bc739bf6ffc077c2116917eabcc901f88da1b9856b210ef63f35", size = 95550 }, + { url = "https://files.pythonhosted.org/packages/21/5b/1b390b03b1d16c7e382b561c5329f83cc06623916aab983e8ab9239c7d5c/charset_normalizer-3.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:6ff8a4a60c227ad87030d76e99cd1698345d4491638dfa6673027c48b3cd395f", size = 102785 }, + { url = "https://files.pythonhosted.org/packages/38/94/ce8e6f63d18049672c76d07d119304e1e2d7c6098f0841b51c666e9f44a0/charset_normalizer-3.4.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:aabfa34badd18f1da5ec1bc2715cadc8dca465868a4e73a0173466b688f29dda", size = 195698 }, + { url = "https://files.pythonhosted.org/packages/24/2e/dfdd9770664aae179a96561cc6952ff08f9a8cd09a908f259a9dfa063568/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22e14b5d70560b8dd51ec22863f370d1e595ac3d024cb8ad7d308b4cd95f8313", size = 140162 }, + { url = "https://files.pythonhosted.org/packages/24/4e/f646b9093cff8fc86f2d60af2de4dc17c759de9d554f130b140ea4738ca6/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8436c508b408b82d87dc5f62496973a1805cd46727c34440b0d29d8a2f50a6c9", size = 150263 }, + { url = "https://files.pythonhosted.org/packages/5e/67/2937f8d548c3ef6e2f9aab0f6e21001056f692d43282b165e7c56023e6dd/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2d074908e1aecee37a7635990b2c6d504cd4766c7bc9fc86d63f9c09af3fa11b", size = 142966 }, + { url = "https://files.pythonhosted.org/packages/52/ed/b7f4f07de100bdb95c1756d3a4d17b90c1a3c53715c1a476f8738058e0fa/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:955f8851919303c92343d2f66165294848d57e9bba6cf6e3625485a70a038d11", size = 144992 }, + { url = "https://files.pythonhosted.org/packages/96/2c/d49710a6dbcd3776265f4c923bb73ebe83933dfbaa841c5da850fe0fd20b/charset_normalizer-3.4.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:44ecbf16649486d4aebafeaa7ec4c9fed8b88101f4dd612dcaf65d5e815f837f", size = 147162 }, + { url = "https://files.pythonhosted.org/packages/b4/41/35ff1f9a6bd380303dea55e44c4933b4cc3c4850988927d4082ada230273/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:0924e81d3d5e70f8126529951dac65c1010cdf117bb75eb02dd12339b57749dd", size = 140972 }, + { url = "https://files.pythonhosted.org/packages/fb/43/c6a0b685fe6910d08ba971f62cd9c3e862a85770395ba5d9cad4fede33ab/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:2967f74ad52c3b98de4c3b32e1a44e32975e008a9cd2a8cc8966d6a5218c5cb2", size = 149095 }, + { url = "https://files.pythonhosted.org/packages/4c/ff/a9a504662452e2d2878512115638966e75633519ec11f25fca3d2049a94a/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c75cb2a3e389853835e84a2d8fb2b81a10645b503eca9bcb98df6b5a43eb8886", size = 152668 }, + { url = "https://files.pythonhosted.org/packages/6c/71/189996b6d9a4b932564701628af5cee6716733e9165af1d5e1b285c530ed/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:09b26ae6b1abf0d27570633b2b078a2a20419c99d66fb2823173d73f188ce601", size = 150073 }, + { url = "https://files.pythonhosted.org/packages/e4/93/946a86ce20790e11312c87c75ba68d5f6ad2208cfb52b2d6a2c32840d922/charset_normalizer-3.4.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fa88b843d6e211393a37219e6a1c1df99d35e8fd90446f1118f4216e307e48cd", size = 145732 }, + { url = "https://files.pythonhosted.org/packages/cd/e5/131d2fb1b0dddafc37be4f3a2fa79aa4c037368be9423061dccadfd90091/charset_normalizer-3.4.1-cp313-cp313-win32.whl", hash = "sha256:eb8178fe3dba6450a3e024e95ac49ed3400e506fd4e9e5c32d30adda88cbd407", size = 95391 }, + { url = "https://files.pythonhosted.org/packages/27/f2/4f9a69cc7712b9b5ad8fdb87039fd89abba997ad5cbe690d1835d40405b0/charset_normalizer-3.4.1-cp313-cp313-win_amd64.whl", hash = "sha256:b1ac5992a838106edb89654e0aebfc24f5848ae2547d22c2c3f66454daa11971", size = 102702 }, + { url = "https://files.pythonhosted.org/packages/0e/f6/65ecc6878a89bb1c23a086ea335ad4bf21a588990c3f535a227b9eea9108/charset_normalizer-3.4.1-py3-none-any.whl", hash = "sha256:d98b1668f06378c6dbefec3b92299716b931cd4e6061f3c875a71ced1780ab85", size = 49767 }, +] + +[[package]] +name = "click" +version = "8.1.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "platform_system == 'Windows'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188 }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 }, +] + +[[package]] +name = "flask" +version = "2.3.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "blinker" }, + { name = "click" }, + { name = "itsdangerous" }, + { name = "jinja2" }, + { name = "werkzeug" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4d/00/ef81c18da32fdfcde6381c315f4b11597fb6691180a330418848efee0ae7/Flask-2.3.2.tar.gz", hash = "sha256:8c2f9abd47a9e8df7f0c3f091ce9497d011dc3b31effcf4c85a6e2b50f4114ef", size = 686251 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/1a/f191d32818e5cd985bdd3f47a6e4f525e2db1ce5e8150045ca0c31813686/Flask-2.3.2-py3-none-any.whl", hash = "sha256:77fd4e1249d8c9923de34907236b747ced06e5467ecac1a7bb7115ae0e9670b0", size = 96867 }, +] + +[[package]] +name = "flask-wtf" +version = "1.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "flask" }, + { name = "itsdangerous" }, + { name = "wtforms" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9b/ef/b6ec35e02f479f6e76e02ede14594c9cfa5e6dcbab6ea0e82fa413993a2a/flask_wtf-1.2.1.tar.gz", hash = "sha256:8bb269eb9bb46b87e7c8233d7e7debdf1f8b74bf90cc1789988c29b37a97b695", size = 42498 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/2b/0f0cf68a2f052ea3dbb8b6c8c2a7e8aea5e6df7410f5e289437fefbeb461/flask_wtf-1.2.1-py3-none-any.whl", hash = "sha256:fa6793f2fb7e812e0fe9743b282118e581fb1b6c45d414b8af05e659bd653287", size = 12725 }, +] + +[[package]] +name = "idna" +version = "3.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442 }, +] + +[[package]] +name = "itsdangerous" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234 }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899 }, +] + +[[package]] +name = "jmespath" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/2a/e867e8531cf3e36b41201936b7fa7ba7b5702dbef42922193f05c8976cd6/jmespath-1.0.1.tar.gz", hash = "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe", size = 25843 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/b4/b9b800c45527aadd64d5b442f9b932b00648617eb5d63d2c7a6587b7cafc/jmespath-1.0.1-py3-none-any.whl", hash = "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", size = 20256 }, +] + +[[package]] +name = "markupsafe" +version = "3.0.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b2/97/5d42485e71dfc078108a86d6de8fa46db44a1a9295e89c5d6d4a06e23a62/markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0", size = 20537 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/22/09/d1f21434c97fc42f09d290cbb6350d44eb12f09cc62c9476effdb33a18aa/MarkupSafe-3.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:9778bd8ab0a994ebf6f84c2b949e65736d5575320a17ae8984a77fab08db94cf", size = 14274 }, + { url = "https://files.pythonhosted.org/packages/6b/b0/18f76bba336fa5aecf79d45dcd6c806c280ec44538b3c13671d49099fdd0/MarkupSafe-3.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:846ade7b71e3536c4e56b386c2a47adf5741d2d8b94ec9dc3e92e5e1ee1e2225", size = 12348 }, + { url = "https://files.pythonhosted.org/packages/e0/25/dd5c0f6ac1311e9b40f4af06c78efde0f3b5cbf02502f8ef9501294c425b/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c99d261bd2d5f6b59325c92c73df481e05e57f19837bdca8413b9eac4bd8028", size = 24149 }, + { url = "https://files.pythonhosted.org/packages/f3/f0/89e7aadfb3749d0f52234a0c8c7867877876e0a20b60e2188e9850794c17/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e17c96c14e19278594aa4841ec148115f9c7615a47382ecb6b82bd8fea3ab0c8", size = 23118 }, + { url = "https://files.pythonhosted.org/packages/d5/da/f2eeb64c723f5e3777bc081da884b414671982008c47dcc1873d81f625b6/MarkupSafe-3.0.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:88416bd1e65dcea10bc7569faacb2c20ce071dd1f87539ca2ab364bf6231393c", size = 22993 }, + { url = "https://files.pythonhosted.org/packages/da/0e/1f32af846df486dce7c227fe0f2398dc7e2e51d4a370508281f3c1c5cddc/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2181e67807fc2fa785d0592dc2d6206c019b9502410671cc905d132a92866557", size = 24178 }, + { url = "https://files.pythonhosted.org/packages/c4/f6/bb3ca0532de8086cbff5f06d137064c8410d10779c4c127e0e47d17c0b71/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:52305740fe773d09cffb16f8ed0427942901f00adedac82ec8b67752f58a1b22", size = 23319 }, + { url = "https://files.pythonhosted.org/packages/a2/82/8be4c96ffee03c5b4a034e60a31294daf481e12c7c43ab8e34a1453ee48b/MarkupSafe-3.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ad10d3ded218f1039f11a75f8091880239651b52e9bb592ca27de44eed242a48", size = 23352 }, + { url = "https://files.pythonhosted.org/packages/51/ae/97827349d3fcffee7e184bdf7f41cd6b88d9919c80f0263ba7acd1bbcb18/MarkupSafe-3.0.2-cp312-cp312-win32.whl", hash = "sha256:0f4ca02bea9a23221c0182836703cbf8930c5e9454bacce27e767509fa286a30", size = 15097 }, + { url = "https://files.pythonhosted.org/packages/c1/80/a61f99dc3a936413c3ee4e1eecac96c0da5ed07ad56fd975f1a9da5bc630/MarkupSafe-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:8e06879fc22a25ca47312fbe7c8264eb0b662f6db27cb2d3bbbc74b1df4b9b87", size = 15601 }, + { url = "https://files.pythonhosted.org/packages/83/0e/67eb10a7ecc77a0c2bbe2b0235765b98d164d81600746914bebada795e97/MarkupSafe-3.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ba9527cdd4c926ed0760bc301f6728ef34d841f405abf9d4f959c478421e4efd", size = 14274 }, + { url = "https://files.pythonhosted.org/packages/2b/6d/9409f3684d3335375d04e5f05744dfe7e9f120062c9857df4ab490a1031a/MarkupSafe-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f8b3d067f2e40fe93e1ccdd6b2e1d16c43140e76f02fb1319a05cf2b79d99430", size = 12352 }, + { url = "https://files.pythonhosted.org/packages/d2/f5/6eadfcd3885ea85fe2a7c128315cc1bb7241e1987443d78c8fe712d03091/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:569511d3b58c8791ab4c2e1285575265991e6d8f8700c7be0e88f86cb0672094", size = 24122 }, + { url = "https://files.pythonhosted.org/packages/0c/91/96cf928db8236f1bfab6ce15ad070dfdd02ed88261c2afafd4b43575e9e9/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:15ab75ef81add55874e7ab7055e9c397312385bd9ced94920f2802310c930396", size = 23085 }, + { url = "https://files.pythonhosted.org/packages/c2/cf/c9d56af24d56ea04daae7ac0940232d31d5a8354f2b457c6d856b2057d69/MarkupSafe-3.0.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3818cb119498c0678015754eba762e0d61e5b52d34c8b13d770f0719f7b1d79", size = 22978 }, + { url = "https://files.pythonhosted.org/packages/2a/9f/8619835cd6a711d6272d62abb78c033bda638fdc54c4e7f4272cf1c0962b/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cdb82a876c47801bb54a690c5ae105a46b392ac6099881cdfb9f6e95e4014c6a", size = 24208 }, + { url = "https://files.pythonhosted.org/packages/f9/bf/176950a1792b2cd2102b8ffeb5133e1ed984547b75db47c25a67d3359f77/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cabc348d87e913db6ab4aa100f01b08f481097838bdddf7c7a84b7575b7309ca", size = 23357 }, + { url = "https://files.pythonhosted.org/packages/ce/4f/9a02c1d335caabe5c4efb90e1b6e8ee944aa245c1aaaab8e8a618987d816/MarkupSafe-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:444dcda765c8a838eaae23112db52f1efaf750daddb2d9ca300bcae1039adc5c", size = 23344 }, + { url = "https://files.pythonhosted.org/packages/ee/55/c271b57db36f748f0e04a759ace9f8f759ccf22b4960c270c78a394f58be/MarkupSafe-3.0.2-cp313-cp313-win32.whl", hash = "sha256:bcf3e58998965654fdaff38e58584d8937aa3096ab5354d493c77d1fdd66d7a1", size = 15101 }, + { url = "https://files.pythonhosted.org/packages/29/88/07df22d2dd4df40aba9f3e402e6dc1b8ee86297dddbad4872bd5e7b0094f/MarkupSafe-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:e6a2a455bd412959b57a172ce6328d2dd1f01cb2135efda2e4576e8a23fa3b0f", size = 15603 }, + { url = "https://files.pythonhosted.org/packages/62/6a/8b89d24db2d32d433dffcd6a8779159da109842434f1dd2f6e71f32f738c/MarkupSafe-3.0.2-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:b5a6b3ada725cea8a5e634536b1b01c30bcdcd7f9c6fff4151548d5bf6b3a36c", size = 14510 }, + { url = "https://files.pythonhosted.org/packages/7a/06/a10f955f70a2e5a9bf78d11a161029d278eeacbd35ef806c3fd17b13060d/MarkupSafe-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:a904af0a6162c73e3edcb969eeeb53a63ceeb5d8cf642fade7d39e7963a22ddb", size = 12486 }, + { url = "https://files.pythonhosted.org/packages/34/cf/65d4a571869a1a9078198ca28f39fba5fbb910f952f9dbc5220afff9f5e6/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4aa4e5faecf353ed117801a068ebab7b7e09ffb6e1d5e412dc852e0da018126c", size = 25480 }, + { url = "https://files.pythonhosted.org/packages/0c/e3/90e9651924c430b885468b56b3d597cabf6d72be4b24a0acd1fa0e12af67/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ef13eaeee5b615fb07c9a7dadb38eac06a0608b41570d8ade51c56539e509d", size = 23914 }, + { url = "https://files.pythonhosted.org/packages/66/8c/6c7cf61f95d63bb866db39085150df1f2a5bd3335298f14a66b48e92659c/MarkupSafe-3.0.2-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d16a81a06776313e817c951135cf7340a3e91e8c1ff2fac444cfd75fffa04afe", size = 23796 }, + { url = "https://files.pythonhosted.org/packages/bb/35/cbe9238ec3f47ac9a7c8b3df7a808e7cb50fe149dc7039f5f454b3fba218/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:6381026f158fdb7c72a168278597a5e3a5222e83ea18f543112b2662a9b699c5", size = 25473 }, + { url = "https://files.pythonhosted.org/packages/e6/32/7621a4382488aa283cc05e8984a9c219abad3bca087be9ec77e89939ded9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:3d79d162e7be8f996986c064d1c7c817f6df3a77fe3d6859f6f9e7be4b8c213a", size = 24114 }, + { url = "https://files.pythonhosted.org/packages/0d/80/0985960e4b89922cb5a0bac0ed39c5b96cbc1a536a99f30e8c220a996ed9/MarkupSafe-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:131a3c7689c85f5ad20f9f6fb1b866f402c445b220c19fe4308c0b147ccd2ad9", size = 24098 }, + { url = "https://files.pythonhosted.org/packages/82/78/fedb03c7d5380df2427038ec8d973587e90561b2d90cd472ce9254cf348b/MarkupSafe-3.0.2-cp313-cp313t-win32.whl", hash = "sha256:ba8062ed2cf21c07a9e295d5b8a2a5ce678b913b45fdf68c32d95d6c1291e0b6", size = 15208 }, + { url = "https://files.pythonhosted.org/packages/4f/65/6079a46068dfceaeabb5dcad6d674f5f5c61a6fa5673746f42a9f4c233b3/MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f", size = 15739 }, +] + +[[package]] +name = "psutil" +version = "5.9.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2d/01/beb7331fc6c8d1c49dd051e3611379bfe379e915c808e1301506027fce9d/psutil-5.9.6.tar.gz", hash = "sha256:e4b92ddcd7dd4cdd3f900180ea1e104932c7bce234fb88976e2a3b296441225a", size = 496866 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/36/35b12441ba1bc6684c9215191f955415196ca57ca85d88e313bec7f2cf8e/psutil-5.9.6-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:c69596f9fc2f8acd574a12d5f8b7b1ba3765a641ea5d60fb4736bf3c08a8214a", size = 246101 }, + { url = "https://files.pythonhosted.org/packages/61/c8/e684dea1912943347922ab5c05efc94b4ff3d7470038e8afbe3941ef9efe/psutil-5.9.6-cp36-abi3-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:92e0cc43c524834af53e9d3369245e6cc3b130e78e26100d1f63cdb0abeb3d3c", size = 280854 }, + { url = "https://files.pythonhosted.org/packages/19/06/4e3fa3c1b79271e933c5ddbad3a48aa2c3d5f592a0fb7c037f3e0f619f4d/psutil-5.9.6-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:748c9dd2583ed86347ed65d0035f45fa8c851e8d90354c122ab72319b5f366f4", size = 283614 }, + { url = "https://files.pythonhosted.org/packages/06/ac/f31a0faf98267e63fc6ed046ad2aca68bd79521380026e92fd4921c869aa/psutil-5.9.6-cp37-abi3-win32.whl", hash = "sha256:a6f01f03bf1843280f4ad16f4bde26b817847b4c1a0db59bf6419807bc5ce05c", size = 248489 }, + { url = "https://files.pythonhosted.org/packages/c5/b2/699c50fe0b0402a1ccb64ad71313bcb740e735008dd3ab9abeddbe148e45/psutil-5.9.6-cp37-abi3-win_amd64.whl", hash = "sha256:6e5fb8dc711a514da83098bc5234264e551ad980cec5f85dabf4d38ed6f15e9a", size = 252327 }, + { url = "https://files.pythonhosted.org/packages/9e/cb/e4b83c27eea66bc255effc967053f6fce7c14906dd9b43a348ead9f0cfea/psutil-5.9.6-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:daecbcbd29b289aac14ece28eca6a3e60aa361754cf6da3dfb20d4d32b6c7f57", size = 246859 }, +] + +[[package]] +name = "pybackupper" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "apscheduler" }, + { name = "boto3" }, + { name = "botocore" }, + { name = "flask" }, + { name = "flask-wtf" }, + { name = "psutil" }, + { name = "requests" }, + { name = "tzlocal" }, +] + +[package.metadata] +requires-dist = [ + { name = "apscheduler", specifier = "==3.10.1" }, + { name = "boto3", specifier = "==1.26.158" }, + { name = "botocore", specifier = "==1.29.165" }, + { name = "flask", specifier = "==2.3.2" }, + { name = "flask-wtf", specifier = "==1.2.1" }, + { name = "psutil", specifier = "==5.9.6" }, + { name = "requests", specifier = "==2.32.3" }, + { name = "tzlocal", specifier = "==5.2" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892 }, +] + +[[package]] +name = "pytz" +version = "2025.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/57/df1c9157c8d5a05117e455d66fd7cf6dbc46974f832b1058ed4856785d8a/pytz-2025.1.tar.gz", hash = "sha256:c2db42be2a2518b28e65f9207c4d05e6ff547d1efa4086469ef855e4ab70178e", size = 319617 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/38/ac33370d784287baa1c3d538978b5e2ea064d4c1b93ffbd12826c190dd10/pytz-2025.1-py2.py3-none-any.whl", hash = "sha256:89dd22dca55b46eac6eda23b2d72721bf1bdfef212645d81513ef5d03038de57", size = 507930 }, +] + +[[package]] +name = "requests" +version = "2.32.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", size = 131218 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", size = 64928 }, +] + +[[package]] +name = "s3transfer" +version = "0.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/47/d676353674e651910085e3537866f093d2b9e9699e95e89d960e78df9ecf/s3transfer-0.6.2.tar.gz", hash = "sha256:cab66d3380cca3e70939ef2255d01cd8aece6a4907a9528740f668c4b0611861", size = 132821 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d9/17/a3b666f5ef9543cfd3c661d39d1e193abb9649d0cfbbfee3cf3b51d5af02/s3transfer-0.6.2-py3-none-any.whl", hash = "sha256:b014be3a8a2aab98cfe1abc7229cc5a9a0cf05eb9c1f2b86b230fd8df3f78084", size = 79765 }, +] + +[[package]] +name = "setuptools" +version = "75.8.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d1/53/43d99d7687e8cdef5ab5f9ec5eaf2c0423c2b35133a2b7e7bc276fc32b21/setuptools-75.8.2.tar.gz", hash = "sha256:4880473a969e5f23f2a2be3646b2dfd84af9028716d398e46192f84bc36900d2", size = 1344083 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/38/7d7362e031bd6dc121e5081d8cb6aa6f6fedf2b67bf889962134c6da4705/setuptools-75.8.2-py3-none-any.whl", hash = "sha256:558e47c15f1811c1fa7adbd0096669bf76c1d3f433f58324df69f3f5ecac4e8f", size = 1229385 }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050 }, +] + +[[package]] +name = "tzdata" +version = "2025.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/0f/fa4723f22942480be4ca9527bbde8d43f6c3f2fe8412f00e7f5f6746bc8b/tzdata-2025.1.tar.gz", hash = "sha256:24894909e88cdb28bd1636c6887801df64cb485bd593f2fd83ef29075a81d694", size = 194950 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0f/dd/84f10e23edd882c6f968c21c2434fe67bd4a528967067515feca9e611e5e/tzdata-2025.1-py2.py3-none-any.whl", hash = "sha256:7e127113816800496f027041c570f50bcd464a020098a3b6b199517772303639", size = 346762 }, +] + +[[package]] +name = "tzlocal" +version = "5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "platform_system == 'Windows'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/d3/c19d65ae67636fe63953b20c2e4a8ced4497ea232c43ff8d01db16de8dc0/tzlocal-5.2.tar.gz", hash = "sha256:8d399205578f1a9342816409cc1e46a93ebd5755e39ea2d85334bea911bf0e6e", size = 30201 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/97/3f/c4c51c55ff8487f2e6d0e618dba917e3c3ee2caae6cf0fbb59c9b1876f2e/tzlocal-5.2-py3-none-any.whl", hash = "sha256:49816ef2fe65ea8ac19d19aa7a1ae0551c834303d5014c6d5a62e4cbda8047b8", size = 17859 }, +] + +[[package]] +name = "urllib3" +version = "1.26.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/e8/6ff5e6bc22095cfc59b6ea711b687e2b7ed4bdb373f7eeec370a97d7392f/urllib3-1.26.20.tar.gz", hash = "sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32", size = 307380 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/cf/8435d5a7159e2a9c83a95896ed596f68cf798005fe107cc655b5c5c14704/urllib3-1.26.20-py2.py3-none-any.whl", hash = "sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e", size = 144225 }, +] + +[[package]] +name = "werkzeug" +version = "3.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/69/83029f1f6300c5fb2471d621ab06f6ec6b3324685a2ce0f9777fd4a8b71e/werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746", size = 806925 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e", size = 224498 }, +] + +[[package]] +name = "wtforms" +version = "3.2.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/01/e4/633d080897e769ed5712dcfad626e55dbd6cf45db0ff4d9884315c6a82da/wtforms-3.2.1.tar.gz", hash = "sha256:df3e6b70f3192e92623128123ec8dca3067df9cfadd43d59681e210cfb8d4682", size = 137801 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/08/c9/2088fb5645cd289c99ebe0d4cdcc723922a1d8e1beaefb0f6f76dff9b21c/wtforms-3.2.1-py3-none-any.whl", hash = "sha256:583bad77ba1dd7286463f21e11aa3043ca4869d03575921d1a1698d0715e0fd4", size = 152454 }, +]