From 8fd33e74f566e51309b52c3c99c61e0fdc89917e Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Tue, 29 Apr 2025 20:53:05 +0000 Subject: [PATCH 01/34] v0.2.1 --- kvprocessor/__init__.py | 5 ++++- kvprocessor/kvdiff.py | 18 ++++++++++++++++ kvprocessor/kvfileexporter.py | 39 +++++++++++++++++++++++++++++++++++ kvprocessor/kvfilemerger.py | 18 ++++++++++++++++ kvprocessor/kvprocessor.py | 2 +- kvprocessor/log.py | 9 +++++++- pyproject.toml | 2 +- 7 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 kvprocessor/kvdiff.py create mode 100644 kvprocessor/kvfileexporter.py create mode 100644 kvprocessor/kvfilemerger.py diff --git a/kvprocessor/__init__.py b/kvprocessor/__init__.py index 409e481..c9b923d 100644 --- a/kvprocessor/__init__.py +++ b/kvprocessor/__init__.py @@ -1,6 +1,9 @@ -__version__ = "0.1.12" +__version__ = "0.2.1" from .kvprocessor import KVProcessor from .kvenvloader import LoadEnv from .kvstructloader import KVStructLoader +from .kvfileexporter import KVFileExporter +from .kvfilemerger import KVFileMerger +from .kvdiff import KVFileDiffChecker from .errors import KVProcessorError, InvalidKVFileError, MissingEnvironmentVariableError, NamespaceNotFoundError, InvalidNamespaceError \ No newline at end of file diff --git a/kvprocessor/kvdiff.py b/kvprocessor/kvdiff.py new file mode 100644 index 0000000..ad3f9d2 --- /dev/null +++ b/kvprocessor/kvdiff.py @@ -0,0 +1,18 @@ +class KVFileDiffChecker: + """Compares two .kv files and highlights differences.""" + def __init__(self, file1: str, file2: str): + self.file1 = file1 + self.file2 = file2 + + def diff(self) -> dict: + with open(self.file1, 'r') as f1, open(self.file2, 'r') as f2: + lines1 = {line.strip() for line in f1 if line.strip() and not line.startswith('#')} + lines2 = {line.strip() for line in f2 if line.strip() and not line.startswith('#')} + + added = lines2 - lines1 + removed = lines1 - lines2 + + return { + 'added': added, + 'removed': removed + } \ No newline at end of file diff --git a/kvprocessor/kvfileexporter.py b/kvprocessor/kvfileexporter.py new file mode 100644 index 0000000..a787bd5 --- /dev/null +++ b/kvprocessor/kvfileexporter.py @@ -0,0 +1,39 @@ +class KVFileExporter: + """Exports processed configuration back into a .kv file format.""" + def __init__(self, output_path: str): + self.output_path = output_path + + def export(self, config: dict): + """Exports the configuration dictionary to a .kv file.""" + try: + with open(self.output_path, 'w') as file: + for key, value in config.items(): + value_type = type(value).__name__ + if value is None: + value = 'none' + file.write(f"{key}<{value_type}>:{value}\n") + except IOError as e: + raise IOError(f"Failed to write to file {self.output_path}: {e}") + + def validate_and_export(self, config: dict): + """Validates the configuration dictionary before exporting.""" + if not isinstance(config, dict): + raise TypeError("Configuration must be a dictionary.") + + for key, value in config.items(): + if not isinstance(key, str): + raise ValueError(f"Invalid key type: {key}. Keys must be strings.") + + self.export(config) + + def append_to_file(self, additional_config: dict): + """Appends additional configuration to the existing .kv file.""" + try: + with open(self.output_path, 'a') as file: + for key, value in additional_config.items(): + value_type = type(value).__name__ + if value is None: + value = 'none' + file.write(f"{key}<{value_type}>:{value}\n") + except IOError as e: + raise IOError(f"Failed to append to file {self.output_path}: {e}") \ No newline at end of file diff --git a/kvprocessor/kvfilemerger.py b/kvprocessor/kvfilemerger.py new file mode 100644 index 0000000..cbf3731 --- /dev/null +++ b/kvprocessor/kvfilemerger.py @@ -0,0 +1,18 @@ +class KVFileMerger: + """Merges multiple .kv files into one.""" + def __init__(self, output_path: str): + self.output_path = output_path + + def merge(self, file_paths: list): + merged_data = {} + for file_path in file_paths: + with open(file_path, 'r') as file: + for line in file: + line = line.strip() + if not line or line.startswith('#'): + continue + key, _, value = line.partition(':') + merged_data[key] = value + with open(self.output_path, 'w') as file: + for key, value in merged_data.items(): + file.write(f"{key}:{value}\n") \ No newline at end of file diff --git a/kvprocessor/kvprocessor.py b/kvprocessor/kvprocessor.py index e8e7141..6a8e5bd 100644 --- a/kvprocessor/kvprocessor.py +++ b/kvprocessor/kvprocessor.py @@ -87,4 +87,4 @@ def process_config(self, config: Dict[str, Any]) -> Dict[str, Any]: def return_names(self) -> list: """Return the names of the keys in the KV file.""" - return list(self.config_spec.keys()) \ No newline at end of file + return list(self.config_spec.keys()) diff --git a/kvprocessor/log.py b/kvprocessor/log.py index 3c0d0bf..8f824a1 100644 --- a/kvprocessor/log.py +++ b/kvprocessor/log.py @@ -2,4 +2,11 @@ def log(message: str): if os.environ.get("DEBUG"): - print(message) \ No newline at end of file + print(f"INFO: {message}") + +def log_error(message: str): + print(f"ERROR: {message}") + +def log_debug(message: str): + if os.environ.get("DEBUG"): + print(f"DEBUG: {message}") \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 3cd103d..95e22f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "kvprocessor" -version = "0.1.12" +version = "0.2.1" description = "A Python package for processing and validating configuration dictionaries against a custom .kv file format" readme = "README.md" authors = [{name = "connor33341", email = "connor@connor33341.dev"}] From 2ab9de48a0f49a0b510a5f4f96e5b4b16a27ace5 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Tue, 29 Apr 2025 20:57:38 +0000 Subject: [PATCH 02/34] version 0.2.2 --- kvprocessor/cli.py | 68 +++++++++++++++++++++++++++++++++ kvprocessor/kvfileutils.py | 23 +++++++++++ kvprocessor/kvmanifestloader.py | 30 ++++++++++++++- kvprocessor/kvvalidator.py | 32 +++++++++++++++- kvprocessor/kvversionmanager.py | 35 +++++++++++++++++ kvprocessor/log.py | 23 ++++++++--- test/test.py | 37 +++++++++++++++++- 7 files changed, 239 insertions(+), 9 deletions(-) create mode 100644 kvprocessor/cli.py create mode 100644 kvprocessor/kvfileutils.py create mode 100644 kvprocessor/kvversionmanager.py diff --git a/kvprocessor/cli.py b/kvprocessor/cli.py new file mode 100644 index 0000000..0a20f60 --- /dev/null +++ b/kvprocessor/cli.py @@ -0,0 +1,68 @@ +import argparse +from kvprocessor.kvvalidator import validate_kv_file +from kvprocessor.kvmanifestloader import KVManifestLoader, NamespaceManager +from kvprocessor.kvprocessor import KVProcessor + +def main(): + parser = argparse.ArgumentParser(description="CLI for kvProcessor") + subparsers = parser.add_subparsers(dest="command") + + # Subcommand: Validate .kv file + validate_parser = subparsers.add_parser("validate", help="Validate a .kv file") + validate_parser.add_argument("file", type=str, help="Path to the .kv file") + + # Subcommand: List namespaces + list_parser = subparsers.add_parser("list-namespaces", help="List all namespaces") + list_parser.add_argument("manifest", type=str, help="Path to the manifest file") + + # Subcommand: Add namespace + add_parser = subparsers.add_parser("add-namespace", help="Add a new namespace") + add_parser.add_argument("manifest", type=str, help="Path to the manifest file") + add_parser.add_argument("key", type=str, help="Namespace key") + add_parser.add_argument("value", type=str, help="Namespace value") + + # Subcommand: Remove namespace + remove_parser = subparsers.add_parser("remove-namespace", help="Remove a namespace") + remove_parser.add_argument("manifest", type=str, help="Path to the manifest file") + remove_parser.add_argument("key", type=str, help="Namespace key") + + args = parser.parse_args() + + if args.command == "validate": + try: + if validate_kv_file(args.file): + print(f"{args.file} is valid.") + except Exception as e: + print(f"Validation failed: {e}") + + elif args.command == "list-namespaces": + try: + manifest_loader = KVManifestLoader(args.manifest) + manager = NamespaceManager(manifest_loader) + namespaces = manager.list_namespaces() + print("Available namespaces:") + for namespace in namespaces: + print(namespace) + except Exception as e: + print(f"Error listing namespaces: {e}") + + elif args.command == "add-namespace": + try: + manifest_loader = KVManifestLoader(args.manifest) + manager = NamespaceManager(manifest_loader) + manager.add_namespace(args.key, args.value) + print(f"Namespace {args.key} added successfully.") + except Exception as e: + print(f"Error adding namespace: {e}") + + elif args.command == "remove-namespace": + try: + manifest_loader = KVManifestLoader(args.manifest) + manager = NamespaceManager(manifest_loader) + manager.remove_namespace(args.key) + print(f"Namespace {args.key} removed successfully.") + except Exception as e: + print(f"Error removing namespace: {e}") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/kvprocessor/kvfileutils.py b/kvprocessor/kvfileutils.py new file mode 100644 index 0000000..5b3924f --- /dev/null +++ b/kvprocessor/kvfileutils.py @@ -0,0 +1,23 @@ +import os +import shutil + +def search_kv_files(directory: str) -> list: + """Search for all .kv files in a directory.""" + kv_files = [] + for root, _, files in os.walk(directory): + for file in files: + if file.endswith(".kv"): + kv_files.append(os.path.join(root, file)) + return kv_files + +def copy_kv_file(source: str, destination: str): + """Copy a .kv file to a new location.""" + if not os.path.exists(source): + raise FileNotFoundError(f"Source file not found: {source}") + shutil.copy(source, destination) + +def delete_kv_file(file_path: str): + """Delete a .kv file.""" + if not os.path.exists(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + os.remove(file_path) \ No newline at end of file diff --git a/kvprocessor/kvmanifestloader.py b/kvprocessor/kvmanifestloader.py index bb71876..34a285c 100644 --- a/kvprocessor/kvmanifestloader.py +++ b/kvprocessor/kvmanifestloader.py @@ -56,4 +56,32 @@ def _parse_manifest(self): self.namespace_overides[key] = value except FileNotFoundError: print(f"Manifest file not found: {self.file_url}") - return None \ No newline at end of file + return None + +class NamespaceManager: + """Utility class for managing namespaces dynamically.""" + + def __init__(self, manifest_loader: KVManifestLoader): + self.manifest_loader = manifest_loader + + def add_namespace(self, key: str, value: str): + """Add a new namespace to the manifest.""" + if key in self.manifest_loader.namespace_overides: + raise ValueError(f"Namespace {key} already exists.") + self.manifest_loader.namespace_overides[key] = value + + def remove_namespace(self, key: str): + """Remove a namespace from the manifest.""" + if key not in self.manifest_loader.namespace_overides: + raise KeyError(f"Namespace {key} does not exist.") + del self.manifest_loader.namespace_overides[key] + + def list_namespaces(self) -> list: + """List all available namespaces.""" + return list(self.manifest_loader.namespace_overides.keys()) + + def update_namespace(self, key: str, new_value: str): + """Update an existing namespace.""" + if key not in self.manifest_loader.namespace_overides: + raise KeyError(f"Namespace {key} does not exist.") + self.manifest_loader.namespace_overides[key] = new_value \ No newline at end of file diff --git a/kvprocessor/kvvalidator.py b/kvprocessor/kvvalidator.py index 3deb087..f8e682f 100644 --- a/kvprocessor/kvvalidator.py +++ b/kvprocessor/kvvalidator.py @@ -16,4 +16,34 @@ def validate_kv_file(file_path: str) -> bool: raise InvalidKVFileError(f"Invalid .kv file format in line {i}: {line}") return True except FileNotFoundError: - raise FileNotFoundError(f"KV file not found: {file_path}") \ No newline at end of file + raise FileNotFoundError(f"KV file not found: {file_path}") + +def validate_kv_key(key: str) -> bool: + """Validate a single key in a .kv file.""" + match = re.match(r'(\w+)<([\w\|]+)>:([\w+]+|none)', key) + if not match: + raise InvalidKVFileError(f"Invalid key format: {key}") + return True + +def validate_kv_value(value: str, expected_types: list) -> bool: + """Validate a value against expected types.""" + type_map = { + 'string': str, + 'int': int, + 'float': float, + 'bool': bool, + 'none': type(None), + 'list': list, + 'dict': dict, + 'tuple': tuple, + 'set': set, + 'object': object, + 'any': object, + 'str': str + } + for type_name in expected_types: + if type_name not in type_map: + raise ValueError(f"Unsupported type: {type_name}") + if isinstance(value, type_map[type_name]): + return True + return False \ No newline at end of file diff --git a/kvprocessor/kvversionmanager.py b/kvprocessor/kvversionmanager.py new file mode 100644 index 0000000..9383c9f --- /dev/null +++ b/kvprocessor/kvversionmanager.py @@ -0,0 +1,35 @@ +import os +import shutil +from datetime import datetime + +class KVVersionManager: + """Manages versions of .kv files.""" + + def __init__(self, version_dir: str = "./versions"): + self.version_dir = version_dir + os.makedirs(self.version_dir, exist_ok=True) + + def save_version(self, file_path: str): + """Save a version of the .kv file with a timestamp.""" + if not os.path.exists(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + timestamp = datetime.now().strftime("%Y%m%d%H%M%S") + file_name = os.path.basename(file_path) + versioned_file = os.path.join(self.version_dir, f"{file_name}.{timestamp}") + shutil.copy(file_path, versioned_file) + return versioned_file + + def list_versions(self, file_name: str) -> list: + """List all saved versions of a .kv file.""" + versions = [] + for file in os.listdir(self.version_dir): + if file.startswith(file_name): + versions.append(file) + return sorted(versions) + + def restore_version(self, file_name: str, timestamp: str, restore_path: str): + """Restore a specific version of a .kv file.""" + versioned_file = os.path.join(self.version_dir, f"{file_name}.{timestamp}") + if not os.path.exists(versioned_file): + raise FileNotFoundError(f"Version not found: {versioned_file}") + shutil.copy(versioned_file, restore_path) \ No newline at end of file diff --git a/kvprocessor/log.py b/kvprocessor/log.py index 8f824a1..f24647c 100644 --- a/kvprocessor/log.py +++ b/kvprocessor/log.py @@ -1,12 +1,23 @@ -import os +import logging + +# Configure logging +logging.basicConfig( + level=logging.DEBUG, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[ + logging.StreamHandler(), + logging.FileHandler("kvprocessor.log", mode="a") + ] +) def log(message: str): - if os.environ.get("DEBUG"): - print(f"INFO: {message}") + logging.info(message) def log_error(message: str): - print(f"ERROR: {message}") + logging.error(message) def log_debug(message: str): - if os.environ.get("DEBUG"): - print(f"DEBUG: {message}") \ No newline at end of file + logging.debug(message) + +def log_warning(message: str): + logging.warning(message) \ No newline at end of file diff --git a/test/test.py b/test/test.py index 63a5e64..e0a0159 100644 --- a/test/test.py +++ b/test/test.py @@ -1,6 +1,8 @@ import os import dotenv from kvprocessor import LoadEnv, KVProcessor, KVStructLoader +from kvprocessor.kvfileutils import search_kv_files, copy_kv_file, delete_kv_file +from kvprocessor.kvversionmanager import KVVersionManager dotenv.load_dotenv() # Load the .env file def test_file(): @@ -24,6 +26,39 @@ def test_struct_loader(): } validated_config = kv_processor.process_config(user_settings) # Verifies that those env varibles exist and are of the correct type print(validated_config) + +def test_file_operations(): + print("Testing file operations") + kv_files = search_kv_files("test") + print("Found .kv files:", kv_files) + + if kv_files: + test_file = kv_files[0] + copy_path = "test/copy_test.kv" + copy_kv_file(test_file, copy_path) + print(f"Copied {test_file} to {copy_path}") + + delete_kv_file(copy_path) + print(f"Deleted {copy_path}") + +def test_version_manager(): + print("Testing version manager") + version_manager = KVVersionManager("test/versions") + + test_file = "test/test.kv" + versioned_file = version_manager.save_version(test_file) + print(f"Saved version: {versioned_file}") + + versions = version_manager.list_versions("test.kv") + print("Available versions:", versions) + + if versions: + restore_path = "test/restored_test.kv" + version_manager.restore_version("test.kv", versions[0].split(".")[-1], restore_path) + print(f"Restored version to: {restore_path}") + if __name__ == "__main__": test_file() - test_struct_loader() \ No newline at end of file + test_struct_loader() + test_file_operations() + test_version_manager() \ No newline at end of file From 99efee3606e6fd2324c71c94110f003964758a91 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Wed, 30 Apr 2025 14:41:27 +0000 Subject: [PATCH 03/34] Move legacy errors --- kvprocessor/errors.py | 18 ++++++++++++++++++ kvprocessor/kvstructloader.py | 18 +----------------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/kvprocessor/errors.py b/kvprocessor/errors.py index a27f711..9602ebd 100644 --- a/kvprocessor/errors.py +++ b/kvprocessor/errors.py @@ -16,4 +16,22 @@ class NamespaceNotFoundError(KVProcessorError): class InvalidNamespaceError(KVProcessorError): """Raised when a namespace is invalid or malformed.""" + pass + +# Legacy errors + +class KVStructLoaderError(Exception): + """Base exception for KVStructLoader.""" + pass + +class ConfigFetchError(KVStructLoaderError): + """Raised when there is an error fetching the configuration.""" + pass + +class KVFetchError(KVStructLoaderError): + """Raised when there is an error fetching a KV file.""" + pass + +class ManifestError(KVStructLoaderError): + """Raised when there is an issue with the manifest.""" pass \ No newline at end of file diff --git a/kvprocessor/kvstructloader.py b/kvprocessor/kvstructloader.py index b66c421..8c5c213 100644 --- a/kvprocessor/kvstructloader.py +++ b/kvprocessor/kvstructloader.py @@ -5,23 +5,7 @@ from kvprocessor.kvprocessor import KVProcessor from kvprocessor.kvmanifestloader import KVManifestLoader from kvprocessor.log import log -from kvprocessor.errors import NamespaceNotFoundError, InvalidNamespaceError - -class KVStructLoaderError(Exception): - """Base exception for KVStructLoader.""" - pass - -class ConfigFetchError(KVStructLoaderError): - """Raised when there is an error fetching the configuration.""" - pass - -class KVFetchError(KVStructLoaderError): - """Raised when there is an error fetching a KV file.""" - pass - -class ManifestError(KVStructLoaderError): - """Raised when there is an issue with the manifest.""" - pass +from kvprocessor.errors import NamespaceNotFoundError, InvalidNamespaceError, ManifestError, ConfigFetchError, KVFetchError class KVStructLoader: def __init__(self, config_file: str, cache_dir: str = "./struct"): From 44a6267674051e434f6daef5b8dbe52d6dc97f96 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Wed, 30 Apr 2025 16:00:00 +0000 Subject: [PATCH 04/34] update init --- kvprocessor/__init__.py | 3 +++ kvprocessor/kvmanifestloader.py | 13 ++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/kvprocessor/__init__.py b/kvprocessor/__init__.py index c9b923d..a8cd067 100644 --- a/kvprocessor/__init__.py +++ b/kvprocessor/__init__.py @@ -6,4 +6,7 @@ from .kvfileexporter import KVFileExporter from .kvfilemerger import KVFileMerger from .kvdiff import KVFileDiffChecker +from .kvvalidator import KVFileValidator +from .kvversionmanager import KVVersionManager +from .kvdiff import KVFileDiffChecker from .errors import KVProcessorError, InvalidKVFileError, MissingEnvironmentVariableError, NamespaceNotFoundError, InvalidNamespaceError \ No newline at end of file diff --git a/kvprocessor/kvmanifestloader.py b/kvprocessor/kvmanifestloader.py index 34a285c..01f2802 100644 --- a/kvprocessor/kvmanifestloader.py +++ b/kvprocessor/kvmanifestloader.py @@ -2,6 +2,7 @@ import requests import re from kvprocessor.kvprocessor import KVProcessor +from kvprocessor.kvstructloader import KVStructLoader from kvprocessor.log import log class KVManifestLoader: @@ -13,6 +14,7 @@ def __init__(self, file_url: str, cache_dir: str = "./struct", root: str = None) self.namespace_overides = {} self._fetch_manifest() self._parse_manifest() + self.manifest_version = "0.1.12" def _fetch_manifest(self): try: @@ -44,11 +46,12 @@ def _parse_manifest(self): continue match: dict = re.match(r'([^:]+):([^:]+)', line) if not match: - if (len(line.split(":")) == 0) and (len(line.split(".") >= 1)): - log("Found namespace") - match.clear() - match[str(line).strip()] = str(line).strip() - raise ValueError(f"Invalid manifest file format in line: {line}") + if str(self.manifest_version).strip().split(".")[1] >= 2: + if (len(line.split(":")) == 0) and (len(line.split(".") >= 1)): + log("Found namespace") + match.clear() + match[str(line).strip()] = str(line).strip() + raise ValueError(f"Invalid manifest file format in line: {line}") else: log("Found namespace overide") key, value = match.groups() From 5d5942b3261a399e78e1c5bc1c305b7b9faf4fde Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Wed, 30 Apr 2025 17:30:40 +0000 Subject: [PATCH 05/34] update kvtypemap --- kvprocessor/__init__.py | 5 +-- kvprocessor/kvenvloader.py | 6 +++- kvprocessor/kvprocessor.py | 16 ++------- kvprocessor/kvtypemap.py | 67 ++++++++++++++++++++++++++++++++++++++ kvprocessor/kvvalidator.py | 16 ++------- pyproject.toml | 2 +- 6 files changed, 80 insertions(+), 32 deletions(-) create mode 100644 kvprocessor/kvtypemap.py diff --git a/kvprocessor/__init__.py b/kvprocessor/__init__.py index a8cd067..0a8aa35 100644 --- a/kvprocessor/__init__.py +++ b/kvprocessor/__init__.py @@ -1,7 +1,7 @@ -__version__ = "0.2.1" +__version__ = "0.2.14" from .kvprocessor import KVProcessor -from .kvenvloader import LoadEnv +from .kvenvloader import load_env, LoadEnv from .kvstructloader import KVStructLoader from .kvfileexporter import KVFileExporter from .kvfilemerger import KVFileMerger @@ -9,4 +9,5 @@ from .kvvalidator import KVFileValidator from .kvversionmanager import KVVersionManager from .kvdiff import KVFileDiffChecker +from .kvtypemap import get_type_map, set_type_map, remove_type_map, has_type_map, clear_type_map, add_type_map from .errors import KVProcessorError, InvalidKVFileError, MissingEnvironmentVariableError, NamespaceNotFoundError, InvalidNamespaceError \ No newline at end of file diff --git a/kvprocessor/kvenvloader.py b/kvprocessor/kvenvloader.py index 38297f4..e2018be 100644 --- a/kvprocessor/kvenvloader.py +++ b/kvprocessor/kvenvloader.py @@ -1,8 +1,12 @@ import os -def LoadEnv(Names: list) -> dict[str, any]: +def load_env(Names: list) -> dict[str, any]: EnvList = {} for Name in Names: Value = os.environ.get(Name) EnvList[Name] = Value return EnvList + +def LoadEnv(Names: list) -> dict[str, any]: + print("Using depricated function, update to load_env") + return load_env(Names) \ No newline at end of file diff --git a/kvprocessor/kvprocessor.py b/kvprocessor/kvprocessor.py index 6a8e5bd..3083ed2 100644 --- a/kvprocessor/kvprocessor.py +++ b/kvprocessor/kvprocessor.py @@ -2,6 +2,7 @@ from typing import Dict, Any, Union from kvprocessor.log import log from kvprocessor.errors import InvalidKVFileError +from kvprocessor.kvtypemap import get_type_map class KVProcessor: def __init__(self, kv_file_path: str): @@ -36,20 +37,7 @@ def _parse_kv_file(self, file_path: str) -> Dict[str, dict]: def _validate_type(self, value: Any, expected_types: list) -> bool: """Validate if the value matches one of the expected types.""" - type_map = { - 'string': str, - 'int': int, - 'float': float, - 'bool': bool, - 'none': type(None), - 'list': list, - 'dict': dict, - 'tuple': tuple, - 'set': set, - 'object': object, - 'any': Any, - 'str': str - } + type_map = get_type_map() for type_name in expected_types: if type_name not in type_map: raise ValueError(f"Unsupported type in .kv file: {type_name}") diff --git a/kvprocessor/kvtypemap.py b/kvprocessor/kvtypemap.py new file mode 100644 index 0000000..eb45608 --- /dev/null +++ b/kvprocessor/kvtypemap.py @@ -0,0 +1,67 @@ +from typing import Any, Dict + +type_map = { + 'string': str, + 'int': int, + 'float': float, + 'bool': bool, + 'none': type(None), + 'list': list, + 'dict': dict, + 'tuple': tuple, + 'set': set, + 'object': object, + 'any': Any, + 'str': str +} + +def get_type_map() -> Dict[str, type]: + """ + Returns the type map for KV types. + + :return: A dictionary mapping KV types to Python types. + """ + return type_map + +def set_type_map(new_type_map: Dict[str, type]) -> None: + """ + Replaces the current type map with a new one. + + :param new_type_map: A dictionary mapping KV types to Python types. + """ + global type_map + type_map = new_type_map + +def add_type_map(key: str, value: type) -> None: + """ + Adds or overrides a single key-value pair in the type map. + + :param key: The key to add or override in the type map. + :param value: The Python type to associate with the key. + """ + type_map[key] = value + +def remove_type_map(key: str) -> None: + """ + Removes a key from the type map if it exists. + + :param key: The key to remove from the type map. + """ + if key in type_map: + del type_map[key] + +def has_type_map(key: str) -> bool: + """ + Checks if a key exists in the type map. + + :param key: The key to check in the type map. + :return: True if the key exists, False otherwise. + """ + return key in type_map + +def clear_type_map() -> None: + """ + Clears all entries in the type map. + """ + global type_map + type_map.clear() diff --git a/kvprocessor/kvvalidator.py b/kvprocessor/kvvalidator.py index f8e682f..c946504 100644 --- a/kvprocessor/kvvalidator.py +++ b/kvprocessor/kvvalidator.py @@ -1,4 +1,5 @@ from kvprocessor.errors import InvalidKVFileError +from kvprocessor.kvtypemap import get_type_map import re def validate_kv_file(file_path: str) -> bool: @@ -27,20 +28,7 @@ def validate_kv_key(key: str) -> bool: def validate_kv_value(value: str, expected_types: list) -> bool: """Validate a value against expected types.""" - type_map = { - 'string': str, - 'int': int, - 'float': float, - 'bool': bool, - 'none': type(None), - 'list': list, - 'dict': dict, - 'tuple': tuple, - 'set': set, - 'object': object, - 'any': object, - 'str': str - } + type_map = get_type_map() for type_name in expected_types: if type_name not in type_map: raise ValueError(f"Unsupported type: {type_name}") diff --git a/pyproject.toml b/pyproject.toml index 95e22f8..62f01e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "kvprocessor" -version = "0.2.1" +version = "0.2.14" description = "A Python package for processing and validating configuration dictionaries against a custom .kv file format" readme = "README.md" authors = [{name = "connor33341", email = "connor@connor33341.dev"}] From ccdef533b33e58463279ffc83c0ea6bd585d4014 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Thu, 1 May 2025 11:46:35 +0000 Subject: [PATCH 06/34] fetch version from KVStructLoader --- kvprocessor/kvmanifestloader.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kvprocessor/kvmanifestloader.py b/kvprocessor/kvmanifestloader.py index 01f2802..7ac535d 100644 --- a/kvprocessor/kvmanifestloader.py +++ b/kvprocessor/kvmanifestloader.py @@ -1,6 +1,8 @@ import os +import urllib.parse import requests import re +from urllib.parse import urlparse, urlunparse from kvprocessor.kvprocessor import KVProcessor from kvprocessor.kvstructloader import KVStructLoader from kvprocessor.log import log @@ -14,7 +16,7 @@ def __init__(self, file_url: str, cache_dir: str = "./struct", root: str = None) self.namespace_overides = {} self._fetch_manifest() self._parse_manifest() - self.manifest_version = "0.1.12" + self.manifest_version = KVStructLoader(urlparse(self.file_url).path.rsplit('/', 1)[0] + '/config.json', self.cache_dir).version def _fetch_manifest(self): try: From d9be0706ac87d07c9a9676a987913df900cdbafe Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Thu, 1 May 2025 11:55:49 +0000 Subject: [PATCH 07/34] add glovel settings --- kvprocessor/__init__.py | 1 + kvprocessor/kvglobalsettings.py | 21 +++++++++++++++++++++ kvprocessor/kvmanifestloader.py | 3 ++- 3 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 kvprocessor/kvglobalsettings.py diff --git a/kvprocessor/__init__.py b/kvprocessor/__init__.py index 0a8aa35..756692d 100644 --- a/kvprocessor/__init__.py +++ b/kvprocessor/__init__.py @@ -10,4 +10,5 @@ from .kvversionmanager import KVVersionManager from .kvdiff import KVFileDiffChecker from .kvtypemap import get_type_map, set_type_map, remove_type_map, has_type_map, clear_type_map, add_type_map +from .kvglobalsettings import set_version, get_version, get_version_tuple, get_version_major, get_version_minor from .errors import KVProcessorError, InvalidKVFileError, MissingEnvironmentVariableError, NamespaceNotFoundError, InvalidNamespaceError \ No newline at end of file diff --git a/kvprocessor/kvglobalsettings.py b/kvprocessor/kvglobalsettings.py new file mode 100644 index 0000000..c3e1027 --- /dev/null +++ b/kvprocessor/kvglobalsettings.py @@ -0,0 +1,21 @@ +import kvprocessor + +version = str(kvprocessor.__version__) + +def set_version(v: str): + """Set the version of the KVProcessor.""" + global version + verison = v + +def get_version() -> str: + """Get the version of the KVProcessor.""" + return version +def get_version_tuple() -> tuple[int, int, int]: + """Get the version of the KVProcessor as a tuple.""" + return tuple(map(int, version.split('.'))) +def get_version_major() -> int: + """Get the major version of the KVProcessor.""" + return int(version.split('.')[0]) +def get_version_minor() -> int: + """Get the minor version of the KVProcessor.""" + return int(version.split('.')[1]) \ No newline at end of file diff --git a/kvprocessor/kvmanifestloader.py b/kvprocessor/kvmanifestloader.py index 7ac535d..31eec82 100644 --- a/kvprocessor/kvmanifestloader.py +++ b/kvprocessor/kvmanifestloader.py @@ -3,6 +3,7 @@ import requests import re from urllib.parse import urlparse, urlunparse +from kvprocessor.kvglobalsettings import get_version_major, get_version_minor, get_version from kvprocessor.kvprocessor import KVProcessor from kvprocessor.kvstructloader import KVStructLoader from kvprocessor.log import log @@ -16,7 +17,7 @@ def __init__(self, file_url: str, cache_dir: str = "./struct", root: str = None) self.namespace_overides = {} self._fetch_manifest() self._parse_manifest() - self.manifest_version = KVStructLoader(urlparse(self.file_url).path.rsplit('/', 1)[0] + '/config.json', self.cache_dir).version + self.manifest_version = KVStructLoader(str(urlparse(self.file_url).path.rsplit('/', 1)[0] + '/config.json'), self.cache_dir).version or get_version() def _fetch_manifest(self): try: From 55e395ded85875d528424740456f01a3092b85ca Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Thu, 1 May 2025 15:16:42 +0000 Subject: [PATCH 08/34] Update TODO.md --- TODO.md | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 TODO.md diff --git a/TODO.md b/TODO.md new file mode 100644 index 0000000..351bfcb --- /dev/null +++ b/TODO.md @@ -0,0 +1,11 @@ +# TODO + +### Documentation: + - Create a doc/wiki with mkdocs + - Update `README.md` to have basic examples for 0.2.x versions of the API + +### API: + - Migrate to `kvprocessor.errors` for custom errors + - Update cli to support 0.2.x verions of the API + - Expand `kvprocessor.kvtypemap` to support more types. + - Update `kvprocessor.kvmanifestloader` to be more feature rich \ No newline at end of file From be9d9ce8b58b653e365ae952aa3e2c5da1332a46 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Thu, 1 May 2025 15:18:05 +0000 Subject: [PATCH 09/34] Update TODO.md --- TODO.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/TODO.md b/TODO.md index 351bfcb..f87ff7d 100644 --- a/TODO.md +++ b/TODO.md @@ -4,6 +4,11 @@ - Create a doc/wiki with mkdocs - Update `README.md` to have basic examples for 0.2.x versions of the API +### Testing: + - Update test.py to test for most usecases with the 0.2.x API + - Use pytest aswell + - Create a GHAction to makesure that all tests are passed. + ### API: - Migrate to `kvprocessor.errors` for custom errors - Update cli to support 0.2.x verions of the API From f74ee64e82e02ed98faff039baaeabd6faf7c384 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Thu, 1 May 2025 19:02:34 +0000 Subject: [PATCH 10/34] add some pytest targets --- test/requirements.txt | 3 +- test/test.py | 67 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/test/requirements.txt b/test/requirements.txt index 40448e6..4524122 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1 +1,2 @@ -dotenv \ No newline at end of file +dotenv +pytest \ No newline at end of file diff --git a/test/test.py b/test/test.py index e0a0159..10cdb56 100644 --- a/test/test.py +++ b/test/test.py @@ -1,5 +1,6 @@ import os import dotenv +import pytest from kvprocessor import LoadEnv, KVProcessor, KVStructLoader from kvprocessor.kvfileutils import search_kv_files, copy_kv_file, delete_kv_file from kvprocessor.kvversionmanager import KVVersionManager @@ -57,6 +58,72 @@ def test_version_manager(): version_manager.restore_version("test.kv", versions[0].split(".")[-1], restore_path) print(f"Restored version to: {restore_path}") +@pytest.fixture +def kv_processor(): + kv_file_path = "test/test.kv" # Directory to .kv file + return KVProcessor(kv_file_path) + +@pytest.fixture +def kv_struct_loader(): + return KVStructLoader("https://github.com/Voxa-Communications/VoxaCommunicaitons-Structures/raw/refs/heads/main/struct/config.json") + +def test_kv_processor_return_names(kv_processor): + kv_keys = kv_processor.return_names() # Gets the keys (VARIBLENAME) from the .kv file + assert isinstance(kv_keys, list) + assert len(kv_keys) > 0 + +def test_kv_processor_process_config(kv_processor): + kv_keys = kv_processor.return_names() + env_list = LoadEnv(kv_keys) # Loads all the ENV variables that match those keys + validated_config = kv_processor.process_config(env_list) # Verifies that those env variables exist and are of the correct type + assert isinstance(validated_config, dict) + +def test_kv_struct_loader_root(kv_struct_loader): + assert kv_struct_loader.root == "voxa" + +def test_kv_struct_loader_url(kv_struct_loader): + assert kv_struct_loader.URL.startswith("https://") + +def test_kv_struct_loader_namespace(kv_struct_loader): + kv_processor = kv_struct_loader.from_namespace("voxa.api.user.user_settings") + user_settings = { + "2FA_ENABLED": True, + "TELEMETRY": False, + "AGE": "25", + "LANGUAGE": "en", + } + validated_config = kv_processor.process_config(user_settings) + assert isinstance(validated_config, dict) + assert validated_config["2FA_ENABLED"] is True + assert validated_config["TELEMETRY"] is False + +def test_file_operations(): + kv_files = search_kv_files("test") + assert len(kv_files) > 0 + + test_file = kv_files[0] + copy_path = "test/copy_test.kv" + copy_kv_file(test_file, copy_path) + assert os.path.exists(copy_path) + + delete_kv_file(copy_path) + assert not os.path.exists(copy_path) + +def test_version_manager(): + version_manager = KVVersionManager("test/versions") + + test_file = "test/test.kv" + versioned_file = version_manager.save_version(test_file) + assert os.path.exists(versioned_file) + + versions = version_manager.list_versions("test.kv") + assert len(versions) > 0 + + restore_path = "test/restored_test.kv" + version_manager.restore_version("test.kv", versions[0].split(".")[-1], restore_path) + assert os.path.exists(restore_path) + delete_kv_file(restore_path) + if __name__ == "__main__": test_file() test_struct_loader() From 8c7f527910c2ad04fce1b68fadac68263b4712ce Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Thu, 1 May 2025 19:47:52 +0000 Subject: [PATCH 11/34] depricate some stuff, and fix a few things --- .gitignore | 3 +- kvprocessor/__init__.py | 9 +++- kvprocessor/kvenvloader.py | 4 +- kvprocessor/kvglobalsettings.py | 8 +++- kvprocessor/kvmanifestloader.py | 36 ++------------- kvprocessor/kvnamespacemanager.py | 29 ++++++++++++ kvprocessor/kvvalidator.py | 75 +++++++++++++++++++------------ kvprocessor/warnings.py | 8 ++++ 8 files changed, 107 insertions(+), 65 deletions(-) create mode 100644 kvprocessor/kvnamespacemanager.py create mode 100644 kvprocessor/warnings.py diff --git a/.gitignore b/.gitignore index e0d8e0a..87c09ca 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ dist .pypirc -struct \ No newline at end of file +struct +*.log \ No newline at end of file diff --git a/kvprocessor/__init__.py b/kvprocessor/__init__.py index 756692d..923680c 100644 --- a/kvprocessor/__init__.py +++ b/kvprocessor/__init__.py @@ -5,10 +5,17 @@ from .kvstructloader import KVStructLoader from .kvfileexporter import KVFileExporter from .kvfilemerger import KVFileMerger +from .kvfileutils import ( + search_kv_files, + copy_kv_file, + delete_kv_file, +) from .kvdiff import KVFileDiffChecker -from .kvvalidator import KVFileValidator +from .kvnamespacemanager import NamespaceManager as KVNamespaceManager +from .kvmanifestloader import KVManifestLoader from .kvversionmanager import KVVersionManager from .kvdiff import KVFileDiffChecker +from .kvvalidator import KVFileValidator, validate_kv_file, validate_kv_key, validate_kv_value from .kvtypemap import get_type_map, set_type_map, remove_type_map, has_type_map, clear_type_map, add_type_map from .kvglobalsettings import set_version, get_version, get_version_tuple, get_version_major, get_version_minor from .errors import KVProcessorError, InvalidKVFileError, MissingEnvironmentVariableError, NamespaceNotFoundError, InvalidNamespaceError \ No newline at end of file diff --git a/kvprocessor/kvenvloader.py b/kvprocessor/kvenvloader.py index e2018be..f1b8a9e 100644 --- a/kvprocessor/kvenvloader.py +++ b/kvprocessor/kvenvloader.py @@ -1,4 +1,6 @@ import os +import warnings +from kvprocessor.warnings import deprecated def load_env(Names: list) -> dict[str, any]: EnvList = {} @@ -7,6 +9,6 @@ def load_env(Names: list) -> dict[str, any]: EnvList[Name] = Value return EnvList +@deprecated def LoadEnv(Names: list) -> dict[str, any]: - print("Using depricated function, update to load_env") return load_env(Names) \ No newline at end of file diff --git a/kvprocessor/kvglobalsettings.py b/kvprocessor/kvglobalsettings.py index c3e1027..c08d694 100644 --- a/kvprocessor/kvglobalsettings.py +++ b/kvprocessor/kvglobalsettings.py @@ -1,4 +1,5 @@ import kvprocessor +from urllib.parse import urlparse, urlunparse version = str(kvprocessor.__version__) @@ -18,4 +19,9 @@ def get_version_major() -> int: return int(version.split('.')[0]) def get_version_minor() -> int: """Get the minor version of the KVProcessor.""" - return int(version.split('.')[1]) \ No newline at end of file + return int(version.split('.')[1]) +@DeprecationWarning +def get_config_version_from_url(url: str) -> str: + """Get the config version of the from a URL.""" + from kvprocessor.kvstructloader import KVStructLoader # Local import to avoid circular dependency + return KVStructLoader(str(urlparse(url).path.rsplit('/', 1)[0] + '/config.json')).version \ No newline at end of file diff --git a/kvprocessor/kvmanifestloader.py b/kvprocessor/kvmanifestloader.py index 31eec82..ea329e3 100644 --- a/kvprocessor/kvmanifestloader.py +++ b/kvprocessor/kvmanifestloader.py @@ -1,15 +1,13 @@ import os -import urllib.parse import requests import re from urllib.parse import urlparse, urlunparse from kvprocessor.kvglobalsettings import get_version_major, get_version_minor, get_version from kvprocessor.kvprocessor import KVProcessor -from kvprocessor.kvstructloader import KVStructLoader from kvprocessor.log import log class KVManifestLoader: - def __init__(self, file_url: str, cache_dir: str = "./struct", root: str = None): + def __init__(self, file_url: str, cache_dir: str = "./struct", root: str = None, manifest_version: str = get_version()): self.file_url = file_url self.cache_dir = cache_dir self.root = root @@ -17,7 +15,7 @@ def __init__(self, file_url: str, cache_dir: str = "./struct", root: str = None) self.namespace_overides = {} self._fetch_manifest() self._parse_manifest() - self.manifest_version = KVStructLoader(str(urlparse(self.file_url).path.rsplit('/', 1)[0] + '/config.json'), self.cache_dir).version or get_version() + self.manifest_version = manifest_version def _fetch_manifest(self): try: @@ -62,32 +60,4 @@ def _parse_manifest(self): self.namespace_overides[key] = value except FileNotFoundError: print(f"Manifest file not found: {self.file_url}") - return None - -class NamespaceManager: - """Utility class for managing namespaces dynamically.""" - - def __init__(self, manifest_loader: KVManifestLoader): - self.manifest_loader = manifest_loader - - def add_namespace(self, key: str, value: str): - """Add a new namespace to the manifest.""" - if key in self.manifest_loader.namespace_overides: - raise ValueError(f"Namespace {key} already exists.") - self.manifest_loader.namespace_overides[key] = value - - def remove_namespace(self, key: str): - """Remove a namespace from the manifest.""" - if key not in self.manifest_loader.namespace_overides: - raise KeyError(f"Namespace {key} does not exist.") - del self.manifest_loader.namespace_overides[key] - - def list_namespaces(self) -> list: - """List all available namespaces.""" - return list(self.manifest_loader.namespace_overides.keys()) - - def update_namespace(self, key: str, new_value: str): - """Update an existing namespace.""" - if key not in self.manifest_loader.namespace_overides: - raise KeyError(f"Namespace {key} does not exist.") - self.manifest_loader.namespace_overides[key] = new_value \ No newline at end of file + return None \ No newline at end of file diff --git a/kvprocessor/kvnamespacemanager.py b/kvprocessor/kvnamespacemanager.py new file mode 100644 index 0000000..c433459 --- /dev/null +++ b/kvprocessor/kvnamespacemanager.py @@ -0,0 +1,29 @@ +from kvprocessor.kvmanifestloader import KVManifestLoader + +class NamespaceManager: + """Utility class for managing namespaces dynamically.""" + + def __init__(self, manifest_loader: KVManifestLoader): + self.manifest_loader = manifest_loader + + def add_namespace(self, key: str, value: str): + """Add a new namespace to the manifest.""" + if key in self.manifest_loader.namespace_overides: + raise ValueError(f"Namespace {key} already exists.") + self.manifest_loader.namespace_overides[key] = value + + def remove_namespace(self, key: str): + """Remove a namespace from the manifest.""" + if key not in self.manifest_loader.namespace_overides: + raise KeyError(f"Namespace {key} does not exist.") + del self.manifest_loader.namespace_overides[key] + + def list_namespaces(self) -> list: + """List all available namespaces.""" + return list(self.manifest_loader.namespace_overides.keys()) + + def update_namespace(self, key: str, new_value: str): + """Update an existing namespace.""" + if key not in self.manifest_loader.namespace_overides: + raise KeyError(f"Namespace {key} does not exist.") + self.manifest_loader.namespace_overides[key] = new_value \ No newline at end of file diff --git a/kvprocessor/kvvalidator.py b/kvprocessor/kvvalidator.py index c946504..9e92cc2 100644 --- a/kvprocessor/kvvalidator.py +++ b/kvprocessor/kvvalidator.py @@ -1,37 +1,56 @@ +import re from kvprocessor.errors import InvalidKVFileError from kvprocessor.kvtypemap import get_type_map -import re +from kvprocessor.warnings import deprecated + +class KVFileValidator(): + def __init__(self): + self.file_path: str = None + + def validate_kv_file(self, file_path: str) -> bool: + if self.file_path is None: + self.file_path = file_path + try: + with open(self.file_path, 'r') as file: + for i, line in enumerate(file, start=1): + line = line.strip() + if not line or line.startswith('#'): + continue + if line.split("#"): + line = line.split("#")[0].strip() + match = re.match(r'(\w+)<([\w\|]+)>:([\w+]+|none)', line) + if not match: + raise InvalidKVFileError(f"Invalid .kv file format in line {i}: {line}") + return True + except FileNotFoundError: + raise FileNotFoundError(f"KV file not found: {self.file_path}") + def validate_kv_key(key: str) -> bool: + """Validate a single key in a .kv file.""" + match = re.match(r'(\w+)<([\w\|]+)>:([\w+]+|none)', key) + if not match: + raise InvalidKVFileError(f"Invalid key format: {key}") + return True + + def validate_kv_value(value: str, expected_types: list) -> bool: + """Validate a value against expected types.""" + type_map = get_type_map() + for type_name in expected_types: + if type_name not in type_map: + raise ValueError(f"Unsupported type: {type_name}") + if isinstance(value, type_map[type_name]): + return True + return False + +@deprecated def validate_kv_file(file_path: str) -> bool: """Validate the syntax of a .kv file.""" - try: - with open(file_path, 'r') as file: - for i, line in enumerate(file, start=1): - line = line.strip() - if not line or line.startswith('#'): - continue - if line.split("#"): - line = line.split("#")[0].strip() - match = re.match(r'(\w+)<([\w\|]+)>:([\w+]+|none)', line) - if not match: - raise InvalidKVFileError(f"Invalid .kv file format in line {i}: {line}") - return True - except FileNotFoundError: - raise FileNotFoundError(f"KV file not found: {file_path}") + return KVFileValidator().validate_kv_file(file_path) +@deprecated def validate_kv_key(key: str) -> bool: - """Validate a single key in a .kv file.""" - match = re.match(r'(\w+)<([\w\|]+)>:([\w+]+|none)', key) - if not match: - raise InvalidKVFileError(f"Invalid key format: {key}") - return True + return KVFileValidator().validate_kv_key(key) +@deprecated def validate_kv_value(value: str, expected_types: list) -> bool: - """Validate a value against expected types.""" - type_map = get_type_map() - for type_name in expected_types: - if type_name not in type_map: - raise ValueError(f"Unsupported type: {type_name}") - if isinstance(value, type_map[type_name]): - return True - return False \ No newline at end of file + return KVFileValidator().validate_kv_value(value, expected_types) \ No newline at end of file diff --git a/kvprocessor/warnings.py b/kvprocessor/warnings.py new file mode 100644 index 0000000..3714a3f --- /dev/null +++ b/kvprocessor/warnings.py @@ -0,0 +1,8 @@ +import warnings + +def deprecated(func): + def wrapper(*args, **kwargs): + warnings.warn(f"{func.__name__} is deprecated and will be removed in a future version.", + DeprecationWarning, stacklevel=2) + return func(*args, **kwargs) + return wrapper \ No newline at end of file From 7f9137148a029e3f43f066d9f004850c36487adf Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Thu, 1 May 2025 20:02:20 +0000 Subject: [PATCH 12/34] update the config.json standard, and refactor some code --- .gitignore | 3 ++- kvprocessor/__init__.py | 4 +++- kvprocessor/kvenvloader.py | 2 +- kvprocessor/kvmanifestloader.py | 2 +- kvprocessor/kvprocessor.py | 4 ++-- kvprocessor/kvstructloader.py | 17 ++++++++++++----- kvprocessor/kvvalidator.py | 4 ++-- kvprocessor/{ => util}/errors.py | 0 kvprocessor/{ => util}/log.py | 0 kvprocessor/{ => util}/warnings.py | 0 test/test.py | 1 + 11 files changed, 24 insertions(+), 13 deletions(-) rename kvprocessor/{ => util}/errors.py (100%) rename kvprocessor/{ => util}/log.py (100%) rename kvprocessor/{ => util}/warnings.py (100%) diff --git a/.gitignore b/.gitignore index 87c09ca..a337f56 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,5 @@ dist .pypirc struct -*.log \ No newline at end of file +*.log +versions \ No newline at end of file diff --git a/kvprocessor/__init__.py b/kvprocessor/__init__.py index 923680c..507d722 100644 --- a/kvprocessor/__init__.py +++ b/kvprocessor/__init__.py @@ -18,4 +18,6 @@ from .kvvalidator import KVFileValidator, validate_kv_file, validate_kv_key, validate_kv_value from .kvtypemap import get_type_map, set_type_map, remove_type_map, has_type_map, clear_type_map, add_type_map from .kvglobalsettings import set_version, get_version, get_version_tuple, get_version_major, get_version_minor -from .errors import KVProcessorError, InvalidKVFileError, MissingEnvironmentVariableError, NamespaceNotFoundError, InvalidNamespaceError \ No newline at end of file +from .util.errors import KVProcessorError, InvalidKVFileError, MissingEnvironmentVariableError, NamespaceNotFoundError, InvalidNamespaceError +from .util.warnings import deprecated as kv_deprecated_warning +from .util.log import log as kv_log \ No newline at end of file diff --git a/kvprocessor/kvenvloader.py b/kvprocessor/kvenvloader.py index f1b8a9e..b7434b6 100644 --- a/kvprocessor/kvenvloader.py +++ b/kvprocessor/kvenvloader.py @@ -1,6 +1,6 @@ import os import warnings -from kvprocessor.warnings import deprecated +from kvprocessor.util.warnings import deprecated def load_env(Names: list) -> dict[str, any]: EnvList = {} diff --git a/kvprocessor/kvmanifestloader.py b/kvprocessor/kvmanifestloader.py index ea329e3..d1312e6 100644 --- a/kvprocessor/kvmanifestloader.py +++ b/kvprocessor/kvmanifestloader.py @@ -4,7 +4,7 @@ from urllib.parse import urlparse, urlunparse from kvprocessor.kvglobalsettings import get_version_major, get_version_minor, get_version from kvprocessor.kvprocessor import KVProcessor -from kvprocessor.log import log +from kvprocessor.util.log import log class KVManifestLoader: def __init__(self, file_url: str, cache_dir: str = "./struct", root: str = None, manifest_version: str = get_version()): diff --git a/kvprocessor/kvprocessor.py b/kvprocessor/kvprocessor.py index 3083ed2..7a169ca 100644 --- a/kvprocessor/kvprocessor.py +++ b/kvprocessor/kvprocessor.py @@ -1,7 +1,7 @@ import re from typing import Dict, Any, Union -from kvprocessor.log import log -from kvprocessor.errors import InvalidKVFileError +from kvprocessor.util.log import log +from kvprocessor.util.errors import InvalidKVFileError from kvprocessor.kvtypemap import get_type_map class KVProcessor: diff --git a/kvprocessor/kvstructloader.py b/kvprocessor/kvstructloader.py index 8c5c213..85c6870 100644 --- a/kvprocessor/kvstructloader.py +++ b/kvprocessor/kvstructloader.py @@ -4,8 +4,9 @@ from pathlib import Path from kvprocessor.kvprocessor import KVProcessor from kvprocessor.kvmanifestloader import KVManifestLoader -from kvprocessor.log import log -from kvprocessor.errors import NamespaceNotFoundError, InvalidNamespaceError, ManifestError, ConfigFetchError, KVFetchError +from kvprocessor.util.log import log +from kvprocessor.util.errors import NamespaceNotFoundError, InvalidNamespaceError, ManifestError, ConfigFetchError, KVFetchError +from kvprocessor.kvglobalsettings import get_version class KVStructLoader: def __init__(self, config_file: str, cache_dir: str = "./struct"): @@ -21,6 +22,7 @@ def __init__(self, config_file: str, cache_dir: str = "./struct"): self.version = self.config["version"] self.root = self.config["root"] self.Manifest = None + self.manifest_version = None if int(str(self.version).split(".")[2]) >= 7: log(f"Version: {self.version} >= 7") self.Platform = self.config.get("platform") @@ -30,11 +32,16 @@ def __init__(self, config_file: str, cache_dir: str = "./struct"): self.Branch = self.config.get("branch") self.Struct = self.config.get("struct") self.URL = f"https://raw.githubusercontent.com/{self.Owner}/{self.Repo}/refs/heads/{self.Branch}/{self.Struct}/" + if (int(str(self.version).split(".")[1]) >= 2) and (int(str(self.version).split(".")[2]) >= 14): + print("Using latest manifest attr") + self.manifest_version = self.config.get("manifest_version") + if not self.manifest_version: + raise ManifestError("Manifest version is missing in the configuration. Add tag 'manifest_version' to the config. 0.2.14+") else: self.URL = self.config.get("URL") - self.Manifest = self.config.get("manifest") - if self.Manifest: - self.Manifest = KVManifestLoader(f"{self.URL}{self.Manifest}", self.cache_dir, self.root) + manifest_file = self.config.get("manifest") + if manifest_file: + self.Manifest = KVManifestLoader(f"{self.URL}{manifest_file}", self.cache_dir, self.root, self.manifest_version or get_version()) else: raise ManifestError("Manifest file is missing in the configuration.") else: diff --git a/kvprocessor/kvvalidator.py b/kvprocessor/kvvalidator.py index 9e92cc2..a7befb9 100644 --- a/kvprocessor/kvvalidator.py +++ b/kvprocessor/kvvalidator.py @@ -1,7 +1,7 @@ import re -from kvprocessor.errors import InvalidKVFileError +from kvprocessor.util.errors import InvalidKVFileError from kvprocessor.kvtypemap import get_type_map -from kvprocessor.warnings import deprecated +from kvprocessor.util.warnings import deprecated class KVFileValidator(): def __init__(self): diff --git a/kvprocessor/errors.py b/kvprocessor/util/errors.py similarity index 100% rename from kvprocessor/errors.py rename to kvprocessor/util/errors.py diff --git a/kvprocessor/log.py b/kvprocessor/util/log.py similarity index 100% rename from kvprocessor/log.py rename to kvprocessor/util/log.py diff --git a/kvprocessor/warnings.py b/kvprocessor/util/warnings.py similarity index 100% rename from kvprocessor/warnings.py rename to kvprocessor/util/warnings.py diff --git a/test/test.py b/test/test.py index 10cdb56..e11cbb9 100644 --- a/test/test.py +++ b/test/test.py @@ -4,6 +4,7 @@ from kvprocessor import LoadEnv, KVProcessor, KVStructLoader from kvprocessor.kvfileutils import search_kv_files, copy_kv_file, delete_kv_file from kvprocessor.kvversionmanager import KVVersionManager +from kvprocessor.util.errors import ManifestError dotenv.load_dotenv() # Load the .env file def test_file(): From 0c91e0e6eec16c2dc99a36b22d6f5a0adbb2eee5 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Thu, 1 May 2025 20:11:41 +0000 Subject: [PATCH 13/34] update docs, and add some stuff --- README.md | 37 ++++++++++++++++++++++++++++ kvprocessor/cli.py | 2 ++ kvprocessor/kvmanifestloader.py | 21 ++++++++++++++-- kvprocessor/kvtypemap.py | 9 +++++++ test/test.py | 43 +++++++++++++++++++++++++++++++++ 5 files changed, 110 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 838e580..14a5067 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,43 @@ Namespace JSON files, have to be on a static host. They cannot be used locally. } ``` +### New Features in 0.2.x + +#### Additional Data Types +The library now supports additional data types such as `datetime`, `date`, `time`, and `decimal`. These can be used in `.kv` files as follows: + +```custom +EVENT_DATE:none +PRICE:none +``` + +#### Manifest Validation +Manifests are now validated for proper structure and required fields. This ensures that namespace mappings are correctly defined. + +### CLI Enhancements +The CLI now includes a `--version` flag to display the library version: + +```bash +python kvprocessor.cli --version +``` + +### Example: Using Additional Data Types +```python +from kvprocessor import KVProcessor + +kv_file_path = "test/test.kv" # Path to your .kv file +kv_processor = KVProcessor(kv_file_path) + +# Example configuration with additional data types +config = { + "EVENT_DATE": "2025-05-01T12:00:00", + "PRICE": "19.99", +} + +validated_config = kv_processor.process_config(config) +print(validated_config) +``` + ## Building For building the library locally \ **Requires**: `python3.8+`, `pip`, `linux system`(if using the predefined shell files) diff --git a/kvprocessor/cli.py b/kvprocessor/cli.py index 0a20f60..8dd2f94 100644 --- a/kvprocessor/cli.py +++ b/kvprocessor/cli.py @@ -2,9 +2,11 @@ from kvprocessor.kvvalidator import validate_kv_file from kvprocessor.kvmanifestloader import KVManifestLoader, NamespaceManager from kvprocessor.kvprocessor import KVProcessor +from kvprocessor.kvglobalsettings import get_version def main(): parser = argparse.ArgumentParser(description="CLI for kvProcessor") + parser.add_argument("--version", action="version", version=f"kvProcessor {get_version()}", help="Show the version of kvProcessor") subparsers = parser.add_subparsers(dest="command") # Subcommand: Validate .kv file diff --git a/kvprocessor/kvmanifestloader.py b/kvprocessor/kvmanifestloader.py index d1312e6..790c91d 100644 --- a/kvprocessor/kvmanifestloader.py +++ b/kvprocessor/kvmanifestloader.py @@ -5,17 +5,20 @@ from kvprocessor.kvglobalsettings import get_version_major, get_version_minor, get_version from kvprocessor.kvprocessor import KVProcessor from kvprocessor.util.log import log +from kvprocessor.util.errors import ManifestError class KVManifestLoader: def __init__(self, file_url: str, cache_dir: str = "./struct", root: str = None, manifest_version: str = get_version()): self.file_url = file_url self.cache_dir = cache_dir + self.manifest_version = manifest_version self.root = root self.manifest = None self.namespace_overides = {} self._fetch_manifest() + if str(self.manifest_version).strip().split(".")[1] >= 2: + self.validate_manifest() self._parse_manifest() - self.manifest_version = manifest_version def _fetch_manifest(self): try: @@ -60,4 +63,18 @@ def _parse_manifest(self): self.namespace_overides[key] = value except FileNotFoundError: print(f"Manifest file not found: {self.file_url}") - return None \ No newline at end of file + return None + + def validate_manifest(self): + """Validates the manifest for required fields and structure.""" + if not self.manifest: + raise ManifestError("Manifest is not loaded.") + + for i, line in enumerate(self.manifest.splitlines(), start=1): + line = line.strip() + if not line or line.startswith('#'): + continue + if ':' not in line: + raise ManifestError(f"Invalid manifest format at line {i}: {line}") + + log("Manifest validation passed.") \ No newline at end of file diff --git a/kvprocessor/kvtypemap.py b/kvprocessor/kvtypemap.py index eb45608..8855344 100644 --- a/kvprocessor/kvtypemap.py +++ b/kvprocessor/kvtypemap.py @@ -1,4 +1,6 @@ from typing import Any, Dict +import datetime +import decimal type_map = { 'string': str, @@ -15,6 +17,13 @@ 'str': str } +type_map.update({ + 'datetime': datetime.datetime, + 'date': datetime.date, + 'time': datetime.time, + 'decimal': decimal.Decimal +}) + def get_type_map() -> Dict[str, type]: """ Returns the type map for KV types. diff --git a/test/test.py b/test/test.py index e11cbb9..70b1321 100644 --- a/test/test.py +++ b/test/test.py @@ -1,10 +1,14 @@ import os import dotenv import pytest +import datetime +import decimal from kvprocessor import LoadEnv, KVProcessor, KVStructLoader from kvprocessor.kvfileutils import search_kv_files, copy_kv_file, delete_kv_file from kvprocessor.kvversionmanager import KVVersionManager from kvprocessor.util.errors import ManifestError +from kvprocessor.kvtypemap import get_type_map +from kvprocessor.kvmanifestloader import KVManifestLoader, ManifestError dotenv.load_dotenv() # Load the .env file def test_file(): @@ -125,6 +129,45 @@ def test_version_manager(): assert os.path.exists(restore_path) delete_kv_file(restore_path) +def test_additional_data_types(): + type_map = get_type_map() + assert 'datetime' in type_map + assert 'date' in type_map + assert 'time' in type_map + assert 'decimal' in type_map + assert type_map['datetime'] == datetime.datetime + assert type_map['decimal'] == decimal.Decimal + +def test_validate_manifest(): + valid_manifest_content = """# A valid manifest + namespace1:namespace2 + namespace3:namespace4 + """ + invalid_manifest_content = """# An invalid manifest + namespace1 namespace2 + """ + + # Write valid manifest to a temporary file + with open("test_valid_manifest.txt", "w") as file: + file.write(valid_manifest_content) + + # Write invalid manifest to a temporary file + with open("test_invalid_manifest.txt", "w") as file: + file.write(invalid_manifest_content) + + try: + loader = KVManifestLoader("test_valid_manifest.txt", root="test") + loader.validate_manifest() # Should pass without exceptions + + loader = KVManifestLoader("test_invalid_manifest.txt", root="test") + try: + loader.validate_manifest() + except ManifestError as e: + assert "Invalid manifest format" in str(e) + finally: + os.remove("test_valid_manifest.txt") + os.remove("test_invalid_manifest.txt") + if __name__ == "__main__": test_file() test_struct_loader() From f7d6b0372a00886d5f2d1bcd1dac13ffb22552ee Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Thu, 1 May 2025 20:18:02 +0000 Subject: [PATCH 14/34] add extra args to CLI, and fix an outdated import --- kvprocessor/cli.py | 11 ++++++++++- test/test.py | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/kvprocessor/cli.py b/kvprocessor/cli.py index 8dd2f94..02154a7 100644 --- a/kvprocessor/cli.py +++ b/kvprocessor/cli.py @@ -1,6 +1,7 @@ import argparse from kvprocessor.kvvalidator import validate_kv_file -from kvprocessor.kvmanifestloader import KVManifestLoader, NamespaceManager +from kvprocessor.kvmanifestloader import KVManifestLoader +from kvprocessor.kvnamespacemanager import NamespaceManager from kvprocessor.kvprocessor import KVProcessor from kvprocessor.kvglobalsettings import get_version @@ -17,6 +18,14 @@ def main(): list_parser = subparsers.add_parser("list-namespaces", help="List all namespaces") list_parser.add_argument("manifest", type=str, help="Path to the manifest file") + # Add a --list-namespaces command + list_namespaces_parser = subparsers.add_parser("list-namespaces", help="List all namespaces in a manifest") + list_namespaces_parser.add_argument("manifest", type=str, help="Path to the manifest file") + + # Add a --validate-manifest command + validate_manifest_parser = subparsers.add_parser("validate-manifest", help="Validate a manifest file") + validate_manifest_parser.add_argument("manifest", type=str, help="Path to the manifest file") + # Subcommand: Add namespace add_parser = subparsers.add_parser("add-namespace", help="Add a new namespace") add_parser.add_argument("manifest", type=str, help="Path to the manifest file") diff --git a/test/test.py b/test/test.py index 70b1321..b3bb01a 100644 --- a/test/test.py +++ b/test/test.py @@ -3,6 +3,7 @@ import pytest import datetime import decimal +import subprocess from kvprocessor import LoadEnv, KVProcessor, KVStructLoader from kvprocessor.kvfileutils import search_kv_files, copy_kv_file, delete_kv_file from kvprocessor.kvversionmanager import KVVersionManager @@ -168,6 +169,41 @@ def test_validate_manifest(): os.remove("test_valid_manifest.txt") os.remove("test_invalid_manifest.txt") +def test_list_namespaces(): + manifest_content = """# Example manifest + namespace1:namespace2 + namespace3:namespace4 + """ + + # Write manifest to a temporary file + with open("test_manifest.txt", "w") as file: + file.write(manifest_content) + + try: + loader = KVManifestLoader("test_manifest.txt", root="test") + namespaces = loader.list_namespaces() + assert namespaces == ["namespace1", "namespace3"] + finally: + os.remove("test_manifest.txt") + +def test_cli_list_namespaces(): + manifest_content = """# Example manifest + namespace1:namespace2 + namespace3:namespace4 + """ + + # Write manifest to a temporary file + with open("test_manifest.txt", "w") as file: + file.write(manifest_content) + + try: + result = subprocess.run(["python3", "-m", "kvprocessor.cli", "list-namespaces", "test_manifest.txt"], capture_output=True, text=True) + assert result.returncode == 0 + assert "namespace1" in result.stdout + assert "namespace3" in result.stdout + finally: + os.remove("test_manifest.txt") + if __name__ == "__main__": test_file() test_struct_loader() From 818e7b0d385c27694a4d5e2d7a87e2200a82882c Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Thu, 1 May 2025 20:22:10 +0000 Subject: [PATCH 15/34] update CLI --- kvprocessor/cli.py | 45 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/kvprocessor/cli.py b/kvprocessor/cli.py index 02154a7..ca4bbd2 100644 --- a/kvprocessor/cli.py +++ b/kvprocessor/cli.py @@ -18,11 +18,7 @@ def main(): list_parser = subparsers.add_parser("list-namespaces", help="List all namespaces") list_parser.add_argument("manifest", type=str, help="Path to the manifest file") - # Add a --list-namespaces command - list_namespaces_parser = subparsers.add_parser("list-namespaces", help="List all namespaces in a manifest") - list_namespaces_parser.add_argument("manifest", type=str, help="Path to the manifest file") - - # Add a --validate-manifest command + # Subcommand: Validate manifest validate_manifest_parser = subparsers.add_parser("validate-manifest", help="Validate a manifest file") validate_manifest_parser.add_argument("manifest", type=str, help="Path to the manifest file") @@ -37,6 +33,16 @@ def main(): remove_parser.add_argument("manifest", type=str, help="Path to the manifest file") remove_parser.add_argument("key", type=str, help="Namespace key") + # Subcommand: Export configuration + export_parser = subparsers.add_parser("export-config", help="Export configuration to a .kv file") + export_parser.add_argument("config", type=str, help="Path to the configuration JSON file") + export_parser.add_argument("output", type=str, help="Path to the output .kv file") + + # Subcommand: Merge .kv files + merge_parser = subparsers.add_parser("merge-kv", help="Merge multiple .kv files into one") + merge_parser.add_argument("files", nargs='+', help="Paths to the .kv files to merge") + merge_parser.add_argument("output", type=str, help="Path to the output .kv file") + args = parser.parse_args() if args.command == "validate": @@ -57,6 +63,14 @@ def main(): except Exception as e: print(f"Error listing namespaces: {e}") + elif args.command == "validate-manifest": + try: + manifest_loader = KVManifestLoader(args.manifest) + manifest_loader.validate_manifest() + print(f"Manifest {args.manifest} is valid.") + except Exception as e: + print(f"Manifest validation failed: {e}") + elif args.command == "add-namespace": try: manifest_loader = KVManifestLoader(args.manifest) @@ -75,5 +89,26 @@ def main(): except Exception as e: print(f"Error removing namespace: {e}") + elif args.command == "export-config": + from kvprocessor.kvfileexporter import KVFileExporter + import json + try: + with open(args.config, 'r') as config_file: + config = json.load(config_file) + exporter = KVFileExporter(args.output) + exporter.validate_and_export(config) + print(f"Configuration exported to {args.output}.") + except Exception as e: + print(f"Error exporting configuration: {e}") + + elif args.command == "merge-kv": + from kvprocessor.kvfilemerger import KVFileMerger + try: + merger = KVFileMerger(args.output) + merger.merge(args.files) + print(f"Merged .kv files into {args.output}.") + except Exception as e: + print(f"Error merging .kv files: {e}") + if __name__ == "__main__": main() \ No newline at end of file From 9ada4b55e2e573e3426054c2f23a0c92c216d960 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Thu, 1 May 2025 20:27:30 +0000 Subject: [PATCH 16/34] update CLI --- kvprocessor/cli.py | 67 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/kvprocessor/cli.py b/kvprocessor/cli.py index ca4bbd2..f1d9441 100644 --- a/kvprocessor/cli.py +++ b/kvprocessor/cli.py @@ -43,6 +43,26 @@ def main(): merge_parser.add_argument("files", nargs='+', help="Paths to the .kv files to merge") merge_parser.add_argument("output", type=str, help="Path to the output .kv file") + # Subcommand: Diff .kv files + diff_parser = subparsers.add_parser("diff-kv", help="Show differences between two .kv files") + diff_parser.add_argument("file1", type=str, help="Path to the first .kv file") + diff_parser.add_argument("file2", type=str, help="Path to the second .kv file") + + # Subcommand: Generate namespace report + report_parser = subparsers.add_parser("namespace-report", help="Generate a report of namespaces") + report_parser.add_argument("manifest", type=str, help="Path to the manifest file") + report_parser.add_argument("output", type=str, help="Path to the output report file") + + # Subcommand: Convert legacy config + convert_parser = subparsers.add_parser("convert-legacy", help="Convert legacy configuration to new format") + convert_parser.add_argument("legacy_config", type=str, help="Path to the legacy configuration file") + convert_parser.add_argument("output", type=str, help="Path to the output configuration file") + + # Subcommand: Backup .kv file + backup_parser = subparsers.add_parser("backup-kv", help="Create a backup of a .kv file") + backup_parser.add_argument("file", type=str, help="Path to the .kv file") + backup_parser.add_argument("backup_dir", type=str, help="Directory to store the backup") + args = parser.parse_args() if args.command == "validate": @@ -110,5 +130,52 @@ def main(): except Exception as e: print(f"Error merging .kv files: {e}") + elif args.command == "diff-kv": + from kvprocessor.kvdiff import KVFileDiffChecker as KVFileDiff + try: + differ = KVFileDiff() + differences = differ.diff(args.file1, args.file2) + print("Differences between files:") + print(differences) + except Exception as e: + print(f"Error diffing .kv files: {e}") + + elif args.command == "namespace-report": + try: + manifest_loader = KVManifestLoader(args.manifest) + manager = NamespaceManager(manifest_loader) + namespaces = manager.list_namespaces() + with open(args.output, 'w') as report_file: + report_file.write("Namespace Report\n") + report_file.write("================\n") + for namespace in namespaces: + report_file.write(f"{namespace}\n") + print(f"Namespace report generated at {args.output}.") + except Exception as e: + print(f"Error generating namespace report: {e}") + + elif args.command == "convert-legacy": + from kvprocessor.kvstructloader import KVStructLoader + try: + struct_loader = KVStructLoader(args.legacy_config) + new_config = struct_loader.convert_to_new_format() + with open(args.output, 'w') as output_file: + output_file.write(new_config) + print(f"Legacy configuration converted and saved to {args.output}.") + except Exception as e: + print(f"Error converting legacy configuration: {e}") + + elif args.command == "backup-kv": + import os + import shutil + try: + if not os.path.exists(args.backup_dir): + os.makedirs(args.backup_dir) + backup_path = os.path.join(args.backup_dir, os.path.basename(args.file)) + shutil.copy2(args.file, backup_path) + print(f"Backup created at {backup_path}.") + except Exception as e: + print(f"Error creating backup: {e}") + if __name__ == "__main__": main() \ No newline at end of file From 85eeb1a97c18ede9317a3d1ed39c3cac6ad30dce Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Thu, 1 May 2025 16:30:55 -0400 Subject: [PATCH 17/34] Update kvprocessor/kvvalidator.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- kvprocessor/kvvalidator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kvprocessor/kvvalidator.py b/kvprocessor/kvvalidator.py index a7befb9..c37c755 100644 --- a/kvprocessor/kvvalidator.py +++ b/kvprocessor/kvvalidator.py @@ -25,14 +25,14 @@ def validate_kv_file(self, file_path: str) -> bool: except FileNotFoundError: raise FileNotFoundError(f"KV file not found: {self.file_path}") - def validate_kv_key(key: str) -> bool: + def validate_kv_key(self, key: str) -> bool: """Validate a single key in a .kv file.""" match = re.match(r'(\w+)<([\w\|]+)>:([\w+]+|none)', key) if not match: raise InvalidKVFileError(f"Invalid key format: {key}") return True - def validate_kv_value(value: str, expected_types: list) -> bool: + def validate_kv_value(self, value: str, expected_types: list) -> bool: """Validate a value against expected types.""" type_map = get_type_map() for type_name in expected_types: From 6b7d6fce3e188d22af52f1edb9bce67bcbe45ce5 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Thu, 1 May 2025 16:31:33 -0400 Subject: [PATCH 18/34] Update kvprocessor/cli.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- kvprocessor/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kvprocessor/cli.py b/kvprocessor/cli.py index f1d9441..71e98c8 100644 --- a/kvprocessor/cli.py +++ b/kvprocessor/cli.py @@ -133,8 +133,8 @@ def main(): elif args.command == "diff-kv": from kvprocessor.kvdiff import KVFileDiffChecker as KVFileDiff try: - differ = KVFileDiff() - differences = differ.diff(args.file1, args.file2) + differ = KVFileDiff(args.file1, args.file2) + differences = differ.diff() print("Differences between files:") print(differences) except Exception as e: From 2192f5b1a885c4e49693da25cc611ee1b15e3f47 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Thu, 1 May 2025 16:31:51 -0400 Subject: [PATCH 19/34] Update kvprocessor/kvglobalsettings.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- kvprocessor/kvglobalsettings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kvprocessor/kvglobalsettings.py b/kvprocessor/kvglobalsettings.py index c08d694..b199d2d 100644 --- a/kvprocessor/kvglobalsettings.py +++ b/kvprocessor/kvglobalsettings.py @@ -6,7 +6,7 @@ def set_version(v: str): """Set the version of the KVProcessor.""" global version - verison = v + version = v def get_version() -> str: """Get the version of the KVProcessor.""" From 652371f178d282e4f542647af16a10e2521ed4d7 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Thu, 1 May 2025 20:39:13 +0000 Subject: [PATCH 20/34] update test --- test/expirament.py | 66 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 61 insertions(+), 5 deletions(-) diff --git a/test/expirament.py b/test/expirament.py index 46d6295..3558bae 100644 --- a/test/expirament.py +++ b/test/expirament.py @@ -1,9 +1,65 @@ from kvprocessor import KVProcessor, KVStructLoader import json +import sys + +def display_menu(): + print("\nMenu:") + print("1. Load Namespace") + print("2. Process Data") + print("3. Export Data") + print("4. Exit") + +def load_namespace(kv_struct_loader): + namespace = input("Enter Namespace: ") + try: + kv_processor = kv_struct_loader.from_namespace(namespace) + print(f"Namespace '{namespace}' loaded successfully.") + return kv_processor + except Exception as e: + print(f"Error loading namespace: {e}") + return None + +def process_data(kv_processor): + if not kv_processor: + print("No namespace loaded. Please load a namespace first.") + return + try: + data = input("Enter data to process (JSON format): ") + parsed_data = json.loads(data) + processed_data = kv_processor.process(parsed_data) + print("Processed Data:") + print(json.dumps(processed_data, indent=4)) + except Exception as e: + print(f"Error processing data: {e}") + +def export_data(kv_processor): + if not kv_processor: + print("No namespace loaded. Please load a namespace first.") + return + try: + export_path = input("Enter export file path: ") + kv_processor.export(export_path) + print(f"Data exported successfully to {export_path}.") + except Exception as e: + print(f"Error exporting data: {e}") if __name__ == "__main__": - print("Init") - Running = True - while Running: - kv_struct_loader = KVStructLoader("https://github.com/Voxa-Communications/VoxaCommunicaitons-Structures/raw/refs/heads/main/struct/config.json") - kv_processor: KVProcessor = kv_struct_loader.from_namespace(str(input("Namespace: "))) \ No newline at end of file + print("KV Processor Initialized") + kv_struct_loader = KVStructLoader("https://github.com/Voxa-Communications/VoxaCommunicaitons-Structures/raw/refs/heads/main/struct/config.json") + kv_processor = None + + while True: + display_menu() + choice = input("Enter your choice: ") + + if choice == "1": + kv_processor = load_namespace(kv_struct_loader) + elif choice == "2": + process_data(kv_processor) + elif choice == "3": + export_data(kv_processor) + elif choice == "4": + print("Exiting KV Processor.") + sys.exit(0) + else: + print("Invalid choice. Please try again.") \ No newline at end of file From 15e57e43e9f986290f096d9b1a9b6bfd2ac6e0a4 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Fri, 2 May 2025 14:31:46 +0000 Subject: [PATCH 21/34] add extra functionality to kvenvloader --- kvprocessor/kvenvloader.py | 45 ++++++++++++++++++++++++++++++++++---- 1 file changed, 41 insertions(+), 4 deletions(-) diff --git a/kvprocessor/kvenvloader.py b/kvprocessor/kvenvloader.py index b7434b6..49b1c74 100644 --- a/kvprocessor/kvenvloader.py +++ b/kvprocessor/kvenvloader.py @@ -2,13 +2,50 @@ import warnings from kvprocessor.util.warnings import deprecated -def load_env(Names: list) -> dict[str, any]: +def load_env(Names: list, defaults: dict = None) -> dict[str, any]: + """ + Load environment variables into a dictionary. + + Args: + Names (list): List of environment variable names to load. + defaults (dict): Optional dictionary of default values for environment variables. + + Returns: + dict: A dictionary containing the environment variable names and their values. + """ EnvList = {} + defaults = defaults or {} for Name in Names: - Value = os.environ.get(Name) + Value = os.environ.get(Name, defaults.get(Name)) + if Value is None: + warnings.warn(f"Environment variable '{Name}' is not set and no default value is provided.", UserWarning) EnvList[Name] = Value return EnvList +def validate_env(Names: list) -> None: + """ + Validate that all required environment variables are set. + + Args: + Names (list): List of environment variable names to validate. + + Raises: + EnvironmentError: If any required environment variable is not set. + """ + missing_vars = [Name for Name in Names if os.environ.get(Name) is None] + if missing_vars: + raise EnvironmentError(f"The following required environment variables are missing: {', '.join(missing_vars)}") + @deprecated -def LoadEnv(Names: list) -> dict[str, any]: - return load_env(Names) \ No newline at end of file +def LoadEnv(Names: list, defaults: dict = None) -> dict[str, any]: + """ + Deprecated wrapper for load_env. + + Args: + Names (list): List of environment variable names to load. + defaults (dict): Optional dictionary of default values for environment variables. + + Returns: + dict: A dictionary containing the environment variable names and their values. + """ + return load_env(Names, defaults) \ No newline at end of file From 6a89c5ea3d6b78045bb096d7a6f513fa9b3e222c Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Fri, 2 May 2025 14:34:54 +0000 Subject: [PATCH 22/34] update test.sh --- test.sh | 86 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 2 deletions(-) diff --git a/test.sh b/test.sh index cbcf990..786ba00 100644 --- a/test.sh +++ b/test.sh @@ -1,4 +1,86 @@ +# Step 1: Build the project +echo "Starting the build process..." bash build.sh -echo "Testing" +if [ $? -ne 0 ]; then + echo "Build failed. Exiting." + exit 1 +fi +echo "Build completed successfully." + +# Step 2: Install dependencies +echo "Installing test dependencies..." +python -m pip install --upgrade pip +if [ $? -ne 0 ]; then + echo "Failed to upgrade pip. Exiting." + exit 1 +fi + python -m pip install -r test/requirements.txt -python test/test.py \ No newline at end of file +if [ $? -ne 0 ]; then + echo "Failed to install dependencies. Exiting." + exit 1 +fi +echo "Dependencies installed successfully." + +# Step 3: Run unit tests +echo "Running unit tests..." +python test/test.py +if [ $? -ne 0 ]; then + echo "Unit tests failed. Exiting." + exit 1 +fi +echo "Unit tests passed successfully." + +# Step 4: Run linting +echo "Running lint checks..." +python -m pip install flake8 +if [ $? -ne 0 ]; then + echo "Failed to install flake8. Exiting." + exit 1 +fi + +flake8 . +if [ $? -ne 0 ]; then + echo "Linting failed. Please fix the issues." + exit 1 +fi +echo "Linting passed successfully." + +# Step 5: Run type checks +echo "Running type checks with mypy..." +python -m pip install mypy +if [ $? -ne 0 ]; then + echo "Failed to install mypy. Exiting." + exit 1 +fi + +mypy . +if [ $? -ne 0 ]; then + echo "Type checks failed. Please fix the issues." + exit 1 +fi +echo "Type checks passed successfully." + +# Step 6: Run security checks +echo "Running security checks with bandit..." +python -m pip install bandit +if [ $? -ne 0 ]; then + echo "Failed to install bandit. Exiting." + exit 1 +fi + +bandit -r . +if [ $? -ne 0 ]; then + echo "Security checks failed. Please fix the issues." + exit 1 +fi +echo "Security checks passed successfully." + +# Step 7: Clean up temporary files +echo "Cleaning up temporary files..." +find . -type f -name "*.pyc" -delete +find . -type d -name "__pycache__" -exec rm -r {} + +echo "Cleanup completed." + +# Final message +echo "All steps completed successfully." \ No newline at end of file From 3b5898b3b9343d1d69e3211082d48e8670723e90 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Fri, 2 May 2025 14:35:58 +0000 Subject: [PATCH 23/34] workflow --- .github/workflows/run-tests.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/run-tests.yml diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml new file mode 100644 index 0000000..bbf0609 --- /dev/null +++ b/.github/workflows/run-tests.yml @@ -0,0 +1,33 @@ +name: Run Tests + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + run-tests: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r test/requirements.txt + + - name: Make test.sh executable + run: chmod +x test.sh + + - name: Run test.sh + run: ./test.sh \ No newline at end of file From b24472634b21194a6750e47ca8136e2893757bed Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Fri, 2 May 2025 14:38:23 +0000 Subject: [PATCH 24/34] patch some bugs --- kvprocessor/kvmanifestloader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kvprocessor/kvmanifestloader.py b/kvprocessor/kvmanifestloader.py index 790c91d..51c73d8 100644 --- a/kvprocessor/kvmanifestloader.py +++ b/kvprocessor/kvmanifestloader.py @@ -16,7 +16,7 @@ def __init__(self, file_url: str, cache_dir: str = "./struct", root: str = None, self.manifest = None self.namespace_overides = {} self._fetch_manifest() - if str(self.manifest_version).strip().split(".")[1] >= 2: + if int(str(self.manifest_version).strip().split(".")[1]) >= 2: self.validate_manifest() self._parse_manifest() @@ -51,7 +51,7 @@ def _parse_manifest(self): match: dict = re.match(r'([^:]+):([^:]+)', line) if not match: if str(self.manifest_version).strip().split(".")[1] >= 2: - if (len(line.split(":")) == 0) and (len(line.split(".") >= 1)): + if (len(line.split(":")) == 0) and (len(line.split(".")) >= 1): log("Found namespace") match.clear() match[str(line).strip()] = str(line).strip() From 3b03be8845165ae6b192504d5c61a038968aa92c Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Fri, 2 May 2025 13:01:52 -0400 Subject: [PATCH 25/34] Update kvprocessor/kvmanifestloader.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- kvprocessor/kvmanifestloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kvprocessor/kvmanifestloader.py b/kvprocessor/kvmanifestloader.py index 51c73d8..9fd653c 100644 --- a/kvprocessor/kvmanifestloader.py +++ b/kvprocessor/kvmanifestloader.py @@ -50,7 +50,7 @@ def _parse_manifest(self): continue match: dict = re.match(r'([^:]+):([^:]+)', line) if not match: - if str(self.manifest_version).strip().split(".")[1] >= 2: + if int(str(self.manifest_version).strip().split(".")[1]) >= 2: if (len(line.split(":")) == 0) and (len(line.split(".")) >= 1): log("Found namespace") match.clear() From b3695c61bf5a018dca87cf98d3029a9314767198 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Fri, 2 May 2025 17:28:56 +0000 Subject: [PATCH 26/34] fix validation bug, and spelling error --- kvprocessor/kvmanifestloader.py | 6 +++--- kvprocessor/kvstructloader.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/kvprocessor/kvmanifestloader.py b/kvprocessor/kvmanifestloader.py index 9fd653c..de2592d 100644 --- a/kvprocessor/kvmanifestloader.py +++ b/kvprocessor/kvmanifestloader.py @@ -14,11 +14,11 @@ def __init__(self, file_url: str, cache_dir: str = "./struct", root: str = None, self.manifest_version = manifest_version self.root = root self.manifest = None - self.namespace_overides = {} + self.namespace_overrides = {} self._fetch_manifest() + self._parse_manifest() if int(str(self.manifest_version).strip().split(".")[1]) >= 2: self.validate_manifest() - self._parse_manifest() def _fetch_manifest(self): try: @@ -60,7 +60,7 @@ def _parse_manifest(self): log("Found namespace overide") key, value = match.groups() log(f"Parsing Line {i} key={key}, value={value}") - self.namespace_overides[key] = value + self.namespace_overrides[key] = value except FileNotFoundError: print(f"Manifest file not found: {self.file_url}") return None diff --git a/kvprocessor/kvstructloader.py b/kvprocessor/kvstructloader.py index 85c6870..81f23ff 100644 --- a/kvprocessor/kvstructloader.py +++ b/kvprocessor/kvstructloader.py @@ -85,8 +85,8 @@ def from_namespace(self, namespace: str) -> KVProcessor: if self.Manifest: log(f"Using Manifest to load KVProcessor from namespace: {namespace}") - if namespace in self.Manifest.namespace_overides: - namespace = self.Manifest.namespace_overides[namespace] + if namespace in self.Manifest.namespace_overrides: + namespace = self.Manifest.namespace_overrides[namespace] log(f"Namespace overridden to: {namespace}") else: log(f"Namespace not found in manifest, using original: {namespace}") From b4413cb0e1a249e076b34ce86918d343a1eba5ee Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Fri, 2 May 2025 17:30:39 +0000 Subject: [PATCH 27/34] update TODO.md --- TODO.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/TODO.md b/TODO.md index f87ff7d..8b28231 100644 --- a/TODO.md +++ b/TODO.md @@ -13,4 +13,5 @@ - Migrate to `kvprocessor.errors` for custom errors - Update cli to support 0.2.x verions of the API - Expand `kvprocessor.kvtypemap` to support more types. - - Update `kvprocessor.kvmanifestloader` to be more feature rich \ No newline at end of file + - Update `kvprocessor.kvmanifestloader` to be more feature rich + - Fix the style, so it will lint correctly \ No newline at end of file From 9e19b24ce4887290b86f39926c6589fde7d9cd34 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Fri, 2 May 2025 18:19:19 +0000 Subject: [PATCH 28/34] Update test.sh, and add a configuration --- kvprocessor/kvmanifestloader.py | 2 +- kvprocessor/kvnamespacemanager.py | 14 ++--- kvprocessor/kvstructloader.py | 2 +- test.sh | 98 ++++++++++++++++++------------- 4 files changed, 66 insertions(+), 50 deletions(-) diff --git a/kvprocessor/kvmanifestloader.py b/kvprocessor/kvmanifestloader.py index de2592d..0c6e581 100644 --- a/kvprocessor/kvmanifestloader.py +++ b/kvprocessor/kvmanifestloader.py @@ -14,7 +14,7 @@ def __init__(self, file_url: str, cache_dir: str = "./struct", root: str = None, self.manifest_version = manifest_version self.root = root self.manifest = None - self.namespace_overrides = {} + self.namespace_overrides: dict[str, str] = {} self._fetch_manifest() self._parse_manifest() if int(str(self.manifest_version).strip().split(".")[1]) >= 2: diff --git a/kvprocessor/kvnamespacemanager.py b/kvprocessor/kvnamespacemanager.py index c433459..efda70b 100644 --- a/kvprocessor/kvnamespacemanager.py +++ b/kvprocessor/kvnamespacemanager.py @@ -8,22 +8,22 @@ def __init__(self, manifest_loader: KVManifestLoader): def add_namespace(self, key: str, value: str): """Add a new namespace to the manifest.""" - if key in self.manifest_loader.namespace_overides: + if key in self.manifest_loader.namespace_overrides: raise ValueError(f"Namespace {key} already exists.") - self.manifest_loader.namespace_overides[key] = value + self.manifest_loader.namespace_overrides[key] = value def remove_namespace(self, key: str): """Remove a namespace from the manifest.""" - if key not in self.manifest_loader.namespace_overides: + if key not in self.manifest_loader.namespace_overrides: raise KeyError(f"Namespace {key} does not exist.") - del self.manifest_loader.namespace_overides[key] + del self.manifest_loader.namespace_overrides[key] def list_namespaces(self) -> list: """List all available namespaces.""" - return list(self.manifest_loader.namespace_overides.keys()) + return list(self.manifest_loader.namespace_overrides.keys()) def update_namespace(self, key: str, new_value: str): """Update an existing namespace.""" - if key not in self.manifest_loader.namespace_overides: + if key not in self.manifest_loader.namespace_overrides: raise KeyError(f"Namespace {key} does not exist.") - self.manifest_loader.namespace_overides[key] = new_value \ No newline at end of file + self.manifest_loader.namespace_overrides[key] = new_value \ No newline at end of file diff --git a/kvprocessor/kvstructloader.py b/kvprocessor/kvstructloader.py index 81f23ff..8a298ed 100644 --- a/kvprocessor/kvstructloader.py +++ b/kvprocessor/kvstructloader.py @@ -127,4 +127,4 @@ def list_available_namespaces(self) -> list: """List all available namespaces from the manifest.""" if not self.Manifest: raise ManifestError("Manifest is not loaded.") - return list(self.Manifest.namespace_overides.keys()) + return list(self.Manifest.namespace_overrides.keys()) diff --git a/test.sh b/test.sh index 786ba00..c87d099 100644 --- a/test.sh +++ b/test.sh @@ -1,3 +1,11 @@ +#!/bin/bash + +# CONFIGURATION +LINTING="false" +TYPE_CHECKING="false" +SECURITY_CHECKING="false" +CLEANUP="true" + # Step 1: Build the project echo "Starting the build process..." bash build.sh @@ -9,13 +17,13 @@ echo "Build completed successfully." # Step 2: Install dependencies echo "Installing test dependencies..." -python -m pip install --upgrade pip +python3 -m pip install --upgrade pip if [ $? -ne 0 ]; then echo "Failed to upgrade pip. Exiting." exit 1 fi -python -m pip install -r test/requirements.txt +python3 -m pip install -r test/requirements.txt if [ $? -ne 0 ]; then echo "Failed to install dependencies. Exiting." exit 1 @@ -24,7 +32,7 @@ echo "Dependencies installed successfully." # Step 3: Run unit tests echo "Running unit tests..." -python test/test.py +python3 test/test.py if [ $? -ne 0 ]; then echo "Unit tests failed. Exiting." exit 1 @@ -32,55 +40,63 @@ fi echo "Unit tests passed successfully." # Step 4: Run linting -echo "Running lint checks..." -python -m pip install flake8 -if [ $? -ne 0 ]; then - echo "Failed to install flake8. Exiting." - exit 1 -fi +if [ "$LINTING" = "true" ]; then + echo "Running lint checks..." + python3 -m pip install flake8 + if [ $? -ne 0 ]; then + echo "Failed to install flake8. Exiting." + exit 1 + fi -flake8 . -if [ $? -ne 0 ]; then - echo "Linting failed. Please fix the issues." - exit 1 -fi -echo "Linting passed successfully." + flake8 . + if [ $? -ne 0 ]; then + echo "Linting failed. Please fix the issues." + exit 1 + fi + echo "Linting passed successfully." +fi # Step 5: Run type checks -echo "Running type checks with mypy..." -python -m pip install mypy -if [ $? -ne 0 ]; then - echo "Failed to install mypy. Exiting." - exit 1 -fi +if [ "$TYPE_CHECKING" = "true" ]; then + echo "Running type checks with mypy..." + python3 -m pip install mypy + if [ $? -ne 0 ]; then + echo "Failed to install mypy. Exiting." + exit 1 + fi -mypy . -if [ $? -ne 0 ]; then - echo "Type checks failed. Please fix the issues." - exit 1 + mypy . + if [ $? -ne 0 ]; then + echo "Type checks failed. Please fix the issues." + exit 1 + fi + echo "Type checks passed successfully." fi -echo "Type checks passed successfully." # Step 6: Run security checks -echo "Running security checks with bandit..." -python -m pip install bandit -if [ $? -ne 0 ]; then - echo "Failed to install bandit. Exiting." - exit 1 -fi +if [ "$SECURITY_CHECKING" = "true" ]; then + echo "Running security checks with bandit..." + python3 -m pip install bandit + if [ $? -ne 0 ]; then + echo "Failed to install bandit. Exiting." + exit 1 + fi -bandit -r . -if [ $? -ne 0 ]; then - echo "Security checks failed. Please fix the issues." - exit 1 + bandit -r . + if [ $? -ne 0 ]; then + echo "Security checks failed. Please fix the issues." + exit 1 + fi + echo "Security checks passed successfully." fi -echo "Security checks passed successfully." # Step 7: Clean up temporary files -echo "Cleaning up temporary files..." -find . -type f -name "*.pyc" -delete -find . -type d -name "__pycache__" -exec rm -r {} + -echo "Cleanup completed." +if [ "$CLEANUP" = "true" ]; then + echo "Cleaning up temporary files..." + find . -type f -name "*.pyc" -delete + find . -type d -name "__pycache__" -exec rm -r {} + + echo "Cleanup completed." +fi # Final message echo "All steps completed successfully." \ No newline at end of file From 16bc819fe5de934b3ce2155121ae28a55219431d Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Fri, 2 May 2025 18:21:30 +0000 Subject: [PATCH 29/34] ignore mypycache --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a337f56..3c92728 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ dist .pypirc struct *.log -versions \ No newline at end of file +versions +.mypy_cache \ No newline at end of file From 8c6c28c56637cd70c4008af5907ae4faf34f52b6 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Fri, 2 May 2025 19:29:22 +0000 Subject: [PATCH 30/34] fix type issues --- kvprocessor/kvenvloader.py | 8 +++++--- kvprocessor/kvglobalsettings.py | 5 +++-- kvprocessor/kvmanifestloader.py | 3 ++- kvprocessor/kvprocessor.py | 2 +- kvprocessor/kvtypemap.py | 19 ++++++++----------- kvprocessor/util/warnings.py | 16 ++++++++++++++++ mypy.ini | 21 +++++++++++++++++++++ test.sh | 8 +++++++- test/test.py | 8 ++++---- 9 files changed, 67 insertions(+), 23 deletions(-) create mode 100644 mypy.ini diff --git a/kvprocessor/kvenvloader.py b/kvprocessor/kvenvloader.py index 49b1c74..d8e964c 100644 --- a/kvprocessor/kvenvloader.py +++ b/kvprocessor/kvenvloader.py @@ -1,8 +1,9 @@ import os import warnings -from kvprocessor.util.warnings import deprecated +from typing import Any, Optional +from kvprocessor.util.warnings import deprecated, ignore_warnings -def load_env(Names: list, defaults: dict = None) -> dict[str, any]: +def load_env(Names: list, defaults: Optional[dict] = None) -> dict[str, Any]: """ Load environment variables into a dictionary. @@ -36,8 +37,9 @@ def validate_env(Names: list) -> None: if missing_vars: raise EnvironmentError(f"The following required environment variables are missing: {', '.join(missing_vars)}") +@ignore_warnings @deprecated -def LoadEnv(Names: list, defaults: dict = None) -> dict[str, any]: +def LoadEnv(Names: list, defaults: Optional[dict] = None) -> dict[str, Any]: """ Deprecated wrapper for load_env. diff --git a/kvprocessor/kvglobalsettings.py b/kvprocessor/kvglobalsettings.py index b199d2d..ebeafc5 100644 --- a/kvprocessor/kvglobalsettings.py +++ b/kvprocessor/kvglobalsettings.py @@ -11,9 +11,10 @@ def set_version(v: str): def get_version() -> str: """Get the version of the KVProcessor.""" return version -def get_version_tuple() -> tuple[int, int, int]: +def get_version_tuple() -> tuple[int, ...]: """Get the version of the KVProcessor as a tuple.""" - return tuple(map(int, version.split('.'))) + version_parts = version.split('.') + return tuple(map(int, version_parts[:3])) def get_version_major() -> int: """Get the major version of the KVProcessor.""" return int(version.split('.')[0]) diff --git a/kvprocessor/kvmanifestloader.py b/kvprocessor/kvmanifestloader.py index 0c6e581..dec23aa 100644 --- a/kvprocessor/kvmanifestloader.py +++ b/kvprocessor/kvmanifestloader.py @@ -6,9 +6,10 @@ from kvprocessor.kvprocessor import KVProcessor from kvprocessor.util.log import log from kvprocessor.util.errors import ManifestError +from typing import Optional class KVManifestLoader: - def __init__(self, file_url: str, cache_dir: str = "./struct", root: str = None, manifest_version: str = get_version()): + def __init__(self, file_url: str, cache_dir: str = "./struct", root: Optional[str] = None, manifest_version: str = get_version()): self.file_url = file_url self.cache_dir = cache_dir self.manifest_version = manifest_version diff --git a/kvprocessor/kvprocessor.py b/kvprocessor/kvprocessor.py index 7a169ca..d839fa7 100644 --- a/kvprocessor/kvprocessor.py +++ b/kvprocessor/kvprocessor.py @@ -41,7 +41,7 @@ def _validate_type(self, value: Any, expected_types: list) -> bool: for type_name in expected_types: if type_name not in type_map: raise ValueError(f"Unsupported type in .kv file: {type_name}") - if isinstance(value, type_map[type_name]): + if isinstance(value, tuple(type_map[type_name] if isinstance(type_map[type_name], tuple) else [type_map[type_name]])): return True return False diff --git a/kvprocessor/kvtypemap.py b/kvprocessor/kvtypemap.py index 8855344..2158c2d 100644 --- a/kvprocessor/kvtypemap.py +++ b/kvprocessor/kvtypemap.py @@ -1,8 +1,8 @@ -from typing import Any, Dict +from typing import Any import datetime import decimal -type_map = { +type_map: dict[str, Any] = { 'string': str, 'int': int, 'float': float, @@ -14,17 +14,14 @@ 'set': set, 'object': object, 'any': Any, - 'str': str -} - -type_map.update({ + 'str': str, 'datetime': datetime.datetime, 'date': datetime.date, 'time': datetime.time, - 'decimal': decimal.Decimal -}) + 'decimal': decimal.Decimal, +} -def get_type_map() -> Dict[str, type]: +def get_type_map() -> dict[str, Any]: """ Returns the type map for KV types. @@ -32,7 +29,7 @@ def get_type_map() -> Dict[str, type]: """ return type_map -def set_type_map(new_type_map: Dict[str, type]) -> None: +def set_type_map(new_type_map: dict[str, Any]) -> None: """ Replaces the current type map with a new one. @@ -41,7 +38,7 @@ def set_type_map(new_type_map: Dict[str, type]) -> None: global type_map type_map = new_type_map -def add_type_map(key: str, value: type) -> None: +def add_type_map(key: str, value: Any) -> None: """ Adds or overrides a single key-value pair in the type map. diff --git a/kvprocessor/util/warnings.py b/kvprocessor/util/warnings.py index 3714a3f..0666e6d 100644 --- a/kvprocessor/util/warnings.py +++ b/kvprocessor/util/warnings.py @@ -5,4 +5,20 @@ def wrapper(*args, **kwargs): warnings.warn(f"{func.__name__} is deprecated and will be removed in a future version.", DeprecationWarning, stacklevel=2) return func(*args, **kwargs) + return wrapper + +def ignore_warnings(func): + """ + Decorator to ignore warnings in a function. + + Args: + func (callable): The function to decorate. + + Returns: + callable: The decorated function. + """ + def wrapper(*args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return func(*args, **kwargs) return wrapper \ No newline at end of file diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..9032648 --- /dev/null +++ b/mypy.ini @@ -0,0 +1,21 @@ +[mypy] +# Specify the files and directories to check +files = . + +# Disallow the use of untyped function definitions +disallow_untyped_defs = False + +# Disallow untyped calls to functions +disallow_untyped_calls = False + +# Require type hints for function arguments and return values +strict = False + +# Ignore missing imports (useful if some dependencies don't have type hints) +ignore_missing_imports = True + +# Show error codes in the output +show_error_codes = True + +# Cache directory for mypy +cache_dir = .mypy_cache \ No newline at end of file diff --git a/test.sh b/test.sh index c87d099..4c1c16c 100644 --- a/test.sh +++ b/test.sh @@ -2,7 +2,7 @@ # CONFIGURATION LINTING="false" -TYPE_CHECKING="false" +TYPE_CHECKING="true" SECURITY_CHECKING="false" CLEANUP="true" @@ -65,6 +65,12 @@ if [ "$TYPE_CHECKING" = "true" ]; then exit 1 fi + mypy --install-types --non-interactive + if [ $? -ne 0 ]; then + echo "Failed to install types for mypy. Exiting." + exit 1 + fi + mypy . if [ $? -ne 0 ]; then echo "Type checks failed. Please fix the issues." diff --git a/test/test.py b/test/test.py index b3bb01a..cd022f8 100644 --- a/test/test.py +++ b/test/test.py @@ -34,7 +34,7 @@ def test_struct_loader(): validated_config = kv_processor.process_config(user_settings) # Verifies that those env varibles exist and are of the correct type print(validated_config) -def test_file_operations(): +def test_file_operations_v2(): print("Testing file operations") kv_files = search_kv_files("test") print("Found .kv files:", kv_files) @@ -48,7 +48,7 @@ def test_file_operations(): delete_kv_file(copy_path) print(f"Deleted {copy_path}") -def test_version_manager(): +def test_version_manager_v2(): print("Testing version manager") version_manager = KVVersionManager("test/versions") @@ -207,5 +207,5 @@ def test_cli_list_namespaces(): if __name__ == "__main__": test_file() test_struct_loader() - test_file_operations() - test_version_manager() \ No newline at end of file + test_file_operations_v2() + test_version_manager_v2() \ No newline at end of file From 3d5ce631219fa32c6ec1ffe8ebc8edc809344b87 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Fri, 2 May 2025 15:32:43 -0400 Subject: [PATCH 31/34] Update kvprocessor/kvprocessor.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- kvprocessor/kvprocessor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kvprocessor/kvprocessor.py b/kvprocessor/kvprocessor.py index d839fa7..1c0c0a8 100644 --- a/kvprocessor/kvprocessor.py +++ b/kvprocessor/kvprocessor.py @@ -41,7 +41,8 @@ def _validate_type(self, value: Any, expected_types: list) -> bool: for type_name in expected_types: if type_name not in type_map: raise ValueError(f"Unsupported type in .kv file: {type_name}") - if isinstance(value, tuple(type_map[type_name] if isinstance(type_map[type_name], tuple) else [type_map[type_name]])): + expected_type = type_map[type_name] if isinstance(type_map[type_name], tuple) else (type_map[type_name],) + if isinstance(value, expected_type): return True return False From 8048e9e07c217bc4fae0f321812a4027817f4620 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Fri, 2 May 2025 19:56:26 +0000 Subject: [PATCH 32/34] update kvmanifest standard --- kvprocessor/__init__.py | 5 ++- kvprocessor/kvmanifestloader.py | 53 ++++++++++++++++++++++-------- kvprocessor/util/filedownloader.py | 11 +++++++ test/restored_test.kv | 8 +++++ 4 files changed, 63 insertions(+), 14 deletions(-) create mode 100644 kvprocessor/util/filedownloader.py create mode 100644 test/restored_test.kv diff --git a/kvprocessor/__init__.py b/kvprocessor/__init__.py index 507d722..c88edd1 100644 --- a/kvprocessor/__init__.py +++ b/kvprocessor/__init__.py @@ -20,4 +20,7 @@ from .kvglobalsettings import set_version, get_version, get_version_tuple, get_version_major, get_version_minor from .util.errors import KVProcessorError, InvalidKVFileError, MissingEnvironmentVariableError, NamespaceNotFoundError, InvalidNamespaceError from .util.warnings import deprecated as kv_deprecated_warning -from .util.log import log as kv_log \ No newline at end of file +from .util.log import log as kv_log + +# CLI +from .cli import main as kv_cli_main # idealy you dont use this, and use cli directly, however, if you are lazy, just import it, then run kvprocessor.kv_cli_main() to run the CLI \ No newline at end of file diff --git a/kvprocessor/kvmanifestloader.py b/kvprocessor/kvmanifestloader.py index dec23aa..39b27e9 100644 --- a/kvprocessor/kvmanifestloader.py +++ b/kvprocessor/kvmanifestloader.py @@ -5,7 +5,9 @@ from kvprocessor.kvglobalsettings import get_version_major, get_version_minor, get_version from kvprocessor.kvprocessor import KVProcessor from kvprocessor.util.log import log +from kvprocessor.util.warnings import ignore_warnings from kvprocessor.util.errors import ManifestError +from kvprocessor.util.filedownloader import download_file from typing import Optional class KVManifestLoader: @@ -37,10 +39,12 @@ def _fetch_manifest(self): except requests.RequestException as e: print(f"Error fetching manifest file: {e}") return None - + + @ignore_warnings def _parse_manifest(self): try: - with open(os.path.join(self.cache_dir, f"{self.root}.txt"), 'r') as file: + manifest_path = os.path.join(self.cache_dir, f"{self.root}.txt") + with open(manifest_path, 'r') as file: self.manifest = file.read() log(f"Manifest loaded: {self.manifest}") i = -1 @@ -49,21 +53,44 @@ def _parse_manifest(self): line = line.strip() if not line or line.startswith('#'): continue - match: dict = re.match(r'([^:]+):([^:]+)', line) + if line.startswith('$import'): + # Handle $import directive + imported_file = line.split(' ', 1)[1].strip() + imported_path = os.path.join(self.cache_dir, imported_file) + + if not os.path.exists(imported_path): + # Attempt to fetch the file from the URL + log(f"Imported manifest file not found locally: {imported_path}") + manifest_url = urlparse(self.file_url)._replace(path=f"/{imported_file}").geturl() + log(f"Attempting to fetch imported manifest from URL: {manifest_url}") + + try: + download_file(manifest_url, imported_path) + log(f"Successfully fetched and saved imported manifest: {imported_path}") + except requests.RequestException as e: + raise FileNotFoundError(f"Failed to fetch imported manifest from URL: {manifest_url}. Error: {e}") + + # Read the imported manifest + with open(imported_path, 'r') as imported_file: + imported_content = imported_file.read() + # Append imported content with a separating comment + self.manifest += f"\n# Imported from {imported_file.name}\n{imported_content}" + continue + + match = re.match(r'([^:]+):([^:]+)', line) if not match: - if int(str(self.manifest_version).strip().split(".")[1]) >= 2: - if (len(line.split(":")) == 0) and (len(line.split(".")) >= 1): - log("Found namespace") - match.clear() - match[str(line).strip()] = str(line).strip() - raise ValueError(f"Invalid manifest file format in line: {line}") + if (len(line.split(":")) == 0) and (len(line.split(".")) >= 1): + log("Found namespace") + match.clear() + match[str(line).strip()] = str(line).strip() + raise ValueError(f"Invalid manifest file format in line: {line}") else: - log("Found namespace overide") + log("Found namespace override") key, value = match.groups() - log(f"Parsing Line {i} key={key}, value={value}") - self.namespace_overrides[key] = value + log(f"Parsing Line {i} key={key}, value={value}") + self.namespace_overrides[key] = value except FileNotFoundError: - print(f"Manifest file not found: {self.file_url}") + print(f"Manifest file not found: {manifest_path}") return None def validate_manifest(self): diff --git a/kvprocessor/util/filedownloader.py b/kvprocessor/util/filedownloader.py new file mode 100644 index 0000000..d534696 --- /dev/null +++ b/kvprocessor/util/filedownloader.py @@ -0,0 +1,11 @@ +import io +import os +import requests + +def download_file(url: str, destination: str): + response = requests.get(url, stream=True) + response.raise_for_status() + os.makedirs(os.path.dirname(destination), exist_ok=True) + with open(destination, 'wb') as fetched_file: + for chunk in response.iter_content(chunk_size=8192): + fetched_file.write(chunk) \ No newline at end of file diff --git a/test/restored_test.kv b/test/restored_test.kv new file mode 100644 index 0000000..490ec8d --- /dev/null +++ b/test/restored_test.kv @@ -0,0 +1,8 @@ +DATABASE_NAME:none +DATABASE_USER:none +DATABASE_PASSWORD:none +DATABASE_HOST:none +DATABASE_PORT:none +DATABASE_DRIVER:mysql+mysqlconnector +DATABASE_DIALECT:none +KV_FILE_PATH:config/env.kv \ No newline at end of file From 50e71b53df94899aa3374462c13f4ecd77774de6 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Fri, 2 May 2025 20:16:59 +0000 Subject: [PATCH 33/34] update README.md --- README.md | 227 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 158 insertions(+), 69 deletions(-) diff --git a/README.md b/README.md index 14a5067..73f64d9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # kvProcessor -[**PYPI Package**](https://pypi.org/project/kvprocessor/) \ +[**PYPI Package**](https://pypi.org/project/kvprocessor/) **•** [**GitHub**](https://github.com/connor33341/kvProcessor) \ A Python package for processing and validating configuration dictionaries against a custom `.kv` file format. @@ -14,94 +14,169 @@ pip install kvprocessor ## File format +The `.kv` file format is a simple key-value configuration format with support for type validation and default values. Each line in a `.kv` file follows this syntax: + +```custom +VARIABLENAME:DEFAULTVALUE +``` + +- **VARIABLENAME**: The name of the variable. +- **TYPE**: The expected type(s) of the variable. Multiple types can be separated by `|`. +- **DEFAULTVALUE**: The default value for the variable. Use `none` if no default value is provided. +- **Comments**: Both comments as a new line, or inline are supprorted, with the `#` character. + +### Example `.kv` file: +```custom +DATABASE_NAME:none +DATABASE_PORT:3306 +ENABLE_LOGGING:true +MAX_CONNECTIONS:none +``` + +## KV Manifests + +A KV manifest is a file that defines namespaces and their relationships. It is used to organize and manage configurations across multiple `.kv` files. Each line in a manifest file follows this syntax: + +```custom +namespace1:namespace2 +``` + +- **namespace1**: The namespace that dosent exist as a file, but has the value as **namespace2**. +- **namespace2**: The full namespace path + +### Example manifest file: ```custom -# Comments are defined by a "#" -VARIBLENAME:DEFAULTVAULE +# A valid manifest +root:database +root:logging +database:connection +``` + +### Validating a manifest: +You can validate a manifest using the `KVManifestLoader`: + +```python +from kvprocessor.kvmanifestloader import KVManifestLoader + +manifest_path = "test/manifest.txt" +loader = KVManifestLoader(manifest_path) +loader.validate_manifest() # Validates the manifest structure +``` + +## Config.json + +The `config.json` file is used by the `KVStructLoader` to define the structure and metadata of the configuration. It includes details such as the version, root namespace, and manifest file. + +### Example `config.json`: +**Note**: This example uses features from `0.1.10`, the current version is `0.2.14+`. Some extra parameters may be needed. +```json +{ + "version": "0.1.10", + "root": "root", + "manifest": "manifest.txt", + "platform": "github", + "owner": "Voxa-Communications", + "repo": "VoxaCommunicaitons-Structures", + "branch": "main" +} +``` + +### Using `KVStructLoader` with `config.json`: +```python +from kvprocessor import KVStructLoader + +kv_config_url = "https://github.com/Voxa-Communications/VoxaCommunicaitons-Structures/raw/refs/heads/main/struct/config.json" +kv_struct_loader = KVStructLoader(kv_config_url) +kv_processor = kv_struct_loader.from_namespace("root.database.connection") ``` ## Usage +### KVProcessor ```python -from kvprocessor import LoadEnv, KVProcessor +from kvprocessor import KVProcessor +from kvprocessor.kvenvloader import load_env -kv_file_path = "test/test.kv" # Directory to .kv file -kv_processor = KVProcessor(kv_file_path) # Create a KV processor class -kv_keys = kv_processor.return_names() # Gets the keys (VARIBLENAME) from the .kv file -env_list = LoadEnv(kv_keys) # Loads all the ENV varibles that match those keys -validated_config = kv_processor.process_config(env_list) # Verifies that those env varibles exist and are of the correct type +kv_file_path = "test/test.kv" # Directory to .kv file +kv_processor = KVProcessor(kv_file_path) # Create a KV processor class +kv_keys = kv_processor.return_names() # Gets the keys (VARIBLENAME) from the .kv file +env_list = load_env(kv_keys) # Loads all the ENV variables that match those keys +validated_config = kv_processor.process_config(env_list) # Verifies that those env variables exist and are of the correct type print(validated_config) ``` -This example mimics the one found in the `/test` directory. With the kv file of: -```custom -# This is a comment -DATABASE_NAME:none -DATABASE_USER:none -DATABASE_PASSWORD:none -DATABASE_HOST:none -DATABASE_PORT:none -DATABASE_DRIVER:mysql+mysqlconnector -DATABASE_DIALECT:none -``` -You **should** get a result of: -`{'DATABASE_NAME': None, 'DATABASE_USER': None, 'DATABASE_PASSWORD': None, 'DATABASE_HOST': None, 'DATABASE_PORT': None, 'DATABASE_DRIVER': None, 'DATABASE_DIALECT': None}` This is because the kvProcessor is taking input from the env, and we dont have these env varibles defined. As a result these values default to the defined default value - -### Using "Namespaces" -This allows you to "import" kv files from a static host. +### KVStructLoader ```python -from kvprocessor import KVProcessor, KVStructLoader +from kvprocessor import KVStructLoader -kv_config_url = "https://github.com/Voxa-Communications/VoxaCommunicaitons-Structures/raw/refs/heads/main/struct/config.json" # STATIC url to json config -kv_struct_loader = KVStructLoader(kv_config_url) # Create a KVStructLoader object with the URL of the config file -kv_processor: KVProcessor = kv_struct_loader.from_namespace("voxa.api.user.user_settings") # Loads the KV file from the URL and returns a KVProcessor object -# For example this loads a file in /api/user/user_settings.kv +kv_config_url = "https://github.com/Voxa-Communications/VoxaCommunicaitons-Structures/raw/refs/heads/main/struct/config.json" +kv_struct_loader = KVStructLoader(kv_config_url) +kv_processor = kv_struct_loader.from_namespace("root.database.connection") user_settings = { - "2FA_ENABLED": True, - "TELEMETRY": False, - "AGE": "25", - "LANGUAGE": "en", -} # Example Dict Structure + "DATABASE_NAME": "test_db", + "DATABASE_PORT": 5432, +} validated_config = kv_processor.process_config(user_settings) print(validated_config) ``` -For an example config.json navigate to `test/config.json`. This file is just what is found on `https://github.com/Voxa-Communications/VoxaCommunicaitons-Structures/blob/main/struct/config.json` which is used in this example. -### Namespace's config.json -Namespace JSON files, have to be on a static host. They cannot be used locally. The easiest way to do this is to make a github repo, and use the raw file. -#### A config.json in a namespace should include at least two parts: - - A "root", the name that preceedes the rest of the namespace. Ex: `voxa` in `voxa.api.user.user_settings` - - A "URL". Ex: `https://mysite.example/kvstructures`, when the namespace `mysite.folder.structure` is used (assuming `root` is set to `mysite`), will fetch `https://mysite.example/kvstructures/folder/structure.kv` +### KVFileMerger +```python +from kvprocessor import KVFileMerger - Here is an example JSON (Note: on 0.7.1+ the URL is not needed, however a `struct` needs to be defined): -```json -{ - "root": "voxa", - "version": "0.1.5", - "URL": "https://raw.githubusercontent.com/Voxa-Communications/VoxaCommunicaitons-Structures/refs/heads/main/struct/" -} +file1 = "test/file1.kv" +file2 = "test/file2.kv" +merger = KVFileMerger(file1, file2) +merged_file = merger.merge("merged.kv") # Merges two KV files into a new file +print(f"Merged file created at: {merged_file}") ``` -### New Features in 0.2.x +### KVFileUtils +```python +from kvprocessor.kvfileutils import search_kv_files, copy_kv_file, delete_kv_file -#### Additional Data Types -The library now supports additional data types such as `datetime`, `date`, `time`, and `decimal`. These can be used in `.kv` files as follows: +# Search for KV files in a directory +kv_files = search_kv_files("test") +print(f"Found KV files: {kv_files}") -```custom -EVENT_DATE:none -PRICE:none +# Copy a KV file +copy_kv_file("test/test.kv", "test/copy_test.kv") +print("KV file copied.") + +# Delete a KV file +delete_kv_file("test/copy_test.kv") +print("KV file deleted.") +``` + +### KVFileDiffChecker +```python +from kvprocessor import KVFileDiffChecker + +file1 = "test/file1.kv" +file2 = "test/file2.kv" +diff_checker = KVFileDiffChecker(file1, file2) +differences = diff_checker.diff() +print(f"Differences between files: {differences}") ``` -#### Manifest Validation -Manifests are now validated for proper structure and required fields. This ensures that namespace mappings are correctly defined. +### KVValidator +```python +from kvprocessor import validate_kv_file + +kv_file_path = "test/test.kv" +is_valid = validate_kv_file(kv_file_path) +print(f"KV file is valid: {is_valid}") +``` -### CLI Enhancements -The CLI now includes a `--version` flag to display the library version: +### Additional Data Types +The library supports additional data types such as `datetime`, `date`, `time`, and `decimal`. These can be used in `.kv` files as follows: -```bash -python kvprocessor.cli --version +```custom +EVENT_DATE:none +PRICE:none ``` -### Example: Using Additional Data Types +Example usage: ```python from kvprocessor import KVProcessor @@ -120,17 +195,31 @@ print(validated_config) ## Building For building the library locally \ -**Requires**: `python3.8+`, `pip`, `linux system`(if using the predefined shell files) +**Requires**: `python3.8+`, `pip`, `linux system` (if using the predefined shell files) + +1. `git clone https://github.com/connor33341/kvProcessor.git` +2. `cd kvProcessor` +3. `bash build.sh` + +`build.sh` will also install kvProcessor as a local package, which you will be able to use. If you add new features to your fork and would like them to be featured on the main repo, submit a Pull Request. + +## CLI +At the current moment, there exists no documentation on this. If you would like to find usage, visit the file `kvprocessor\cli.py`. \ +\ +Basic Usage: +```bash +python kvprocessor/cli.py --version +``` - 1. `git clone https://github.com/connor33341/kvProcessor.git` - 2. `cd kvProcessor` - 3. `bash build.sh` +## Library Modules +For a complete list, visit `kvprocessor\__init__.py`. A breif list of main modules, will be listed here. + - `kvprocessor.kvprocessor`, Exports: `KVProcessor` + - `kvprocessor.kvstructloader`, Exports: `KVStructLoader` + - `kvprocessor.kvmanifestloader`, Exports: `KVManifestLoader` -`build.sh` will also install kvProcessor as a local package, which you will be able to use. -If you add new features to your fork, and would like them to be featured on the main repo. Submit a Pull Request ## For the nerds -The syntax was already mentioned, however if you would like to see how it parses. The following regex is used to determine the: `name`, `type`, and `default`: +The syntax was already mentioned, however, if you would like to see how it parses, the following regex is used to determine the: `name`, `type`, and `default`: ```re (\w+)<([\w\|]+)>:([\w+]+|none) ``` -With this knowlege, you probably can figure out a way to write .kv files in a weird way. Out of typical standard. \ No newline at end of file +With this knowledge, you probably can figure out a way to write `.kv` files in a weird way, out of typical standard. \ No newline at end of file From 135f6bfbaab06ebd54a0e6b8bf0e45e55dfa7b64 Mon Sep 17 00:00:00 2001 From: Connor <107011324+connor33341@users.noreply.github.com> Date: Fri, 2 May 2025 16:39:43 -0400 Subject: [PATCH 34/34] Create SECURITY.md --- SECURITY.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 SECURITY.md diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..e583b1e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,20 @@ +# Security Policy + +## Supported Versions + +Versions below `0.1.7` are no longer supproted. `0.2.12`, and `0.2.14-alpha` are both not supproted. Migrate to `0.2.14` w/o the alpha designation, for a stable release. + +| Version | Supported | +| ------- | ------------------ | +| 0.2.14+ | :white_check_mark: | +| 0.2.12 | :x: | +| 0.2.1-2 | :x: | +| 0.1.10+ | :white_check_mark: | +| < 0.1.7 | :x: | + +## Reporting a Vulnerability + +Contact the developer: \ +\ +Vunerabilites are rare, on this library. However, there is still a potental for them to exist. Please report these, so that they can be delt with accordingly. +There is no set style inwhich you have to submit a Vunerability. However, I would recomend being detailed, and having the information of what file or library is causing the issue.