From 1a90b76822cb1bd9ef148e6a7590720ac77af667 Mon Sep 17 00:00:00 2001 From: sheldy Date: Sat, 28 Jun 2025 17:10:21 +0200 Subject: [PATCH 01/36] Remove old code --- .flake8 | 2 - .gitlab-ci.yml | 38 -- .vscode/settings.json | 3 + example/MRE.py | 19 +- fluentogram/__init__.py | 17 +- fluentogram/{src => }/abc/__init__.py | 4 - fluentogram/abc/runner.py | 12 + fluentogram/{src => }/abc/transformer.py | 14 +- fluentogram/{src => }/abc/translator.py | 11 +- fluentogram/abc/translator_hub.py | 20 + fluentogram/cli/__init__.py | 5 - fluentogram/cli/cli.py | 80 ---- fluentogram/exceptions.py | 24 + fluentogram/exceptions/__init__.py | 5 - .../exceptions/root_locale_translator.py | 18 - fluentogram/impl/__init__.py | 12 + fluentogram/impl/runner.py | 27 ++ .../{src => }/impl/transformers/__init__.py | 4 +- .../impl/transformers/datetime_transformer.py | 25 + .../impl/transformers/money_transformer.py | 36 +- fluentogram/impl/translator.py | 35 ++ fluentogram/impl/translator_hub.py | 53 +++ fluentogram/misc/__init__.py | 4 - fluentogram/misc/timezones.py | 445 ------------------ fluentogram/src/__init__.py | 6 - fluentogram/src/abc/misc.py | 33 -- fluentogram/src/abc/runner.py | 27 -- fluentogram/src/abc/storage.py | 33 -- fluentogram/src/abc/translator_hub.py | 50 -- fluentogram/src/impl/__init__.py | 19 - fluentogram/src/impl/attrib_tracer.py | 25 - fluentogram/src/impl/filter.py | 62 --- fluentogram/src/impl/runner.py | 39 -- fluentogram/src/impl/storages/__init__.py | 1 - fluentogram/src/impl/storages/nats_storage.py | 134 ------ .../src/impl/stubs_translator_runner.py | 12 - .../src/impl/transator_hubs/__init__.py | 2 - .../impl/transator_hubs/kv_translator_hub.py | 103 ---- .../src/impl/transator_hubs/translator_hub.py | 67 --- .../impl/transformers/datetime_transformer.py | 26 - fluentogram/src/impl/translator.py | 28 -- fluentogram/tests/__init__.py | 4 - fluentogram/tests/test_stub_generation.py | 236 ---------- fluentogram/tests/test_usage.py | 46 -- fluentogram/typing_generator/__init__.py | 16 - fluentogram/typing_generator/parsed_ftl.py | 180 ------- .../typing_generator/renderable_items.py | 92 ---- fluentogram/typing_generator/stubs.py | 51 -- .../typing_generator/translation_dto.py | 9 - fluentogram/typing_generator/tree.py | 59 --- pyproject.toml | 50 +- 51 files changed, 302 insertions(+), 2021 deletions(-) delete mode 100644 .flake8 delete mode 100644 .gitlab-ci.yml create mode 100644 .vscode/settings.json rename fluentogram/{src => }/abc/__init__.py (74%) create mode 100644 fluentogram/abc/runner.py rename fluentogram/{src => }/abc/transformer.py (61%) rename fluentogram/{src => }/abc/translator.py (77%) create mode 100644 fluentogram/abc/translator_hub.py delete mode 100644 fluentogram/cli/__init__.py delete mode 100644 fluentogram/cli/cli.py create mode 100644 fluentogram/exceptions.py delete mode 100644 fluentogram/exceptions/__init__.py delete mode 100644 fluentogram/exceptions/root_locale_translator.py create mode 100644 fluentogram/impl/__init__.py create mode 100644 fluentogram/impl/runner.py rename fluentogram/{src => }/impl/transformers/__init__.py (58%) create mode 100644 fluentogram/impl/transformers/datetime_transformer.py rename fluentogram/{src => }/impl/transformers/money_transformer.py (51%) create mode 100644 fluentogram/impl/translator.py create mode 100644 fluentogram/impl/translator_hub.py delete mode 100644 fluentogram/misc/__init__.py delete mode 100644 fluentogram/misc/timezones.py delete mode 100644 fluentogram/src/__init__.py delete mode 100644 fluentogram/src/abc/misc.py delete mode 100644 fluentogram/src/abc/runner.py delete mode 100644 fluentogram/src/abc/storage.py delete mode 100644 fluentogram/src/abc/translator_hub.py delete mode 100644 fluentogram/src/impl/__init__.py delete mode 100644 fluentogram/src/impl/attrib_tracer.py delete mode 100644 fluentogram/src/impl/filter.py delete mode 100644 fluentogram/src/impl/runner.py delete mode 100644 fluentogram/src/impl/storages/__init__.py delete mode 100644 fluentogram/src/impl/storages/nats_storage.py delete mode 100644 fluentogram/src/impl/stubs_translator_runner.py delete mode 100644 fluentogram/src/impl/transator_hubs/__init__.py delete mode 100644 fluentogram/src/impl/transator_hubs/kv_translator_hub.py delete mode 100644 fluentogram/src/impl/transator_hubs/translator_hub.py delete mode 100644 fluentogram/src/impl/transformers/datetime_transformer.py delete mode 100644 fluentogram/src/impl/translator.py delete mode 100644 fluentogram/tests/__init__.py delete mode 100644 fluentogram/tests/test_stub_generation.py delete mode 100644 fluentogram/tests/test_usage.py delete mode 100644 fluentogram/typing_generator/__init__.py delete mode 100644 fluentogram/typing_generator/parsed_ftl.py delete mode 100644 fluentogram/typing_generator/renderable_items.py delete mode 100644 fluentogram/typing_generator/stubs.py delete mode 100644 fluentogram/typing_generator/translation_dto.py delete mode 100644 fluentogram/typing_generator/tree.py diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 79a16af..0000000 --- a/.flake8 +++ /dev/null @@ -1,2 +0,0 @@ -[flake8] -max-line-length = 120 \ No newline at end of file diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 0c5e461..0000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,38 +0,0 @@ -# This file is a template, and might need editing before it works on your project. -# To contribute improvements to CI/CD templates, please follow the Development guide at: -# https://docs.gitlab.com/ee/development/cicd/templates.html -# This specific template is located at: -# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml - -# This is a sample GitLab CI/CD configuration file that should run without any modifications. -# It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts, -# it uses echo commands to simulate the pipeline execution. -# -# A pipeline is composed of independent jobs that run scripts, grouped into stages. -# Stages run in sequential order, but jobs within stages run in parallel. -# -# For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages - -stages: # List of stages for jobs, and their order of execution - - build - - test - - deploy - - -unit-test-job: # This job runs in the test stage. - stage: test # It only starts when the job in the build stage completes successfully. - script: - - echo "Running unit tests" - - poetry env use python3.10 - - poetry install - - poetry run python3 -m unittest $CI_PROJECT_NAME.tests.test_usage - -deploy-job: # This job runs in the deploy stage. - stage: deploy # It only runs when *both* jobs in the test stage complete successfully. - script: - - python3 -m build - - twine upload --repository test.pypi.org --username __token__ --password $PYPI_TEST_TOKEN dist/* - - sleep 60 - - pip3 install --index-url https://test.pypi.org/simple/ fluentogram - - echo "Application successfully deployed." - - twine upload --username __token__ --password $PYPI_PROD_TOKEN dist/* diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..825efbe --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,3 @@ +{ + "cSpell.words": ["fluentogram"] +} diff --git a/example/MRE.py b/example/MRE.py index 40e3f61..d5d6c9e 100644 --- a/example/MRE.py +++ b/example/MRE.py @@ -1,8 +1,9 @@ import asyncio import os -from typing import TYPE_CHECKING, Callable, Dict, Any, Awaitable +from collections.abc import Awaitable, Callable +from typing import TYPE_CHECKING, Any -from aiogram import Bot, Dispatcher, Router, BaseMiddleware +from aiogram import BaseMiddleware, Bot, Dispatcher, Router from aiogram.types import Message from fluent_compiler.bundle import FluentBundle @@ -15,13 +16,13 @@ class TranslatorRunnerMiddleware(BaseMiddleware): async def __call__( self, - handler: Callable[[Message, Dict[str, Any]], Awaitable[Any]], + handler: Callable[[Message, dict[str, Any]], Awaitable[Any]], event: Message, - data: Dict[str, Any] + data: dict[str, Any], ) -> Any: - hub: TranslatorHub = data.get('_translator_hub') + hub: TranslatorHub = data.get("_translator_hub") # There you can ask your database for locale - data['i18n'] = hub.get_translator_by_locale(event.from_user.language_code) + data["i18n"] = hub.get_translator_by_locale(event.from_user.language_code) return await handler(event, data) @@ -38,11 +39,11 @@ async def main(): translator_hub = TranslatorHub( { "ru": ("ru", "en"), - "en": ("en",) + "en": ("en",), }, [ FluentTranslator("en", translator=FluentBundle.from_string("en-US", "start-hello = Hello, { $username }")), - FluentTranslator("ru", translator=FluentBundle.from_string("ru", "start-hello = Привет, { $username }")) + FluentTranslator("ru", translator=FluentBundle.from_string("ru", "start-hello = Привет, { $username }")), ], ) bot = Bot(token=os.getenv("TOKEN")) @@ -52,5 +53,5 @@ async def main(): await dp.start_polling(bot, _translator_hub=translator_hub) -if __name__ == '__main__': +if __name__ == "__main__": asyncio.run(main()) diff --git a/fluentogram/__init__.py b/fluentogram/__init__.py index af213e2..61f5f7b 100644 --- a/fluentogram/__init__.py +++ b/fluentogram/__init__.py @@ -1,24 +1,15 @@ -# coding=utf-8 -from . import misc -from .src.impl import ( - AttribTracer, +from .impl import ( + DateTimeTransformer, FluentTranslator, - TranslatorRunner, - TranslatorHub, - KvTranslatorHub, MoneyTransformer, - DateTimeTransformer, - NatsStorage, + TranslatorHub, + TranslatorRunner, ) __all__ = [ - "AttribTracer", "DateTimeTransformer", "FluentTranslator", "MoneyTransformer", "TranslatorHub", "TranslatorRunner", - "KvTranslatorHub", - "misc", - "NatsStorage", ] diff --git a/fluentogram/src/abc/__init__.py b/fluentogram/abc/__init__.py similarity index 74% rename from fluentogram/src/abc/__init__.py rename to fluentogram/abc/__init__.py index 4db1d9b..4092097 100644 --- a/fluentogram/src/abc/__init__.py +++ b/fluentogram/abc/__init__.py @@ -1,12 +1,8 @@ -# coding=utf-8 -from .misc import AbstractAttribTracer from .transformer import AbstractDataTransformer from .translator import AbstractTranslator from .translator_hub import AbstractTranslatorsHub - __all__ = [ - "AbstractAttribTracer", "AbstractDataTransformer", "AbstractTranslator", "AbstractTranslatorsHub", diff --git a/fluentogram/abc/runner.py b/fluentogram/abc/runner.py new file mode 100644 index 0000000..ca50764 --- /dev/null +++ b/fluentogram/abc/runner.py @@ -0,0 +1,12 @@ +"""An abstract translator runner""" + +from abc import ABC, abstractmethod +from typing import Any + + +class AbstractTranslatorRunner(ABC): + """This is one-shot per Telegram event translator with attrib tracer access way.""" + + @abstractmethod + def get(self, key: str, **kwargs: Any) -> str: + """Fastest, direct way to use translator, without sugar-like typing supported attribute access way""" diff --git a/fluentogram/src/abc/transformer.py b/fluentogram/abc/transformer.py similarity index 61% rename from fluentogram/src/abc/transformer.py rename to fluentogram/abc/transformer.py index 40b2957..21cea37 100644 --- a/fluentogram/src/abc/transformer.py +++ b/fluentogram/abc/transformer.py @@ -1,14 +1,11 @@ -# coding=utf-8 -""" -An AbstractDataTransformer object, using to transform any data before being passed to translator directly. -""" +"""An AbstractDataTransformer object, using to transform any data before being passed to translator directly.""" + from abc import ABC, abstractmethod from typing import Any class AbstractDataTransformer(ABC): - """ - These transformers inspired by Functions of Project Fluent by Mozilla. + """These transformers inspired by Functions of Project Fluent by Mozilla. Of course, it's a simple function, like a def function(money: Union[int, float], **kwargs) -> str: ... @@ -17,7 +14,8 @@ def function(money: Union[int, float], **kwargs) -> str: ... """ @abstractmethod - def __new__(cls, data: Any, **kwargs) -> Any: + def __new__(cls, data: Any, **kwargs: Any) -> Any: """Using incoming data, create an object representation of these data for your translator via all needed - parameters using kwargs""" + parameters using kwargs + """ raise NotImplementedError diff --git a/fluentogram/src/abc/translator.py b/fluentogram/abc/translator.py similarity index 77% rename from fluentogram/src/abc/translator.py rename to fluentogram/abc/translator.py index 225c59a..f080e00 100644 --- a/fluentogram/src/abc/translator.py +++ b/fluentogram/abc/translator.py @@ -1,7 +1,5 @@ -# coding=utf-8 -""" -Translator as itself -""" +from __future__ import annotations + from abc import ABC, abstractmethod from typing import Any @@ -16,9 +14,8 @@ def __init__(self, locale: str, translator: Any, separator: str = "-") -> None: self.translator = translator @abstractmethod - def get(self, key: str, **kwargs) -> str: - """ - Convert a translation key to a translated text string. + def get(self, key: str, **kwargs: Any) -> str | None: + """Convert a translation key to a translated text string. Use kwargs dict to pass external data to the translator. Expects to be fast and furious. """ diff --git a/fluentogram/abc/translator_hub.py b/fluentogram/abc/translator_hub.py new file mode 100644 index 0000000..3557e3a --- /dev/null +++ b/fluentogram/abc/translator_hub.py @@ -0,0 +1,20 @@ +"""An abstract base for the Translator Hub and Key/Value Translator Hub objects""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Iterable + +from fluentogram.abc.runner import AbstractTranslatorRunner + + +class AbstractTranslatorsHub(ABC): + """This class should contain a couple of translator objects, usually one object per one locale.""" + + def normalize_locales_map(self, locales_map: dict[str, str | Iterable[str]]) -> dict[str, Iterable[str]]: + return {key: (value,) if isinstance(value, str) else value for key, value in locales_map.items()} + + @abstractmethod + def get_translator_by_locale(self, locale: str) -> AbstractTranslatorRunner: + """Returns a Translator object by selected locale""" + raise NotImplementedError diff --git a/fluentogram/cli/__init__.py b/fluentogram/cli/__init__.py deleted file mode 100644 index a851800..0000000 --- a/fluentogram/cli/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# coding=utf-8 -"""A couple of CLI-access functions""" -from .cli import cli - -__all__ = ["cli"] diff --git a/fluentogram/cli/cli.py b/fluentogram/cli/cli.py deleted file mode 100644 index f09f2ac..0000000 --- a/fluentogram/cli/cli.py +++ /dev/null @@ -1,80 +0,0 @@ -import argparse -import time -from pathlib import Path - -from watchdog.events import FileModifiedEvent -from watchdog.events import FileSystemEventHandler -from watchdog.observers import Observer - -from fluentogram.typing_generator import ParsedRawFTL, Stubs, Tree - - -class FtlFileEventHandler(FileSystemEventHandler): - def __init__(self, track_path: str, stub_path: str): - self.track_path = track_path - self.stub_path = stub_path - - def on_modified(self, event: FileModifiedEvent): - print('event type: %s, path: %s' % (event.event_type, event.src_path)) - if not event.is_directory: - messages = parse_ftl_dir(self.track_path) - tree = Tree(messages) - stubs = Stubs(tree) - stubs.to_file(self.stub_path) - - -def parse_ftl(ftl_path: str | Path) -> dict: - with open(ftl_path, "r", encoding="utf-8") as input_f: - raw = ParsedRawFTL(input_f.read()) - messages = raw.get_messages() - return messages - - -def parse_ftl_dir(dir_path: str) -> dict: - messages = {} - for file in Path(dir_path).glob("*.ftl"): - messages.update(parse_ftl(file)) - return messages - - -def watch_ftl_dir(track_path: str, stub_path: str) -> None: - observer = Observer() - observer.schedule(FtlFileEventHandler(track_path, stub_path), track_path, recursive=True) - observer.start() - try: - while True: - time.sleep(1) - finally: - observer.stop() - observer.join() - - -def cli() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("-ftl", dest="ftl_path", required=False) - parser.add_argument("-track-ftl", dest="track_path", required=False) - parser.add_argument("-dir-ftl", dest="dir_path", required=False) - parser.add_argument("-stub", dest="stub_path", required=False) - - args = parser.parse_args() - - if not args.ftl_path and not args.track_path and not args.dir_path: - print("Use 'i18n --help' to see help message") - return - - if args.track_path: - print("Watching for changes in %s" % args.track_path) - watch_ftl_dir(args.track_path, args.stub_path) - return - - elif args.dir_path: - messages = parse_ftl_dir(args.dir_path) - else: - messages = parse_ftl(args.ftl_path) - - tree = Tree(messages) - stubs = Stubs(tree) - if args.stub_path: - stubs.to_file(args.stub_path) - else: - print(stubs.echo()) diff --git a/fluentogram/exceptions.py b/fluentogram/exceptions.py new file mode 100644 index 0000000..c932358 --- /dev/null +++ b/fluentogram/exceptions.py @@ -0,0 +1,24 @@ +class FluentogramError(Exception): + pass + + +class RootTranslatorNotFoundError(FluentogramError): + def __init__(self, root_locale: str) -> None: + self.root_locale = root_locale + super().__init__( + f"TranslatorHub does not have a root locale translator. " + f"Root locale is {self.root_locale!r}, provide translator with this locale.", + ) + + +class KeyNotFoundError(FluentogramError): + def __init__(self, key: str) -> None: + self.key = key + super().__init__(f"Key {self.key!r} not found in translators") + + +class FormatError(FluentogramError): + def __init__(self, original_error: Exception, key: str) -> None: + self.original_error = original_error + self.key = key + super().__init__(f"Error formatting key: {self.key!r}: {self.original_error!r}") diff --git a/fluentogram/exceptions/__init__.py b/fluentogram/exceptions/__init__.py deleted file mode 100644 index 32923ea..0000000 --- a/fluentogram/exceptions/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# coding=utf-8 -from .root_locale_translator import NotImplementedRootLocaleTranslator - - -__all__ = ["NotImplementedRootLocaleTranslator", ] diff --git a/fluentogram/exceptions/root_locale_translator.py b/fluentogram/exceptions/root_locale_translator.py deleted file mode 100644 index 350e045..0000000 --- a/fluentogram/exceptions/root_locale_translator.py +++ /dev/null @@ -1,18 +0,0 @@ -# coding=utf-8 -"""Custom exception, raises when main, core translator does not exist""" - - -class NotImplementedRootLocaleTranslator(Exception): - """ - This exception is raised when TranslatorHub has no translator for root locale and being impossible to work. - """ - - def __init__(self, root_locale) -> None: - super().__init__( - f"""\n - You do not have a root locale translator. - Root locale is "{root_locale}" - Please, fix it! - Just provide the data! - """ - ) diff --git a/fluentogram/impl/__init__.py b/fluentogram/impl/__init__.py new file mode 100644 index 0000000..8ddb35b --- /dev/null +++ b/fluentogram/impl/__init__.py @@ -0,0 +1,12 @@ +from .runner import TranslatorRunner +from .transformers import DateTimeTransformer, MoneyTransformer +from .translator import FluentTranslator +from .translator_hub import TranslatorHub + +__all__ = [ + "DateTimeTransformer", + "FluentTranslator", + "MoneyTransformer", + "TranslatorHub", + "TranslatorRunner", +] diff --git a/fluentogram/impl/runner.py b/fluentogram/impl/runner.py new file mode 100644 index 0000000..803411d --- /dev/null +++ b/fluentogram/impl/runner.py @@ -0,0 +1,27 @@ +"""A translator runner by itself""" + +from collections.abc import Iterable +from typing import Any + +from fluentogram.abc import AbstractTranslator +from fluentogram.abc.runner import AbstractTranslatorRunner +from fluentogram.exceptions import KeyNotFoundError + + +class TranslatorRunner(AbstractTranslatorRunner): + def __init__(self, translators: Iterable[AbstractTranslator], separator: str = "-") -> None: + self.translators = translators + self.separator = separator + self.request_line = "" + + def get(self, key: str, **kwargs: Any) -> str: + """Fastest, direct way to use translator, without sugar-like typing supported attribute access way""" + return self._get_translation(key, **kwargs) + + def _get_translation(self, key: str, **kwargs: Any) -> str: + for translator in self.translators: + text = translator.get(key, **kwargs) + if text is not None: + return text + + raise KeyNotFoundError(key) diff --git a/fluentogram/src/impl/transformers/__init__.py b/fluentogram/impl/transformers/__init__.py similarity index 58% rename from fluentogram/src/impl/transformers/__init__.py rename to fluentogram/impl/transformers/__init__.py index 4ba013d..ae984ce 100644 --- a/fluentogram/src/impl/transformers/__init__.py +++ b/fluentogram/impl/transformers/__init__.py @@ -1,6 +1,4 @@ -# coding=utf-8 from .datetime_transformer import DateTimeTransformer from .money_transformer import MoneyTransformer - -__all__ = ["DateTimeTransformer", "MoneyTransformer", ] +__all__ = ["DateTimeTransformer", "MoneyTransformer"] diff --git a/fluentogram/impl/transformers/datetime_transformer.py b/fluentogram/impl/transformers/datetime_transformer.py new file mode 100644 index 0000000..e1de29e --- /dev/null +++ b/fluentogram/impl/transformers/datetime_transformer.py @@ -0,0 +1,25 @@ +"""A DateTimeTransformer itself""" + +from __future__ import annotations + +from datetime import datetime + +from fluent_compiler.types import FluentDateType, FluentNone, fluent_date + +from fluentogram.abc import AbstractDataTransformer + + +class DateTimeTransformer(AbstractDataTransformer): + """This transformer converts a default python datetime object to FluentDate + Typings refer to https://github.com/tc39/ecma402 + """ + + def __new__( + cls, + date: datetime, + **kwargs, + ) -> FluentDateType | FluentNone: + return fluent_date( + date, + **kwargs, + ) diff --git a/fluentogram/src/impl/transformers/money_transformer.py b/fluentogram/impl/transformers/money_transformer.py similarity index 51% rename from fluentogram/src/impl/transformers/money_transformer.py rename to fluentogram/impl/transformers/money_transformer.py index fea33cf..5bf5a61 100644 --- a/fluentogram/src/impl/transformers/money_transformer.py +++ b/fluentogram/impl/transformers/money_transformer.py @@ -1,13 +1,13 @@ -# coding=utf-8 -""" -A MoneyTransformer by itself -""" +"""A MoneyTransformer by itself""" + +from __future__ import annotations + from decimal import Decimal -from typing import Literal, Union, Optional +from typing import Literal -from fluent_compiler.types import fluent_number, FluentNumber, FluentNone +from fluent_compiler.types import FluentNone, FluentNumber, fluent_number -from fluentogram.src.abc import AbstractDataTransformer +from fluentogram.abc import AbstractDataTransformer class MoneyTransformer(AbstractDataTransformer): @@ -15,20 +15,18 @@ class MoneyTransformer(AbstractDataTransformer): Typings refer to https://github.com/tc39/ecma402 """ - def __new__( + def __new__( # noqa: PLR0913 cls, amount: Decimal, currency: str, - currency_display: Union[ - Literal["code"], Literal["symbol"], Literal["name"] - ] = "code", - use_grouping: bool = False, - minimum_significant_digits: Optional[int] = None, - maximum_significant_digits: Optional[int] = None, - minimum_fraction_digits: Optional[int] = None, - maximum_fraction_digits: Optional[int] = None, - **kwargs - ) -> Union[FluentNumber, FluentNone]: + currency_display: Literal["code", "symbol", "name"] = "code", + use_grouping: bool = False, # noqa: FBT002 + minimum_significant_digits: int | None = None, + maximum_significant_digits: int | None = None, + minimum_fraction_digits: int | None = None, + maximum_fraction_digits: int | None = None, + **kwargs, + ) -> FluentNumber | FluentNone: return fluent_number( amount, style="currency", @@ -39,5 +37,5 @@ def __new__( maximumSignificantDigits=maximum_significant_digits, minimumFractionDigits=minimum_fraction_digits, maximumFractionDigits=maximum_fraction_digits, - **kwargs + **kwargs, ) diff --git a/fluentogram/impl/translator.py b/fluentogram/impl/translator.py new file mode 100644 index 0000000..017ee73 --- /dev/null +++ b/fluentogram/impl/translator.py @@ -0,0 +1,35 @@ +"""Fluent implementation of AbstractTranslator""" + +from __future__ import annotations + +from typing import Any + +from fluent_compiler.bundle import FluentBundle + +from fluentogram.abc import AbstractTranslator +from fluentogram.exceptions import FormatError + + +class FluentTranslator(AbstractTranslator): + """Single-locale Translator, implemented with fluent_compiler Bundles""" + + def __init__(self, locale: str, translator: FluentBundle, separator: str = "-") -> None: + self.locale = locale + self.translator = translator + self.separator = separator + + def get(self, key: str, **kwargs: Any) -> str | None: + """STR100: Calling format with insecure string. + Route questions to --> https://github.com/django-ftl/fluent-compiler + """ + try: + text, errors = self.translator.format(key, kwargs) + if errors: + raise FormatError(errors.pop(), key) + except KeyError: + return None + + return text + + def __repr__(self) -> str: + return f"" diff --git a/fluentogram/impl/translator_hub.py b/fluentogram/impl/translator_hub.py new file mode 100644 index 0000000..2e75d04 --- /dev/null +++ b/fluentogram/impl/translator_hub.py @@ -0,0 +1,53 @@ +"""A Translator Hub, using as factory for Translator objects""" + +from __future__ import annotations + +from collections.abc import Iterable + +from fluentogram.abc import AbstractTranslator, AbstractTranslatorsHub +from fluentogram.exceptions import RootTranslatorNotFoundError +from fluentogram.impl import TranslatorRunner + + +class TranslatorHub(AbstractTranslatorsHub): + """This class implements a storage for all single-locale translators.""" + + def __init__( + self, + locales_map: dict[str, str | Iterable[str]], + translators: list[AbstractTranslator], + root_locale: str = "en", + separator: str = "-", + ) -> None: + self.locales_map = self.normalize_locales_map(locales_map) + self.translators = translators + self.root_locale = root_locale + self.separator = separator + self.storage: dict[str, AbstractTranslator] = dict( + zip([translator.locale for translator in translators], translators, strict=False), + ) + if not self.storage.get(root_locale): + raise RootTranslatorNotFoundError(self.root_locale) + + self.translators_map: dict[str, Iterable[AbstractTranslator]] = self._locales_map_parser(self.locales_map) + + def _locales_map_parser( + self, + locales_map: dict[str, str | Iterable[str]], + ) -> dict[str, Iterable[AbstractTranslator]]: + return { + lang: tuple(self.storage[locale] for locale in translator_locales if locale in self.storage) + for lang, translator_locales in locales_map.items() + } + + def get_translator_by_locale(self, locale: str) -> TranslatorRunner: + """Here is a little tricky moment. + There should be like a one-shot scheme. + For proper isolation, function returns TranslatorRunner new instance every time, not the same translator. + This trick makes "obj.attribute1.attribute2" access to be able. + You are able to do what you want, refer to the abstract class. + """ + return TranslatorRunner( + translators=self.translators_map.get(locale) or self.translators_map[self.root_locale], + separator=self.separator, + ) diff --git a/fluentogram/misc/__init__.py b/fluentogram/misc/__init__.py deleted file mode 100644 index bdb5da0..0000000 --- a/fluentogram/misc/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# coding=utf-8 -from .timezones import timezones - -__all__ = ["timezones"] diff --git a/fluentogram/misc/timezones.py b/fluentogram/misc/timezones.py deleted file mode 100644 index 12cf932..0000000 --- a/fluentogram/misc/timezones.py +++ /dev/null @@ -1,445 +0,0 @@ -# coding=utf-8 -"""A timezones typing""" -from typing import Literal - -timezones = Literal[ - "Africa/Abidjan", - "Africa/Accra", - "Africa/Addis_Ababa", - "Africa/Algiers", - "Africa/Asmara", - "Africa/Bamako", - "Africa/Bangui", - "Africa/Banjul", - "Africa/Bissau", - "Africa/Blantyre", - "Africa/Brazzaville", - "Africa/Bujumbura", - "Africa/Cairo", - "Africa/Casablanca", - "Africa/Ceuta", - "Africa/Conakry", - "Africa/Dakar", - "Africa/Dar_es_Salaam", - "Africa/Djibouti", - "Africa/Douala", - "Africa/El_Aaiun", - "Africa/Freetown", - "Africa/Gaborone", - "Africa/Harare", - "Africa/Johannesburg", - "Africa/Juba", - "Africa/Kampala", - "Africa/Khartoum", - "Africa/Kigali", - "Africa/Kinshasa", - "Africa/Lagos", - "Africa/Libreville", - "Africa/Lome", - "Africa/Luanda", - "Africa/Lubumbashi", - "Africa/Lusaka", - "Africa/Malabo", - "Africa/Maputo", - "Africa/Maseru", - "Africa/Mbabane", - "Africa/Mogadishu", - "Africa/Monrovia", - "Africa/Nairobi", - "Africa/Ndjamena", - "Africa/Niamey", - "Africa/Nouakchott", - "Africa/Ouagadougou", - "Africa/Porto-Novo", - "Africa/Sao_Tome", - "Africa/Tripoli", - "Africa/Tunis", - "Africa/Windhoek", - "America/Adak", - "America/Anchorage", - "America/Anguilla", - "America/Antigua", - "America/Araguaina", - "America/Argentina/Buenos_Aires", - "America/Argentina/Catamarca", - "America/Argentina/Cordoba", - "America/Argentina/Jujuy", - "America/Argentina/La_Rioja", - "America/Argentina/Mendoza", - "America/Argentina/Rio_Gallegos", - "America/Argentina/Salta", - "America/Argentina/San_Juan", - "America/Argentina/San_Luis", - "America/Argentina/Tucuman", - "America/Argentina/Ushuaia", - "America/Aruba", - "America/Asuncion", - "America/Atikokan", - "America/Bahia", - "America/Bahia_Banderas", - "America/Barbados", - "America/Belem", - "America/Belize", - "America/Blanc-Sablon", - "America/Boa_Vista", - "America/Bogota", - "America/Boise", - "America/Cambridge_Bay", - "America/Campo_Grande", - "America/Cancun", - "America/Caracas", - "America/Cayenne", - "America/Cayman", - "America/Chicago", - "America/Chihuahua", - "America/Costa_Rica", - "America/Creston", - "America/Cuiaba", - "America/Curacao", - "America/Danmarkshavn", - "America/Dawson", - "America/Dawson_Creek", - "America/Denver", - "America/Detroit", - "America/Dominica", - "America/Edmonton", - "America/Eirunepe", - "America/El_Salvador", - "America/Fort_Nelson", - "America/Fortaleza", - "America/Glace_Bay", - "America/Goose_Bay", - "America/Grand_Turk", - "America/Grenada", - "America/Guadeloupe", - "America/Guatemala", - "America/Guayaquil", - "America/Guyana", - "America/Halifax", - "America/Havana", - "America/Hermosillo", - "America/Indiana/Indianapolis", - "America/Indiana/Knox", - "America/Indiana/Marengo", - "America/Indiana/Petersburg", - "America/Indiana/Tell_City", - "America/Indiana/Vevay", - "America/Indiana/Vincennes", - "America/Indiana/Winamac", - "America/Inuvik", - "America/Iqaluit", - "America/Jamaica", - "America/Juneau", - "America/Kentucky/Louisville", - "America/Kentucky/Monticello", - "America/Kralendijk", - "America/La_Paz", - "America/Lima", - "America/Los_Angeles", - "America/Lower_Princes", - "America/Maceio", - "America/Managua", - "America/Manaus", - "America/Marigot", - "America/Martinique", - "America/Matamoros", - "America/Mazatlan", - "America/Menominee", - "America/Merida", - "America/Metlakatla", - "America/Mexico_City", - "America/Miquelon", - "America/Moncton", - "America/Monterrey", - "America/Montevideo", - "America/Montserrat", - "America/Nassau", - "America/New_York", - "America/Nipigon", - "America/Nome", - "America/Noronha", - "America/North_Dakota/Beulah", - "America/North_Dakota/Center", - "America/North_Dakota/New_Salem", - "America/Nuuk", - "America/Ojinaga", - "America/Panama", - "America/Pangnirtung", - "America/Paramaribo", - "America/Phoenix", - "America/Port-au-Prince", - "America/Port_of_Spain", - "America/Porto_Velho", - "America/Puerto_Rico", - "America/Punta_Arenas", - "America/Rainy_River", - "America/Rankin_Inlet", - "America/Recife", - "America/Regina", - "America/Resolute", - "America/Rio_Branco", - "America/Santarem", - "America/Santiago", - "America/Santo_Domingo", - "America/Sao_Paulo", - "America/Scoresbysund", - "America/Sitka", - "America/St_Barthelemy", - "America/St_Johns", - "America/St_Kitts", - "America/St_Lucia", - "America/St_Thomas", - "America/St_Vincent", - "America/Swift_Current", - "America/Tegucigalpa", - "America/Thule", - "America/Thunder_Bay", - "America/Tijuana", - "America/Toronto", - "America/Tortola", - "America/Vancouver", - "America/Whitehorse", - "America/Winnipeg", - "America/Yakutat", - "America/Yellowknife", - "Antarctica/Casey", - "Antarctica/Davis", - "Antarctica/DumontDUrville", - "Antarctica/Macquarie", - "Antarctica/Mawson", - "Antarctica/McMurdo", - "Antarctica/Palmer", - "Antarctica/Rothera", - "Antarctica/Syowa", - "Antarctica/Troll", - "Antarctica/Vostok", - "Arctic/Longyearbyen", - "Asia/Aden", - "Asia/Almaty", - "Asia/Amman", - "Asia/Anadyr", - "Asia/Aqtau", - "Asia/Aqtobe", - "Asia/Ashgabat", - "Asia/Atyrau", - "Asia/Baghdad", - "Asia/Bahrain", - "Asia/Baku", - "Asia/Bangkok", - "Asia/Barnaul", - "Asia/Beirut", - "Asia/Bishkek", - "Asia/Brunei", - "Asia/Chita", - "Asia/Choibalsan", - "Asia/Colombo", - "Asia/Damascus", - "Asia/Dhaka", - "Asia/Dili", - "Asia/Dubai", - "Asia/Dushanbe", - "Asia/Famagusta", - "Asia/Gaza", - "Asia/Hebron", - "Asia/Ho_Chi_Minh", - "Asia/Hong_Kong", - "Asia/Hovd", - "Asia/Irkutsk", - "Asia/Jakarta", - "Asia/Jayapura", - "Asia/Jerusalem", - "Asia/Kabul", - "Asia/Kamchatka", - "Asia/Karachi", - "Asia/Kathmandu", - "Asia/Khandyga", - "Asia/Kolkata", - "Asia/Krasnoyarsk", - "Asia/Kuala_Lumpur", - "Asia/Kuching", - "Asia/Kuwait", - "Asia/Macau", - "Asia/Magadan", - "Asia/Makassar", - "Asia/Manila", - "Asia/Muscat", - "Asia/Nicosia", - "Asia/Novokuznetsk", - "Asia/Novosibirsk", - "Asia/Omsk", - "Asia/Oral", - "Asia/Phnom_Penh", - "Asia/Pontianak", - "Asia/Pyongyang", - "Asia/Qatar", - "Asia/Qostanay", - "Asia/Qyzylorda", - "Asia/Riyadh", - "Asia/Sakhalin", - "Asia/Samarkand", - "Asia/Seoul", - "Asia/Shanghai", - "Asia/Singapore", - "Asia/Srednekolymsk", - "Asia/Taipei", - "Asia/Tashkent", - "Asia/Tbilisi", - "Asia/Tehran", - "Asia/Thimphu", - "Asia/Tokyo", - "Asia/Tomsk", - "Asia/Ulaanbaatar", - "Asia/Urumqi", - "Asia/Ust-Nera", - "Asia/Vientiane", - "Asia/Vladivostok", - "Asia/Yakutsk", - "Asia/Yangon", - "Asia/Yekaterinburg", - "Asia/Yerevan", - "Atlantic/Azores", - "Atlantic/Bermuda", - "Atlantic/Canary", - "Atlantic/Cape_Verde", - "Atlantic/Faroe", - "Atlantic/Madeira", - "Atlantic/Reykjavik", - "Atlantic/South_Georgia", - "Atlantic/St_Helena", - "Atlantic/Stanley", - "Australia/Adelaide", - "Australia/Brisbane", - "Australia/Broken_Hill", - "Australia/Darwin", - "Australia/Eucla", - "Australia/Hobart", - "Australia/Lindeman", - "Australia/Lord_Howe", - "Australia/Melbourne", - "Australia/Perth", - "Australia/Sydney", - "Canada/Atlantic", - "Canada/Central", - "Canada/Eastern", - "Canada/Mountain", - "Canada/Newfoundland", - "Canada/Pacific", - "Europe/Amsterdam", - "Europe/Andorra", - "Europe/Astrakhan", - "Europe/Athens", - "Europe/Belgrade", - "Europe/Berlin", - "Europe/Bratislava", - "Europe/Brussels", - "Europe/Bucharest", - "Europe/Budapest", - "Europe/Busingen", - "Europe/Chisinau", - "Europe/Copenhagen", - "Europe/Dublin", - "Europe/Gibraltar", - "Europe/Guernsey", - "Europe/Helsinki", - "Europe/Isle_of_Man", - "Europe/Istanbul", - "Europe/Jersey", - "Europe/Kaliningrad", - "Europe/Kiev", - "Europe/Kirov", - "Europe/Lisbon", - "Europe/Ljubljana", - "Europe/London", - "Europe/Luxembourg", - "Europe/Madrid", - "Europe/Malta", - "Europe/Mariehamn", - "Europe/Minsk", - "Europe/Monaco", - "Europe/Moscow", - "Europe/Oslo", - "Europe/Paris", - "Europe/Podgorica", - "Europe/Prague", - "Europe/Riga", - "Europe/Rome", - "Europe/Samara", - "Europe/San_Marino", - "Europe/Sarajevo", - "Europe/Saratov", - "Europe/Simferopol", - "Europe/Skopje", - "Europe/Sofia", - "Europe/Stockholm", - "Europe/Tallinn", - "Europe/Tirane", - "Europe/Ulyanovsk", - "Europe/Uzhgorod", - "Europe/Vaduz", - "Europe/Vatican", - "Europe/Vienna", - "Europe/Vilnius", - "Europe/Volgograd", - "Europe/Warsaw", - "Europe/Zagreb", - "Europe/Zaporozhye", - "Europe/Zurich", - "GMT", - "Indian/Antananarivo", - "Indian/Chagos", - "Indian/Christmas", - "Indian/Cocos", - "Indian/Comoro", - "Indian/Kerguelen", - "Indian/Mahe", - "Indian/Maldives", - "Indian/Mauritius", - "Indian/Mayotte", - "Indian/Reunion", - "Pacific/Apia", - "Pacific/Auckland", - "Pacific/Bougainville", - "Pacific/Chatham", - "Pacific/Chuuk", - "Pacific/Easter", - "Pacific/Efate", - "Pacific/Fakaofo", - "Pacific/Fiji", - "Pacific/Funafuti", - "Pacific/Galapagos", - "Pacific/Gambier", - "Pacific/Guadalcanal", - "Pacific/Guam", - "Pacific/Honolulu", - "Pacific/Kanton", - "Pacific/Kiritimati", - "Pacific/Kosrae", - "Pacific/Kwajalein", - "Pacific/Majuro", - "Pacific/Marquesas", - "Pacific/Midway", - "Pacific/Nauru", - "Pacific/Niue", - "Pacific/Norfolk", - "Pacific/Noumea", - "Pacific/Pago_Pago", - "Pacific/Palau", - "Pacific/Pitcairn", - "Pacific/Pohnpei", - "Pacific/Port_Moresby", - "Pacific/Rarotonga", - "Pacific/Saipan", - "Pacific/Tahiti", - "Pacific/Tarawa", - "Pacific/Tongatapu", - "Pacific/Wake", - "Pacific/Wallis", - "US/Alaska", - "US/Arizona", - "US/Central", - "US/Eastern", - "US/Hawaii", - "US/Mountain", - "US/Pacific", - "UTC", -] diff --git a/fluentogram/src/__init__.py b/fluentogram/src/__init__.py deleted file mode 100644 index df424ed..0000000 --- a/fluentogram/src/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# coding=utf-8 -from . import abc -from . import impl - - -__all__ = ["abc", "impl", ] diff --git a/fluentogram/src/abc/misc.py b/fluentogram/src/abc/misc.py deleted file mode 100644 index 1f48458..0000000 --- a/fluentogram/src/abc/misc.py +++ /dev/null @@ -1,33 +0,0 @@ -# coding=utf-8 -""" -Some miscellaneous -""" - -from abc import ABC, abstractmethod - - -class AbstractAttribTracer(ABC): - """Implements a mechanism for tracing attributes access way. - - Like a pretty simple, external-typing supported version of the translator.get("some-key-for-translation") - - Equivalent to obj.some.key.for.translation(**some_kwargs) - """ - - @abstractmethod - def __init__(self) -> None: - self.request_line = "" - - @abstractmethod - def _get_request_line(self) -> str: - request_line = self.request_line - self.request_line = "" - return request_line - - @abstractmethod - def __getattr__(self, item) -> 'AbstractAttribTracer': - """ - This method exists to map the "obj.attrib1.attrib2" access to "attrib1-attrib2" key. - """ - self.request_line += f"{item}{self.separator}" - return self diff --git a/fluentogram/src/abc/runner.py b/fluentogram/src/abc/runner.py deleted file mode 100644 index 2d8f049..0000000 --- a/fluentogram/src/abc/runner.py +++ /dev/null @@ -1,27 +0,0 @@ -# coding=utf-8 -""" -An abstract translator runner -""" -from abc import ABC, abstractmethod - -from fluentogram.src.abc import AbstractAttribTracer - - -class AbstractTranslatorRunner(AbstractAttribTracer, ABC): - """This is one-shot per Telegram event translator with attrib tracer access way.""" - - @abstractmethod - def get(self, key: str, **kwargs) -> str: - """Fastest, direct way to use translator, without sugar-like typing supported attribute access way""" - - @abstractmethod - def _get_translation(self, key, **kwargs) -> str: - ... - - @abstractmethod - def __call__(self, **kwargs) -> str: - ... - - @abstractmethod - def __getattr__(self, item: str) -> 'AbstractTranslatorRunner': - ... diff --git a/fluentogram/src/abc/storage.py b/fluentogram/src/abc/storage.py deleted file mode 100644 index 7400fc3..0000000 --- a/fluentogram/src/abc/storage.py +++ /dev/null @@ -1,33 +0,0 @@ -# coding=utf-8 -""" -An abstract base for the Storage object -""" -from abc import ABC, abstractmethod -from typing import Any - - -class AbstractStorage(ABC): - - @abstractmethod - def __init__(self, kv, *args, **kwargs): - raise NotImplementedError - - @abstractmethod - def put(self, locale: str, key: str, value: Any, mapping_values: dict[str, Any]): - """Creates or replaces an existing key/value""" - raise NotImplementedError - - @abstractmethod - def create(self, locale: str, key: str, value: Any, mapping_values: dict[str, Any]): - """Create will add the key/value pair iff it does not exist.""" - raise NotImplementedError - - @abstractmethod - def delete(self, locale: str, *keys): - """Deletes all transmitted keys""" - raise NotImplementedError - - @abstractmethod - def listen(self, messages: dict[str, dict]) -> None: - """Listen for new keys/values and updates them in TranslatorHub""" - raise NotImplementedError \ No newline at end of file diff --git a/fluentogram/src/abc/translator_hub.py b/fluentogram/src/abc/translator_hub.py deleted file mode 100644 index b3d4591..0000000 --- a/fluentogram/src/abc/translator_hub.py +++ /dev/null @@ -1,50 +0,0 @@ -# coding=utf-8 -""" -An abstract base for the Translator Hub and Key/Value Translator Hub objects -""" -import sys -from abc import ABC, abstractmethod -if sys.version_info >= (3, 11): - from typing import Self, Any -else: - from typing import Any - from typing_extensions import Self - -from fluentogram.src.abc.runner import AbstractTranslatorRunner -from fluentogram.src.abc.storage import AbstractStorage - - -class AbstractTranslatorsHub(ABC): - """This class should contain a couple of translator objects, usually one object per one locale.""" - - @abstractmethod - def __init__(self): - raise NotImplementedError - - @abstractmethod - def get_translator_by_locale(self, locale: str) -> AbstractTranslatorRunner: - """ - Returns a Translator object by selected locale - """ - raise NotImplementedError - -class AbstractKvTranslatorHub(AbstractTranslatorsHub, ABC): - @abstractmethod - def from_storage(self, kv_storage: AbstractStorage) -> Self: - """ - Initializes the Translator Hub with the provided storage - """ - raise NotImplementedError - - @abstractmethod - def put(self, locale: str, key: str, value: Any, mapping_values: dict[str, Any]): - raise NotImplementedError - - @abstractmethod - def create(self, locale: str, key: str, value: Any, mapping_values: dict[str, Any]): - raise NotImplementedError - - @abstractmethod - def delete(self, locale: str, *keys): - raise NotImplementedError - diff --git a/fluentogram/src/impl/__init__.py b/fluentogram/src/impl/__init__.py deleted file mode 100644 index 32cf323..0000000 --- a/fluentogram/src/impl/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -# coding=utf-8 -from .attrib_tracer import AttribTracer -from .translator import FluentTranslator -from .runner import TranslatorRunner -from .transformers import MoneyTransformer, DateTimeTransformer -from .transator_hubs.translator_hub import TranslatorHub -from .transator_hubs.kv_translator_hub import KvTranslatorHub -from .storages import NatsStorage - -__all__ = [ - "AttribTracer", - "DateTimeTransformer", - "FluentTranslator", - "MoneyTransformer", - "TranslatorRunner", - "NatsStorage", - "TranslatorHub", - "KvTranslatorHub", -] diff --git a/fluentogram/src/impl/attrib_tracer.py b/fluentogram/src/impl/attrib_tracer.py deleted file mode 100644 index 445f516..0000000 --- a/fluentogram/src/impl/attrib_tracer.py +++ /dev/null @@ -1,25 +0,0 @@ -# coding=utf-8 -""" -An AttribTracer implementation -""" - -from fluentogram.src.abc import AbstractAttribTracer - - -class AttribTracer(AbstractAttribTracer): - """Attribute tracer class for obj.attrib1.attrib2 access""" - - def __init__(self) -> None: - self.request_line = "" - - def _get_request_line(self) -> str: - request_line = self.request_line - self.request_line = "" - return request_line - - def __getattr__(self, item) -> 'AttribTracer': - """ - This method exists to map the "obj.attrib1.attrib2" access to "attrib1-attrib2" key. - """ - self.request_line += f"{item}{self.separator}" - return self diff --git a/fluentogram/src/impl/filter.py b/fluentogram/src/impl/filter.py deleted file mode 100644 index 58aa83a..0000000 --- a/fluentogram/src/impl/filter.py +++ /dev/null @@ -1,62 +0,0 @@ -from typing import Optional, cast, Any - -from aiogram.filters import BaseFilter -from aiogram.types import Message -from fluentogram import TranslatorRunner - - -DEFAULT_TRANSLATOR_KEY = "i18n" - - -class FText(BaseFilter): - translator_key = DEFAULT_TRANSLATOR_KEY - - def __init__( - self, *, - equals: Optional[str] = None, - contains: Optional[str] = None, - startswith: Optional[str] = None, - endswith: Optional[str] = None, - ignore_case: bool = False, - translator_key: Optional[str] = None - ) -> None: - """ - Fluentogram text filter in the spirit of the deprecated aiogram Text filter. - - Usage example: - @router.message(FText(equals="command-help")) - async def helpCommand(message: Message, i18n: TranslatorRunner) -> TelegramMethod[Any]: - return message.answer(i18n.help()) - """ - self.equals = equals - self.contains = contains - self.startswith = startswith - self.endswith = endswith - self.ignore_case = ignore_case - - if translator_key: - self.translator_key = translator_key - - async def __call__( - self, event: Message, **data: Any - ) -> bool: - text = event.text or event.caption - if text is not None: - i18n: Optional[TranslatorRunner] = data.get(self.translator_key) - if i18n is None: - raise RuntimeError( - f"TranslatorRunner not found for key '{self.translator_key}'." - ) - if self.ignore_case: - text = text.casefold() - - if self.equals: - return cast(bool, text == i18n.get(self.equals)) # lmao idk what he wants but mypy can't see bool here - if self.contains: - return i18n.get(self.contains) in text - if self.startswith: - return text.startswith(i18n.get(self.startswith)) - if self.endswith: - return text.endswith(i18n.get(self.endswith)) - - return False diff --git a/fluentogram/src/impl/runner.py b/fluentogram/src/impl/runner.py deleted file mode 100644 index 79e5a5e..0000000 --- a/fluentogram/src/impl/runner.py +++ /dev/null @@ -1,39 +0,0 @@ -# coding=utf-8 -""" -A translator runner by itself -""" -from typing import Iterable - -from fluentogram.src.abc import AbstractTranslator -from fluentogram.src.abc.runner import AbstractTranslatorRunner -from fluentogram.src.impl import AttribTracer - - -class TranslatorRunner(AbstractTranslatorRunner, AttribTracer): - """This is one-shot per Telegram event translator with attrib tracer access way.""" - - def __init__(self, translators: Iterable[AbstractTranslator], separator: str = "-") -> None: - super().__init__() - self.translators = translators - self.separator = separator - self.request_line = "" - - def get(self, key: str, **kwargs) -> str: - """Fastest, direct way to use translator, without sugar-like typing supported attribute access way""" - return self._get_translation(key, **kwargs) - - def _get_translation(self, key, **kwargs): - for translator in self.translators: - try: - return translator.get(key, **kwargs) - except KeyError: - continue - - def __call__(self, **kwargs) -> str: - text = self._get_translation(self.request_line[:-1], **kwargs) - self.request_line = "" - return text - - def __getattr__(self, item: str) -> 'TranslatorRunner': - self.request_line += f"{item}{self.separator}" - return self diff --git a/fluentogram/src/impl/storages/__init__.py b/fluentogram/src/impl/storages/__init__.py deleted file mode 100644 index d0372c4..0000000 --- a/fluentogram/src/impl/storages/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from .nats_storage import NatsStorage \ No newline at end of file diff --git a/fluentogram/src/impl/storages/nats_storage.py b/fluentogram/src/impl/storages/nats_storage.py deleted file mode 100644 index c8bacd1..0000000 --- a/fluentogram/src/impl/storages/nats_storage.py +++ /dev/null @@ -1,134 +0,0 @@ -# coding=utf-8 -""" -NATS-based storage -""" -import asyncio -import json -import sys -from collections import defaultdict -if sys.version_info >= (3, 10): - from typing import Any, NoReturn, TypeAlias, Optional -else: - from typing import Any, NoReturn, Optional - from typing_extensions import TypeAlias - -from fluent_compiler.compiler import compile_messages -from fluent_compiler.resource import FtlResource -from nats.aio.msg import Msg -from nats.js import JetStreamContext -from nats.js.kv import KeyValue, KV_OP, KV_DEL, KV_PURGE - -from fluentogram.src.abc.storage import AbstractStorage - -KeyType: TypeAlias = Optional[str] -ValueType: TypeAlias = Optional[Any] -MappingValuesType: TypeAlias = Optional[dict[KeyType, ValueType]] - - -class NatsStorage(AbstractStorage): - def __init__( - self, - kv: KeyValue, - js: JetStreamContext, - separator: str = '.', - serializer=lambda data: json.dumps(data).encode('utf-8'), - deserializer=json.loads, - consume_timeout: float = 1.0 - ): - self._kv = kv - self._js = js - self.separator = separator - self.messages = None - self.serializer = serializer - self.deserializer = deserializer - self.consume_timeout = consume_timeout - - async def put( - self, - locale: str, - key: KeyType, - value: ValueType, - mapping_values: MappingValuesType - ): - await self._interaction( - func=self._kv.put, - locale=locale, - key=key, - value=value, - mapping_values=mapping_values - ) - - async def create( - self, - locale: str, - key: KeyType, - value: ValueType, - mapping_values: MappingValuesType - ): - await self._interaction( - func=self._kv.create, - locale=locale, - key=key, - value=value, - mapping_values=mapping_values - ) - - async def delete(self, locale: str, *keys): - await asyncio.gather( - *[ - self._kv.purge(f'{locale}{self.separator}{key}') - for key in keys - ] - ) - - async def _interaction( - self, - func, - locale: str, - key: KeyType, - value: ValueType, - mapping_values: MappingValuesType - ): - if key and value: - await func(f'{locale}{self.separator}{key}', self.serializer(value)) - if mapping_values: - await asyncio.gather( - *[ - func(f'{locale}{self.separator}{m_key}', self.serializer(m_value)) - for m_key, m_value in mapping_values.items() - ] - ) - - async def listen(self, messages: dict[str, dict]) -> NoReturn: - self.messages = messages - stream = await self._js.stream_info(self._kv._stream) - stream_name = stream.config.name - subject_name = stream_name.replace("_", self.separator, 1) - subject = f'${subject_name}.>' - consumer = await self._js.pull_subscribe(subject=subject, stream=stream_name) - while True: - try: - messages: list[Msg] = await consumer.fetch(50, timeout=self.consume_timeout) - except TimeoutError: - pass - else: - await self._update_compiled_messages(messages) - - async def _update_compiled_messages(self, messages: list[Msg]): - changes = defaultdict(list) - for m in messages: - kind = m.headers.get(KV_OP) if m.headers is not None else None - *args, locale, key = m.subject.split(self.separator) - if kind in (KV_DEL, KV_PURGE): - self.messages[locale].pop(key, None) - else: - value = self.deserializer(m.data) - changes[locale].append(f'{key} = {value}') - await m.ack() - self._set_new_compiled_messages(changes) - - def _set_new_compiled_messages(self, new_messages: dict[str, list[str]]) -> None: - for locale, messages in new_messages.items(): - resources = [FtlResource.from_string(message) for message in messages] - compiled_ftl = compile_messages(locale, resources) - self.messages[locale].update(compiled_ftl.message_functions) diff --git a/fluentogram/src/impl/stubs_translator_runner.py b/fluentogram/src/impl/stubs_translator_runner.py deleted file mode 100644 index ca7d713..0000000 --- a/fluentogram/src/impl/stubs_translator_runner.py +++ /dev/null @@ -1,12 +0,0 @@ -from fluentogram import AttribTracer - - -class StubsTranslatorRunner(AttribTracer): - def __init__(self): - super().__init__() - self.kwargs = {} - - def __call__(self, **kwargs): - out = self._get_request_line()[:-1], kwargs - self.request_line = "" - return out diff --git a/fluentogram/src/impl/transator_hubs/__init__.py b/fluentogram/src/impl/transator_hubs/__init__.py deleted file mode 100644 index 97fe9ec..0000000 --- a/fluentogram/src/impl/transator_hubs/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .kv_translator_hub import KvTranslatorHub -from .translator_hub import TranslatorHub diff --git a/fluentogram/src/impl/transator_hubs/kv_translator_hub.py b/fluentogram/src/impl/transator_hubs/kv_translator_hub.py deleted file mode 100644 index 112c4b8..0000000 --- a/fluentogram/src/impl/transator_hubs/kv_translator_hub.py +++ /dev/null @@ -1,103 +0,0 @@ -# coding=utf-8 -""" -A Key/Value Translator Hub, using as factory for Translator objects -""" -import asyncio - -from typing import Dict, Iterable, Union, Optional, List, Any - -from fluent_compiler.bundle import FluentBundle - -from fluentogram.exceptions import NotImplementedRootLocaleTranslator -from fluentogram.src.abc import AbstractTranslator -from fluentogram.src.abc.storage import AbstractStorage -from fluentogram.src.abc.translator_hub import AbstractKvTranslatorHub -from fluentogram.src.impl import FluentTranslator -from fluentogram.src.impl.transator_hubs.translator_hub import TranslatorHub - -class KvTranslatorHub(TranslatorHub, AbstractKvTranslatorHub): - def __init__( - self, - locales_map: Dict[str, Union[str, Iterable[str]]], - translators: Optional[List[AbstractTranslator]] = None, - root_locale: str = "en", - separator: str = "-" - ) -> None: - self.kv_storage = None - self.messages = None - if translators is not None: - super().__init__(locales_map, translators, root_locale, separator) - return - - self.locales_map = dict( - zip( - locales_map.keys(), - map( - lambda lang: tuple([lang]) if isinstance(lang, str) else lang, - locales_map.values() - ) - ) - ) - self.translators = translators - self.root_locale = root_locale - self.separator = separator - self.storage = None - self.translators_map = None - - async def from_storage(self, kv_storage: AbstractStorage): - self.kv_storage = kv_storage - if self.translators is None: - self.translators = self._create_translators() - - self.storage: Dict[str, AbstractTranslator] = dict( - zip([translator.locale for translator in self.translators], self.translators) - ) - if not self.storage.get(self.root_locale): - raise NotImplementedRootLocaleTranslator(self.root_locale) - self.translators_map: Dict[str, Iterable[AbstractTranslator]] = self._locales_map_parser(self.locales_map) - - self.messages = self._get_translators_messages() - - asyncio.create_task(self._on_update()) - - return self - - async def put(self, - locale: str, - key: Optional[str] = None, - value: Optional[Any] = None, - mapping_values: Optional[dict[str, Any]] = None): - await self.kv_storage.put(locale=locale, - key=key, - value=value, - mapping_values=mapping_values) - - async def create(self, - locale: str, - key: Optional[str] = None, - value: Optional[Any] = None, - mapping_values: Optional[dict[str, Any]] = None): - await self.kv_storage.create(locale, - key, - value, - mapping_values) - - async def delete(self, locale: str, *keys): - await self.kv_storage.delete(locale, *keys) - - def _create_translators(self) -> list[FluentTranslator]: - translators = [] - for locale in self.locales_map: - bundle = FluentBundle(locale=locale, resources=[]) - translators.append(FluentTranslator(locale=locale, - translator=bundle)) - return translators - - def _get_translators_messages(self) -> dict[str, dict]: - messages = {} - for translator in self.translators: - messages[translator.locale] = translator.translator._compiled_messages - return messages - - async def _on_update(self): - await asyncio.shield(self.kv_storage.listen(self.messages)) \ No newline at end of file diff --git a/fluentogram/src/impl/transator_hubs/translator_hub.py b/fluentogram/src/impl/transator_hubs/translator_hub.py deleted file mode 100644 index 0021502..0000000 --- a/fluentogram/src/impl/transator_hubs/translator_hub.py +++ /dev/null @@ -1,67 +0,0 @@ -# coding=utf-8 -""" -A Translator Hub, using as factory for Translator objects -""" -from typing import Dict, Iterable, List, Union - -from fluentogram.exceptions import NotImplementedRootLocaleTranslator -from fluentogram.src.abc import AbstractTranslator, AbstractTranslatorsHub -from fluentogram.src.impl import TranslatorRunner - - -class TranslatorHub(AbstractTranslatorsHub): - """ - This class implements a storage for all single-locale translators. - """ - - def __init__( - self, - locales_map: Dict[str, Union[str, Iterable[str]]], - translators: List[AbstractTranslator], - root_locale: str = "en", - separator: str = "-", - ) -> None: - self.locales_map = dict( - zip( - locales_map.keys(), - map( - lambda lang: tuple([lang]) if isinstance(lang, str) else lang, - locales_map.values() - ) - ) - ) - self.translators = translators - self.root_locale = root_locale - self.separator = separator - self.storage: Dict[str, AbstractTranslator] = dict( - zip([translator.locale for translator in translators], translators) - ) - if not self.storage.get(root_locale): - raise NotImplementedRootLocaleTranslator(self.root_locale) - self.translators_map: Dict[str, Iterable[AbstractTranslator]] = self._locales_map_parser(self.locales_map) - - def _locales_map_parser( - self, - locales_map: Dict[str, Union[str, Iterable[str]]] - ) -> Dict[str, Iterable[AbstractTranslator]]: - return { - lang: tuple( - [self.storage.get(locale) - for locale in translator_locales if locale in self.storage.keys()] - ) - for lang, translator_locales in - locales_map.items() - } - - def get_translator_by_locale(self, locale: str) -> TranslatorRunner: - """ - Here is a little tricky moment. - There should be like a one-shot scheme. - For proper isolation, function returns TranslatorRunner new instance every time, not the same translator. - This trick makes "obj.attribute1.attribute2" access to be able. - You are able to do what you want, refer to the abstract class. - """ - return TranslatorRunner( - translators=self.translators_map.get(locale) or self.translators_map[self.root_locale], - separator=self.separator - ) diff --git a/fluentogram/src/impl/transformers/datetime_transformer.py b/fluentogram/src/impl/transformers/datetime_transformer.py deleted file mode 100644 index 1788edc..0000000 --- a/fluentogram/src/impl/transformers/datetime_transformer.py +++ /dev/null @@ -1,26 +0,0 @@ -# coding=utf-8 -""" -A DateTimeTransformer itself -""" -from datetime import datetime -from typing import Union - -from fluent_compiler.types import fluent_date, FluentDateType, FluentNone - -from fluentogram.src.abc import AbstractDataTransformer - - -class DateTimeTransformer(AbstractDataTransformer): - """This transformer converts a default python datetime object to FluentDate - Typings refer to https://github.com/tc39/ecma402 - """ - - def __new__( - cls, - date: datetime, - **kwargs - ) -> Union[FluentDateType, FluentNone]: - return fluent_date( - date, - **kwargs - ) diff --git a/fluentogram/src/impl/translator.py b/fluentogram/src/impl/translator.py deleted file mode 100644 index 59dd4b0..0000000 --- a/fluentogram/src/impl/translator.py +++ /dev/null @@ -1,28 +0,0 @@ -# coding=utf-8 -""" -Fluent implementation of AbstractTranslator -""" - -from fluent_compiler.bundle import FluentBundle - -from fluentogram.src.abc import AbstractTranslator - - -class FluentTranslator(AbstractTranslator): - """Single-locale Translator, implemented with fluent_compiler Bundles""" - - def __init__(self, locale: str, translator: FluentBundle, separator: str = "-"): - self.locale = locale - self.translator = translator - self.separator = separator - - def get(self, key: str, **kwargs): - """STR100: Calling format with insecure string. - Route questions to --> https://github.com/django-ftl/fluent-compiler""" - text, errors = self.translator.format(key, kwargs) - if errors: - raise errors.pop() - return text - - def __repr__(self): - return f"" diff --git a/fluentogram/tests/__init__.py b/fluentogram/tests/__init__.py deleted file mode 100644 index f8a505e..0000000 --- a/fluentogram/tests/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# coding=utf-8 -from . import test_usage - -__all__ = ["test_usage", ] diff --git a/fluentogram/tests/test_stub_generation.py b/fluentogram/tests/test_stub_generation.py deleted file mode 100644 index 36c117e..0000000 --- a/fluentogram/tests/test_stub_generation.py +++ /dev/null @@ -1,236 +0,0 @@ -# coding=utf-8 -import unittest - -from fluentogram.typing_generator import ParsedRawFTL, Tree, Stubs - - -class StubGeneration(unittest.TestCase): - DEFAULT_STUB_TEXT = """from typing import Literal - - -class TranslatorRunner: - def get(self, path: str, **kwargs) -> str: ... - """ - - def _gen_stub_text(self, raw_text): - raw = ParsedRawFTL(raw_text) - messages = raw.get_messages() - tree = Tree(messages) - stubs = Stubs(tree) - return stubs.echo() - - def test_text_element(self): - self.assertEquals( - self._gen_stub_text("welcome = Welcome to the fluent aiogram addon!"), - self.DEFAULT_STUB_TEXT - + ''' - @staticmethod - def welcome() -> Literal["""Welcome to the fluent aiogram addon!"""]: ... - -''', - ) - - def test_variable_reference(self): - self.assertEquals( - self._gen_stub_text( - "greet-by-name = Hello, { $user }... You name is { $user }, isn't it?" - ), - self.DEFAULT_STUB_TEXT - + ''' - greet: Greet - - -class Greet: - by: GreetBy - - -class GreetBy: - @staticmethod - def name(*, user) -> Literal["""Hello, { $user }... You name is { $user }, isn't it?"""]: ... - -''', - ) - - def test_variable_reference_with_two_args(self): - self.assertEquals( - self._gen_stub_text( - "shop-success-payment = Your money, { $amount }, has been sent successfully at { $dt }." - ), - self.DEFAULT_STUB_TEXT - + ''' - shop: Shop - - -class Shop: - success: ShopSuccess - - -class ShopSuccess: - @staticmethod - def payment(*, amount, dt) -> Literal["""Your money, { $amount }, has been sent successfully at { $dt }."""]: ... - -''', - ) - - def test_selector(self): - self.assertEquals( - self._gen_stub_text( - """test-bool_indicator = { $is_true -> - [one] ✅ - *[other] ❌ - } """ - ), - self.DEFAULT_STUB_TEXT - + ''' - test: Test - - -class Test: - @staticmethod - def bool_indicator(*, is_true) -> Literal["""{ $is_true -> -[one] ✅ -*[other] ❌ -}"""]: ... - -''', - ) - - def test_selector_num_key(self): - self.assertEquals( - self._gen_stub_text( - """test-bool_indicator = { $is_true -> - [0] ✅ - *[other] ❌ - } """ - ), - self.DEFAULT_STUB_TEXT - + ''' - test: Test - - -class Test: - @staticmethod - def bool_indicator(*, is_true) -> Literal["""{ $is_true -> -[0] ✅ -*[other] ❌ -}"""]: ... - -''', - ) - - def test_recursion(self): - self.assertEquals( - self._gen_stub_text( - """recursion = { $is_true -> -[one] one -*[other] Recursion { $is_true -> - [one] one - *[other] Recursion { $is_true -> - [one] one - *[other] Recursion - } - } -}""" - ), - self.DEFAULT_STUB_TEXT - + ''' - @staticmethod - def recursion(*, is_true) -> Literal["""{ $is_true -> -[one] one -*[other] Recursion -*[other] { $is_true -> -[one] one -*[other] Recursion -*[other] { $is_true -> -[one] one -*[other] Recursion -} -} -}"""]: ... - -''', - ) - - def test_function_reference(self): - self.assertEquals( - self._gen_stub_text("test-number = { NUMBER($num, useGrouping: 0) }"), - self.DEFAULT_STUB_TEXT - + ''' - test: Test - - -class Test: - @staticmethod - def number(*, num) -> Literal["""{ NUMBER({ $num }, useGrouping: 0) }"""]: ... - -''', - ) - - def test_message_reference_to_text(self): - self.assertEquals( - self._gen_stub_text( - """simple = text -ref = { simple } - """ - ), - self.DEFAULT_STUB_TEXT - + ''' - @staticmethod - def simple() -> Literal["""text"""]: ... - - @staticmethod - def ref() -> Literal["""text"""]: ... - -''', - ) - - def test_message_reference_to_var(self): - self.assertEquals( - self._gen_stub_text( - """var = { $name } -ref = { var } - """ - ), - self.DEFAULT_STUB_TEXT - + ''' - @staticmethod - def var(*, name) -> Literal["""{ $name }"""]: ... - - @staticmethod - def ref(*, name) -> Literal["""{ $name }"""]: ... - -''', - ) - - def test_message_reference_in_selector(self): - self.assertEquals( - self._gen_stub_text( - """foo = { $var -> -[test] { test } -*[any] any text -} -test = { $is_true -> -[one] true -*[other] false -} -""" - ), - self.DEFAULT_STUB_TEXT - + ''' - @staticmethod - def test(*, is_true) -> Literal["""{ $is_true -> -[one] true -*[other] false -}"""]: ... - - @staticmethod - def foo(*, var, is_true) -> Literal["""{ $var -> -[test] { $is_true -> -[one] true -*[other] false -} -*[any] any text -}"""]: ... - -''', - ) diff --git a/fluentogram/tests/test_usage.py b/fluentogram/tests/test_usage.py deleted file mode 100644 index 7f911bb..0000000 --- a/fluentogram/tests/test_usage.py +++ /dev/null @@ -1,46 +0,0 @@ -# coding=utf-8 -import unittest -from datetime import datetime -from decimal import Decimal - -from fluent_compiler.bundle import FluentBundle - -from fluentogram import ( - FluentTranslator, - TranslatorHub, - TranslatorRunner, - MoneyTransformer, - DateTimeTransformer, -) - - -class BasicUsage(unittest.TestCase): - def test_full_usage(self): - example_ftl_file_content = """ -welcome = Welcome to the fluent aiogram addon! -greet-by-name = Hello, { $user }! -shop-success-payment = Your money, { $amount }, has been sent successfully at { $dt }. - """ - t_hub = TranslatorHub( - {"ua": ("ua", "ru", "en"), "ru": ("ru", "en"), "en": ("en",)}, - translators=[ - FluentTranslator( - locale="en", - translator=FluentBundle.from_string( - "en-US", example_ftl_file_content, use_isolating=False - ), - ) - ], - root_locale="en", - ) - translator_runner: TranslatorRunner = t_hub.get_translator_by_locale("en") - print( - translator_runner.welcome(), - "\n", - translator_runner.greet.by.name(user="Alex"), - "\n", - translator_runner.shop.success.payment( - amount=MoneyTransformer(currency="$", amount=Decimal("500")), - dt=DateTimeTransformer(datetime.now()), - ), - ) diff --git a/fluentogram/typing_generator/__init__.py b/fluentogram/typing_generator/__init__.py deleted file mode 100644 index 0ebf246..0000000 --- a/fluentogram/typing_generator/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -from .parsed_ftl import ParsedRawFTL -from .renderable_items import Knot, InternalMethod, Method -from .stubs import Stubs -from .translation_dto import Translation -from .tree import Tree, TreeNode - -__all__ = [ - "InternalMethod", - "Knot", - "Method", - "ParsedRawFTL", - "Stubs", - "Translation", - "Tree", - "TreeNode", -] diff --git a/fluentogram/typing_generator/parsed_ftl.py b/fluentogram/typing_generator/parsed_ftl.py deleted file mode 100644 index 613668b..0000000 --- a/fluentogram/typing_generator/parsed_ftl.py +++ /dev/null @@ -1,180 +0,0 @@ -from dataclasses import dataclass -from typing import Dict - -from fluent.syntax import FluentParser -from fluent.syntax.ast import ( - Message, - Placeable, - Literal, - TextElement, - VariableReference, - SelectExpression, - MessageReference, - StringLiteral, - NumberLiteral, - TermReference, - FunctionReference, - InlineExpression, - NamedArgument, - Identifier, -) -from ordered_set import OrderedSet - -from fluentogram.typing_generator.translation_dto import Translation - - -@dataclass -class Node: - value: str - args: list[str] - - -class ReferenceNotExists(Exception): - pass - - -class ParsedRawFTL: - def __init__(self, ftl_data: str, parser=FluentParser()) -> None: - self.parsed_ftl = parser.parse(ftl_data) - self.nodes: dict[str, Node] - - def _parse_literal(self, obj: Literal) -> Node: - return Node(value=obj.parse()["value"], args=[]) - - def _parse_string_literal(self, obj: StringLiteral) -> Node: - return self._parse_literal(obj) - - def _parse_number_literal(self, obj: NumberLiteral) -> Node: - data = obj.parse() - return Node(value=str(f"{data['value']:.{data['precision']}f}"), args=[]) - - def _parse_message_reference(self, obj: MessageReference) -> Node: - if obj.id.name not in self.nodes: - raise ReferenceNotExists - return self.nodes[obj.id.name] - - def _parse_text_element(self, obj: TextElement) -> Node: - return Node(value=obj.value, args=[]) - - def _parse_variable_reference(self, obj: VariableReference) -> Node: - return Node(value=f"{{ ${obj.id.name} }}", args=[obj.id.name]) - - def _parse_named_argument(self, obj: NamedArgument) -> Node: - if isinstance(obj.value, NumberLiteral): - value = self._parse_number_literal(obj.value) - else: # StringLiteral - value = self._parse_string_literal(obj.value) - return Node(value=f"{obj.name.name}: {value.value}", args=[]) - - def _parse_function_reference(self, obj: FunctionReference) -> Node: - named_args = [] - for named_arg in obj.arguments.named: - named_args.append(self._parse_named_argument(named_arg)) - positionals = [] - for positional in obj.arguments.positional: - if isinstance(positional, Placeable): - positionals.append(self._parse_placeable(positional)) - else: # InlineExpression - positionals.append(self._parse_inline_expression(positional)) - - named_args_string = ", ".join([named_arg.value for named_arg in named_args]) - positional_string = ", ".join([positional.value for positional in positionals]) - - positional_args = [] - for positional in positionals: - positional_args += positional.args - - return Node( - value=f"{{ {obj.id.name}({positional_string}{',' if named_args_string else ''} {named_args_string}) }}", - args=positional_args, - ) - - def _parse_inline_expression(self, obj: InlineExpression) -> Node: - if isinstance(obj, NumberLiteral): - return self._parse_number_literal(obj) - elif isinstance(obj, StringLiteral): - return self._parse_string_literal(obj) - elif isinstance(obj, MessageReference): - return self._parse_message_reference(obj) - elif isinstance(obj, TermReference): - ... - # TODO: implementation - elif isinstance(obj, VariableReference): - return self._parse_variable_reference(obj) - elif isinstance(obj, FunctionReference): - return self._parse_function_reference(obj) - return Node(value="", args=[]) - - def _parse_select_expression(self, obj: SelectExpression) -> Node: - selector = self._parse_inline_expression(obj.selector) - value = f"{{ {', '.join([f'${s}' for s in selector.args])} ->" - args = selector.args - - for variant in obj.variants: - for element in variant.value.elements: - if isinstance(element, TextElement): - variant_node = self._parse_text_element(element) - elif isinstance(element, Placeable): - variant_node = self._parse_placeable(element) - else: - continue - if isinstance(variant.key, Identifier): - key_name = variant.key.name - elif isinstance(variant.key, NumberLiteral): - key_name = self._parse_number_literal(variant.key).value - else: - continue - value += f"\n{'*' if variant.default else ''}[{key_name}] {variant_node.value}" - args += variant_node.args - - value += "\n}" - return Node(value=value, args=args) - - def _parse_placeable(self, obj: Placeable) -> Node: - ex = obj.expression - if isinstance(ex, VariableReference): - return self._parse_variable_reference(ex) - elif isinstance(ex, SelectExpression): - return self._parse_select_expression(ex) - elif isinstance(ex, InlineExpression): - return self._parse_inline_expression(ex) - elif isinstance(ex, Placeable): - return self._parse_placeable(ex) - return Node(value="", args=[]) - - def _parse_message(self, obj: Message) -> Node: - nodes: list[Node] = [] - for element in obj.value.elements: - if isinstance(element, TextElement): - nodes.append(self._parse_text_element(element)) - elif isinstance(element, Placeable): - nodes.append(self._parse_placeable(element)) - else: - continue - - node_value, node_args = "", [] - for sub_node in nodes: - node_value += sub_node.value - node_args += sub_node.args - return Node(value=node_value, args=node_args) - - def _parse_body(self) -> dict[str, Node]: - self.nodes: dict[str, Node] = {} - for message in self.parsed_ftl.body: - if not isinstance(message, Message): - # TODO: other Entry - continue - if message.value is None: - continue - try: - self.nodes[message.id.name] = self._parse_message(message) - except ReferenceNotExists: - self.parsed_ftl.body.append(message) - continue - return self.nodes - - def get_messages(self) -> Dict[str, Translation]: - return { - name: Translation(node.value, args=OrderedSet(node.args)) - for name, node in self._parse_body().items() - } diff --git a/fluentogram/typing_generator/renderable_items.py b/fluentogram/typing_generator/renderable_items.py deleted file mode 100644 index 149136b..0000000 --- a/fluentogram/typing_generator/renderable_items.py +++ /dev/null @@ -1,92 +0,0 @@ -# coding=utf-8 -from typing import Optional, List, Iterable - -try: - from jinja2 import Template -except ModuleNotFoundError: - raise ModuleNotFoundError("You should install Jinja2 package to use cli tools") - - -class RenderAble: - render_pattern: Template - - def __init__(self, **kwargs) -> None: - self.kwargs = kwargs - - def render(self) -> str: - return self.render_pattern.render(**self.kwargs) + "\n" - - -class Import(RenderAble): - render_pattern = Template("from typing import Literal", autoescape=True) - - -class Method(RenderAble): - render_pattern = Template( - " @staticmethod\n" - ' def {{ method_name }}({{ args }}) -> Literal["""{{ translation }}"""]: ...', - autoescape=True, - ) - - def __init__( - self, method_name: str, translation: str, args: Optional[Iterable[str]] = None - ) -> None: - if args: - formatted_args = "*, " + ", ".join(args) - else: - formatted_args = "" - super().__init__(translation=translation, args=formatted_args) - self.kwargs["method_name"] = method_name - - -class InternalMethod(Method): - def __init__(self, translation: str, args: Optional[Iterable[str]] = None) -> None: - super().__init__(method_name="__call__", translation=translation, args=args) - - -class Var(RenderAble): - render_pattern = Template( - " {{ var_name }}: {{ var_full_name }}", autoescape=True - ) - - def __init__(self, var_name: str, var_full_name: Optional[str] = None) -> None: - super().__init__( - var_name=var_name, - var_full_name=var_name if not var_full_name else var_full_name, - ) - - -class Knot(RenderAble): - render_pattern = Template("\nclass {{ class_name }}:\n", autoescape=True) - - def __init__(self, class_name: str) -> None: - super().__init__() - self.class_name = class_name - self.variables: List[Var] = [] - self.methods: List[Method] = [] - - def render(self) -> str: - text = self.render_pattern.render(class_name=self.class_name) + "\n" - for var in self.variables: - text += var.render() - if self.variables: - text += "\n" - for method in self.methods: - text += method.render() + "\n" - return text - - def add_var(self, var: Var) -> None: - self.variables.append(var) - - def add_method(self, method: Method) -> None: - self.methods.append(method) - - -class Runner(Knot): - render_pattern = Template( - "\nclass {{ class_name }}:\n def get(self, path: str, **kwargs) -> str: ...\n ", - autoescape=True, - ) - - def __init__(self, name: str = "TranslatorRunner") -> None: - super().__init__(name) diff --git a/fluentogram/typing_generator/stubs.py b/fluentogram/typing_generator/stubs.py deleted file mode 100644 index c126b88..0000000 --- a/fluentogram/typing_generator/stubs.py +++ /dev/null @@ -1,51 +0,0 @@ -from typing import Iterator - -from fluentogram.typing_generator.renderable_items import ( - Var, - Knot, - InternalMethod, - Method, - Runner, -) -from fluentogram.typing_generator.tree import Tree - - -class Stubs: - def __init__(self, tree: Tree, root: str = "TranslatorRunner") -> None: - self.root = root - self.nodes = tree.elements - self.content: str = "from typing import Literal\n\n " - for stub in self._gen_stubs(): - self.content += stub - - def _gen_stubs(self) -> Iterator[str]: - for path, node in self.nodes.items(): - if node.is_leaf: - continue - if node.path: - knot = Knot(node.path) - else: - knot = Runner(self.root) - if node.children: - if node.value: - knot.add_method( - InternalMethod(node.value, args=node.translation_vars) - ) - for name, sub_node in node.children.items(): - if sub_node.is_leaf: - if sub_node.value: - knot.add_method( - Method( - name, sub_node.value, args=sub_node.translation_vars - ) - ) - else: - knot.add_var(Var(name, sub_node.path)) - yield knot.render() - - def to_file(self, file_name: str) -> None: - with open(file_name, "w", encoding="utf-8") as f: - f.write(self.content) - - def echo(self) -> str: - return self.content diff --git a/fluentogram/typing_generator/translation_dto.py b/fluentogram/typing_generator/translation_dto.py deleted file mode 100644 index 5c78825..0000000 --- a/fluentogram/typing_generator/translation_dto.py +++ /dev/null @@ -1,9 +0,0 @@ -from dataclasses import dataclass - -from ordered_set import OrderedSet - - -@dataclass -class Translation: - text: str - args: OrderedSet diff --git a/fluentogram/typing_generator/tree.py b/fluentogram/typing_generator/tree.py deleted file mode 100644 index 900ae29..0000000 --- a/fluentogram/typing_generator/tree.py +++ /dev/null @@ -1,59 +0,0 @@ -from dataclasses import dataclass -from typing import Optional - -from ordered_set import OrderedSet - -from fluentogram.typing_generator.translation_dto import Translation - - -@dataclass -class TreeNode: - path: str - children: dict[str, "TreeNode"] - name: str - value: Optional[str] = None - translation_vars: Optional[OrderedSet] = None - - @property - def is_leaf(self) -> bool: - if not self.children: - return True - return False - - -class Tree: - def __init__( - self, - ftl_syntax: dict[str, Translation], - separator: str = "-", - safe_separator: str = "", - ) -> None: - self.safe_separator = safe_separator - self.ftl_syntax = ftl_syntax - self.separator = separator - self.elements: dict[tuple[str, ...], TreeNode] = {} - for path, translation in ftl_syntax.items(): - *point_path, name = path.split("-") - point_path.insert(0, "") - self._build(tuple(point_path), name, translation) - - def path_to_str(self, path: tuple) -> str: - clean_path = map(lambda s: s[0].capitalize() + s[1:], filter(lambda x: x, path)) - return self.safe_separator.join(clean_path) - - def _build(self, path: tuple[str, ...], name: str, value=None) -> None: - own_class_def = TreeNode( - path=self.path_to_str(path + (name,)), - name=name, - value=value.text if value else "", - children={}, - translation_vars=value.args if value else OrderedSet(), - ) - - if path: - if path not in self.elements: - self._build(path[:-1], path[-1]) - - self.elements[path].children[name] = own_class_def - - self.elements.setdefault(path + (name,), own_class_def) diff --git a/pyproject.toml b/pyproject.toml index 01c15af..4bd5171 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,11 +7,14 @@ requires-python = "~=3.9" license = "MIT" dependencies = [ "fluent-compiler~=1.1", - "watchdog>=2.3.0,<3", "ordered-set>=4.1.0,<5", "nats-py>=2.9.0,<3", "typing-extensions>=4.12.2,<5", ] +[project.optional-dependencies] +cli = ["watchdog>=2.3.0,<3"] +dev = ["ruff~=0.11.2"] +aiogram = ["aiogram~=3.20.0"] [project.urls] Repository = "https://github.com/Arustinal/fluentogram" @@ -22,3 +25,48 @@ i18n = "fluentogram.cli:cli" [build-system] requires = ["setuptools>=42"] build-backend = "setuptools.build_meta" + +[tool.ruff] +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", + "tmp" +] +include = ["fluentogram/**/*.py"] +line-length = 120 +indent-width = 4 + +[tool.ruff.lint] +select = ["ALL"] +ignore = ["D", "RET504", "PGH003", "FBT001", "EM101", "TRY003", "TC003", "ANN401", "ANN003", "TC001", "TC002"] + +[tool.ruff.lint.mccabe] +max-complexity = 5 + +[tool.ruff.format] +quote-style = "double" +indent-style = "space" \ No newline at end of file From 46956554aaa6ce94899afc1244ea759758f4cb2c Mon Sep 17 00:00:00 2001 From: sheldy Date: Sat, 28 Jun 2025 17:20:15 +0200 Subject: [PATCH 02/36] Return ability to get translation by `_getattr__`. --- fluentogram/impl/runner.py | 11 ++++++++++- pyproject.toml | 2 -- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/fluentogram/impl/runner.py b/fluentogram/impl/runner.py index 803411d..b8ba1a7 100644 --- a/fluentogram/impl/runner.py +++ b/fluentogram/impl/runner.py @@ -12,7 +12,7 @@ class TranslatorRunner(AbstractTranslatorRunner): def __init__(self, translators: Iterable[AbstractTranslator], separator: str = "-") -> None: self.translators = translators self.separator = separator - self.request_line = "" + self._request_line = "" def get(self, key: str, **kwargs: Any) -> str: """Fastest, direct way to use translator, without sugar-like typing supported attribute access way""" @@ -25,3 +25,12 @@ def _get_translation(self, key: str, **kwargs: Any) -> str: return text raise KeyNotFoundError(key) + + def __getattr__(self, item: str) -> "TranslatorRunner": + self._request_line += f"{item}{self.separator}" + return self + + def __call__(self, **kwargs: Any) -> str: + text = self._get_translation(self._request_line.rstrip(self.separator), **kwargs) + self._request_line = "" + return text diff --git a/pyproject.toml b/pyproject.toml index 4bd5171..4dfb1de 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,8 +7,6 @@ requires-python = "~=3.9" license = "MIT" dependencies = [ "fluent-compiler~=1.1", - "ordered-set>=4.1.0,<5", - "nats-py>=2.9.0,<3", "typing-extensions>=4.12.2,<5", ] [project.optional-dependencies] From 239e522484fa626cbbb35b4c8879699f5eaee152 Mon Sep 17 00:00:00 2001 From: sheldy Date: Sat, 28 Jun 2025 21:41:41 +0200 Subject: [PATCH 03/36] Update dependencies. --- pyproject.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4dfb1de..9379192 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,16 +3,16 @@ name = "fluentogram" version = "1.1.13" description = "A proper way to use an i18n mechanism with Aiogram3." authors = [{ name = "Aleks" }] -requires-python = "~=3.9" +requires-python = ">=3.9" license = "MIT" dependencies = [ "fluent-compiler~=1.1", - "typing-extensions>=4.12.2,<5", ] [project.optional-dependencies] cli = ["watchdog>=2.3.0,<3"] -dev = ["ruff~=0.11.2"] +dev = ["ruff~=0.11.2", "pytest~=8.4.1"] aiogram = ["aiogram~=3.20.0"] +nats = ["nats-py~=2.10.0"] [project.urls] Repository = "https://github.com/Arustinal/fluentogram" From 44adae24ef57aae38d4d20c13bceae5a772f2849 Mon Sep 17 00:00:00 2001 From: sheldy Date: Sat, 28 Jun 2025 21:41:59 +0200 Subject: [PATCH 04/36] Update .gitignore. --- .gitignore | 1 + .vscode/settings.json | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) delete mode 100644 .vscode/settings.json diff --git a/.gitignore b/.gitignore index a2787e6..05f307e 100644 --- a/.gitignore +++ b/.gitignore @@ -74,3 +74,4 @@ dmypy.json .pytype/ cython_debug/ .idea +.vscode \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100644 index 825efbe..0000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "cSpell.words": ["fluentogram"] -} From fa67856d689795cc3706997fc8ca9ffbdc939ed2 Mon Sep 17 00:00:00 2001 From: sheldy Date: Sat, 28 Jun 2025 21:42:16 +0200 Subject: [PATCH 05/36] Remove docs. --- README.md | 29 ---------- example/BaseUsage.md | 45 ---------------- example/KeyValueTranslatorHubUsage.md | 76 --------------------------- example/MRE.py | 57 -------------------- example/TranslatorHub.md | 56 -------------------- example/TranslatorRunner.md | 30 ----------- example/TypingGenerator.md | 29 ---------- 7 files changed, 322 deletions(-) delete mode 100644 README.md delete mode 100644 example/BaseUsage.md delete mode 100644 example/KeyValueTranslatorHubUsage.md delete mode 100644 example/MRE.py delete mode 100644 example/TranslatorHub.md delete mode 100644 example/TranslatorRunner.md delete mode 100644 example/TypingGenerator.md diff --git a/README.md b/README.md deleted file mode 100644 index 3aea1e6..0000000 --- a/README.md +++ /dev/null @@ -1,29 +0,0 @@ -# fluentogram - -A proper way to use an i18n mechanism with Aiogram3. Using Project Fluent by Mozilla -https://projectfluent.org/fluent/guide/ - -Short example: - -```py -# Somewhere in middleware.Grab language_code from telegram user object, or database, etc. -translator_runner: TranslatorRunner = t_hub.get_translator_by_locale("en") - -# In message handler: -async def message_handler(message: Message, ..., i18n: TranslatorRunner): - ... - await message.answer(i18n.welcome()) - await message.answer(i18n.greet.by.name(user="Alex")) #aka message.from_user.username - await message.answer(i18n.shop.success.payment( - amount=MoneyTransformer(currency="$", amount=Decimal("500")), - dt=DateTimeTransformer(datetime.now())) - -# Going to be like: - """ - Welcome to the fluent aiogram addon! - Hello, Alex! - Your money, $500.00, has been sent successfully at Dec 4, 2022. - """ -``` - -Check [*Examples*](example) folder. diff --git a/example/BaseUsage.md b/example/BaseUsage.md deleted file mode 100644 index 280d17c..0000000 --- a/example/BaseUsage.md +++ /dev/null @@ -1,45 +0,0 @@ -```py -example_ftl_file_content = """ -welcome = Welcome to the fluent aiogram addon! -greet-by-name = Hello, { $user }! -shop-success-payment = Your money, { $amount }, has been sent successfully. -""" - -# main.py of bot: -example_ftl_file_content = """ -welcome = Welcome to the fluent aiogram addon! -greet-by-name = Hello, { $user }! -shop-success-payment = Your money, { $amount }, has been sent successfully at { $dt }. -""" - -t_hub = TranslatorHub( - {"ua": ("ua", "ru", "en"), - "ru": ("ru", "en"), - "en": ("en",)}, - translators=[ - FluentTranslator(locale="en", - translator=FluentBundle.from_string("en-US", example_ftl_file_content, - use_isolating=False))], - FluentTranslator(locale="ru", - translator=...)] - root_locale="en", -) - -# Somewhere in middleware.Grab language_code from telegram user object, or database, etc. -translator_runner: TranslatorRunner = t_hub.get_translator_by_locale("en") - -# In message handler: -async def message_handler(message: Message, ..., i18n: TranslatorRunner): - ... - await message.answer(i18n.welcome()) - await message.answer(i18n.greet.by.name(user="Alex")) #aka message.from_user.username - await message.answer(i18n.shop.success.payment( - amount=MoneyTransformer(currency="$", amount=Decimal("500")), - dt=DateTimeTransformer(datetime.now())) - -# Going to be like: - """ - Welcome to the fluent aiogram addon! - Hello, Alex! - Your money, $500.00, has been sent successfully at Dec 4, 2022. - """ \ No newline at end of file diff --git a/example/KeyValueTranslatorHubUsage.md b/example/KeyValueTranslatorHubUsage.md deleted file mode 100644 index c46d319..0000000 --- a/example/KeyValueTranslatorHubUsage.md +++ /dev/null @@ -1,76 +0,0 @@ -```py -import asyncio - -import nats -from fluent_compiler.bundle import FluentBundle -from nats.js.api import KeyValueConfig, StorageType - -from fluentogram import FluentTranslator -from fluentogram import NatsStorage -from fluentogram.src.impl.transator_hubs import KvTranslatorHub - -async def main(): - # initialising the NATS connection and JetStream - nc = await nats.connect(servers = ["nats://localhost:4222"]) - js = nc.jetstream() - - # key/value store creation - kv_config = KeyValueConfig( - bucket='my_bucket', - storage=StorageType.FILE, - ) - kv = await js.create_key_value(config=kv_config) - storage = NatsStorage(kv=kv, js=js) - - translator_hub = KvTranslatorHub( - { - 'ru': ('ru', 'en'), - 'en': ('en',), - - }, - translators=[ - FluentTranslator(locale='en', - translator=FluentBundle.from_string( - locale='en-US', - text='hello = Hello {$name}!') - ), - FluentTranslator(locale='ru', - translator=FluentBundle.from_string( - locale='ru', - text='hello = Привет {$name}!') - ) - ] - ) - - # initialising storage in the translator hub - await translator_hub.from_storage(kv_storage=storage) - - i18n_ru = translator_hub.get_translator_by_locale(locale='ru') - i18n_en = translator_hub.get_translator_by_locale(locale='en') - - # retrieving texts from passed translators - print(i18n_ru.hello(name='Александр')) # Привет Александр! - print(i18n_en.hello(name='Alex')) # Hello Alex! - - # creating and updating keys/values - await translator_hub.put(locale='ru', key='put_key', value='вставленное или обновленное значение №{$number}') # key insertion or replacement - await translator_hub.create(locale='en', key='create_key', value='create value №{$number}') # only if this key does not exist - await translator_hub.put(locale='en', mapping_values={ - "mapping_key1": "mapping value №1", - "mapping_key2": "mapping value №2", - }) - - # it may take a few seconds for a listener running in the background to catch and process incoming updates - await asyncio.sleep(5) - - print(i18n_ru.put_key(number=1)) # вставленное или обновленное значение №1 - print(i18n_en.create_key(number=1)) # create value №1 - print(i18n_en.mapping_key1()) # mapping value №1 - print(i18n_en.mapping_key2()) # mapping value №2 - - # deletion keys/values - await translator_hub.delete('en', "mapping_key1", "create_key") - -if __name__ == '__main__': - asyncio.run(main()) -``` \ No newline at end of file diff --git a/example/MRE.py b/example/MRE.py deleted file mode 100644 index d5d6c9e..0000000 --- a/example/MRE.py +++ /dev/null @@ -1,57 +0,0 @@ -import asyncio -import os -from collections.abc import Awaitable, Callable -from typing import TYPE_CHECKING, Any - -from aiogram import BaseMiddleware, Bot, Dispatcher, Router -from aiogram.types import Message -from fluent_compiler.bundle import FluentBundle - -from fluentogram import FluentTranslator, TranslatorHub, TranslatorRunner - -if TYPE_CHECKING: - from stub import TranslatorRunner # you haven't this file until you use TypingGenerator - - -class TranslatorRunnerMiddleware(BaseMiddleware): - async def __call__( - self, - handler: Callable[[Message, dict[str, Any]], Awaitable[Any]], - event: Message, - data: dict[str, Any], - ) -> Any: - hub: TranslatorHub = data.get("_translator_hub") - # There you can ask your database for locale - data["i18n"] = hub.get_translator_by_locale(event.from_user.language_code) - return await handler(event, data) - - -main_router = Router() -main_router.message.middleware(TranslatorRunnerMiddleware()) - - -@main_router.message() -async def handler(message: Message, i18n: TranslatorRunner): - await message.answer(i18n.start.hello(username=message.from_user.username)) - - -async def main(): - translator_hub = TranslatorHub( - { - "ru": ("ru", "en"), - "en": ("en",), - }, - [ - FluentTranslator("en", translator=FluentBundle.from_string("en-US", "start-hello = Hello, { $username }")), - FluentTranslator("ru", translator=FluentBundle.from_string("ru", "start-hello = Привет, { $username }")), - ], - ) - bot = Bot(token=os.getenv("TOKEN")) - dp = Dispatcher() - dp.include_router(main_router) - print("bot is ready") - await dp.start_polling(bot, _translator_hub=translator_hub) - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/example/TranslatorHub.md b/example/TranslatorHub.md deleted file mode 100644 index b051f20..0000000 --- a/example/TranslatorHub.md +++ /dev/null @@ -1,56 +0,0 @@ -# TranslatorHub - -TranslatorHub is unit of distribution TranslatorRunner's. - -Init: - -```python -def __init__( - self, - locales_map: Dict[str, Union[str, Iterable[str]]], - translators: List[TAbstractTranslator], - root_locale: str = "en", -) -> None: -``` - -*Locales map* - that's like a configuration map for "Rollback" feature. If you haven't configured translation for -current locale - first in collection, -"Rollback" will look to others locales' data and try to find a translation - -For example: - -```python -locales_map = { - "ua": ("ua", "de", "en"), - "de": ("de", "en"), - "en": ("en",) -} -``` - -Let's look at this example - -If translator does not find a translation for "ua" locale in "ua" data, next stop is "de" data. If it's failed too - it -will look to "en" translations. - -*Translators* - List of translator instances. Every translator has only one locale. Refer to "Translator" doc page. -First parameter - telegram locale. Pay attention to format of them. Hub's -method `def get_translator_by_locale(self, locale: str)` will use this parameter as key to find translator. - -Example: - -```python -FluentTranslator("en", translator=FluentBundle.from_files("en-US", filenames=[".../main.ftl"])), -``` - -"Wait, what about no-files configuration?" may you ask. This is OK too, because you should just choose another option -from FluentBundle: - -```python -FluentTranslator("de", translator=FluentBundle.from_string("your*ftl*content")) -``` - -Get your strings from anywhere - Databases, Files, no matter the source. - -*Root locale* - if fluentogram will meet unknown locale - this locale will be used for getting translation. - -Pretty simple \ No newline at end of file diff --git a/example/TranslatorRunner.md b/example/TranslatorRunner.md deleted file mode 100644 index 61daeae..0000000 --- a/example/TranslatorRunner.md +++ /dev/null @@ -1,30 +0,0 @@ -# TranslatorRunner - -TranslatorRunner is a key component of Fluentogram - -This single-per-message unit executes translation request - -Like that: - -```python -@router.message() -async def handler(message: Message, i18n: TranslatorRunner): - await message.answer( - i18n.say.hello(username=message.from_user.username) - ) -``` - -FTL content: - -```text -say-hello = "Hello { $username}!" -``` - -So, as can you see, i18n is instance of TranslatorRunner, created in middleware before the message handler. - -Note: *Any variables for TranslatorRunner should be passed like key-word arguments. This is means using "=" symbol -between attribute name and content* - -Remember to be careful with count of subkeys (in example - "say" and "hello"). Very big count can slow things down. If -it needed - you can use `i18n.get("say-hello", username=...)` -instead of classic sugar-typed dot access method. diff --git a/example/TypingGenerator.md b/example/TypingGenerator.md deleted file mode 100644 index 1e28519..0000000 --- a/example/TypingGenerator.md +++ /dev/null @@ -1,29 +0,0 @@ -After installation, use: - -`i18n -ftl example.ftl -stub stub.pyi` - -By default, `stub.py` will contain `TranslatorRunner` class with type hints for translation keys. - -Usage in files: - -```py -from typing import TYPE_CHECKING - -from aiogram import Router -from aiogram.types import Message -from fluentogram import TranslatorRunner - -if TYPE_CHECKING: - from stub import TranslatorRunner - -router = Router() - - -@router.message() -async def handler(message: Message, i18n: TranslatorRunner): - await message.answer( - i18n.hello(username=message.from_user.username) - ) -``` - -stub.pyi - result file after stub generator \ No newline at end of file From 6ebb360341de12916fa61cd9078d4cd1b82e5eec Mon Sep 17 00:00:00 2001 From: sheldy Date: Sat, 28 Jun 2025 22:15:15 +0200 Subject: [PATCH 06/36] Add tests and add CI for that. --- .github/workflows/tests.yml | 31 +++++++ pyproject.toml | 3 + tests/__init__.py | 0 tests/test_basic_usage.py | 163 ++++++++++++++++++++++++++++++++++++ 4 files changed, 197 insertions(+) create mode 100644 .github/workflows/tests.yml create mode 100644 tests/__init__.py create mode 100644 tests/test_basic_usage.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..7d84e30 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,31 @@ +name: fluentogram-tests + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install uv + uv pip install .[dev] + + - name: Run tests + run: pytest tests \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 9379192..57a4fdb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,6 +62,9 @@ indent-width = 4 select = ["ALL"] ignore = ["D", "RET504", "PGH003", "FBT001", "EM101", "TRY003", "TC003", "ANN401", "ANN003", "TC001", "TC002"] +[tool.ruff.lint.per-file-ignores] +"tests/**/*.py" = ["S101"] + [tool.ruff.lint.mccabe] max-complexity = 5 diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_basic_usage.py b/tests/test_basic_usage.py new file mode 100644 index 0000000..b010ce5 --- /dev/null +++ b/tests/test_basic_usage.py @@ -0,0 +1,163 @@ +import pytest +from fluent_compiler.bundle import FluentBundle + +from fluentogram import FluentTranslator, TranslatorHub +from fluentogram.exceptions import FormatError, KeyNotFoundError, RootTranslatorNotFoundError + + +def test_basic_usage() -> None: + translator_hub = TranslatorHub( + { + "en": "en", + }, + [ + FluentTranslator( + "en", + translator=FluentBundle.from_string("en-US", "start-hello = Hello"), + ), + ], + ) + translator = translator_hub.get_translator_by_locale("en") + assert translator.get("start-hello") == "Hello" + + +def test_basic_usage_with_args() -> None: + translator_hub = TranslatorHub( + { + "en": "en", + }, + [ + FluentTranslator( + "en", + translator=FluentBundle.from_string("en-US", "start-hello = Hello, { $username }", use_isolating=False), + ), + ], + ) + translator = translator_hub.get_translator_by_locale("en") + assert translator.get("start-hello", username="Alex") == "Hello, Alex" + + +def test_fallback_to_default_locale() -> None: + translator_hub = TranslatorHub( + { + "en": "en", + "ru": ("ru", "en"), + }, + [ + FluentTranslator( + "en", + translator=FluentBundle.from_string("en-US", "start-hello = Hello, { $username }", use_isolating=False), + ), + ], + ) + translator = translator_hub.get_translator_by_locale("ru") + assert translator.get("start-hello", username="Alex") == "Hello, Alex" + + +def test_with_full_locales() -> None: + translator_hub = TranslatorHub( + { + "en": "en", + "ru": ("ru", "en"), + }, + [ + FluentTranslator( + "en", + translator=FluentBundle.from_string( + "en-US", + "start-hello = Hello, { $username }", + use_isolating=False, + ), + ), + FluentTranslator( + "ru", + translator=FluentBundle.from_string( + "ru-RU", + "start-hello = Привет, { $username }", + use_isolating=False, + ), + ), + ], + ) + translator = translator_hub.get_translator_by_locale("ru") + assert translator.get("start-hello", username="Alex") == "Привет, Alex" + + +def test_when_translation_not_found() -> None: + translator_hub = TranslatorHub( + { + "en": "en", + }, + [ + FluentTranslator( + "en", + translator=FluentBundle.from_string("en-US", "start-hello = Hello", use_isolating=False), + ), + ], + ) + translator = translator_hub.get_translator_by_locale("en") + with pytest.raises(KeyNotFoundError): + translator.get("start-hello1", username="Alex") + + +def test_when_root_translator_not_provided() -> None: + with pytest.raises(RootTranslatorNotFoundError): + TranslatorHub( + { + "en": "en", + }, + [ + FluentTranslator( + "ru", + translator=FluentBundle.from_string("ru-RU", "start-hello = Привет", use_isolating=False), + ), + ], + ) + + +def test_formatting_error() -> None: + translator_hub = TranslatorHub( + { + "en": "en", + }, + [ + FluentTranslator( + "en", + translator=FluentBundle.from_string("en-US", "start-hello = Hello, { $username }", use_isolating=False), + ), + ], + ) + translator = translator_hub.get_translator_by_locale("en") + with pytest.raises(FormatError): + translator.get("start-hello", username1="Alex") + +def test_get_text_by_attribute() -> None: + translator_hub = TranslatorHub( + { + "en": "en", + }, + [ + FluentTranslator( + "en", + translator=FluentBundle.from_string("en-US", "start-hello = Hello", use_isolating=False), + ), + ], + ) + translator = translator_hub.get_translator_by_locale("en") + assert translator.start.hello() == "Hello" + +def test_get_text_by_attribute_not_found() -> None: + translator_hub = TranslatorHub( + { + "en": "en", + }, + [ + FluentTranslator( + "en", + translator=FluentBundle.from_string("en-US", "start-hello = Hello", use_isolating=False), + ), + ], + ) + translator = translator_hub.get_translator_by_locale("en") + with pytest.raises(KeyNotFoundError): + translator.start.hello1() From 2c4b9ee1c8cfd085150d30ea6d8b66f3942bcbcd Mon Sep 17 00:00:00 2001 From: sheldy Date: Sat, 28 Jun 2025 22:16:50 +0200 Subject: [PATCH 07/36] Fix CI. --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7d84e30..3e20a68 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -25,7 +25,7 @@ jobs: run: | python -m pip install --upgrade pip pip install uv - uv pip install .[dev] + uv pip install --system .[dev] - name: Run tests run: pytest tests \ No newline at end of file From 0f05ad5e05384f1699b7c70cb8c076a318ed66e9 Mon Sep 17 00:00:00 2001 From: sheldy Date: Sat, 28 Jun 2025 22:23:24 +0200 Subject: [PATCH 08/36] Add pre-commit. --- .github/workflows/tests.yml | 16 +++++ .pre-commit-config.yaml | 28 ++++++++ .secrets.baseline | 127 ++++++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- scripts/lint-pre-commit.sh | 28 ++++++++ scripts/lint.sh | 8 +++ 6 files changed, 208 insertions(+), 1 deletion(-) create mode 100644 .pre-commit-config.yaml create mode 100644 .secrets.baseline create mode 100755 scripts/lint-pre-commit.sh create mode 100755 scripts/lint.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3e20a68..8cf9db9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -9,6 +9,22 @@ on: - main jobs: + pre-commit-check: + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Set $PY environment variable + run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV + - uses: actions/cache@v4 + with: + path: ~/.cache/pre-commit + key: pre-commit|${{ hashFiles('.pre-commit-config.yaml') }} + - uses: pre-commit/action@v3.0.1 + test: runs-on: ubuntu-latest diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..a2e1ec8 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,28 @@ +default_stages: [pre-commit, pre-merge-commit] +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: no-commit-to-branch + args: [ '--branch', 'master' ] + - id: end-of-file-fixer + files: ^fluentogram/ + - id: trailing-whitespace + files: ^fluentogram/ + + - repo: local + hooks: + - id: lint + name: Linter + entry: "scripts/lint-pre-commit.sh" + language: python + types: [python] + require_serial: true + verbose: true + + - repo: https://github.com/Yelp/detect-secrets + rev: v1.5.0 + hooks: + - id: detect-secrets + args: ['--baseline', '.secrets.baseline'] + stages: [pre-commit, pre-merge-commit, manual] \ No newline at end of file diff --git a/.secrets.baseline b/.secrets.baseline new file mode 100644 index 0000000..572c8da --- /dev/null +++ b/.secrets.baseline @@ -0,0 +1,127 @@ +{ + "version": "1.5.0", + "plugins_used": [ + { + "name": "ArtifactoryDetector" + }, + { + "name": "AWSKeyDetector" + }, + { + "name": "AzureStorageKeyDetector" + }, + { + "name": "Base64HighEntropyString", + "limit": 4.5 + }, + { + "name": "BasicAuthDetector" + }, + { + "name": "CloudantDetector" + }, + { + "name": "DiscordBotTokenDetector" + }, + { + "name": "GitHubTokenDetector" + }, + { + "name": "GitLabTokenDetector" + }, + { + "name": "HexHighEntropyString", + "limit": 3.0 + }, + { + "name": "IbmCloudIamDetector" + }, + { + "name": "IbmCosHmacDetector" + }, + { + "name": "IPPublicDetector" + }, + { + "name": "JwtTokenDetector" + }, + { + "name": "KeywordDetector", + "keyword_exclude": "" + }, + { + "name": "MailchimpDetector" + }, + { + "name": "NpmDetector" + }, + { + "name": "OpenAIDetector" + }, + { + "name": "PrivateKeyDetector" + }, + { + "name": "PypiTokenDetector" + }, + { + "name": "SendGridDetector" + }, + { + "name": "SlackDetector" + }, + { + "name": "SoftlayerDetector" + }, + { + "name": "SquareOAuthDetector" + }, + { + "name": "StripeDetector" + }, + { + "name": "TelegramBotTokenDetector" + }, + { + "name": "TwilioKeyDetector" + } + ], + "filters_used": [ + { + "path": "detect_secrets.filters.allowlist.is_line_allowlisted" + }, + { + "path": "detect_secrets.filters.common.is_ignored_due_to_verification_policies", + "min_level": 2 + }, + { + "path": "detect_secrets.filters.heuristic.is_indirect_reference" + }, + { + "path": "detect_secrets.filters.heuristic.is_likely_id_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_lock_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_not_alphanumeric_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_potential_uuid" + }, + { + "path": "detect_secrets.filters.heuristic.is_prefixed_with_dollar_sign" + }, + { + "path": "detect_secrets.filters.heuristic.is_sequential_string" + }, + { + "path": "detect_secrets.filters.heuristic.is_swagger_file" + }, + { + "path": "detect_secrets.filters.heuristic.is_templated_secret" + } + ], + "results": {}, + "generated_at": "2025-06-28T20:21:22Z" +} diff --git a/pyproject.toml b/pyproject.toml index 57a4fdb..0b40591 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ ] [project.optional-dependencies] cli = ["watchdog>=2.3.0,<3"] -dev = ["ruff~=0.11.2", "pytest~=8.4.1"] +dev = ["ruff~=0.11.2", "pytest~=8.4.1", "pre-commit~=4.2.0", "detect-secrets~=1.5.0"] aiogram = ["aiogram~=3.20.0"] nats = ["nats-py~=2.10.0"] diff --git a/scripts/lint-pre-commit.sh b/scripts/lint-pre-commit.sh new file mode 100755 index 0000000..cbb6679 --- /dev/null +++ b/scripts/lint-pre-commit.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# from: https://jaredkhan.com/blog/mypy-pre-commit + +# A script for running mypy, +# with all its dependencies installed. + +set -o errexit + +# Change directory to the project root directory. +cd "$(dirname "$0")"/.. + +# Install the dependencies into the mypy env. +# Note that this can take seconds to run. +# In my case, I need to use a custom index URL. +# Avoid pip spending time quietly retrying since +# likely cause of failure is lack of VPN connection. +pip install ruff + +# Run on all files, +# ignoring the paths passed to this script, +# so as not to miss type errors. +# My repo makes use of namespace packages. +# Use the namespace-packages flag +# and specify the package to run on explicitly. +# Note that we do not use --ignore-missing-imports, +# as this can give us false confidence in our results. +./scripts/lint.sh diff --git a/scripts/lint.sh b/scripts/lint.sh new file mode 100755 index 0000000..4cec36d --- /dev/null +++ b/scripts/lint.sh @@ -0,0 +1,8 @@ +#!/bin/bash +set -e + +echo "Running ruff linter (isort, flake, pyupgrade, etc. replacement)..." +ruff check --fix --exit-non-zero-on-fix + +echo "Running ruff formatter (black replacement)..." +ruff format From b21781a64f8e43b950015f42b69dc0287d178ced Mon Sep 17 00:00:00 2001 From: sheldy Date: Sun, 29 Jun 2025 00:04:32 +0200 Subject: [PATCH 09/36] Remove useless abc. --- fluentogram/__init__.py | 11 ++- fluentogram/abc/__init__.py | 9 --- fluentogram/abc/runner.py | 12 ---- fluentogram/abc/translator.py | 22 ------ fluentogram/abc/translator_hub.py | 20 ------ fluentogram/impl/__init__.py | 12 ---- fluentogram/impl/transformers/__init__.py | 4 -- fluentogram/impl/translator_hub.py | 53 --------------- fluentogram/{impl => }/runner.py | 7 +- fluentogram/storage/__init__.py | 0 fluentogram/storage/base.py | 64 +++++++++++++++++ fluentogram/storage/memory.py | 68 +++++++++++++++++++ fluentogram/transformers/__init__.py | 7 ++ .../transformer.py => transformers/base.py} | 0 .../datetime.py} | 2 +- .../money.py} | 2 +- fluentogram/{impl => }/translator.py | 3 +- fluentogram/translator_hub.py | 61 +++++++++++++++++ 18 files changed, 210 insertions(+), 147 deletions(-) delete mode 100644 fluentogram/abc/__init__.py delete mode 100644 fluentogram/abc/runner.py delete mode 100644 fluentogram/abc/translator.py delete mode 100644 fluentogram/abc/translator_hub.py delete mode 100644 fluentogram/impl/__init__.py delete mode 100644 fluentogram/impl/transformers/__init__.py delete mode 100644 fluentogram/impl/translator_hub.py rename fluentogram/{impl => }/runner.py (80%) create mode 100644 fluentogram/storage/__init__.py create mode 100644 fluentogram/storage/base.py create mode 100644 fluentogram/storage/memory.py create mode 100644 fluentogram/transformers/__init__.py rename fluentogram/{abc/transformer.py => transformers/base.py} (100%) rename fluentogram/{impl/transformers/datetime_transformer.py => transformers/datetime.py} (89%) rename fluentogram/{impl/transformers/money_transformer.py => transformers/money.py} (95%) rename fluentogram/{impl => }/translator.py (91%) create mode 100644 fluentogram/translator_hub.py diff --git a/fluentogram/__init__.py b/fluentogram/__init__.py index 61f5f7b..8ddb35b 100644 --- a/fluentogram/__init__.py +++ b/fluentogram/__init__.py @@ -1,10 +1,7 @@ -from .impl import ( - DateTimeTransformer, - FluentTranslator, - MoneyTransformer, - TranslatorHub, - TranslatorRunner, -) +from .runner import TranslatorRunner +from .transformers import DateTimeTransformer, MoneyTransformer +from .translator import FluentTranslator +from .translator_hub import TranslatorHub __all__ = [ "DateTimeTransformer", diff --git a/fluentogram/abc/__init__.py b/fluentogram/abc/__init__.py deleted file mode 100644 index 4092097..0000000 --- a/fluentogram/abc/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from .transformer import AbstractDataTransformer -from .translator import AbstractTranslator -from .translator_hub import AbstractTranslatorsHub - -__all__ = [ - "AbstractDataTransformer", - "AbstractTranslator", - "AbstractTranslatorsHub", -] diff --git a/fluentogram/abc/runner.py b/fluentogram/abc/runner.py deleted file mode 100644 index ca50764..0000000 --- a/fluentogram/abc/runner.py +++ /dev/null @@ -1,12 +0,0 @@ -"""An abstract translator runner""" - -from abc import ABC, abstractmethod -from typing import Any - - -class AbstractTranslatorRunner(ABC): - """This is one-shot per Telegram event translator with attrib tracer access way.""" - - @abstractmethod - def get(self, key: str, **kwargs: Any) -> str: - """Fastest, direct way to use translator, without sugar-like typing supported attribute access way""" diff --git a/fluentogram/abc/translator.py b/fluentogram/abc/translator.py deleted file mode 100644 index f080e00..0000000 --- a/fluentogram/abc/translator.py +++ /dev/null @@ -1,22 +0,0 @@ -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import Any - - -class AbstractTranslator(ABC): - """A translator class, implements key-value interface for your translator mechanism.""" - - @abstractmethod - def __init__(self, locale: str, translator: Any, separator: str = "-") -> None: - self.locale = locale - self.separator = separator - self.translator = translator - - @abstractmethod - def get(self, key: str, **kwargs: Any) -> str | None: - """Convert a translation key to a translated text string. - Use kwargs dict to pass external data to the translator. - Expects to be fast and furious. - """ - raise NotImplementedError diff --git a/fluentogram/abc/translator_hub.py b/fluentogram/abc/translator_hub.py deleted file mode 100644 index 3557e3a..0000000 --- a/fluentogram/abc/translator_hub.py +++ /dev/null @@ -1,20 +0,0 @@ -"""An abstract base for the Translator Hub and Key/Value Translator Hub objects""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from collections.abc import Iterable - -from fluentogram.abc.runner import AbstractTranslatorRunner - - -class AbstractTranslatorsHub(ABC): - """This class should contain a couple of translator objects, usually one object per one locale.""" - - def normalize_locales_map(self, locales_map: dict[str, str | Iterable[str]]) -> dict[str, Iterable[str]]: - return {key: (value,) if isinstance(value, str) else value for key, value in locales_map.items()} - - @abstractmethod - def get_translator_by_locale(self, locale: str) -> AbstractTranslatorRunner: - """Returns a Translator object by selected locale""" - raise NotImplementedError diff --git a/fluentogram/impl/__init__.py b/fluentogram/impl/__init__.py deleted file mode 100644 index 8ddb35b..0000000 --- a/fluentogram/impl/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -from .runner import TranslatorRunner -from .transformers import DateTimeTransformer, MoneyTransformer -from .translator import FluentTranslator -from .translator_hub import TranslatorHub - -__all__ = [ - "DateTimeTransformer", - "FluentTranslator", - "MoneyTransformer", - "TranslatorHub", - "TranslatorRunner", -] diff --git a/fluentogram/impl/transformers/__init__.py b/fluentogram/impl/transformers/__init__.py deleted file mode 100644 index ae984ce..0000000 --- a/fluentogram/impl/transformers/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .datetime_transformer import DateTimeTransformer -from .money_transformer import MoneyTransformer - -__all__ = ["DateTimeTransformer", "MoneyTransformer"] diff --git a/fluentogram/impl/translator_hub.py b/fluentogram/impl/translator_hub.py deleted file mode 100644 index 2e75d04..0000000 --- a/fluentogram/impl/translator_hub.py +++ /dev/null @@ -1,53 +0,0 @@ -"""A Translator Hub, using as factory for Translator objects""" - -from __future__ import annotations - -from collections.abc import Iterable - -from fluentogram.abc import AbstractTranslator, AbstractTranslatorsHub -from fluentogram.exceptions import RootTranslatorNotFoundError -from fluentogram.impl import TranslatorRunner - - -class TranslatorHub(AbstractTranslatorsHub): - """This class implements a storage for all single-locale translators.""" - - def __init__( - self, - locales_map: dict[str, str | Iterable[str]], - translators: list[AbstractTranslator], - root_locale: str = "en", - separator: str = "-", - ) -> None: - self.locales_map = self.normalize_locales_map(locales_map) - self.translators = translators - self.root_locale = root_locale - self.separator = separator - self.storage: dict[str, AbstractTranslator] = dict( - zip([translator.locale for translator in translators], translators, strict=False), - ) - if not self.storage.get(root_locale): - raise RootTranslatorNotFoundError(self.root_locale) - - self.translators_map: dict[str, Iterable[AbstractTranslator]] = self._locales_map_parser(self.locales_map) - - def _locales_map_parser( - self, - locales_map: dict[str, str | Iterable[str]], - ) -> dict[str, Iterable[AbstractTranslator]]: - return { - lang: tuple(self.storage[locale] for locale in translator_locales if locale in self.storage) - for lang, translator_locales in locales_map.items() - } - - def get_translator_by_locale(self, locale: str) -> TranslatorRunner: - """Here is a little tricky moment. - There should be like a one-shot scheme. - For proper isolation, function returns TranslatorRunner new instance every time, not the same translator. - This trick makes "obj.attribute1.attribute2" access to be able. - You are able to do what you want, refer to the abstract class. - """ - return TranslatorRunner( - translators=self.translators_map.get(locale) or self.translators_map[self.root_locale], - separator=self.separator, - ) diff --git a/fluentogram/impl/runner.py b/fluentogram/runner.py similarity index 80% rename from fluentogram/impl/runner.py rename to fluentogram/runner.py index b8ba1a7..d8eaca7 100644 --- a/fluentogram/impl/runner.py +++ b/fluentogram/runner.py @@ -3,13 +3,12 @@ from collections.abc import Iterable from typing import Any -from fluentogram.abc import AbstractTranslator -from fluentogram.abc.runner import AbstractTranslatorRunner from fluentogram.exceptions import KeyNotFoundError +from fluentogram.translator import FluentTranslator -class TranslatorRunner(AbstractTranslatorRunner): - def __init__(self, translators: Iterable[AbstractTranslator], separator: str = "-") -> None: +class TranslatorRunner: + def __init__(self, translators: Iterable[FluentTranslator], separator: str = "-") -> None: self.translators = translators self.separator = separator self._request_line = "" diff --git a/fluentogram/storage/__init__.py b/fluentogram/storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fluentogram/storage/base.py b/fluentogram/storage/base.py new file mode 100644 index 0000000..f3e1200 --- /dev/null +++ b/fluentogram/storage/base.py @@ -0,0 +1,64 @@ +"""An abstract base for storage implementations""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from collections.abc import Iterable +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from fluentogram.translator import FluentTranslator + + +class AbstractStorage(ABC): + """Abstract storage for translators by locale.""" + + @abstractmethod + def add_translator(self, translator: FluentTranslator) -> None: + """Add a translator to storage.""" + raise NotImplementedError + + @abstractmethod + def add_translators(self, translators: Iterable[FluentTranslator]) -> None: + """Add multiple translators to storage.""" + raise NotImplementedError + + @abstractmethod + def get_translator(self, locale: str) -> FluentTranslator | None: + """Get translator by locale.""" + raise NotImplementedError + + @abstractmethod + def has_translator(self, locale: str) -> bool: + """Check if translator exists for given locale.""" + raise NotImplementedError + + @abstractmethod + def get_all_translators(self) -> Iterable[FluentTranslator]: + """Get all translators from storage.""" + raise NotImplementedError + + @abstractmethod + def get_translators_by_locales(self, locales: Iterable[str]) -> Iterable[FluentTranslator]: + """Get translators by list of locales.""" + raise NotImplementedError + + @abstractmethod + def get_translators_list(self) -> list[FluentTranslator]: + """Get all translators as a list.""" + raise NotImplementedError + + @abstractmethod + def set_locales_map(self, locales_map: dict[str, str | Iterable[str]]) -> None: + """Set the locales mapping configuration.""" + raise NotImplementedError + + @abstractmethod + def get_translators_map(self) -> dict[str, Iterable[FluentTranslator]]: + """Get the translators map based on locales configuration.""" + raise NotImplementedError + + @abstractmethod + def get_translators_for_language(self, language: str) -> Iterable[FluentTranslator]: + """Get translators for a specific language based on locales map.""" + raise NotImplementedError diff --git a/fluentogram/storage/memory.py b/fluentogram/storage/memory.py new file mode 100644 index 0000000..4183d46 --- /dev/null +++ b/fluentogram/storage/memory.py @@ -0,0 +1,68 @@ +"""Default storage implementation""" + +from __future__ import annotations + +from collections.abc import Iterable + +from fluentogram.storage.base import AbstractStorage +from fluentogram.translator import FluentTranslator + + +class MemoryStorage(AbstractStorage): + """Default storage implementation using dictionary.""" + + def __init__(self, translators: dict[str, FluentTranslator] | None = None) -> None: + self._storage: dict[str, FluentTranslator] = translators or {} + self._locales_map: dict[str, Iterable[str]] = {} + self._translators_map: dict[str, Iterable[FluentTranslator]] = {} + + def add_translator(self, translator: FluentTranslator) -> None: + """Add a translator to storage.""" + self._storage[translator.locale] = translator + + def add_translators(self, translators: Iterable[FluentTranslator]) -> None: + """Add multiple translators to storage.""" + for translator in translators: + self.add_translator(translator) + + def get_translator(self, locale: str) -> FluentTranslator | None: + """Get translator by locale.""" + return self._storage.get(locale) + + def has_translator(self, locale: str) -> bool: + """Check if translator exists for given locale.""" + return locale in self._storage + + def get_all_translators(self) -> Iterable[FluentTranslator]: + """Get all translators from storage.""" + return self._storage.values() + + def get_translators_by_locales(self, locales: Iterable[str]) -> Iterable[FluentTranslator]: + """Get translators by list of locales.""" + return tuple(self._storage[locale] for locale in locales if locale in self._storage) + + def get_translators_list(self) -> list[FluentTranslator]: + """Get all translators as a list.""" + return list(self._storage.values()) + + def set_locales_map(self, locales_map: dict[str, str | Iterable[str]]) -> None: + """Set the locales mapping configuration.""" + # Normalize locales map (convert single strings to tuples) + self._locales_map = {key: (value,) if isinstance(value, str) else value for key, value in locales_map.items()} + # Rebuild translators map + self._build_translators_map() + + def get_translators_map(self) -> dict[str, Iterable[FluentTranslator]]: + """Get the translators map based on locales configuration.""" + return self._translators_map + + def get_translators_for_language(self, language: str) -> Iterable[FluentTranslator]: + """Get translators for a specific language based on locales map.""" + return self._translators_map.get(language, ()) + + def _build_translators_map(self) -> None: + """Build the translators map based on locales configuration.""" + self._translators_map = { + lang: self.get_translators_by_locales(translator_locales) + for lang, translator_locales in self._locales_map.items() + } diff --git a/fluentogram/transformers/__init__.py b/fluentogram/transformers/__init__.py new file mode 100644 index 0000000..d2f36f7 --- /dev/null +++ b/fluentogram/transformers/__init__.py @@ -0,0 +1,7 @@ +from .datetime import DateTimeTransformer +from .money import MoneyTransformer + +__all__ = [ + "DateTimeTransformer", + "MoneyTransformer", +] diff --git a/fluentogram/abc/transformer.py b/fluentogram/transformers/base.py similarity index 100% rename from fluentogram/abc/transformer.py rename to fluentogram/transformers/base.py diff --git a/fluentogram/impl/transformers/datetime_transformer.py b/fluentogram/transformers/datetime.py similarity index 89% rename from fluentogram/impl/transformers/datetime_transformer.py rename to fluentogram/transformers/datetime.py index e1de29e..49882b2 100644 --- a/fluentogram/impl/transformers/datetime_transformer.py +++ b/fluentogram/transformers/datetime.py @@ -6,7 +6,7 @@ from fluent_compiler.types import FluentDateType, FluentNone, fluent_date -from fluentogram.abc import AbstractDataTransformer +from fluentogram.transformers.base import AbstractDataTransformer class DateTimeTransformer(AbstractDataTransformer): diff --git a/fluentogram/impl/transformers/money_transformer.py b/fluentogram/transformers/money.py similarity index 95% rename from fluentogram/impl/transformers/money_transformer.py rename to fluentogram/transformers/money.py index 5bf5a61..c3de617 100644 --- a/fluentogram/impl/transformers/money_transformer.py +++ b/fluentogram/transformers/money.py @@ -7,7 +7,7 @@ from fluent_compiler.types import FluentNone, FluentNumber, fluent_number -from fluentogram.abc import AbstractDataTransformer +from fluentogram.transformers.base import AbstractDataTransformer class MoneyTransformer(AbstractDataTransformer): diff --git a/fluentogram/impl/translator.py b/fluentogram/translator.py similarity index 91% rename from fluentogram/impl/translator.py rename to fluentogram/translator.py index 017ee73..6112ae4 100644 --- a/fluentogram/impl/translator.py +++ b/fluentogram/translator.py @@ -6,11 +6,10 @@ from fluent_compiler.bundle import FluentBundle -from fluentogram.abc import AbstractTranslator from fluentogram.exceptions import FormatError -class FluentTranslator(AbstractTranslator): +class FluentTranslator: """Single-locale Translator, implemented with fluent_compiler Bundles""" def __init__(self, locale: str, translator: FluentBundle, separator: str = "-") -> None: diff --git a/fluentogram/translator_hub.py b/fluentogram/translator_hub.py new file mode 100644 index 0000000..0fb0018 --- /dev/null +++ b/fluentogram/translator_hub.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from collections.abc import Iterable + +from fluentogram.exceptions import RootTranslatorNotFoundError +from fluentogram.runner import TranslatorRunner +from fluentogram.storage.memory import MemoryStorage +from fluentogram.translator import FluentTranslator + + +class TranslatorHub: + """This class implements a storage for all single-locale translators.""" + + def __init__( + self, + locales_map: dict[str, str | Iterable[str]], + translators: list[FluentTranslator], + root_locale: str = "en", + separator: str = "-", + storage: MemoryStorage | None = None, + ) -> None: + self.root_locale = root_locale + self.separator = separator + + self.storage = storage or MemoryStorage() + + # Add translators to storage + self.storage.add_translators(translators) + + # Set locales map in storage + self.storage.set_locales_map(locales_map) + + if not self.storage.has_translator(root_locale): + raise RootTranslatorNotFoundError(self.root_locale) + + def get_translator_by_locale(self, locale: str) -> TranslatorRunner: + """Here is a little tricky moment. + There should be like a one-shot scheme. + For proper isolation, function returns TranslatorRunner new instance every time, not the same translator. + This trick makes "obj.attribute1.attribute2" access to be able. + You are able to do what you want, refer to the abstract class. + """ + translators = self.storage.get_translators_for_language(locale) + if not translators: + # Fallback to root locale + translators = self.storage.get_translators_for_language(self.root_locale) + + return TranslatorRunner( + translators=translators, + separator=self.separator, + ) + + @property + def translators(self) -> list[FluentTranslator]: + """Get all translators from storage.""" + return self.storage.get_translators_list() + + @property + def translators_map(self) -> dict[str, Iterable[FluentTranslator]]: + """Get translators map from storage.""" + return self.storage.get_translators_map() From 1ea9954ae27caa51b44e9e4cbce4b31bb942150c Mon Sep 17 00:00:00 2001 From: sheldy Date: Sun, 29 Jun 2025 01:37:34 +0200 Subject: [PATCH 10/36] Add ability to update locales and add nats kv storage. --- fluentogram/nats/__init__.py | 0 fluentogram/nats/storage.py | 100 ++++++++++++++++++++++++++++++++ fluentogram/storage/__init__.py | 5 ++ fluentogram/storage/base.py | 59 ++++++++++++------- fluentogram/storage/memory.py | 68 ---------------------- fluentogram/translator.py | 9 +++ fluentogram/translator_hub.py | 6 +- tests/test_update.py | 46 +++++++++++++++ 8 files changed, 200 insertions(+), 93 deletions(-) create mode 100644 fluentogram/nats/__init__.py create mode 100644 fluentogram/nats/storage.py delete mode 100644 fluentogram/storage/memory.py create mode 100644 tests/test_update.py diff --git a/fluentogram/nats/__init__.py b/fluentogram/nats/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fluentogram/nats/storage.py b/fluentogram/nats/storage.py new file mode 100644 index 0000000..37e7c09 --- /dev/null +++ b/fluentogram/nats/storage.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import json +from collections import defaultdict +from typing import Any, Callable + +from nats.aio.msg import Msg +from nats.js import JetStreamContext +from nats.js.kv import KV_DEL, KV_OP, KV_PURGE, KeyValue + +from fluentogram.storage.base import BaseStorage + +_JsonLoads = Callable[..., Any] +_JsonDumps = Callable[..., bytes] + + +class NatsKvStorage(BaseStorage): + def __init__( # noqa: PLR0913 + self, + kv: KeyValue, + js: JetStreamContext, + separator: str = ".", + serializer: _JsonDumps = lambda data: json.dumps(data).encode("utf-8"), + deserializer: _JsonLoads = json.loads, + consume_timeout: float = 1.0, + ) -> None: + self._kv = kv + self._js = js + self.separator = separator + self.serializer = serializer + self.deserializer = deserializer + self.consume_timeout = consume_timeout + super().__init__() + + # NATS-specific methods for async operations + async def put_translation( + self, + locale: str, + key: str, + value: Any, + ) -> None: + """Put translation to NATS KV store.""" + await self._kv.put(f"{locale}{self.separator}{key}", self.serializer(value)) + + async def create_translation( + self, + locale: str, + key: str, + value: Any, + ) -> None: + """Create translation in NATS KV store.""" + await self._kv.create(f"{locale}{self.separator}{key}", self.serializer(value)) + + async def delete_translation(self, locale: str, key: str) -> None: + """Delete translations from NATS KV store.""" + await self._kv.purge(f"{locale}{self.separator}{key}") + + async def listen_for_changes(self) -> None: + """Listen for changes in NATS KV store and update local storage.""" + stream = await self._js.stream_info(self._kv._stream) # noqa: SLF001 + stream_name = stream.config.name + if stream_name is None: + raise ValueError("Stream name is None") + subject_name = stream_name.replace("_", self.separator, 1) + subject = f"${subject_name}.>" + consumer = await self._js.pull_subscribe(subject=subject, stream=stream_name) + while True: + try: + messages: list[Msg] = await consumer.fetch(50, timeout=self.consume_timeout) + except TimeoutError: # noqa: PERF203 + pass + else: + await self._update_compiled_messages(messages) + + async def _update_compiled_messages(self, messages: list[Msg]) -> None: + """Update compiled messages based on NATS KV changes.""" + changes = defaultdict(list) + for m in messages: + kind = m.headers.get(KV_OP) if m.headers is not None else None + *_, locale, key = m.subject.split(self.separator) + if kind in (KV_DEL, KV_PURGE): + # Remove translation from local storage + translator = self._storage.get(locale) + if translator: + print(f"Removing translation: {key}") # noqa: T201 + else: + value = self.deserializer(m.data) + changes[locale].append((key, value)) + await m.ack() + self._set_new_compiled_messages(changes) + + def _set_new_compiled_messages(self, new_messages: dict[str, list[str]]) -> None: + """Set new compiled messages for translators.""" + for locale, messages in new_messages.items(): + translator = self._storage.get(locale) + if translator is None: + continue + + for key, value in messages: + translator.update_translation(key, value) diff --git a/fluentogram/storage/__init__.py b/fluentogram/storage/__init__.py index e69de29..5559fd6 100644 --- a/fluentogram/storage/__init__.py +++ b/fluentogram/storage/__init__.py @@ -0,0 +1,5 @@ +"""Storage implementations for fluentogram.""" + +from fluentogram.storage.base import BaseStorage + +__all__ = ["BaseStorage"] diff --git a/fluentogram/storage/base.py b/fluentogram/storage/base.py index f3e1200..d98091a 100644 --- a/fluentogram/storage/base.py +++ b/fluentogram/storage/base.py @@ -2,7 +2,6 @@ from __future__ import annotations -from abc import ABC, abstractmethod from collections.abc import Iterable from typing import TYPE_CHECKING @@ -10,55 +9,71 @@ from fluentogram.translator import FluentTranslator -class AbstractStorage(ABC): +class BaseStorage: """Abstract storage for translators by locale.""" - @abstractmethod + def __init__(self) -> None: + """Initialize storage with empty containers.""" + self._storage: dict[str, FluentTranslator] = {} + self._locales_map: dict[str, Iterable[str]] = {} + self._translators_map: dict[str, Iterable[FluentTranslator]] = {} + def add_translator(self, translator: FluentTranslator) -> None: """Add a translator to storage.""" - raise NotImplementedError + self._storage[translator.locale] = translator - @abstractmethod def add_translators(self, translators: Iterable[FluentTranslator]) -> None: """Add multiple translators to storage.""" - raise NotImplementedError + for translator in translators: + self.add_translator(translator) - @abstractmethod def get_translator(self, locale: str) -> FluentTranslator | None: """Get translator by locale.""" - raise NotImplementedError + return self._storage.get(locale) - @abstractmethod def has_translator(self, locale: str) -> bool: """Check if translator exists for given locale.""" - raise NotImplementedError + return locale in self._storage - @abstractmethod def get_all_translators(self) -> Iterable[FluentTranslator]: """Get all translators from storage.""" - raise NotImplementedError + return self._storage.values() - @abstractmethod def get_translators_by_locales(self, locales: Iterable[str]) -> Iterable[FluentTranslator]: """Get translators by list of locales.""" - raise NotImplementedError + return tuple(self._storage[locale] for locale in locales if locale in self._storage) - @abstractmethod def get_translators_list(self) -> list[FluentTranslator]: """Get all translators as a list.""" - raise NotImplementedError + return list(self._storage.values()) - @abstractmethod def set_locales_map(self, locales_map: dict[str, str | Iterable[str]]) -> None: """Set the locales mapping configuration.""" - raise NotImplementedError + # Normalize locales map (convert single strings to tuples) + self._locales_map = {key: (value,) if isinstance(value, str) else value for key, value in locales_map.items()} + # Rebuild translators map + self._build_translators_map() - @abstractmethod def get_translators_map(self) -> dict[str, Iterable[FluentTranslator]]: """Get the translators map based on locales configuration.""" - raise NotImplementedError + return self._translators_map - @abstractmethod def get_translators_for_language(self, language: str) -> Iterable[FluentTranslator]: """Get translators for a specific language based on locales map.""" - raise NotImplementedError + return self._translators_map.get(language, ()) + + def _build_translators_map(self) -> None: + """Build the translators map based on locales configuration.""" + self._translators_map = { + lang: self.get_translators_by_locales(translator_locales) + for lang, translator_locales in self._locales_map.items() + } + + def update_translation(self, locale: str, key: str, value: str) -> bool: + """Update a translation key for a specific locale.""" + translator = self._storage.get(locale) + if translator is None: + return False + + translator.update_translation(key, value) + return True diff --git a/fluentogram/storage/memory.py b/fluentogram/storage/memory.py deleted file mode 100644 index 4183d46..0000000 --- a/fluentogram/storage/memory.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Default storage implementation""" - -from __future__ import annotations - -from collections.abc import Iterable - -from fluentogram.storage.base import AbstractStorage -from fluentogram.translator import FluentTranslator - - -class MemoryStorage(AbstractStorage): - """Default storage implementation using dictionary.""" - - def __init__(self, translators: dict[str, FluentTranslator] | None = None) -> None: - self._storage: dict[str, FluentTranslator] = translators or {} - self._locales_map: dict[str, Iterable[str]] = {} - self._translators_map: dict[str, Iterable[FluentTranslator]] = {} - - def add_translator(self, translator: FluentTranslator) -> None: - """Add a translator to storage.""" - self._storage[translator.locale] = translator - - def add_translators(self, translators: Iterable[FluentTranslator]) -> None: - """Add multiple translators to storage.""" - for translator in translators: - self.add_translator(translator) - - def get_translator(self, locale: str) -> FluentTranslator | None: - """Get translator by locale.""" - return self._storage.get(locale) - - def has_translator(self, locale: str) -> bool: - """Check if translator exists for given locale.""" - return locale in self._storage - - def get_all_translators(self) -> Iterable[FluentTranslator]: - """Get all translators from storage.""" - return self._storage.values() - - def get_translators_by_locales(self, locales: Iterable[str]) -> Iterable[FluentTranslator]: - """Get translators by list of locales.""" - return tuple(self._storage[locale] for locale in locales if locale in self._storage) - - def get_translators_list(self) -> list[FluentTranslator]: - """Get all translators as a list.""" - return list(self._storage.values()) - - def set_locales_map(self, locales_map: dict[str, str | Iterable[str]]) -> None: - """Set the locales mapping configuration.""" - # Normalize locales map (convert single strings to tuples) - self._locales_map = {key: (value,) if isinstance(value, str) else value for key, value in locales_map.items()} - # Rebuild translators map - self._build_translators_map() - - def get_translators_map(self) -> dict[str, Iterable[FluentTranslator]]: - """Get the translators map based on locales configuration.""" - return self._translators_map - - def get_translators_for_language(self, language: str) -> Iterable[FluentTranslator]: - """Get translators for a specific language based on locales map.""" - return self._translators_map.get(language, ()) - - def _build_translators_map(self) -> None: - """Build the translators map based on locales configuration.""" - self._translators_map = { - lang: self.get_translators_by_locales(translator_locales) - for lang, translator_locales in self._locales_map.items() - } diff --git a/fluentogram/translator.py b/fluentogram/translator.py index 6112ae4..8aba78c 100644 --- a/fluentogram/translator.py +++ b/fluentogram/translator.py @@ -5,6 +5,8 @@ from typing import Any from fluent_compiler.bundle import FluentBundle +from fluent_compiler.compiler import compile_messages +from fluent_compiler.resource import FtlResource from fluentogram.exceptions import FormatError @@ -30,5 +32,12 @@ def get(self, key: str, **kwargs: Any) -> str | None: return text + def update_translation(self, key: str, value: str) -> None: + """Update a translation key for a specific locale.""" + self.translator._compiled_messages[key] = compile_messages( # noqa: SLF001 + self.locale, + [FtlResource.from_string(f"{key} = {value}")], + ).message_functions[key] + def __repr__(self) -> str: return f"" diff --git a/fluentogram/translator_hub.py b/fluentogram/translator_hub.py index 0fb0018..013a50b 100644 --- a/fluentogram/translator_hub.py +++ b/fluentogram/translator_hub.py @@ -4,7 +4,7 @@ from fluentogram.exceptions import RootTranslatorNotFoundError from fluentogram.runner import TranslatorRunner -from fluentogram.storage.memory import MemoryStorage +from fluentogram.storage.base import BaseStorage from fluentogram.translator import FluentTranslator @@ -17,12 +17,12 @@ def __init__( translators: list[FluentTranslator], root_locale: str = "en", separator: str = "-", - storage: MemoryStorage | None = None, + storage: BaseStorage | None = None, ) -> None: self.root_locale = root_locale self.separator = separator - self.storage = storage or MemoryStorage() + self.storage = storage or BaseStorage() # Add translators to storage self.storage.add_translators(translators) diff --git a/tests/test_update.py b/tests/test_update.py new file mode 100644 index 0000000..8ad69d5 --- /dev/null +++ b/tests/test_update.py @@ -0,0 +1,46 @@ +from fluent_compiler.bundle import FluentBundle + +from fluentogram import FluentTranslator, TranslatorHub + + +def test_update_translation() -> None: + translator_hub = TranslatorHub( + { + "en": "en", + }, + [ + FluentTranslator( + "en", + translator=FluentBundle.from_string("en-US", "start-hello = Hello"), + ), + ], + ) + translator = translator_hub.get_translator_by_locale("en") + assert translator.get("start-hello") == "Hello" + + translator_hub.storage.update_translation("en", "start-hello1", "Hello123") + assert translator.get("start-hello1") == "Hello123" + + +def test_update_translation_after_fallback() -> None: + translator_hub = TranslatorHub( + { + "en": "en", + "ru": ("ru", "en"), + }, + [ + FluentTranslator( + "en", + translator=FluentBundle.from_string("en-US", "start-hello1 = Hello1"), + ), + FluentTranslator( + "ru", + translator=FluentBundle.from_string("ru-RU", "start-hello = Привет"), + ), + ], + ) + translator = translator_hub.get_translator_by_locale("ru") + assert translator.get("start-hello1") == "Hello1" + + translator_hub.storage.update_translation("ru", "start-hello1", "Привет1") + assert translator.get("start-hello1") == "Привет1" From f88b7e78c6c41bfa96569cce3b94b14e930c5293 Mon Sep 17 00:00:00 2001 From: sheldy Date: Sun, 29 Jun 2025 14:28:10 +0200 Subject: [PATCH 11/36] Update base and nats kv storages, update tests. --- fluentogram/nats/storage.py | 93 ++++++++++++++++++++++----------- fluentogram/storage/__init__.py | 3 +- fluentogram/storage/base.py | 9 +++- fluentogram/storage/memory.py | 9 ++++ fluentogram/translator_hub.py | 4 +- pyproject.toml | 2 +- tests/test_update.py | 12 +++-- 7 files changed, 92 insertions(+), 40 deletions(-) create mode 100644 fluentogram/storage/memory.py diff --git a/fluentogram/nats/storage.py b/fluentogram/nats/storage.py index 37e7c09..3959902 100644 --- a/fluentogram/nats/storage.py +++ b/fluentogram/nats/storage.py @@ -1,11 +1,17 @@ from __future__ import annotations +import asyncio import json +import logging from collections import defaultdict +from contextlib import suppress from typing import Any, Callable +from nats import connect +from nats.aio.client import Client from nats.aio.msg import Msg from nats.js import JetStreamContext +from nats.js.api import KeyValueConfig from nats.js.kv import KV_DEL, KV_OP, KV_PURGE, KeyValue from fluentogram.storage.base import BaseStorage @@ -13,64 +19,81 @@ _JsonLoads = Callable[..., Any] _JsonDumps = Callable[..., bytes] +logger = logging.getLogger(__name__) + class NatsKvStorage(BaseStorage): def __init__( # noqa: PLR0913 self, + nc: Client, kv: KeyValue, - js: JetStreamContext, separator: str = ".", serializer: _JsonDumps = lambda data: json.dumps(data).encode("utf-8"), deserializer: _JsonLoads = json.loads, consume_timeout: float = 1.0, ) -> None: + self._nc = nc + self._js = nc.jetstream() self._kv = kv - self._js = js + self._stream_name = kv._stream # noqa: SLF001 self.separator = separator self.serializer = serializer self.deserializer = deserializer self.consume_timeout = consume_timeout + self._listen_for_changes_task: asyncio.Task | None = None + self._listen_for_changes_task = asyncio.create_task(self.listen_for_changes()) + self._stop_event = asyncio.Event() super().__init__() - # NATS-specific methods for async operations - async def put_translation( + @classmethod + async def from_servers( + cls, + servers: list[str], + kv_config: KeyValueConfig, + separator: str = ".", + serializer: _JsonDumps = lambda data: json.dumps(data).encode("utf-8"), + deserializer: _JsonLoads = json.loads, + ) -> NatsKvStorage: + nc = await connect(servers=servers) + js = nc.jetstream() + kv = await js.create_key_value(config=kv_config) + return cls(nc, kv, separator, serializer, deserializer) + + async def update_translation( self, locale: str, key: str, - value: Any, + value: str, ) -> None: """Put translation to NATS KV store.""" await self._kv.put(f"{locale}{self.separator}{key}", self.serializer(value)) - async def create_translation( - self, - locale: str, - key: str, - value: Any, - ) -> None: - """Create translation in NATS KV store.""" - await self._kv.create(f"{locale}{self.separator}{key}", self.serializer(value)) - - async def delete_translation(self, locale: str, key: str) -> None: - """Delete translations from NATS KV store.""" - await self._kv.purge(f"{locale}{self.separator}{key}") - - async def listen_for_changes(self) -> None: - """Listen for changes in NATS KV store and update local storage.""" - stream = await self._js.stream_info(self._kv._stream) # noqa: SLF001 + async def _create_consumer(self) -> JetStreamContext.PullSubscription: + stream = await self._js.stream_info(self._stream_name) stream_name = stream.config.name if stream_name is None: raise ValueError("Stream name is None") subject_name = stream_name.replace("_", self.separator, 1) subject = f"${subject_name}.>" - consumer = await self._js.pull_subscribe(subject=subject, stream=stream_name) - while True: - try: - messages: list[Msg] = await consumer.fetch(50, timeout=self.consume_timeout) - except TimeoutError: # noqa: PERF203 - pass - else: - await self._update_compiled_messages(messages) + return await self._js.pull_subscribe(subject=subject, stream=stream_name) + + async def _process_messages(self, consumer: JetStreamContext.PullSubscription) -> None: + """Process messages from the consumer.""" + try: + messages: list[Msg] = await consumer.fetch(50, timeout=self.consume_timeout) + logger.debug("Received %d messages: %s", len(messages), messages) + await self._update_compiled_messages(messages) + except TimeoutError: + pass + except Exception as e: + if not self._stop_event.is_set(): + logger.exception("Error in listen_for_changes: %s", exc_info=e) + + async def listen_for_changes(self) -> None: + """Listen for changes in NATS KV store and update local storage.""" + consumer = await self._create_consumer() + while not self._stop_event.is_set(): + await self._process_messages(consumer) async def _update_compiled_messages(self, messages: list[Msg]) -> None: """Update compiled messages based on NATS KV changes.""" @@ -82,7 +105,7 @@ async def _update_compiled_messages(self, messages: list[Msg]) -> None: # Remove translation from local storage translator = self._storage.get(locale) if translator: - print(f"Removing translation: {key}") # noqa: T201 + logger.debug("Removing translation: %s", key) else: value = self.deserializer(m.data) changes[locale].append((key, value)) @@ -98,3 +121,13 @@ def _set_new_compiled_messages(self, new_messages: dict[str, list[str]]) -> None for key, value in messages: translator.update_translation(key, value) + + async def close(self) -> None: + """Close the storage.""" + self._stop_event.set() + if self._listen_for_changes_task is not None: + self._listen_for_changes_task.cancel() + with suppress(asyncio.CancelledError): + await self._listen_for_changes_task + + await self._nc.close() diff --git a/fluentogram/storage/__init__.py b/fluentogram/storage/__init__.py index 5559fd6..f5235b5 100644 --- a/fluentogram/storage/__init__.py +++ b/fluentogram/storage/__init__.py @@ -1,5 +1,6 @@ """Storage implementations for fluentogram.""" from fluentogram.storage.base import BaseStorage +from fluentogram.storage.memory import MemoryStorage -__all__ = ["BaseStorage"] +__all__ = ["BaseStorage", "MemoryStorage"] diff --git a/fluentogram/storage/base.py b/fluentogram/storage/base.py index d98091a..0966238 100644 --- a/fluentogram/storage/base.py +++ b/fluentogram/storage/base.py @@ -2,6 +2,7 @@ from __future__ import annotations +from abc import ABC, abstractmethod from collections.abc import Iterable from typing import TYPE_CHECKING @@ -9,7 +10,7 @@ from fluentogram.translator import FluentTranslator -class BaseStorage: +class BaseStorage(ABC): """Abstract storage for translators by locale.""" def __init__(self) -> None: @@ -69,7 +70,7 @@ def _build_translators_map(self) -> None: for lang, translator_locales in self._locales_map.items() } - def update_translation(self, locale: str, key: str, value: str) -> bool: + async def update_translation(self, locale: str, key: str, value: str) -> bool: """Update a translation key for a specific locale.""" translator = self._storage.get(locale) if translator is None: @@ -77,3 +78,7 @@ def update_translation(self, locale: str, key: str, value: str) -> bool: translator.update_translation(key, value) return True + + @abstractmethod + async def close(self) -> None: + pass diff --git a/fluentogram/storage/memory.py b/fluentogram/storage/memory.py new file mode 100644 index 0000000..28fa59e --- /dev/null +++ b/fluentogram/storage/memory.py @@ -0,0 +1,9 @@ +from fluentogram.storage.base import BaseStorage + + +class MemoryStorage(BaseStorage): + def __init__(self) -> None: + super().__init__() + + async def close(self) -> None: + pass diff --git a/fluentogram/translator_hub.py b/fluentogram/translator_hub.py index 013a50b..26d90de 100644 --- a/fluentogram/translator_hub.py +++ b/fluentogram/translator_hub.py @@ -4,7 +4,7 @@ from fluentogram.exceptions import RootTranslatorNotFoundError from fluentogram.runner import TranslatorRunner -from fluentogram.storage.base import BaseStorage +from fluentogram.storage import BaseStorage, MemoryStorage from fluentogram.translator import FluentTranslator @@ -22,7 +22,7 @@ def __init__( self.root_locale = root_locale self.separator = separator - self.storage = storage or BaseStorage() + self.storage = storage or MemoryStorage() # Add translators to storage self.storage.add_translators(translators) diff --git a/pyproject.toml b/pyproject.toml index 0b40591..ac45c33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ dependencies = [ ] [project.optional-dependencies] cli = ["watchdog>=2.3.0,<3"] -dev = ["ruff~=0.11.2", "pytest~=8.4.1", "pre-commit~=4.2.0", "detect-secrets~=1.5.0"] +dev = ["ruff~=0.11.2", "pytest~=8.4.1", "pre-commit~=4.2.0", "detect-secrets~=1.5.0", "pytest-asyncio~=1.0.0"] aiogram = ["aiogram~=3.20.0"] nats = ["nats-py~=2.10.0"] diff --git a/tests/test_update.py b/tests/test_update.py index 8ad69d5..b47a5a3 100644 --- a/tests/test_update.py +++ b/tests/test_update.py @@ -1,9 +1,12 @@ +import pytest + from fluent_compiler.bundle import FluentBundle from fluentogram import FluentTranslator, TranslatorHub -def test_update_translation() -> None: +@pytest.mark.asyncio +async def test_update_translation() -> None: translator_hub = TranslatorHub( { "en": "en", @@ -18,11 +21,12 @@ def test_update_translation() -> None: translator = translator_hub.get_translator_by_locale("en") assert translator.get("start-hello") == "Hello" - translator_hub.storage.update_translation("en", "start-hello1", "Hello123") + await translator_hub.storage.update_translation("en", "start-hello1", "Hello123") assert translator.get("start-hello1") == "Hello123" -def test_update_translation_after_fallback() -> None: +@pytest.mark.asyncio +async def test_update_translation_after_fallback() -> None: translator_hub = TranslatorHub( { "en": "en", @@ -42,5 +46,5 @@ def test_update_translation_after_fallback() -> None: translator = translator_hub.get_translator_by_locale("ru") assert translator.get("start-hello1") == "Hello1" - translator_hub.storage.update_translation("ru", "start-hello1", "Привет1") + await translator_hub.storage.update_translation("ru", "start-hello1", "Привет1") assert translator.get("start-hello1") == "Привет1" From 91801a68390236f4cec4e95647c2457a252903db Mon Sep 17 00:00:00 2001 From: sheldy Date: Sun, 29 Jun 2025 15:43:24 +0200 Subject: [PATCH 12/36] Add tests for transformers. --- tests/test_transformers.py | 53 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 tests/test_transformers.py diff --git a/tests/test_transformers.py b/tests/test_transformers.py new file mode 100644 index 0000000..c792efe --- /dev/null +++ b/tests/test_transformers.py @@ -0,0 +1,53 @@ +from datetime import UTC, datetime +from decimal import Decimal + +from fluent_compiler.bundle import FluentBundle + +from fluentogram import DateTimeTransformer, FluentTranslator, MoneyTransformer, TranslatorHub + + +def test_date_transformer() -> None: + translators = [ + FluentTranslator( + "en", + translator=FluentBundle.from_string( + "en-US", + "meeting-time = Meeting scheduled for { $date }", + use_isolating=False, + ), + ), + ] + + hub = TranslatorHub({"en": "en"}, translators) + translator = hub.get_translator_by_locale("en") + + # Use DateTimeTransformer for proper date formatting + meeting_date = datetime(2024, 1, 15, 14, 30, tzinfo=UTC) + formatted_date = DateTimeTransformer(meeting_date, dateStyle="full", timeStyle="short") + assert ( + translator.get( + "meeting-time", + date=formatted_date, + ) + == "Meeting scheduled for Monday, January 15, 2024, 2:30 PM" + ) + + +def test_money_transformer() -> None: + translators = [ + FluentTranslator( + "en", + translator=FluentBundle.from_string( + "en-US", + "amount-message = You have { $amount }", + use_isolating=False, + ), + ), + ] + + hub = TranslatorHub({"en": ("en")}, translators) + translator = hub.get_translator_by_locale("en") + + amount = Decimal("123456.78") + formatted_amount = MoneyTransformer(amount, currency="USD", currency_display="symbol") + assert translator.get("amount-message", amount=formatted_amount) == "You have $123456.78" From f665fcabd58d6baa2f31cacc87b3d4e32e191252 Mon Sep 17 00:00:00 2001 From: sheldy Date: Sun, 29 Jun 2025 15:43:31 +0200 Subject: [PATCH 13/36] Add README. --- README.md | 141 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..5982b17 --- /dev/null +++ b/README.md @@ -0,0 +1,141 @@ +# Fluentogram + +Fluentogram is easy way to use i18n (Fluent) mechanism in any python app. + +## Features + +- Customizable storage: You can implement your own storage to save translates. +- Fallback support: Automatic fallback to root locale when translations are missing. +- Precompiled fluent messages using fluent_compiler makes formatting messages faster. +- Dot access to messages: `translator.hello(name='Alex')` + +## Installation + +```bash +pip install fluentogram +``` + +## Quick Start + +### Basic Usage + +```python +from fluent_compiler.bundle import FluentBundle +from fluentogram import FluentTranslator, TranslatorHub + +# Create translators for different locales +translators = [ + FluentTranslator( + "en", + translator=FluentBundle.from_string( + "en-US", + "welcome = Welcome, { $username }!\n" + "items-count = You have { $count } items", + ), + ), +] + +# Configure locale mapping with fallbacks +locales_map = { + "en": "en", +} + +# Create the translator hub +hub = TranslatorHub(locales_map, translators) + +# Get a translator for a specific locale +translator = hub.get_translator_by_locale("en") + +# Use translations +print(translator.get("welcome", username="Alice")) # "Welcome, Alice!" +``` + +### Attribute-based Access + +Fluentogram supports a convenient attribute-based syntax for accessing translations: + +```python +print(translator.welcome(username="Alice")) # "Welcome, Alice!" +print(translator.items.count(count=5)) # "You have 5 items" +``` + +### Dynamic Storage with NATS KV + +Fluentogram supports real-time translation updates using NATS KV storage: + +```python +import asyncio + +from fluent_compiler.bundle import FluentBundle +from nats.js.api import KeyValueConfig, StorageType + +from fluentogram import FluentTranslator, TranslatorHub +from fluentogram.nats.storage import NatsKvStorage + + +async def main(): + # Configure NATS KV storage + kv_config = KeyValueConfig( + bucket="fluentogram", + storage=StorageType.FILE, + ) + + # Create NATS storage + storage = await NatsKvStorage.from_servers( + servers=["nats://localhost:4222"], + kv_config=kv_config, + ) + + # Create translators + translators = [ + FluentTranslator( + "en", + translator=FluentBundle.from_string( + "en-US", + "greeting = Hello, { $name }!", + ), + ), + ] + + # Create hub with NATS storage + hub = TranslatorHub( + {"en": "en"}, + translators, + storage=storage, + ) + + translator = hub.get_translator_by_locale("en") + print(translator.get("greeting", name="World")) # "Hello, World!" + + # Update translation dynamically + await storage.update_translation("en", "greeting", "Hi there, { $name }!") + + # Wait for the update to propagate + await asyncio.sleep(1) + + # Get updated translation + print(translator.get("greeting", name="World")) # "Hi there, World!" + + await storage.close() + + +asyncio.run(main()) +``` + +## Error Handling + +Fluentogram provides comprehensive error handling: + +```python +from fluentogram.exceptions import KeyNotFoundError, FormatError, RootTranslatorNotFoundError + +try: + translator = hub.get_translator_by_locale("fr") + result = translator.get("nonexistent-key") +except KeyNotFoundError as e: + print(f"Translation key not found: {e.key}") +except RootTranslatorNotFoundError as e: + print(f"Root locale translator missing: {e.root_locale}") +except FormatError as e: + print(f"Formatting error for key {e.key}: {e.original_error}") +``` \ No newline at end of file From f059ee9d5d3a0081004adc09e3d367abda1da532 Mon Sep 17 00:00:00 2001 From: sheldy Date: Sun, 29 Jun 2025 17:14:07 +0200 Subject: [PATCH 14/36] Add parser and stub generator with cli. --- fluentogram/__main__.py | 4 + fluentogram/cli/__init__.py | 0 fluentogram/cli/main.py | 16 ++++ fluentogram/exceptions.py | 4 + fluentogram/stub_generator/__init__.py | 0 fluentogram/stub_generator/generator.py | 117 ++++++++++++++++++++++++ fluentogram/stub_generator/parser.py | 38 ++++++++ pyproject.toml | 32 +++++-- tests/assets/broken.ftl | 3 + tests/assets/simple.ftl | 2 + tests/assets/test.ftl | 75 +++++++++++++++ tests/test_generator.py | 65 +++++++++++++ 12 files changed, 348 insertions(+), 8 deletions(-) create mode 100644 fluentogram/__main__.py create mode 100644 fluentogram/cli/__init__.py create mode 100644 fluentogram/cli/main.py create mode 100644 fluentogram/stub_generator/__init__.py create mode 100644 fluentogram/stub_generator/generator.py create mode 100644 fluentogram/stub_generator/parser.py create mode 100644 tests/assets/broken.ftl create mode 100644 tests/assets/simple.ftl create mode 100644 tests/assets/test.ftl create mode 100644 tests/test_generator.py diff --git a/fluentogram/__main__.py b/fluentogram/__main__.py new file mode 100644 index 0000000..65c7fc1 --- /dev/null +++ b/fluentogram/__main__.py @@ -0,0 +1,4 @@ +from fluentogram.cli.main import cli + +if __name__ == "__main__": + cli() diff --git a/fluentogram/cli/__init__.py b/fluentogram/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fluentogram/cli/main.py b/fluentogram/cli/main.py new file mode 100644 index 0000000..4e40a9a --- /dev/null +++ b/fluentogram/cli/main.py @@ -0,0 +1,16 @@ +import argparse + +from fluentogram.stub_generator.generator import generate + + +def cli() -> None: + parser = argparse.ArgumentParser(prog="fluentogram") + parser.add_argument("-o", dest="output_file", required=True) + parser.add_argument("-f", dest="file_path", required=False) + parser.add_argument("-d", dest="dir_path", required=False) + + args = parser.parse_args() + if args.file_path is None and args.dir_path is None: + raise ValueError("You must provide either a file or a directory") + + generate(args.output_file, args.file_path, args.dir_path) diff --git a/fluentogram/exceptions.py b/fluentogram/exceptions.py index c932358..cb023f6 100644 --- a/fluentogram/exceptions.py +++ b/fluentogram/exceptions.py @@ -22,3 +22,7 @@ def __init__(self, original_error: Exception, key: str) -> None: self.original_error = original_error self.key = key super().__init__(f"Error formatting key: {self.key!r}: {self.original_error!r}") + + +class StubGeneratorKeyConflictError(FluentogramError): + pass diff --git a/fluentogram/stub_generator/__init__.py b/fluentogram/stub_generator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fluentogram/stub_generator/generator.py b/fluentogram/stub_generator/generator.py new file mode 100644 index 0000000..a0fa5bd --- /dev/null +++ b/fluentogram/stub_generator/generator.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +from pathlib import Path + +from fluentogram.exceptions import StubGeneratorKeyConflictError +from fluentogram.stub_generator.parser import get_messages + + +class Generator: + def __init__( + self, + output_file: str, + file_path: str | None = None, + directory: str | None = None, + ) -> None: + self.output_file = Path(output_file) + if self.output_file.suffix != ".pyi": + raise ValueError("Output file must have .pyi extension") + + if file_path is None and directory is None: + raise ValueError("Either file_path or directory must be provided") + + self.files = set() # set of Path objects to skip duplicates + if file_path: + self.files.add(Path(file_path)) + if directory: + self.files.update(Path(directory).glob("*.ftl")) + + self.messages = {} + + def _generate_class_name(self, name: str) -> str: + """Generate class name from message name.""" + if "-" in name: + parts = name.split("-") + return parts[0].title() + return name.title() + + def _generate_method_signature(self, name: str, params: set[str]) -> str: + """Generate method signature for a message.""" + if not params: + return f" def {name}(self) -> str: ..." + + param_list = ", ".join(f"{param}: str" for param in sorted(params)) + return f" def {name}(self, {param_list}) -> str: ..." + + def _group_messages(self) -> tuple[dict[str, dict[str, set[str]]], dict[str, set[str]]]: + grouped_messages = {} + simple_messages = {} + for name, params in self.messages.items(): + if "-" in name: + base_name = name.split("-")[0] + if base_name not in grouped_messages: + grouped_messages[base_name] = {} + grouped_messages[base_name][name] = params + else: + simple_messages[name] = params + + return grouped_messages, simple_messages + + def _check_for_conflict( + self, + grouped_messages: dict[str, dict[str, set[str]]], + simple_messages: dict[str, set[str]], + ) -> None: + for name, messages in grouped_messages.items(): + if name in simple_messages: + conflicting_simple = name + conflicting_grouped = ", ".join(key for key, _ in messages.items()) + + raise StubGeneratorKeyConflictError( + f"You have conflicting keys in your .ftl file: " # noqa: EM102 + f"{conflicting_simple} and {conflicting_grouped}. " + f"You can't have a simple key '{conflicting_simple}' " + f"and compound keys with prefix '{conflicting_simple}-'.", + ) + + def generate(self) -> None: # noqa: C901 + for file in self.files: + messages = get_messages(file.read_text()) + self.messages.update(messages) + + # Generate .pyi content + content = [] + + # Group messages by their base name (before dash) + grouped_messages, simple_messages = self._group_messages() + self._check_for_conflict(grouped_messages, simple_messages) + + # Generate TranslatorRunner class + content.append("class TranslatorRunner:\n def get(self, path: str, **kwargs) -> str: ...") + + # Add simple messages as methods + for name, params in simple_messages.items(): + content.append(self._generate_method_signature(name, params)) + + # Add grouped messages as attributes + for base_name in grouped_messages: + class_name = self._generate_class_name(base_name) + content.append(f" {base_name}: {class_name}\n") + + # Generate classes for grouped messages + for base_name, messages_dict in grouped_messages.items(): + class_name = self._generate_class_name(base_name) + content.append(f"class {class_name}:") + + for name, params in messages_dict.items(): + method_name = name.split("-")[1] # Get part after dash + content.append(self._generate_method_signature(method_name, params)) + content.append("") + + # Write to file + self.output_file.write_text("\n".join(content)) + + +def generate(output_file: str, file_path: str | None = None, directory: str | None = None) -> None: + generator = Generator(output_file, file_path, directory) + generator.generate() diff --git a/fluentogram/stub_generator/parser.py b/fluentogram/stub_generator/parser.py new file mode 100644 index 0000000..ca90386 --- /dev/null +++ b/fluentogram/stub_generator/parser.py @@ -0,0 +1,38 @@ +from collections.abc import Generator + +from fluent.syntax import FluentParser, ast +from fluent.syntax.visitor import Visitor + + +class FluentVisitor(Visitor): + def __init__(self) -> None: + self.messages: dict[str, set[str]] = {} + + def _get_placeholders(self, element: ast.BaseNode) -> Generator[str, None, None]: # noqa: C901 + if isinstance(element, ast.VariableReference): + yield element.id.name + elif isinstance(element, ast.Placeable): + yield from self._get_placeholders(element.expression) + elif isinstance(element, ast.SelectExpression): + yield from self._get_placeholders(element.selector) + elif isinstance(element, ast.FunctionReference): + for pos_arg in element.arguments.positional: + yield from self._get_placeholders(pos_arg) + + def visit_Message(self, message: ast.Message) -> None: # noqa: N802 + m = self.messages[message.id.name] = set() + + if not message.value: + return self.generic_visit(message) + + for element in message.value.elements: + if isinstance(element, ast.Placeable): + m.update(self._get_placeholders(element)) + + return self.generic_visit(message) + + +def get_messages(text: str) -> dict[str, set[str]]: + visitor = FluentVisitor() + visitor.visit(FluentParser().parse(text)) + return visitor.messages diff --git a/pyproject.toml b/pyproject.toml index ac45c33..48ca2ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,12 +5,16 @@ description = "A proper way to use an i18n mechanism with Aiogram3." authors = [{ name = "Aleks" }] requires-python = ">=3.9" license = "MIT" -dependencies = [ - "fluent-compiler~=1.1", -] +dependencies = ["fluent-compiler~=1.1"] [project.optional-dependencies] cli = ["watchdog>=2.3.0,<3"] -dev = ["ruff~=0.11.2", "pytest~=8.4.1", "pre-commit~=4.2.0", "detect-secrets~=1.5.0", "pytest-asyncio~=1.0.0"] +dev = [ + "ruff~=0.11.2", + "pytest~=8.4.1", + "pre-commit~=4.2.0", + "detect-secrets~=1.5.0", + "pytest-asyncio~=1.0.0", +] aiogram = ["aiogram~=3.20.0"] nats = ["nats-py~=2.10.0"] @@ -18,7 +22,7 @@ nats = ["nats-py~=2.10.0"] Repository = "https://github.com/Arustinal/fluentogram" [project.scripts] -i18n = "fluentogram.cli:cli" +fluentogram = "fluentogram.cli.main:cli" [build-system] requires = ["setuptools>=42"] @@ -52,7 +56,7 @@ exclude = [ "node_modules", "site-packages", "venv", - "tmp" + "tmp", ] include = ["fluentogram/**/*.py"] line-length = 120 @@ -60,7 +64,19 @@ indent-width = 4 [tool.ruff.lint] select = ["ALL"] -ignore = ["D", "RET504", "PGH003", "FBT001", "EM101", "TRY003", "TC003", "ANN401", "ANN003", "TC001", "TC002"] +ignore = [ + "D", + "RET504", + "PGH003", + "FBT001", + "EM101", + "TRY003", + "TC003", + "ANN401", + "ANN003", + "TC001", + "TC002", +] [tool.ruff.lint.per-file-ignores] "tests/**/*.py" = ["S101"] @@ -70,4 +86,4 @@ max-complexity = 5 [tool.ruff.format] quote-style = "double" -indent-style = "space" \ No newline at end of file +indent-style = "space" diff --git a/tests/assets/broken.ftl b/tests/assets/broken.ftl new file mode 100644 index 0000000..fd8bdb1 --- /dev/null +++ b/tests/assets/broken.ftl @@ -0,0 +1,3 @@ +button = Button + +button-text = Button text diff --git a/tests/assets/simple.ftl b/tests/assets/simple.ftl new file mode 100644 index 0000000..0c3df70 --- /dev/null +++ b/tests/assets/simple.ftl @@ -0,0 +1,2 @@ +hello = Hello, world! +button = Button \ No newline at end of file diff --git a/tests/assets/test.ftl b/tests/assets/test.ftl new file mode 100644 index 0000000..7c33ecd --- /dev/null +++ b/tests/assets/test.ftl @@ -0,0 +1,75 @@ +# Простой перевод +hello = Привет, мир! + +# Многострочное значение +multiline = + Это многострочное сообщение. + Оно продолжается на нескольких строках. + +# Переменные +welcome = Привет, { $name }! + +# Атрибуты +button = + .label = Отправить + .accesskey = O + +# Варианты (селекторы) +email-status = + { $unreadCount -> + [0] У вас нет новых писем. + [one] У вас { $unreadCount } новое письмо. + [few] У вас { $unreadCount } новых письма. + *[other] У вас { $unreadCount } новых писем. + } + +# Использование другого сообщения (message reference) +greeting = { hello } Это фраза с другим сообщением. + +# Селектор по состоянию +task-state = + { $state -> + [new] Новая задача + [in-progress] В процессе + [done] Завершена + *[other] Неизвестное состояние + } + +# Вызов функций (например, для форматирования даты/чисел) +formatted-date = Сегодня: { DATETIME($date, month: "long", year: "numeric", day: "numeric") } + +# Пример с NUMBER и использование параметров +score = Вы набрали { NUMBER($points, minimumFractionDigits: 1) } очков + +# Пример с вложенными сообщениями +outer-message = Вложение: { inner-message } +inner-message = Это вложенное сообщение. + +# Escaping фигурных скобок +escaped = Это не переменная: {{ $notAVar }} + +# Использование term-ов (терминов, начинающихся с -) +-brand-name = Приложение X +about = Информация о { -brand-name } + +# Атрибут у термина +-icon = + .src = /images/icon.svg + .alt = Иконка + +# Комментарии +# Это обычный комментарий +## Это групповой комментарий +### Это документирующий комментарий + +# Комбинация всего +complex-message = + Добро пожаловать, { $name }! + Сегодня { DATETIME($date, weekday: "long") }. + У вас { $unreadCount -> + [0] нет новых писем. + [one] одно новое письмо. + [few] { $unreadCount } новых письма. + *[other] { $unreadCount } новых писем. + } + Спасибо, что используете { -brand-name }! \ No newline at end of file diff --git a/tests/test_generator.py b/tests/test_generator.py new file mode 100644 index 0000000..4a37167 --- /dev/null +++ b/tests/test_generator.py @@ -0,0 +1,65 @@ +import tempfile +from pathlib import Path + +import pytest + +from fluentogram.exceptions import StubGeneratorKeyConflictError +from fluentogram.stub_generator.generator import generate + + +def test_conflict_error() -> None: + with tempfile.NamedTemporaryFile(suffix=".pyi", delete=False) as tmp_file: + output_path = tmp_file.name + + with pytest.raises(StubGeneratorKeyConflictError): + generate(output_path, file_path="tests/assets/broken.ftl") + + +def test_correctly_generated_stub_for_simple_file() -> None: + with tempfile.NamedTemporaryFile(suffix=".pyi", delete=False) as tmp_file: + output_path = tmp_file.name + + generate(output_path, file_path="tests/assets/simple.ftl") + + assert Path(output_path).exists() + content = Path(output_path).read_text() + + assert "class TranslatorRunner:" in content + assert "def hello(self) -> str: ..." in content + assert "def button(self) -> str: ..." in content + + +def test_correctly_generated_stub() -> None: + with tempfile.NamedTemporaryFile(suffix=".pyi", delete=False) as tmp_file: + output_path = tmp_file.name + + generate(output_path, file_path="tests/assets/test.ftl") + + assert Path(output_path).exists() + content = Path(output_path).read_text() + + assert "class TranslatorRunner:" in content + assert "def get(self, path: str, **kwargs) -> str: ..." in content + assert "def hello(self) -> str: ..." in content + assert "def button(self) -> str: ..." in content + assert "def multiline(self) -> str: ..." in content + assert "def welcome(self, name: str) -> str: ..." in content + assert "def button(self) -> str: ..." in content + assert "def multiline(self) -> str: ..." in content + assert "def score(self, points: str) -> str: ..." in content + assert "def escaped(self, notAVar: str) -> str: ..." in content + assert "def about(self) -> str: ..." in content + assert "class Email:" in content + assert "def status(self, unreadCount: str) -> str: ..." in content + assert "class Task:" in content + assert "def state(self, state: str) -> str: ..." in content + assert "class Formatted:" in content + assert "def date(self, date: str) -> str: ..." in content + assert "class Outer:" in content + assert "class Inner:" in content + assert "class Complex:" in content + assert "def multiline(self) -> str: ..." in content + assert "def welcome(self, name: str) -> str: ..." in content + assert "def button(self) -> str: ..." in content + assert "def multiline(self) -> str: ..." in content + assert "def score(self, points: str) -> str: ..." in content From 7f0d9806f7240377ac9235f5d427255b6896d0a0 Mon Sep 17 00:00:00 2001 From: sheldy Date: Sun, 29 Jun 2025 17:14:50 +0200 Subject: [PATCH 15/36] Bump version and update description. --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 48ca2ed..095d1b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "fluentogram" -version = "1.1.13" -description = "A proper way to use an i18n mechanism with Aiogram3." +version = "1.2.0" +description = "Fluentogram is easy way to use i18n (Fluent) mechanism in any python app." authors = [{ name = "Aleks" }] requires-python = ">=3.9" license = "MIT" From 452bc36f9af805644f3a6ac8357bcc8ee031a19e Mon Sep 17 00:00:00 2001 From: sheldy Date: Sun, 29 Jun 2025 18:09:21 +0200 Subject: [PATCH 16/36] Workaround for conflicts in ftl files. --- fluentogram/exceptions.py | 4 -- fluentogram/stub_generator/generator.py | 56 ++++++++++++++++--------- fluentogram/stub_generator/parser.py | 2 +- pyproject.toml | 2 +- 4 files changed, 38 insertions(+), 26 deletions(-) diff --git a/fluentogram/exceptions.py b/fluentogram/exceptions.py index cb023f6..c932358 100644 --- a/fluentogram/exceptions.py +++ b/fluentogram/exceptions.py @@ -22,7 +22,3 @@ def __init__(self, original_error: Exception, key: str) -> None: self.original_error = original_error self.key = key super().__init__(f"Error formatting key: {self.key!r}: {self.original_error!r}") - - -class StubGeneratorKeyConflictError(FluentogramError): - pass diff --git a/fluentogram/stub_generator/generator.py b/fluentogram/stub_generator/generator.py index a0fa5bd..c86828b 100644 --- a/fluentogram/stub_generator/generator.py +++ b/fluentogram/stub_generator/generator.py @@ -2,7 +2,6 @@ from pathlib import Path -from fluentogram.exceptions import StubGeneratorKeyConflictError from fluentogram.stub_generator.parser import get_messages @@ -43,9 +42,13 @@ def _generate_method_signature(self, name: str, params: set[str]) -> str: param_list = ", ".join(f"{param}: str" for param in sorted(params)) return f" def {name}(self, {param_list}) -> str: ..." - def _group_messages(self) -> tuple[dict[str, dict[str, set[str]]], dict[str, set[str]]]: + def _group_messages( + self, + ) -> tuple[dict[str, dict[str, set[str]]], dict[str, set[str]], dict[str, dict[str, set[str]]]]: grouped_messages = {} simple_messages = {} + conflict_classes = {} + for name, params in self.messages.items(): if "-" in name: base_name = name.split("-")[0] @@ -55,24 +58,17 @@ def _group_messages(self) -> tuple[dict[str, dict[str, set[str]]], dict[str, set else: simple_messages[name] = params - return grouped_messages, simple_messages - - def _check_for_conflict( - self, - grouped_messages: dict[str, dict[str, set[str]]], - simple_messages: dict[str, set[str]], - ) -> None: - for name, messages in grouped_messages.items(): + # If there is a conflict, move it to conflict_classes and remove from grouped_messages and simple_messages + for name, messages in list(grouped_messages.items()): if name in simple_messages: - conflicting_simple = name - conflicting_grouped = ", ".join(key for key, _ in messages.items()) + conflict_classes[name] = {} + conflict_classes[name][name] = simple_messages[name] + for compound_name, compound_params in messages.items(): + conflict_classes[name][compound_name] = compound_params + grouped_messages.pop(name) + simple_messages.pop(name) - raise StubGeneratorKeyConflictError( - f"You have conflicting keys in your .ftl file: " # noqa: EM102 - f"{conflicting_simple} and {conflicting_grouped}. " - f"You can't have a simple key '{conflicting_simple}' " - f"and compound keys with prefix '{conflicting_simple}-'.", - ) + return grouped_messages, simple_messages, conflict_classes def generate(self) -> None: # noqa: C901 for file in self.files: @@ -83,8 +79,7 @@ def generate(self) -> None: # noqa: C901 content = [] # Group messages by their base name (before dash) - grouped_messages, simple_messages = self._group_messages() - self._check_for_conflict(grouped_messages, simple_messages) + grouped_messages, simple_messages, conflict_classes = self._group_messages() # Generate TranslatorRunner class content.append("class TranslatorRunner:\n def get(self, path: str, **kwargs) -> str: ...") @@ -98,6 +93,11 @@ def generate(self) -> None: # noqa: C901 class_name = self._generate_class_name(base_name) content.append(f" {base_name}: {class_name}\n") + # Add conflict classes as attributes + for base_name in conflict_classes: + class_name = self._generate_class_name(base_name) + content.append(f" {base_name}: {class_name}\n") + # Generate classes for grouped messages for base_name, messages_dict in grouped_messages.items(): class_name = self._generate_class_name(base_name) @@ -108,6 +108,22 @@ def generate(self) -> None: # noqa: C901 content.append(self._generate_method_signature(method_name, params)) content.append("") + # Generate classes for conflict messages + for base_name, messages_dict in conflict_classes.items(): + class_name = self._generate_class_name(base_name) + content.append(f"class {class_name}:") + + # Add __call__ for simple key + if base_name in messages_dict: + content.append(" def __call__(self) -> str: ...") + + # Add methods for compound keys + for name, params in messages_dict.items(): + if name != base_name: # Skip simple key, it's already handled as __call__ + method_name = name.split("-")[1] # Get part after dash + content.append(self._generate_method_signature(method_name, params)) + content.append("") + # Write to file self.output_file.write_text("\n".join(content)) diff --git a/fluentogram/stub_generator/parser.py b/fluentogram/stub_generator/parser.py index ca90386..0329bc3 100644 --- a/fluentogram/stub_generator/parser.py +++ b/fluentogram/stub_generator/parser.py @@ -8,7 +8,7 @@ class FluentVisitor(Visitor): def __init__(self) -> None: self.messages: dict[str, set[str]] = {} - def _get_placeholders(self, element: ast.BaseNode) -> Generator[str, None, None]: # noqa: C901 + def _get_placeholders(self, element: ast.BaseNode) -> Generator[str, None, None]: if isinstance(element, ast.VariableReference): yield element.id.name elif isinstance(element, ast.Placeable): diff --git a/pyproject.toml b/pyproject.toml index 095d1b5..1de85f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,7 @@ ignore = [ "tests/**/*.py" = ["S101"] [tool.ruff.lint.mccabe] -max-complexity = 5 +max-complexity = 8 [tool.ruff.format] quote-style = "double" From a955d358b0ade648b313ecc27467aafbc09dd95a Mon Sep 17 00:00:00 2001 From: sheldy Date: Sun, 29 Jun 2025 18:14:55 +0200 Subject: [PATCH 17/36] Fix tests. --- tests/assets/{broken.ftl => conflict.ftl} | 0 tests/test_generator.py | 24 +++++++++++++---------- 2 files changed, 14 insertions(+), 10 deletions(-) rename tests/assets/{broken.ftl => conflict.ftl} (100%) diff --git a/tests/assets/broken.ftl b/tests/assets/conflict.ftl similarity index 100% rename from tests/assets/broken.ftl rename to tests/assets/conflict.ftl diff --git a/tests/test_generator.py b/tests/test_generator.py index 4a37167..5ff3af8 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -1,32 +1,36 @@ import tempfile from pathlib import Path -import pytest - -from fluentogram.exceptions import StubGeneratorKeyConflictError from fluentogram.stub_generator.generator import generate -def test_conflict_error() -> None: +def test_correctly_generated_stub_for_simple_file() -> None: with tempfile.NamedTemporaryFile(suffix=".pyi", delete=False) as tmp_file: output_path = tmp_file.name - with pytest.raises(StubGeneratorKeyConflictError): - generate(output_path, file_path="tests/assets/broken.ftl") + generate(output_path, file_path="tests/assets/simple.ftl") + assert Path(output_path).exists() + content = Path(output_path).read_text() -def test_correctly_generated_stub_for_simple_file() -> None: + assert "class TranslatorRunner:" in content + assert "def hello(self) -> str: ..." in content + assert "def button(self) -> str: ..." in content + + +def test_generator_if_conflict() -> None: with tempfile.NamedTemporaryFile(suffix=".pyi", delete=False) as tmp_file: output_path = tmp_file.name - generate(output_path, file_path="tests/assets/simple.ftl") + generate(output_path, file_path="tests/assets/conflict.ftl") assert Path(output_path).exists() content = Path(output_path).read_text() assert "class TranslatorRunner:" in content - assert "def hello(self) -> str: ..." in content - assert "def button(self) -> str: ..." in content + assert "class Button" in content + assert "def __call__(self) -> str: ..." in content + assert "def text(self) -> str: ..." in content def test_correctly_generated_stub() -> None: From 1b1954f83af7b4ca37ca64ee15ef164e11841110 Mon Sep 17 00:00:00 2001 From: sheldy Date: Mon, 30 Jun 2025 02:18:37 +0200 Subject: [PATCH 18/36] Update stub generator. --- fluentogram/stub_generator/generator.py | 53 ++++--- fluentogram/stub_generator/parser.py | 179 ++++++++++++++++++++---- tests/assets/test.ftl | 96 ++++++------- tests/test_generator.py | 46 +++--- 4 files changed, 258 insertions(+), 116 deletions(-) diff --git a/fluentogram/stub_generator/generator.py b/fluentogram/stub_generator/generator.py index c86828b..68cf689 100644 --- a/fluentogram/stub_generator/generator.py +++ b/fluentogram/stub_generator/generator.py @@ -2,7 +2,7 @@ from pathlib import Path -from fluentogram.stub_generator.parser import get_messages +from fluentogram.stub_generator.parser import Message, get_messages class Generator: @@ -25,7 +25,7 @@ def __init__( if directory: self.files.update(Path(directory).glob("*.ftl")) - self.messages = {} + self.messages: dict[str, Message] = {} def _generate_class_name(self, name: str) -> str: """Generate class name from message name.""" @@ -34,17 +34,27 @@ def _generate_class_name(self, name: str) -> str: return parts[0].title() return name.title() - def _generate_method_signature(self, name: str, params: set[str]) -> str: - """Generate method signature for a message.""" - if not params: - return f" def {name}(self) -> str: ..." + def _generate_func_signature(self, message: Message, is_method: bool = False, is_call: bool = False) -> str: # noqa: FBT002 + """Generate function signature for a message.""" - param_list = ", ".join(f"{param}: str" for param in sorted(params)) - return f" def {name}(self, {param_list}) -> str: ..." + if is_call: + name = "__call__" + elif is_method: + name = message.name.split("-")[1] + else: + name = message.name + + formatted_text = f'"""{message.result_text}"""' + + if not message.placeholders: + return f" def {name}(self) -> Literal[{formatted_text}]: ..." + + param_list = ", ".join(f"{param}: str" for param in sorted(message.placeholders)) + return f" def {name}(self, *, {param_list}) -> Literal[{formatted_text}]: ..." def _group_messages( self, - ) -> tuple[dict[str, dict[str, set[str]]], dict[str, set[str]], dict[str, dict[str, set[str]]]]: + ) -> tuple[dict[str, dict[str, Message]], dict[str, Message], dict[str, dict[str, Message]]]: grouped_messages = {} simple_messages = {} conflict_classes = {} @@ -82,11 +92,12 @@ def generate(self) -> None: # noqa: C901 grouped_messages, simple_messages, conflict_classes = self._group_messages() # Generate TranslatorRunner class - content.append("class TranslatorRunner:\n def get(self, path: str, **kwargs) -> str: ...") + content.append( + "from typing import Literal\n\nclass TranslatorRunner:\n def get(self, path: str, **kwargs) -> str: ...", + ) # Add simple messages as methods - for name, params in simple_messages.items(): - content.append(self._generate_method_signature(name, params)) + content.extend(self._generate_func_signature(message) for message in simple_messages.values()) # Add grouped messages as attributes for base_name in grouped_messages: @@ -103,9 +114,8 @@ def generate(self) -> None: # noqa: C901 class_name = self._generate_class_name(base_name) content.append(f"class {class_name}:") - for name, params in messages_dict.items(): - method_name = name.split("-")[1] # Get part after dash - content.append(self._generate_method_signature(method_name, params)) + for message in messages_dict.values(): + content.append(self._generate_func_signature(message, is_method=True)) content.append("") # Generate classes for conflict messages @@ -115,19 +125,22 @@ def generate(self) -> None: # noqa: C901 # Add __call__ for simple key if base_name in messages_dict: - content.append(" def __call__(self) -> str: ...") + content.append(self._generate_func_signature(messages_dict[base_name], is_call=True)) # Add methods for compound keys - for name, params in messages_dict.items(): + for name, message in messages_dict.items(): if name != base_name: # Skip simple key, it's already handled as __call__ - method_name = name.split("-")[1] # Get part after dash - content.append(self._generate_method_signature(method_name, params)) + content.append(self._generate_func_signature(message, is_method=True)) content.append("") # Write to file self.output_file.write_text("\n".join(content)) -def generate(output_file: str, file_path: str | None = None, directory: str | None = None) -> None: +def generate( + output_file: str, + file_path: str | None = None, + directory: str | None = None, +) -> None: generator = Generator(output_file, file_path, directory) generator.generate() diff --git a/fluentogram/stub_generator/parser.py b/fluentogram/stub_generator/parser.py index 0329bc3..0cc4905 100644 --- a/fluentogram/stub_generator/parser.py +++ b/fluentogram/stub_generator/parser.py @@ -1,38 +1,163 @@ -from collections.abc import Generator +from __future__ import annotations + +import logging +from dataclasses import dataclass from fluent.syntax import FluentParser, ast -from fluent.syntax.visitor import Visitor + +logger = logging.getLogger(__name__) + + +@dataclass +class Message: + name: str + result_text: str + placeholders: list[str] + raw_elements: list[ast.TextElement | ast.Placeable] # Store raw elements for later processing -class FluentVisitor(Visitor): +class Parser: def __init__(self) -> None: - self.messages: dict[str, set[str]] = {} - - def _get_placeholders(self, element: ast.BaseNode) -> Generator[str, None, None]: - if isinstance(element, ast.VariableReference): - yield element.id.name - elif isinstance(element, ast.Placeable): - yield from self._get_placeholders(element.expression) - elif isinstance(element, ast.SelectExpression): - yield from self._get_placeholders(element.selector) - elif isinstance(element, ast.FunctionReference): - for pos_arg in element.arguments.positional: - yield from self._get_placeholders(pos_arg) - - def visit_Message(self, message: ast.Message) -> None: # noqa: N802 - m = self.messages[message.id.name] = set() + self.messages: dict[str, Message] = {} + self.terms: dict[str, str] = {} + + def process_term(self, term: ast.Term) -> None: + for element in term.value.elements: + if isinstance(element, ast.TextElement): + self.terms[term.id.name] = element.value + elif isinstance(element, ast.TermReference): + self.terms[term.id.name] = self.terms[element.id.name] + else: + logger.warning("Unknown element type in term %s: %s", term.id.name, type(element)) + + def _parse_variable_reference(self, message_obj: Message, element: ast.VariableReference) -> None: + message_obj.placeholders.append(element.id.name) + message_obj.result_text += f"{{ ${element.id.name} }}" + + def _parse_term_reference(self, message_obj: Message, element: ast.TermReference) -> None: + message_obj.result_text += self.terms[element.id.name] + + def _parse_message_reference(self, message_obj: Message, element: ast.MessageReference) -> None: + if element.id.name in self.messages: + # Resolve the referenced message recursively + referenced_message = self.messages[element.id.name] + if not referenced_message.result_text: + # If the referenced message hasn't been processed yet, process it now + self._process_message_elements(referenced_message) + message_obj.result_text += referenced_message.result_text + else: + logger.warning("Message reference %s not found", element.id.name) + + def _parse_placeable(self, message_obj: Message, element: ast.Placeable) -> None: + expression = element.expression + if isinstance(expression, ast.VariableReference): + self._parse_variable_reference(message_obj, expression) + elif isinstance(expression, ast.TermReference): + self._parse_term_reference(message_obj, expression) + elif isinstance(expression, ast.SelectExpression): + self._parse_select_expression(message_obj, expression) + elif isinstance(expression, ast.FunctionReference): + self._parse_function_reference(message_obj, expression) + elif isinstance(expression, ast.MessageReference): + self._parse_message_reference(message_obj, expression) + elif isinstance(expression, ast.Placeable): + self._parse_placeable(message_obj, expression) # recurse + else: + logger.warning("Unknown expression type in placeable %s: %s", message_obj.name, type(expression)) + + def _parse_function_reference(self, message_obj: Message, element: ast.FunctionReference) -> None: + arguments = element.arguments + for pos_arg in arguments.positional: + if isinstance(pos_arg, ast.VariableReference): + self._parse_variable_reference(message_obj, pos_arg) + else: + logger.warning( + "Unknown positional argument type in function reference %s: %s", + message_obj.name, + type(pos_arg), + ) + + for named_arg in arguments.named: + if isinstance(named_arg, ast.VariableReference): + self._parse_variable_reference(message_obj, named_arg) + elif isinstance(named_arg, ast.NamedArgument): + logger.warning( + "Skipping named argument '%s' in function reference '%s'", + named_arg.name.name, + message_obj.name, + ) + else: + logger.warning( + "Unknown named argument type in function reference %s: %s", + message_obj.name, + type(named_arg), + ) + + def _parse_select_expression(self, message_obj: Message, element: ast.SelectExpression) -> None: + """ + Is selector can be many results, so we parse only default variant. + """ + for variant in element.variants: + if variant.value is None or not variant.default: + continue + for variant_element in variant.value.elements: + if isinstance(variant_element, ast.TextElement): + message_obj.result_text += variant_element.value + elif isinstance(variant_element, ast.Placeable): + self._parse_placeable(message_obj, variant_element) + else: + logger.warning( + "Unknown element type in select expression %s: %s", + message_obj.name, + type(variant_element), + ) + + def process_message(self, message: ast.Message) -> None: + # First pass: just collect the message structure + message_obj = Message( + name=message.id.name, + result_text="", + placeholders=[], + raw_elements=[], + ) if not message.value: - return self.generic_visit(message) + logger.warning("Message %s has no value", message.id.name) + self.messages[message.id.name] = message_obj + return + + # Store raw elements for later processing + message_obj.raw_elements = message.value.elements + self.messages[message.id.name] = message_obj + + def _process_message_elements(self, message_obj: Message) -> None: + """Process the raw elements of a message to generate result_text and placeholders.""" + # Skip if already processed + if message_obj.result_text and message_obj.placeholders: + return + + for element in message_obj.raw_elements: + if isinstance(element, ast.TextElement): + message_obj.result_text += element.value + elif isinstance(element, ast.Placeable): + self._parse_placeable(message_obj, element) + else: + logger.warning("Unknown element type in message %s: %s", message_obj.name, type(element)) - for element in message.value.elements: - if isinstance(element, ast.Placeable): - m.update(self._get_placeholders(element)) + def parse(self, resource: ast.Resource) -> None: + # First pass: collect all terms and message structures + for entry in resource.body: + if isinstance(entry, ast.Term): + self.process_term(entry) + elif isinstance(entry, ast.Message): + self.process_message(entry) - return self.generic_visit(message) + # Second pass: process all message elements (now all messages are available) + for message_obj in self.messages.values(): + self._process_message_elements(message_obj) -def get_messages(text: str) -> dict[str, set[str]]: - visitor = FluentVisitor() - visitor.visit(FluentParser().parse(text)) - return visitor.messages +def get_messages(text: str) -> dict[str, Message]: + parser = Parser() + parser.parse(FluentParser(with_spans=False).parse(text)) + return parser.messages diff --git a/tests/assets/test.ftl b/tests/assets/test.ftl index 7c33ecd..14a4061 100644 --- a/tests/assets/test.ftl +++ b/tests/assets/test.ftl @@ -1,75 +1,75 @@ -# Простой перевод -hello = Привет, мир! +# Simple message +hello = Hello, world! -# Многострочное значение +# Multiline value multiline = - Это многострочное сообщение. - Оно продолжается на нескольких строках. + This is a multiline message. + It continues on multiple lines. -# Переменные -welcome = Привет, { $name }! +# Variables +welcome = Hello, { $name }! -# Атрибуты +# Attributes button = - .label = Отправить + .label = Send .accesskey = O -# Варианты (селекторы) +# Selectors email-status = { $unreadCount -> - [0] У вас нет новых писем. - [one] У вас { $unreadCount } новое письмо. - [few] У вас { $unreadCount } новых письма. - *[other] У вас { $unreadCount } новых писем. + [0] You have no new emails. + [one] You have { $unreadCount } new email. + [few] You have { $unreadCount } new emails. + *[other] You have { $unreadCount } new emails. } -# Использование другого сообщения (message reference) -greeting = { hello } Это фраза с другим сообщением. +# Message reference +greeting = { hello } This is a phrase with another message. -# Селектор по состоянию +# Selector by state task-state = { $state -> - [new] Новая задача - [in-progress] В процессе - [done] Завершена - *[other] Неизвестное состояние + [new] New task + [in-progress] In progress + [done] Done + *[other] Unknown state } -# Вызов функций (например, для форматирования даты/чисел) -formatted-date = Сегодня: { DATETIME($date, month: "long", year: "numeric", day: "numeric") } +# Function call (for formatting date/numbers) +formatted-date = Today: { DATETIME($date, month: "long", year: "numeric", day: "numeric") } -# Пример с NUMBER и использование параметров -score = Вы набрали { NUMBER($points, minimumFractionDigits: 1) } очков +# Example with NUMBER and using parameters +score = You scored { NUMBER($points, minimumFractionDigits: 1) } points -# Пример с вложенными сообщениями -outer-message = Вложение: { inner-message } -inner-message = Это вложенное сообщение. +# Example with nested messages +outer-message = Attachment: { inner-message } +inner-message = This is a nested message. -# Escaping фигурных скобок -escaped = Это не переменная: {{ $notAVar }} +# Escaping curly braces +escaped = This is not a variable: {{ $notAVar }} -# Использование term-ов (терминов, начинающихся с -) --brand-name = Приложение X -about = Информация о { -brand-name } +# Using terms (terms, starting with -) +-brand-name = Application X +about = Information about { -brand-name } -# Атрибут у термина +# Term attribute -icon = .src = /images/icon.svg - .alt = Иконка + .alt = Icon -# Комментарии -# Это обычный комментарий -## Это групповой комментарий -### Это документирующий комментарий +# Comments +# This is a regular comment +## This is a group comment +### This is a documenting comment -# Комбинация всего +# Combination of everything complex-message = - Добро пожаловать, { $name }! - Сегодня { DATETIME($date, weekday: "long") }. - У вас { $unreadCount -> - [0] нет новых писем. - [one] одно новое письмо. - [few] { $unreadCount } новых письма. - *[other] { $unreadCount } новых писем. + Welcome, { $name }! + Today { DATETIME($date, weekday: "long") }. + You have { $unreadCount -> + [0] no new emails. + [one] one new email. + [few] { $unreadCount } new emails. + *[other] { $unreadCount } new emails. } - Спасибо, что используете { -brand-name }! \ No newline at end of file + Thank you for using { -brand-name }! \ No newline at end of file diff --git a/tests/test_generator.py b/tests/test_generator.py index 5ff3af8..e7d8b6e 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -14,8 +14,8 @@ def test_correctly_generated_stub_for_simple_file() -> None: content = Path(output_path).read_text() assert "class TranslatorRunner:" in content - assert "def hello(self) -> str: ..." in content - assert "def button(self) -> str: ..." in content + assert 'def hello(self) -> Literal["""Hello, world!"""]: ...' in content + assert 'def button(self) -> Literal["""Button"""]: ...' in content def test_generator_if_conflict() -> None: @@ -29,8 +29,8 @@ def test_generator_if_conflict() -> None: assert "class TranslatorRunner:" in content assert "class Button" in content - assert "def __call__(self) -> str: ..." in content - assert "def text(self) -> str: ..." in content + assert 'def __call__(self) -> Literal["""Button"""]: ...' in content + assert 'def text(self) -> Literal["""Button text"""]: ...' in content def test_correctly_generated_stub() -> None: @@ -44,26 +44,30 @@ def test_correctly_generated_stub() -> None: assert "class TranslatorRunner:" in content assert "def get(self, path: str, **kwargs) -> str: ..." in content - assert "def hello(self) -> str: ..." in content - assert "def button(self) -> str: ..." in content - assert "def multiline(self) -> str: ..." in content - assert "def welcome(self, name: str) -> str: ..." in content - assert "def button(self) -> str: ..." in content - assert "def multiline(self) -> str: ..." in content - assert "def score(self, points: str) -> str: ..." in content - assert "def escaped(self, notAVar: str) -> str: ..." in content - assert "def about(self) -> str: ..." in content + assert 'def hello(self) -> Literal["""Hello, world!"""]: ...' in content + assert 'def multiline(self) -> Literal["""This is a multiline message."""' in content + assert 'def welcome(self, *, name: str) -> Literal["""Hello, { $name }!"""]: ...' in content assert "class Email:" in content - assert "def status(self, unreadCount: str) -> str: ..." in content + assert ( + 'def status(self, *, unreadCount: str) -> Literal["""You have { $unreadCount } new emails."""]: ...' in content + ) + assert 'def greeting(self) -> Literal["""Hello, world! This is a phrase with another message."""]: ...' in content assert "class Task:" in content - assert "def state(self, state: str) -> str: ..." in content + assert 'def state(self) -> Literal["""Unknown state"""]: ...' in content assert "class Formatted:" in content - assert "def date(self, date: str) -> str: ..." in content + assert 'def date(self, *, date: str) -> Literal["""Today: { $date }"""]: ...' in content + assert 'def score(self, *, points: str) -> Literal["""You scored { $points } points"""]: ...' in content assert "class Outer:" in content + assert 'def message(self) -> Literal["""Attachment: This is a nested message."""]: ...' in content assert "class Inner:" in content + assert 'def message(self) -> Literal["""This is a nested message.This is a nested message."""]: ...' in content + assert 'def escaped(self, *, notAVar: str) -> Literal["""This is not a variable: { $notAVar }"""]: ...' in content + assert 'def about(self) -> Literal["""Information about Application X"""]: ...' in content assert "class Complex:" in content - assert "def multiline(self) -> str: ..." in content - assert "def welcome(self, name: str) -> str: ..." in content - assert "def button(self) -> str: ..." in content - assert "def multiline(self) -> str: ..." in content - assert "def score(self, points: str) -> str: ..." in content + assert ( + '''def message(self, *, date: str, name: str, unreadCount: str) -> Literal["""Welcome, { $name }! +Today { $date }. +You have { $unreadCount } new emails. +Thank you for using Application X!"""]: ...''' + in content + ) From 85c3db10fbf9407dbb2acdbd845c1c0d9438f491 Mon Sep 17 00:00:00 2001 From: sheldy Date: Mon, 30 Jun 2025 02:20:55 +0200 Subject: [PATCH 19/36] Fix tests. --- tests/test_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_generator.py b/tests/test_generator.py index e7d8b6e..f0dbe4b 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -45,7 +45,7 @@ def test_correctly_generated_stub() -> None: assert "class TranslatorRunner:" in content assert "def get(self, path: str, **kwargs) -> str: ..." in content assert 'def hello(self) -> Literal["""Hello, world!"""]: ...' in content - assert 'def multiline(self) -> Literal["""This is a multiline message."""' in content + assert 'def multiline(self) -> Literal["""This is a multiline message.' in content assert 'def welcome(self, *, name: str) -> Literal["""Hello, { $name }!"""]: ...' in content assert "class Email:" in content assert ( From 19f105e92a3c00dfd58d0db12bd7e9d9c7ff49c9 Mon Sep 17 00:00:00 2001 From: sheldy Date: Mon, 30 Jun 2025 17:26:57 +0200 Subject: [PATCH 20/36] Rework generator and update tests. --- fluentogram/stub_generator/generator.py | 110 ++--------------------- fluentogram/stub_generator/renderable.py | 90 +++++++++++++++++++ fluentogram/stub_generator/stubs.py | 35 ++++++++ fluentogram/stub_generator/tree.py | 57 ++++++++++++ tests/assets/test.ftl | 4 +- tests/test_generator.py | 37 ++++---- 6 files changed, 209 insertions(+), 124 deletions(-) create mode 100644 fluentogram/stub_generator/renderable.py create mode 100644 fluentogram/stub_generator/stubs.py create mode 100644 fluentogram/stub_generator/tree.py diff --git a/fluentogram/stub_generator/generator.py b/fluentogram/stub_generator/generator.py index 68cf689..37085ce 100644 --- a/fluentogram/stub_generator/generator.py +++ b/fluentogram/stub_generator/generator.py @@ -3,6 +3,8 @@ from pathlib import Path from fluentogram.stub_generator.parser import Message, get_messages +from fluentogram.stub_generator.stubs import generate_stubs +from fluentogram.stub_generator.tree import Tree class Generator: @@ -27,114 +29,14 @@ def __init__( self.messages: dict[str, Message] = {} - def _generate_class_name(self, name: str) -> str: - """Generate class name from message name.""" - if "-" in name: - parts = name.split("-") - return parts[0].title() - return name.title() - - def _generate_func_signature(self, message: Message, is_method: bool = False, is_call: bool = False) -> str: # noqa: FBT002 - """Generate function signature for a message.""" - - if is_call: - name = "__call__" - elif is_method: - name = message.name.split("-")[1] - else: - name = message.name - - formatted_text = f'"""{message.result_text}"""' - - if not message.placeholders: - return f" def {name}(self) -> Literal[{formatted_text}]: ..." - - param_list = ", ".join(f"{param}: str" for param in sorted(message.placeholders)) - return f" def {name}(self, *, {param_list}) -> Literal[{formatted_text}]: ..." - - def _group_messages( - self, - ) -> tuple[dict[str, dict[str, Message]], dict[str, Message], dict[str, dict[str, Message]]]: - grouped_messages = {} - simple_messages = {} - conflict_classes = {} - - for name, params in self.messages.items(): - if "-" in name: - base_name = name.split("-")[0] - if base_name not in grouped_messages: - grouped_messages[base_name] = {} - grouped_messages[base_name][name] = params - else: - simple_messages[name] = params - - # If there is a conflict, move it to conflict_classes and remove from grouped_messages and simple_messages - for name, messages in list(grouped_messages.items()): - if name in simple_messages: - conflict_classes[name] = {} - conflict_classes[name][name] = simple_messages[name] - for compound_name, compound_params in messages.items(): - conflict_classes[name][compound_name] = compound_params - grouped_messages.pop(name) - simple_messages.pop(name) - - return grouped_messages, simple_messages, conflict_classes - - def generate(self) -> None: # noqa: C901 + def generate(self) -> None: for file in self.files: messages = get_messages(file.read_text()) self.messages.update(messages) - # Generate .pyi content - content = [] - - # Group messages by their base name (before dash) - grouped_messages, simple_messages, conflict_classes = self._group_messages() - - # Generate TranslatorRunner class - content.append( - "from typing import Literal\n\nclass TranslatorRunner:\n def get(self, path: str, **kwargs) -> str: ...", - ) - - # Add simple messages as methods - content.extend(self._generate_func_signature(message) for message in simple_messages.values()) - - # Add grouped messages as attributes - for base_name in grouped_messages: - class_name = self._generate_class_name(base_name) - content.append(f" {base_name}: {class_name}\n") - - # Add conflict classes as attributes - for base_name in conflict_classes: - class_name = self._generate_class_name(base_name) - content.append(f" {base_name}: {class_name}\n") - - # Generate classes for grouped messages - for base_name, messages_dict in grouped_messages.items(): - class_name = self._generate_class_name(base_name) - content.append(f"class {class_name}:") - - for message in messages_dict.values(): - content.append(self._generate_func_signature(message, is_method=True)) - content.append("") - - # Generate classes for conflict messages - for base_name, messages_dict in conflict_classes.items(): - class_name = self._generate_class_name(base_name) - content.append(f"class {class_name}:") - - # Add __call__ for simple key - if base_name in messages_dict: - content.append(self._generate_func_signature(messages_dict[base_name], is_call=True)) - - # Add methods for compound keys - for name, message in messages_dict.items(): - if name != base_name: # Skip simple key, it's already handled as __call__ - content.append(self._generate_func_signature(message, is_method=True)) - content.append("") - - # Write to file - self.output_file.write_text("\n".join(content)) + tree = Tree(self.messages) + content = generate_stubs(tree) + self.output_file.write_text(content) def generate( diff --git a/fluentogram/stub_generator/renderable.py b/fluentogram/stub_generator/renderable.py new file mode 100644 index 0000000..64afcac --- /dev/null +++ b/fluentogram/stub_generator/renderable.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from collections.abc import Iterable + +from jinja2 import Template + + +class RenderAble: + render_pattern: Template + + def __init__(self, **kwargs) -> None: + self.kwargs = kwargs + + def render(self) -> str: + return self.render_pattern.render(**self.kwargs) + "\n" + + +class Import(RenderAble): + render_pattern = Template("from typing import Literal", autoescape=True) + + +class Method(RenderAble): + render_pattern = Template( + ' @staticmethod\n def {{ method_name }}({{ args }}) -> Literal["""{{ translation }}"""]: ...', + autoescape=True, + ) + + def __init__( + self, + method_name: str, + translation: str, + args: Iterable[str] | None = None, + ) -> None: + formatted_args = "*, " + ", ".join(args) if args else "" + super().__init__(translation=translation, args=formatted_args) + self.kwargs["method_name"] = method_name + + +class InternalMethod(Method): + def __init__(self, translation: str, args: Iterable[str] | None = None) -> None: + super().__init__(method_name="__call__", translation=translation, args=args) + + +class Var(RenderAble): + render_pattern = Template( + " {{ var_name }}: {{ var_full_name }}", + autoescape=True, + ) + + def __init__(self, var_name: str, var_full_name: str | None = None) -> None: + super().__init__( + var_name=var_name, + var_full_name=var_full_name or var_name, + ) + + +class Knot(RenderAble): + render_pattern = Template("\nclass {{ class_name }}:\n", autoescape=True) + + def __init__(self, class_name: str) -> None: + super().__init__() + self.class_name = class_name + self.variables: list[Var] = [] + self.methods: list[Method] = [] + + def render(self) -> str: + text = self.render_pattern.render(class_name=self.class_name) + "\n" + for var in self.variables: + text += var.render() + if self.variables: + text += "\n" + for method in self.methods: + text += method.render() + "\n" + return text + + def add_var(self, var: Var) -> None: + self.variables.append(var) + + def add_method(self, method: Method) -> None: + self.methods.append(method) + + +class Runner(Knot): + render_pattern = Template( + "\nclass {{ class_name }}:\n def get(self, path: str, **kwargs) -> str: ...\n ", + autoescape=True, + ) + + def __init__(self, name: str = "TranslatorRunner") -> None: + super().__init__(name) diff --git a/fluentogram/stub_generator/stubs.py b/fluentogram/stub_generator/stubs.py new file mode 100644 index 0000000..3927062 --- /dev/null +++ b/fluentogram/stub_generator/stubs.py @@ -0,0 +1,35 @@ +from fluentogram.stub_generator.renderable import ( + InternalMethod, + Knot, + Method, + Runner, + Var, +) +from fluentogram.stub_generator.tree import Tree + + +def generate_stubs(tree: Tree) -> str: + content = "from typing import Literal\n" + for node in tree.elements.values(): + if node.is_leaf: + continue + knot = Knot(node.path) if node.path else Runner() + if node.children: + if node.value: + knot.add_method( + InternalMethod(node.value, args=node.placeholders), + ) + for name, sub_node in node.children.items(): + if sub_node.is_leaf: + if sub_node.value: + knot.add_method( + Method( + name, + sub_node.value, + args=sub_node.placeholders, + ), + ) + else: + knot.add_var(Var(name, sub_node.path)) + content += knot.render() + return content diff --git a/fluentogram/stub_generator/tree.py b/fluentogram/stub_generator/tree.py new file mode 100644 index 0000000..1902fec --- /dev/null +++ b/fluentogram/stub_generator/tree.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from dataclasses import dataclass, field + +from fluentogram.stub_generator.parser import Message + + +@dataclass +class TreeNode: + path: str + children: dict[str, TreeNode] + name: str + value: str | None = None + placeholders: list[str] = field(default_factory=list) + + @property + def is_leaf(self) -> bool: + return not self.children + + +class Tree: + def __init__( + self, + ftl_syntax: dict[str, Message], + separator: str = "-", + safe_separator: str = "", + ) -> None: + self.safe_separator = safe_separator + self.ftl_syntax = ftl_syntax + self.separator = separator + self.elements: dict[tuple[str, ...], TreeNode] = {} + + for path, translation in ftl_syntax.items(): + *point_path, name = path.split(self.separator) + point_path.insert(0, "") + self._build(tuple(point_path), name, translation) + + def path_to_str(self, path: tuple[str, ...]) -> str: + clean_path = (s[0].capitalize() + s[1:] for s in filter(lambda x: x, path)) + return self.safe_separator.join(clean_path) + + def _build(self, path: tuple[str, ...], name: str, value: Message | None = None) -> None: + own_class_def = TreeNode( + path=self.path_to_str((*path, name)), + name=name, + value=value.result_text if value else "", + children={}, + placeholders=value.placeholders if value else [], + ) + + if path: + if path not in self.elements: + self._build(path[:-1], path[-1]) + + self.elements[path].children[name] = own_class_def + + self.elements.setdefault((*path, name), own_class_def) diff --git a/tests/assets/test.ftl b/tests/assets/test.ftl index 14a4061..36f9c22 100644 --- a/tests/assets/test.ftl +++ b/tests/assets/test.ftl @@ -72,4 +72,6 @@ complex-message = [few] { $unreadCount } new emails. *[other] { $unreadCount } new emails. } - Thank you for using { -brand-name }! \ No newline at end of file + Thank you for using { -brand-name }! + +must-be-shielded = "Must be shielded" \ No newline at end of file diff --git a/tests/test_generator.py b/tests/test_generator.py index f0dbe4b..afd9ee3 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -14,8 +14,8 @@ def test_correctly_generated_stub_for_simple_file() -> None: content = Path(output_path).read_text() assert "class TranslatorRunner:" in content - assert 'def hello(self) -> Literal["""Hello, world!"""]: ...' in content - assert 'def button(self) -> Literal["""Button"""]: ...' in content + assert 'def hello() -> Literal["""Hello, world!"""]: ...' in content + assert 'def button() -> Literal["""Button"""]: ...' in content def test_generator_if_conflict() -> None: @@ -29,8 +29,8 @@ def test_generator_if_conflict() -> None: assert "class TranslatorRunner:" in content assert "class Button" in content - assert 'def __call__(self) -> Literal["""Button"""]: ...' in content - assert 'def text(self) -> Literal["""Button text"""]: ...' in content + assert 'def __call__() -> Literal["""Button"""]: ...' in content + assert 'def text() -> Literal["""Button text"""]: ...' in content def test_correctly_generated_stub() -> None: @@ -44,30 +44,29 @@ def test_correctly_generated_stub() -> None: assert "class TranslatorRunner:" in content assert "def get(self, path: str, **kwargs) -> str: ..." in content - assert 'def hello(self) -> Literal["""Hello, world!"""]: ...' in content - assert 'def multiline(self) -> Literal["""This is a multiline message.' in content - assert 'def welcome(self, *, name: str) -> Literal["""Hello, { $name }!"""]: ...' in content + assert 'def hello() -> Literal["""Hello, world!"""]: ...' in content + assert 'def multiline() -> Literal["""This is a multiline message.' in content + assert 'def welcome(*, name) -> Literal["""Hello, { $name }!"""]: ...' in content assert "class Email:" in content - assert ( - 'def status(self, *, unreadCount: str) -> Literal["""You have { $unreadCount } new emails."""]: ...' in content - ) - assert 'def greeting(self) -> Literal["""Hello, world! This is a phrase with another message."""]: ...' in content + assert 'def status(*, unreadCount) -> Literal["""You have { $unreadCount } new emails."""]: ...' in content + assert 'def greeting() -> Literal["""Hello, world! This is a phrase with another message."""]: ...' in content assert "class Task:" in content - assert 'def state(self) -> Literal["""Unknown state"""]: ...' in content + assert 'def state() -> Literal["""Unknown state"""]: ...' in content assert "class Formatted:" in content - assert 'def date(self, *, date: str) -> Literal["""Today: { $date }"""]: ...' in content - assert 'def score(self, *, points: str) -> Literal["""You scored { $points } points"""]: ...' in content + assert 'def date(*, date) -> Literal["""Today: { $date }"""]: ...' in content + assert 'def score(*, points) -> Literal["""You scored { $points } points"""]: ...' in content assert "class Outer:" in content - assert 'def message(self) -> Literal["""Attachment: This is a nested message."""]: ...' in content + assert 'def message() -> Literal["""Attachment: This is a nested message."""]: ...' in content assert "class Inner:" in content - assert 'def message(self) -> Literal["""This is a nested message.This is a nested message."""]: ...' in content - assert 'def escaped(self, *, notAVar: str) -> Literal["""This is not a variable: { $notAVar }"""]: ...' in content - assert 'def about(self) -> Literal["""Information about Application X"""]: ...' in content + assert 'def message() -> Literal["""This is a nested message.This is a nested message."""]: ...' in content + assert 'def escaped(*, notAVar) -> Literal["""This is not a variable: { $notAVar }"""]: ...' in content + assert 'def about() -> Literal["""Information about Application X"""]: ...' in content assert "class Complex:" in content assert ( - '''def message(self, *, date: str, name: str, unreadCount: str) -> Literal["""Welcome, { $name }! + '''def message(*, name, date, unreadCount) -> Literal["""Welcome, { $name }! Today { $date }. You have { $unreadCount } new emails. Thank you for using Application X!"""]: ...''' in content ) + assert 'def shielded() -> Literal[""""Must be shielded""""]: ...' in content From 2ae54b8a3303940deb54aaaf90a1dfcd6996da7b Mon Sep 17 00:00:00 2001 From: sheldy Date: Mon, 30 Jun 2025 17:37:06 +0200 Subject: [PATCH 21/36] Update pyptoject.toml --- pyproject.toml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1de85f8..90df0c3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,7 @@ +[build-system] +requires = ["setuptools>=42"] +build-backend = "setuptools.build_meta" + [project] name = "fluentogram" version = "1.2.0" @@ -6,17 +10,21 @@ authors = [{ name = "Aleks" }] requires-python = ">=3.9" license = "MIT" dependencies = ["fluent-compiler~=1.1"] + [project.optional-dependencies] -cli = ["watchdog>=2.3.0,<3"] +cli = ["jinja2>=3.1.0,<4"] +aiogram = ["aiogram~=3.20.0"] +nats = ["nats-py~=2.10.0"] + dev = [ + "fluentogram[cli,aiogram,nats]", "ruff~=0.11.2", "pytest~=8.4.1", "pre-commit~=4.2.0", "detect-secrets~=1.5.0", "pytest-asyncio~=1.0.0", ] -aiogram = ["aiogram~=3.20.0"] -nats = ["nats-py~=2.10.0"] + [project.urls] Repository = "https://github.com/Arustinal/fluentogram" @@ -24,10 +32,6 @@ Repository = "https://github.com/Arustinal/fluentogram" [project.scripts] fluentogram = "fluentogram.cli.main:cli" -[build-system] -requires = ["setuptools>=42"] -build-backend = "setuptools.build_meta" - [tool.ruff] exclude = [ ".bzr", From 17507af24be33e6894d0f399dc701cd76f8c56b8 Mon Sep 17 00:00:00 2001 From: sheldy Date: Mon, 30 Jun 2025 21:33:18 +0200 Subject: [PATCH 22/36] Add CLI info to README. --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5982b17..725f812 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,7 @@ Fluentogram is easy way to use i18n (Fluent) mechanism in any python app. - Fallback support: Automatic fallback to root locale when translations are missing. - Precompiled fluent messages using fluent_compiler makes formatting messages faster. - Dot access to messages: `translator.hello(name='Alex')` +- Stub generator ## Installation @@ -59,6 +60,20 @@ print(translator.welcome(username="Alice")) # "Welcome, Alice!" print(translator.items.count(count=5)) # "You have 5 items" ``` +### Stub generator with CLI + +#### Install with CLI dependencies + +```sh +pip install fluentogram[cli] +``` + +#### Run generator + +```sh +fluentogram -f tests/assets/test.ftl -o test.pyi +``` + ### Dynamic Storage with NATS KV Fluentogram supports real-time translation updates using NATS KV storage: @@ -138,4 +153,4 @@ except RootTranslatorNotFoundError as e: print(f"Root locale translator missing: {e.root_locale}") except FormatError as e: print(f"Formatting error for key {e.key}: {e.original_error}") -``` \ No newline at end of file +``` From 94ea8308a32c1c2ba483bbc3d3cd5c36e557001b Mon Sep 17 00:00:00 2001 From: sheldy Date: Tue, 1 Jul 2025 15:15:06 +0200 Subject: [PATCH 23/36] Rework generator (again :D) --- fluentogram/stub_generator/generator.py | 4 +- fluentogram/stub_generator/parser.py | 12 +++- fluentogram/stub_generator/renderable.py | 62 +++++++++++++------ fluentogram/stub_generator/stubs.py | 78 ++++++++++++++++-------- fluentogram/stub_generator/tree.py | 77 ++++++++++++----------- 5 files changed, 149 insertions(+), 84 deletions(-) diff --git a/fluentogram/stub_generator/generator.py b/fluentogram/stub_generator/generator.py index 37085ce..e28b076 100644 --- a/fluentogram/stub_generator/generator.py +++ b/fluentogram/stub_generator/generator.py @@ -4,7 +4,7 @@ from fluentogram.stub_generator.parser import Message, get_messages from fluentogram.stub_generator.stubs import generate_stubs -from fluentogram.stub_generator.tree import Tree +from fluentogram.stub_generator.tree import build_tree class Generator: @@ -34,7 +34,7 @@ def generate(self) -> None: messages = get_messages(file.read_text()) self.messages.update(messages) - tree = Tree(self.messages) + tree = build_tree(self.messages) content = generate_stubs(tree) self.output_file.write_text(content) diff --git a/fluentogram/stub_generator/parser.py b/fluentogram/stub_generator/parser.py index 0cc4905..cf1af56 100644 --- a/fluentogram/stub_generator/parser.py +++ b/fluentogram/stub_generator/parser.py @@ -144,7 +144,7 @@ def _process_message_elements(self, message_obj: Message) -> None: else: logger.warning("Unknown element type in message %s: %s", message_obj.name, type(element)) - def parse(self, resource: ast.Resource) -> None: + def parse(self, resource: ast.Resource) -> dict[str, Message]: # First pass: collect all terms and message structures for entry in resource.body: if isinstance(entry, ast.Term): @@ -156,8 +156,14 @@ def parse(self, resource: ast.Resource) -> None: for message_obj in self.messages.values(): self._process_message_elements(message_obj) + return self._get_processed_messages() + + def _get_processed_messages(self) -> dict[str, Message]: + return { + message.name: message for message in self.messages.values() if message.result_text or message.placeholders + } + def get_messages(text: str) -> dict[str, Message]: parser = Parser() - parser.parse(FluentParser(with_spans=False).parse(text)) - return parser.messages + return parser.parse(FluentParser(with_spans=False).parse(text)) diff --git a/fluentogram/stub_generator/renderable.py b/fluentogram/stub_generator/renderable.py index 64afcac..3e98757 100644 --- a/fluentogram/stub_generator/renderable.py +++ b/fluentogram/stub_generator/renderable.py @@ -2,7 +2,22 @@ from collections.abc import Iterable -from jinja2 import Template +from jinja2 import Environment, Template + + +def create_jinja_env() -> Environment: + """Create Jinja2 environment with custom filters""" + env = Environment(autoescape=True) + + def title(value: str) -> str: + """Convert string to title case""" + return value.title() + + env.filters["title"] = title + return env + + +jinja_env = create_jinja_env() class RenderAble: @@ -16,13 +31,12 @@ def render(self) -> str: class Import(RenderAble): - render_pattern = Template("from typing import Literal", autoescape=True) + render_pattern = jinja_env.from_string("from typing import Literal") class Method(RenderAble): - render_pattern = Template( + render_pattern = jinja_env.from_string( ' @staticmethod\n def {{ method_name }}({{ args }}) -> Literal["""{{ translation }}"""]: ...', - autoescape=True, ) def __init__( @@ -41,10 +55,9 @@ def __init__(self, translation: str, args: Iterable[str] | None = None) -> None: super().__init__(method_name="__call__", translation=translation, args=args) -class Var(RenderAble): - render_pattern = Template( - " {{ var_name }}: {{ var_full_name }}", - autoescape=True, +class ClassRef(RenderAble): + render_pattern = jinja_env.from_string( + " {{ var_name }}: {{ var_full_name | title }}", ) def __init__(self, var_name: str, var_full_name: str | None = None) -> None: @@ -54,37 +67,46 @@ def __init__(self, var_name: str, var_full_name: str | None = None) -> None: ) -class Knot(RenderAble): - render_pattern = Template("\nclass {{ class_name }}:\n", autoescape=True) +class Class(RenderAble): + render_pattern = jinja_env.from_string("\nclass {{ class_name | title }}:\n") def __init__(self, class_name: str) -> None: super().__init__() self.class_name = class_name - self.variables: list[Var] = [] + self.class_refs: list[ClassRef] = [] self.methods: list[Method] = [] def render(self) -> str: text = self.render_pattern.render(class_name=self.class_name) + "\n" - for var in self.variables: - text += var.render() - if self.variables: + for class_ref in self.class_refs: + text += class_ref.render() + if self.class_refs: text += "\n" for method in self.methods: text += method.render() + "\n" return text - def add_var(self, var: Var) -> None: - self.variables.append(var) + def add_class_ref(self, class_ref: ClassRef) -> None: + self.class_refs.append(class_ref) def add_method(self, method: Method) -> None: self.methods.append(method) -class Runner(Knot): - render_pattern = Template( +class Runner(Class): + render_pattern = jinja_env.from_string( "\nclass {{ class_name }}:\n def get(self, path: str, **kwargs) -> str: ...\n ", - autoescape=True, ) - def __init__(self, name: str = "TranslatorRunner") -> None: + def __init__(self, knots: list[Class], name: str = "TranslatorRunner") -> None: super().__init__(name) + self.knots = knots + + def render(self) -> str: + text = super().render() + for knot in self.knots: + text += knot.render() + return text + + def add_knot(self, knot: Class) -> None: + self.knots.append(knot) diff --git a/fluentogram/stub_generator/stubs.py b/fluentogram/stub_generator/stubs.py index 3927062..0b19b45 100644 --- a/fluentogram/stub_generator/stubs.py +++ b/fluentogram/stub_generator/stubs.py @@ -1,35 +1,65 @@ from fluentogram.stub_generator.renderable import ( + Class, + ClassRef, InternalMethod, - Knot, Method, Runner, - Var, ) -from fluentogram.stub_generator.tree import Tree +from fluentogram.stub_generator.tree import TreeNode -def generate_stubs(tree: Tree) -> str: +def _process_node(node: TreeNode, runner: Runner) -> Class: + """Recursively processes tree node and creates corresponding class""" + knot = Class(node.name) + + # If node has value (translation), add method __call__ + if node.value is not None: + knot.add_method( + InternalMethod(node.value, args=node.placeholders), + ) + + # Process child nodes + for name, sub_node in node.children.items(): + if sub_node.is_leaf: + # If child node is leaf with value, add method + if sub_node.value is not None: + knot.add_method( + Method( + name, + sub_node.value, + args=sub_node.placeholders, + ), + ) + else: + # If child node is not leaf, create class reference + sub_class = _process_node(sub_node, runner) + runner.add_knot(sub_class) + knot.add_class_ref(ClassRef(name, sub_class.class_name)) + + return knot + + +def generate_stubs(tree: TreeNode) -> str: content = "from typing import Literal\n" - for node in tree.elements.values(): + runner = Runner(knots=[]) + + # Process root nodes + for node in tree.children.values(): if node.is_leaf: - continue - knot = Knot(node.path) if node.path else Runner() - if node.children: - if node.value: - knot.add_method( - InternalMethod(node.value, args=node.placeholders), + # If root node is leaf, add it as method to Runner + if node.value is not None: + runner.add_method( + Method( + node.name, + node.value, + args=node.placeholders, + ), ) - for name, sub_node in node.children.items(): - if sub_node.is_leaf: - if sub_node.value: - knot.add_method( - Method( - name, - sub_node.value, - args=sub_node.placeholders, - ), - ) - else: - knot.add_var(Var(name, sub_node.path)) - content += knot.render() + else: + # If root node is not leaf, create class + knot = _process_node(node, runner) + runner.add_class_ref(ClassRef(node.name, knot.class_name)) + runner.add_knot(knot) + + content += runner.render() return content diff --git a/fluentogram/stub_generator/tree.py b/fluentogram/stub_generator/tree.py index 1902fec..3237671 100644 --- a/fluentogram/stub_generator/tree.py +++ b/fluentogram/stub_generator/tree.py @@ -7,51 +7,58 @@ @dataclass class TreeNode: - path: str - children: dict[str, TreeNode] + """Translation tree node""" + name: str value: str | None = None placeholders: list[str] = field(default_factory=list) + children: dict[str, TreeNode] = field(default_factory=dict) @property def is_leaf(self) -> bool: + """Is node a leaf (has no children)""" return not self.children + @property + def has_value(self) -> bool: + """Does node have value (translation)""" + return self.value is not None + + +def _build_node(key: str, message: Message, root: TreeNode, separator: str = "-") -> TreeNode: + parts = key.split(separator) + + # Start with root + current_node = root + + # Process all parts of the path, except the last one + for part in parts[:-1]: + # If child node does not exist, create it + if part not in current_node.children: + current_node.children[part] = TreeNode(name=part) + current_node = current_node.children[part] + + # Last part is the node name with value + final_name = parts[-1] -class Tree: - def __init__( - self, - ftl_syntax: dict[str, Message], - separator: str = "-", - safe_separator: str = "", - ) -> None: - self.safe_separator = safe_separator - self.ftl_syntax = ftl_syntax - self.separator = separator - self.elements: dict[tuple[str, ...], TreeNode] = {} - - for path, translation in ftl_syntax.items(): - *point_path, name = path.split(self.separator) - point_path.insert(0, "") - self._build(tuple(point_path), name, translation) - - def path_to_str(self, path: tuple[str, ...]) -> str: - clean_path = (s[0].capitalize() + s[1:] for s in filter(lambda x: x, path)) - return self.safe_separator.join(clean_path) - - def _build(self, path: tuple[str, ...], name: str, value: Message | None = None) -> None: - own_class_def = TreeNode( - path=self.path_to_str((*path, name)), - name=name, - value=value.result_text if value else "", - children={}, - placeholders=value.placeholders if value else [], + # Create or update final node + if final_name not in current_node.children: + current_node.children[final_name] = TreeNode( + name=final_name, + value=message.result_text, + placeholders=message.placeholders, ) + else: + # If node already exists, update its value + existing_node = current_node.children[final_name] + existing_node.value = message.result_text + existing_node.placeholders = message.placeholders - if path: - if path not in self.elements: - self._build(path[:-1], path[-1]) + return current_node - self.elements[path].children[name] = own_class_def - self.elements.setdefault((*path, name), own_class_def) +def build_tree(messages: dict[str, Message], separator: str = "-") -> TreeNode: + root = TreeNode(name="root") + for key, message in messages.items(): + _build_node(key, message, root, separator) + return root From af70dc6213be4ee37b034acda9d154574bd5c2ca Mon Sep 17 00:00:00 2001 From: sheldy Date: Tue, 1 Jul 2025 15:29:43 +0200 Subject: [PATCH 24/36] placeholders as set --- fluentogram/stub_generator/parser.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fluentogram/stub_generator/parser.py b/fluentogram/stub_generator/parser.py index cf1af56..60c2438 100644 --- a/fluentogram/stub_generator/parser.py +++ b/fluentogram/stub_generator/parser.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from dataclasses import dataclass +from dataclasses import dataclass, field from fluent.syntax import FluentParser, ast @@ -12,8 +12,8 @@ class Message: name: str result_text: str - placeholders: list[str] raw_elements: list[ast.TextElement | ast.Placeable] # Store raw elements for later processing + placeholders: set[str] = field(default_factory=set) class Parser: @@ -31,7 +31,7 @@ def process_term(self, term: ast.Term) -> None: logger.warning("Unknown element type in term %s: %s", term.id.name, type(element)) def _parse_variable_reference(self, message_obj: Message, element: ast.VariableReference) -> None: - message_obj.placeholders.append(element.id.name) + message_obj.placeholders.add(element.id.name) message_obj.result_text += f"{{ ${element.id.name} }}" def _parse_term_reference(self, message_obj: Message, element: ast.TermReference) -> None: @@ -117,7 +117,6 @@ def process_message(self, message: ast.Message) -> None: message_obj = Message( name=message.id.name, result_text="", - placeholders=[], raw_elements=[], ) From ff5d7849e6dc9f798e555154d103152ec61af540 Mon Sep 17 00:00:00 2001 From: sheldy Date: Tue, 1 Jul 2025 16:52:21 +0200 Subject: [PATCH 25/36] No, better to check in value already in list. --- fluentogram/stub_generator/parser.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/fluentogram/stub_generator/parser.py b/fluentogram/stub_generator/parser.py index 60c2438..8bbfa7a 100644 --- a/fluentogram/stub_generator/parser.py +++ b/fluentogram/stub_generator/parser.py @@ -13,7 +13,7 @@ class Message: name: str result_text: str raw_elements: list[ast.TextElement | ast.Placeable] # Store raw elements for later processing - placeholders: set[str] = field(default_factory=set) + placeholders: list[str] = field(default_factory=list) class Parser: @@ -31,7 +31,8 @@ def process_term(self, term: ast.Term) -> None: logger.warning("Unknown element type in term %s: %s", term.id.name, type(element)) def _parse_variable_reference(self, message_obj: Message, element: ast.VariableReference) -> None: - message_obj.placeholders.add(element.id.name) + if element.id.name not in message_obj.placeholders: + message_obj.placeholders.append(element.id.name) message_obj.result_text += f"{{ ${element.id.name} }}" def _parse_term_reference(self, message_obj: Message, element: ast.TermReference) -> None: @@ -129,6 +130,9 @@ def process_message(self, message: ast.Message) -> None: message_obj.raw_elements = message.value.elements self.messages[message.id.name] = message_obj + def _sort_placeholders(self, message_obj: Message) -> None: + message_obj.placeholders = sorted(message_obj.placeholders) + def _process_message_elements(self, message_obj: Message) -> None: """Process the raw elements of a message to generate result_text and placeholders.""" # Skip if already processed @@ -143,6 +147,8 @@ def _process_message_elements(self, message_obj: Message) -> None: else: logger.warning("Unknown element type in message %s: %s", message_obj.name, type(element)) + self._sort_placeholders(message_obj) + def parse(self, resource: ast.Resource) -> dict[str, Message]: # First pass: collect all terms and message structures for entry in resource.body: From 707382331521595276cc865cbce8fb4e6b9f7a99 Mon Sep 17 00:00:00 2001 From: sheldy Date: Tue, 1 Jul 2025 16:53:57 +0200 Subject: [PATCH 26/36] Fix tests. --- tests/test_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_generator.py b/tests/test_generator.py index afd9ee3..20d2e1c 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -63,7 +63,7 @@ def test_correctly_generated_stub() -> None: assert 'def about() -> Literal["""Information about Application X"""]: ...' in content assert "class Complex:" in content assert ( - '''def message(*, name, date, unreadCount) -> Literal["""Welcome, { $name }! + '''def message(*, date, name, unreadCount) -> Literal["""Welcome, { $name }! Today { $date }. You have { $unreadCount } new emails. Thank you for using Application X!"""]: ...''' From a91880a364f99f325cdc062541b265767e4042e1 Mon Sep 17 00:00:00 2001 From: sheldy Date: Tue, 1 Jul 2025 19:29:24 +0200 Subject: [PATCH 27/36] Handle conflict when two translates has same prefix. --- fluentogram/stub_generator/stubs.py | 13 ++++++---- .../{renderable.py => templates.py} | 26 ++++++++++++------- tests/assets/conflict_in_prefix.ftl | 3 +++ tests/test_generator.py | 16 ++++++++++++ 4 files changed, 43 insertions(+), 15 deletions(-) rename fluentogram/stub_generator/{renderable.py => templates.py} (79%) create mode 100644 tests/assets/conflict_in_prefix.ftl diff --git a/fluentogram/stub_generator/stubs.py b/fluentogram/stub_generator/stubs.py index 0b19b45..0b25114 100644 --- a/fluentogram/stub_generator/stubs.py +++ b/fluentogram/stub_generator/stubs.py @@ -1,4 +1,4 @@ -from fluentogram.stub_generator.renderable import ( +from fluentogram.stub_generator.templates import ( Class, ClassRef, InternalMethod, @@ -8,9 +8,12 @@ from fluentogram.stub_generator.tree import TreeNode -def _process_node(node: TreeNode, runner: Runner) -> Class: +def _process_node(node: TreeNode, runner: Runner, parent_path: str = "") -> Class: """Recursively processes tree node and creates corresponding class""" - knot = Class(node.name) + # Create unique class name by combining parent path with node name + current_path = f"{parent_path}-{node.name}" if parent_path else node.name + + knot = Class(current_path) # If node has value (translation), add method __call__ if node.value is not None: @@ -32,7 +35,7 @@ def _process_node(node: TreeNode, runner: Runner) -> Class: ) else: # If child node is not leaf, create class reference - sub_class = _process_node(sub_node, runner) + sub_class = _process_node(sub_node, runner, current_path) runner.add_knot(sub_class) knot.add_class_ref(ClassRef(name, sub_class.class_name)) @@ -40,7 +43,7 @@ def _process_node(node: TreeNode, runner: Runner) -> Class: def generate_stubs(tree: TreeNode) -> str: - content = "from typing import Literal\n" + content = "" runner = Runner(knots=[]) # Process root nodes diff --git a/fluentogram/stub_generator/renderable.py b/fluentogram/stub_generator/templates.py similarity index 79% rename from fluentogram/stub_generator/renderable.py rename to fluentogram/stub_generator/templates.py index 3e98757..42d69e2 100644 --- a/fluentogram/stub_generator/renderable.py +++ b/fluentogram/stub_generator/templates.py @@ -9,11 +9,11 @@ def create_jinja_env() -> Environment: """Create Jinja2 environment with custom filters""" env = Environment(autoescape=True) - def title(value: str) -> str: - """Convert string to title case""" - return value.title() + def camelcase(value: str) -> str: + """Convert dash/underscore separated string to CamelCase""" + return "".join(word.capitalize() for word in value.replace("_", "-").split("-")) - env.filters["title"] = title + env.filters["camelcase"] = camelcase return env @@ -27,7 +27,7 @@ def __init__(self, **kwargs) -> None: self.kwargs = kwargs def render(self) -> str: - return self.render_pattern.render(**self.kwargs) + "\n" + return self.render_pattern.render(**self.kwargs) class Import(RenderAble): @@ -49,6 +49,9 @@ def __init__( super().__init__(translation=translation, args=formatted_args) self.kwargs["method_name"] = method_name + def render(self) -> str: + return super().render() + "\n" + class InternalMethod(Method): def __init__(self, translation: str, args: Iterable[str] | None = None) -> None: @@ -57,7 +60,7 @@ def __init__(self, translation: str, args: Iterable[str] | None = None) -> None: class ClassRef(RenderAble): render_pattern = jinja_env.from_string( - " {{ var_name }}: {{ var_full_name | title }}", + " {{ var_name }}: {{ var_full_name | camelcase }}", ) def __init__(self, var_name: str, var_full_name: str | None = None) -> None: @@ -66,9 +69,12 @@ def __init__(self, var_name: str, var_full_name: str | None = None) -> None: var_full_name=var_full_name or var_name, ) + def render(self) -> str: + return super().render() + "\n" + class Class(RenderAble): - render_pattern = jinja_env.from_string("\nclass {{ class_name | title }}:\n") + render_pattern = jinja_env.from_string("\nclass {{ class_name | camelcase }}:") def __init__(self, class_name: str) -> None: super().__init__() @@ -83,8 +89,8 @@ def render(self) -> str: if self.class_refs: text += "\n" for method in self.methods: - text += method.render() + "\n" - return text + text += method.render() + return text.rstrip() + "\n" def add_class_ref(self, class_ref: ClassRef) -> None: self.class_refs.append(class_ref) @@ -95,7 +101,7 @@ def add_method(self, method: Method) -> None: class Runner(Class): render_pattern = jinja_env.from_string( - "\nclass {{ class_name }}:\n def get(self, path: str, **kwargs) -> str: ...\n ", + "from typing import Literal\n\nclass {{ class_name }}:\n def get(self, path: str, **kwargs) -> str: ...", ) def __init__(self, knots: list[Class], name: str = "TranslatorRunner") -> None: diff --git a/tests/assets/conflict_in_prefix.ftl b/tests/assets/conflict_in_prefix.ftl new file mode 100644 index 0000000..ba56868 --- /dev/null +++ b/tests/assets/conflict_in_prefix.ftl @@ -0,0 +1,3 @@ +first-unknown-error = first-unknown-error + +another-unknown-error = another-unknown-error diff --git a/tests/test_generator.py b/tests/test_generator.py index 20d2e1c..0bdc2bd 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -70,3 +70,19 @@ def test_correctly_generated_stub() -> None: in content ) assert 'def shielded() -> Literal[""""Must be shielded""""]: ...' in content + + +def test_generator_if_conflict_in_prefix() -> None: + with tempfile.NamedTemporaryFile(suffix=".pyi", delete=False) as tmp_file: + output_path = tmp_file.name + + generate(output_path, file_path="tests/assets/conflict_in_prefix.ftl") + + assert Path(output_path).exists() + content = Path(output_path).read_text() + + assert "class TranslatorRunner:" in content + assert "class FirstUnknown" in content + assert "class AnotherUnknown" in content + assert 'def error() -> Literal["""first-unknown-error"""]: ...' in content + assert 'def error() -> Literal["""another-unknown-error"""]: ...' in content From 40905cec6efe55b7b4801f93ba7fdf762e09839b Mon Sep 17 00:00:00 2001 From: sheldy Date: Tue, 1 Jul 2025 21:23:20 +0200 Subject: [PATCH 28/36] Add types for arguments. --- fluentogram/stub_generator/stubs.py | 10 +++++----- fluentogram/stub_generator/templates.py | 22 +++++++++++++++------- tests/test_generator.py | 14 +++++++------- 3 files changed, 27 insertions(+), 19 deletions(-) diff --git a/fluentogram/stub_generator/stubs.py b/fluentogram/stub_generator/stubs.py index 0b25114..9b1a902 100644 --- a/fluentogram/stub_generator/stubs.py +++ b/fluentogram/stub_generator/stubs.py @@ -18,7 +18,7 @@ def _process_node(node: TreeNode, runner: Runner, parent_path: str = "") -> Clas # If node has value (translation), add method __call__ if node.value is not None: knot.add_method( - InternalMethod(node.value, args=node.placeholders), + InternalMethod(result_text=node.value, args=node.placeholders), ) # Process child nodes @@ -28,8 +28,8 @@ def _process_node(node: TreeNode, runner: Runner, parent_path: str = "") -> Clas if sub_node.value is not None: knot.add_method( Method( - name, - sub_node.value, + method_name=name, + result_text=sub_node.value, args=sub_node.placeholders, ), ) @@ -53,8 +53,8 @@ def generate_stubs(tree: TreeNode) -> str: if node.value is not None: runner.add_method( Method( - node.name, - node.value, + method_name=node.name, + result_text=node.value, args=node.placeholders, ), ) diff --git a/fluentogram/stub_generator/templates.py b/fluentogram/stub_generator/templates.py index 42d69e2..df4f0db 100644 --- a/fluentogram/stub_generator/templates.py +++ b/fluentogram/stub_generator/templates.py @@ -36,17 +36,17 @@ class Import(RenderAble): class Method(RenderAble): render_pattern = jinja_env.from_string( - ' @staticmethod\n def {{ method_name }}({{ args }}) -> Literal["""{{ translation }}"""]: ...', + ' @staticmethod\n def {{ method_name }}({{ args }}) -> Literal["""{{ result_text }}"""]: ...', ) def __init__( self, method_name: str, - translation: str, + result_text: str, args: Iterable[str] | None = None, ) -> None: - formatted_args = "*, " + ", ".join(args) if args else "" - super().__init__(translation=translation, args=formatted_args) + formatted_args = "*, " + ", ".join(f"{arg}: PossibleValue" for arg in args) if args else "" + super().__init__(result_text=result_text, args=formatted_args) self.kwargs["method_name"] = method_name def render(self) -> str: @@ -54,8 +54,8 @@ def render(self) -> str: class InternalMethod(Method): - def __init__(self, translation: str, args: Iterable[str] | None = None) -> None: - super().__init__(method_name="__call__", translation=translation, args=args) + def __init__(self, result_text: str, args: Iterable[str] | None = None) -> None: + super().__init__(method_name="__call__", result_text=result_text, args=args) class ClassRef(RenderAble): @@ -101,7 +101,15 @@ def add_method(self, method: Method) -> None: class Runner(Class): render_pattern = jinja_env.from_string( - "from typing import Literal\n\nclass {{ class_name }}:\n def get(self, path: str, **kwargs) -> str: ...", + """from decimal import Decimal +from typing import Literal + +from typing_extensions import TypeAlias + +PossibleValue: TypeAlias = str | int | float | Decimal | bool + +class {{ class_name }}: + def get(self, path: str, **kwargs: PossibleValue) -> str: ...""", ) def __init__(self, knots: list[Class], name: str = "TranslatorRunner") -> None: diff --git a/tests/test_generator.py b/tests/test_generator.py index 0bdc2bd..a14e7d3 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -43,27 +43,27 @@ def test_correctly_generated_stub() -> None: content = Path(output_path).read_text() assert "class TranslatorRunner:" in content - assert "def get(self, path: str, **kwargs) -> str: ..." in content + assert "def get(self, path: str, **kwargs: PossibleValue) -> str: ..." in content assert 'def hello() -> Literal["""Hello, world!"""]: ...' in content assert 'def multiline() -> Literal["""This is a multiline message.' in content - assert 'def welcome(*, name) -> Literal["""Hello, { $name }!"""]: ...' in content + assert 'def welcome(*, name: PossibleValue) -> Literal["""Hello, { $name }!"""]: ...' in content assert "class Email:" in content - assert 'def status(*, unreadCount) -> Literal["""You have { $unreadCount } new emails."""]: ...' in content + assert 'def status(*, unreadCount: PossibleValue) -> Literal["""You have { $unreadCount } new emails."""]: ...' in content assert 'def greeting() -> Literal["""Hello, world! This is a phrase with another message."""]: ...' in content assert "class Task:" in content assert 'def state() -> Literal["""Unknown state"""]: ...' in content assert "class Formatted:" in content - assert 'def date(*, date) -> Literal["""Today: { $date }"""]: ...' in content - assert 'def score(*, points) -> Literal["""You scored { $points } points"""]: ...' in content + assert 'def date(*, date: PossibleValue) -> Literal["""Today: { $date }"""]: ...' in content + assert 'def score(*, points: PossibleValue) -> Literal["""You scored { $points } points"""]: ...' in content assert "class Outer:" in content assert 'def message() -> Literal["""Attachment: This is a nested message."""]: ...' in content assert "class Inner:" in content assert 'def message() -> Literal["""This is a nested message.This is a nested message."""]: ...' in content - assert 'def escaped(*, notAVar) -> Literal["""This is not a variable: { $notAVar }"""]: ...' in content + assert 'def escaped(*, notAVar: PossibleValue) -> Literal["""This is not a variable: { $notAVar }"""]: ...' in content assert 'def about() -> Literal["""Information about Application X"""]: ...' in content assert "class Complex:" in content assert ( - '''def message(*, date, name, unreadCount) -> Literal["""Welcome, { $name }! + '''def message(*, date: PossibleValue, name: PossibleValue, unreadCount: PossibleValue) -> Literal["""Welcome, { $name }! Today { $date }. You have { $unreadCount } new emails. Thank you for using Application X!"""]: ...''' From 94e2c026d67c6d50cbac8ba4e1c18b8c78f3986f Mon Sep 17 00:00:00 2001 From: sheldy Date: Tue, 1 Jul 2025 22:02:52 +0200 Subject: [PATCH 29/36] Handle in name is not valid python name --- fluentogram/stub_generator/stubs.py | 55 +++++++++++++++++++++++++++++ pyproject.toml | 1 + tests/test_generator.py | 9 +++-- 3 files changed, 63 insertions(+), 2 deletions(-) diff --git a/fluentogram/stub_generator/stubs.py b/fluentogram/stub_generator/stubs.py index 9b1a902..aa2a177 100644 --- a/fluentogram/stub_generator/stubs.py +++ b/fluentogram/stub_generator/stubs.py @@ -7,6 +7,52 @@ ) from fluentogram.stub_generator.tree import TreeNode +PYTHON_RESERVED_KEYWORDS = { + "class", + "def", + "if", + "else", + "elif", + "not", + "in", + "is", + "and", + "or", + "as", + "assert", + "break", + "continue", + "return", + "yield", + "from", + "import", + "pass", + "raise", + "try", + "except", + "finally", + "with", + "while", + "for", + "lambda", + "match", + "async", + "await", + "True", + "False", + "None", + "self", + "super", +} + + +def _is_reserved_keyword(name: str) -> bool: + return name in PYTHON_RESERVED_KEYWORDS + + +def _is_valid_python_name(name: str) -> bool: + return not name.isdigit() and not _is_reserved_keyword(name) + def _process_node(node: TreeNode, runner: Runner, parent_path: str = "") -> Class: """Recursively processes tree node and creates corresponding class""" @@ -26,6 +72,9 @@ def _process_node(node: TreeNode, runner: Runner, parent_path: str = "") -> Clas if sub_node.is_leaf: # If child node is leaf with value, add method if sub_node.value is not None: + if not _is_valid_python_name(name): + print(f"{name} is not a valid Python name") + continue knot.add_method( Method( method_name=name, @@ -34,6 +83,9 @@ def _process_node(node: TreeNode, runner: Runner, parent_path: str = "") -> Clas ), ) else: + if not _is_valid_python_name(name): + print(f"{name} is not a valid Python name") + continue # If child node is not leaf, create class reference sub_class = _process_node(sub_node, runner, current_path) runner.add_knot(sub_class) @@ -61,6 +113,9 @@ def generate_stubs(tree: TreeNode) -> str: else: # If root node is not leaf, create class knot = _process_node(node, runner) + if not _is_valid_python_name(node.name): + print(f"{node.name} is not a valid Python name") + continue runner.add_class_ref(ClassRef(node.name, knot.class_name)) runner.add_knot(knot) diff --git a/pyproject.toml b/pyproject.toml index 90df0c3..334f4a9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,7 @@ ignore = [ [tool.ruff.lint.per-file-ignores] "tests/**/*.py" = ["S101"] +"fluentogram/stub_generator/**/*.py" = ["T201"] [tool.ruff.lint.mccabe] max-complexity = 8 diff --git a/tests/test_generator.py b/tests/test_generator.py index a14e7d3..69d8b60 100644 --- a/tests/test_generator.py +++ b/tests/test_generator.py @@ -48,7 +48,10 @@ def test_correctly_generated_stub() -> None: assert 'def multiline() -> Literal["""This is a multiline message.' in content assert 'def welcome(*, name: PossibleValue) -> Literal["""Hello, { $name }!"""]: ...' in content assert "class Email:" in content - assert 'def status(*, unreadCount: PossibleValue) -> Literal["""You have { $unreadCount } new emails."""]: ...' in content + assert ( + 'def status(*, unreadCount: PossibleValue) -> Literal["""You have { $unreadCount } new emails."""]: ...' + in content + ) assert 'def greeting() -> Literal["""Hello, world! This is a phrase with another message."""]: ...' in content assert "class Task:" in content assert 'def state() -> Literal["""Unknown state"""]: ...' in content @@ -59,7 +62,9 @@ def test_correctly_generated_stub() -> None: assert 'def message() -> Literal["""Attachment: This is a nested message."""]: ...' in content assert "class Inner:" in content assert 'def message() -> Literal["""This is a nested message.This is a nested message."""]: ...' in content - assert 'def escaped(*, notAVar: PossibleValue) -> Literal["""This is not a variable: { $notAVar }"""]: ...' in content + assert ( + 'def escaped(*, notAVar: PossibleValue) -> Literal["""This is not a variable: { $notAVar }"""]: ...' in content + ) assert 'def about() -> Literal["""Information about Application X"""]: ...' in content assert "class Complex:" in content assert ( From f4ad5059a727740e99cd52a68c1a2367b270572e Mon Sep 17 00:00:00 2001 From: sheldy Date: Tue, 1 Jul 2025 23:42:26 +0200 Subject: [PATCH 30/36] Add `FileStorage`. --- .gitignore | 3 +- README.md | 22 +++++++++ example_file_storage.py | 83 +++++++++++++++++++++++++++++++++ fluentogram/exceptions.py | 7 +++ fluentogram/storage/__init__.py | 3 +- fluentogram/storage/base.py | 4 +- fluentogram/storage/file.py | 67 ++++++++++++++++++++++++++ fluentogram/translator_hub.py | 9 ++-- tests/test_file_storage.py | 42 +++++++++++++++++ 9 files changed, 232 insertions(+), 8 deletions(-) create mode 100644 example_file_storage.py create mode 100644 fluentogram/storage/file.py create mode 100644 tests/test_file_storage.py diff --git a/.gitignore b/.gitignore index 05f307e..2753763 100644 --- a/.gitignore +++ b/.gitignore @@ -74,4 +74,5 @@ dmypy.json .pytype/ cython_debug/ .idea -.vscode \ No newline at end of file +.vscode +locales/ \ No newline at end of file diff --git a/README.md b/README.md index 725f812..a6977ca 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,28 @@ pip install fluentogram[cli] fluentogram -f tests/assets/test.ftl -o test.pyi ``` +## Storages + +### File storage + +```python +from fluentogram import TranslatorHub +from fluentogram.storage.file import FileStorage + +# Create FileStorage with custom path +storage = FileStorage("my_translations/{locale}/") + +locales_map = { + "en": "en", +} + +hub = TranslatorHub(locales_map, storage=storage) + +translator = hub.get_translator_by_locale("en") + +print(translator.get("hello")) # Hello, world! +``` + ### Dynamic Storage with NATS KV Fluentogram supports real-time translation updates using NATS KV storage: diff --git a/example_file_storage.py b/example_file_storage.py new file mode 100644 index 0000000..9eb5712 --- /dev/null +++ b/example_file_storage.py @@ -0,0 +1,83 @@ +"""Example usage of FileStorage with fluentogram""" + +import asyncio + +from fluent_compiler.bundle import FluentBundle + +from fluentogram import FluentTranslator, TranslatorHub +from fluentogram.storage.file import FileStorage + + +async def main(): + # Create FileStorage with custom path + storage = FileStorage("my_translations") + + # Create translators for different locales + translators = [ + FluentTranslator( + "en", + translator=FluentBundle.from_string( + "en-US", + "welcome = Welcome, { $username }!\nitems-count = You have { $count } items\nhello = Hello!", + ), + ), + FluentTranslator( + "ru", + translator=FluentBundle.from_string( + "ru-RU", + "welcome = Добро пожаловать, { $username }!\nitems-count = У вас { $count } элементов\nhello = Привет!", + ), + ), + ] + + # Add translators to storage (this will save them to files) + storage.add_translators(translators) + + # Configure locale mapping with fallbacks + locales_map = { + "en": "en", + "ru": ("ru", "en"), + } + + # Set locales map in storage + storage.set_locales_map(locales_map) + + # Create the translator hub using storage + hub = TranslatorHub(locales_map, storage.get_translators_list()) + + # Get translators and use them + en_translator = hub.get_translator_by_locale("en") + ru_translator = hub.get_translator_by_locale("ru") + + print("English translations:") + print(en_translator.get("welcome", username="Alice")) + print(en_translator.get("items-count", count=5)) + print(en_translator.get("hello")) + + print("\nRussian translations:") + print(ru_translator.get("welcome", username="Алиса")) + print(ru_translator.get("items-count", count=5)) + print(ru_translator.get("hello")) + + # Update a translation + print("\nUpdating translation...") + success = await storage.update_translation("en", "hello", "Updated Hello!") + if success: + print("Translation updated successfully!") + + # Get updated translator + updated_en_translator = hub.get_translator_by_locale("en") + print(f"Updated: {updated_en_translator.get('hello')}") + + # Get storage information + info = storage.get_storage_info() + print(f"\nStorage info: {info}") + + # Close storage (saves any pending changes) + await storage.close() + + print("\nTranslations saved to files in 'my_translations' directory!") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/fluentogram/exceptions.py b/fluentogram/exceptions.py index c932358..cf9a3d5 100644 --- a/fluentogram/exceptions.py +++ b/fluentogram/exceptions.py @@ -22,3 +22,10 @@ def __init__(self, original_error: Exception, key: str) -> None: self.original_error = original_error self.key = key super().__init__(f"Error formatting key: {self.key!r}: {self.original_error!r}") + + +class LocalesNotFoundError(FluentogramError): + def __init__(self, locales: list[str], path: str) -> None: + self.locales = locales + self.path = path + super().__init__(f"No locales found in {self.path}") diff --git a/fluentogram/storage/__init__.py b/fluentogram/storage/__init__.py index f5235b5..9924b54 100644 --- a/fluentogram/storage/__init__.py +++ b/fluentogram/storage/__init__.py @@ -1,6 +1,7 @@ """Storage implementations for fluentogram.""" from fluentogram.storage.base import BaseStorage +from fluentogram.storage.file import FileStorage from fluentogram.storage.memory import MemoryStorage -__all__ = ["BaseStorage", "MemoryStorage"] +__all__ = ["BaseStorage", "FileStorage", "MemoryStorage"] diff --git a/fluentogram/storage/base.py b/fluentogram/storage/base.py index 0966238..ccf84e9 100644 --- a/fluentogram/storage/base.py +++ b/fluentogram/storage/base.py @@ -3,7 +3,7 @@ from __future__ import annotations from abc import ABC, abstractmethod -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from typing import TYPE_CHECKING if TYPE_CHECKING: @@ -48,7 +48,7 @@ def get_translators_list(self) -> list[FluentTranslator]: """Get all translators as a list.""" return list(self._storage.values()) - def set_locales_map(self, locales_map: dict[str, str | Iterable[str]]) -> None: + def set_locales_map(self, locales_map: Mapping[str, str | Iterable[str]]) -> None: """Set the locales mapping configuration.""" # Normalize locales map (convert single strings to tuples) self._locales_map = {key: (value,) if isinstance(value, str) else value for key, value in locales_map.items()} diff --git a/fluentogram/storage/file.py b/fluentogram/storage/file.py new file mode 100644 index 0000000..f3ea159 --- /dev/null +++ b/fluentogram/storage/file.py @@ -0,0 +1,67 @@ +from __future__ import annotations + +from pathlib import Path + +from fluent_compiler.bundle import FluentBundle + +from fluentogram.exceptions import LocalesNotFoundError +from fluentogram.storage.base import BaseStorage +from fluentogram.translator import FluentTranslator + + +class FileStorage(BaseStorage): + def __init__(self, path: str | Path, use_isolating: bool = False) -> None: # noqa: FBT002 + super().__init__() + self.path = Path(path) + self.use_isolating = use_isolating + self._load_translations() + + def _extract_locales(self, path: Path) -> list[str]: + if "{locale}" in path.parts: + path = Path(*path.parts[: path.parts.index("{locale}")]) + + locales: list[str] = [file_path.name for file_path in path.iterdir() if file_path.is_dir()] + + if not locales: + raise LocalesNotFoundError(locales=[], path=path.as_posix()) + + return locales + + @staticmethod + def _find_locales( + path: Path, + locales: list[str], + ) -> dict[str, list[Path]]: + paths: dict[str, list[Path]] = {} + + if "{locale}" not in path.as_posix(): + path = path.joinpath("{locale}") + + for locale in locales: + locale_path = Path(path.as_posix().format(locale=locale)) + recursive_paths = locale_path.rglob("*.ftl") # Will recursively search for files + paths.setdefault(locale, []).extend(recursive_paths) + + if not paths[locale]: + raise LocalesNotFoundError(locales=locales, path=locale_path.as_posix()) + + return paths + + def _load_translations(self) -> None: + locales = self._extract_locales(self.path) + for locale, paths in self._find_locales(self.path, locales).items(): + texts = [path.read_text(encoding="utf8") for path in paths] + + self.add_translator( + FluentTranslator( + locale=locale, + translator=FluentBundle.from_string( + text="\n".join(texts), + locale=locale, + use_isolating=self.use_isolating, + ), + ), + ) + + async def close(self) -> None: + pass diff --git a/fluentogram/translator_hub.py b/fluentogram/translator_hub.py index 26d90de..d2052d5 100644 --- a/fluentogram/translator_hub.py +++ b/fluentogram/translator_hub.py @@ -1,6 +1,6 @@ from __future__ import annotations -from collections.abc import Iterable +from collections.abc import Iterable, Mapping from fluentogram.exceptions import RootTranslatorNotFoundError from fluentogram.runner import TranslatorRunner @@ -13,8 +13,8 @@ class TranslatorHub: def __init__( self, - locales_map: dict[str, str | Iterable[str]], - translators: list[FluentTranslator], + locales_map: Mapping[str, str | Iterable[str]], + translators: list[FluentTranslator] | None = None, root_locale: str = "en", separator: str = "-", storage: BaseStorage | None = None, @@ -25,7 +25,8 @@ def __init__( self.storage = storage or MemoryStorage() # Add translators to storage - self.storage.add_translators(translators) + if translators is not None: + self.storage.add_translators(translators) # Set locales map in storage self.storage.set_locales_map(locales_map) diff --git a/tests/test_file_storage.py b/tests/test_file_storage.py new file mode 100644 index 0000000..34afd83 --- /dev/null +++ b/tests/test_file_storage.py @@ -0,0 +1,42 @@ +import pytest +from fluent_compiler.bundle import FluentBundle + +from fluentogram.exceptions import LocalesNotFoundError +from fluentogram.storage import FileStorage +from fluentogram.translator import FluentTranslator +from fluentogram.translator_hub import TranslatorHub + + +def test_file_storage() -> None: + storage = FileStorage("tests/assets/locales/{locale}/") + hub = TranslatorHub( + { + "en": "en", + }, + storage=storage, + ) + translator = hub.get_translator_by_locale("en") + assert translator.get("hello") == "Hello, world!" + + +def test_file_storage_if_translators_passed() -> None: + storage = FileStorage("tests/assets/locales/{locale}/") + hub = TranslatorHub( + { + "en": "en", + }, + translators=[ + FluentTranslator( + locale="en", + translator=FluentBundle.from_string("en", "hello = Hello, world1!"), + ), + ], + storage=storage, + ) + translator = hub.get_translator_by_locale("en") + assert translator.get("hello") == "Hello, world1!" + + +def test_file_storage_if_no_locales_found() -> None: + with pytest.raises(LocalesNotFoundError): + FileStorage("tests") From 6f4c2c1708f76366e5a13c4de0b715141224b530 Mon Sep 17 00:00:00 2001 From: sheldy Date: Tue, 1 Jul 2025 23:45:51 +0200 Subject: [PATCH 31/36] Push missed locales. --- .gitignore | 3 +-- tests/assets/locales/en/main.ftl | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 tests/assets/locales/en/main.ftl diff --git a/.gitignore b/.gitignore index 2753763..05f307e 100644 --- a/.gitignore +++ b/.gitignore @@ -74,5 +74,4 @@ dmypy.json .pytype/ cython_debug/ .idea -.vscode -locales/ \ No newline at end of file +.vscode \ No newline at end of file diff --git a/tests/assets/locales/en/main.ftl b/tests/assets/locales/en/main.ftl new file mode 100644 index 0000000..5a04424 --- /dev/null +++ b/tests/assets/locales/en/main.ftl @@ -0,0 +1 @@ +hello = Hello, world! \ No newline at end of file From 4877088e281343f5d79eed656c1a804d40065eb5 Mon Sep 17 00:00:00 2001 From: sheldy Date: Wed, 2 Jul 2025 15:00:58 +0200 Subject: [PATCH 32/36] Backward compatibility for nats storage. --- README.md | 12 ++- example_file_storage.py | 83 ------------------- fluentogram/__init__.py | 14 ++++ fluentogram/nats/hub.py | 57 +++++++++++++ fluentogram/nats/mock.py | 11 +++ fluentogram/nats/storage.py | 14 ++-- .../src/impl/transator_hubs/__init__.py | 3 + fluentogram/storage/base.py | 4 + fluentogram/translator_hub.py | 4 + tests/test_deprecated_kv_translator.py | 25 ++++++ 10 files changed, 135 insertions(+), 92 deletions(-) delete mode 100644 example_file_storage.py create mode 100644 fluentogram/nats/hub.py create mode 100644 fluentogram/nats/mock.py create mode 100644 fluentogram/src/impl/transator_hubs/__init__.py create mode 100644 tests/test_deprecated_kv_translator.py diff --git a/README.md b/README.md index a6977ca..9cc3845 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ -# Fluentogram +# fluentogram -Fluentogram is easy way to use i18n (Fluent) mechanism in any python app. +Fluentogram is easy way to use i18n (Fluent) mechanism in any python app. ## Features @@ -96,9 +96,13 @@ translator = hub.get_translator_by_locale("en") print(translator.get("hello")) # Hello, world! ``` -### Dynamic Storage with NATS KV +## Fluentogram supports real-time translation updates using NATS KV storage: + +Install: +```sh +pip install fluentogram[nats] +``` -Fluentogram supports real-time translation updates using NATS KV storage: ```python import asyncio diff --git a/example_file_storage.py b/example_file_storage.py deleted file mode 100644 index 9eb5712..0000000 --- a/example_file_storage.py +++ /dev/null @@ -1,83 +0,0 @@ -"""Example usage of FileStorage with fluentogram""" - -import asyncio - -from fluent_compiler.bundle import FluentBundle - -from fluentogram import FluentTranslator, TranslatorHub -from fluentogram.storage.file import FileStorage - - -async def main(): - # Create FileStorage with custom path - storage = FileStorage("my_translations") - - # Create translators for different locales - translators = [ - FluentTranslator( - "en", - translator=FluentBundle.from_string( - "en-US", - "welcome = Welcome, { $username }!\nitems-count = You have { $count } items\nhello = Hello!", - ), - ), - FluentTranslator( - "ru", - translator=FluentBundle.from_string( - "ru-RU", - "welcome = Добро пожаловать, { $username }!\nitems-count = У вас { $count } элементов\nhello = Привет!", - ), - ), - ] - - # Add translators to storage (this will save them to files) - storage.add_translators(translators) - - # Configure locale mapping with fallbacks - locales_map = { - "en": "en", - "ru": ("ru", "en"), - } - - # Set locales map in storage - storage.set_locales_map(locales_map) - - # Create the translator hub using storage - hub = TranslatorHub(locales_map, storage.get_translators_list()) - - # Get translators and use them - en_translator = hub.get_translator_by_locale("en") - ru_translator = hub.get_translator_by_locale("ru") - - print("English translations:") - print(en_translator.get("welcome", username="Alice")) - print(en_translator.get("items-count", count=5)) - print(en_translator.get("hello")) - - print("\nRussian translations:") - print(ru_translator.get("welcome", username="Алиса")) - print(ru_translator.get("items-count", count=5)) - print(ru_translator.get("hello")) - - # Update a translation - print("\nUpdating translation...") - success = await storage.update_translation("en", "hello", "Updated Hello!") - if success: - print("Translation updated successfully!") - - # Get updated translator - updated_en_translator = hub.get_translator_by_locale("en") - print(f"Updated: {updated_en_translator.get('hello')}") - - # Get storage information - info = storage.get_storage_info() - print(f"\nStorage info: {info}") - - # Close storage (saves any pending changes) - await storage.close() - - print("\nTranslations saved to files in 'my_translations' directory!") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/fluentogram/__init__.py b/fluentogram/__init__.py index 8ddb35b..d0baae9 100644 --- a/fluentogram/__init__.py +++ b/fluentogram/__init__.py @@ -1,3 +1,15 @@ +try: + import nats +except ImportError: + nats = None + +if nats is None: + from .nats.mock import KvTranslatorHubMock as KvTranslatorHub + from .nats.mock import NatsStorageMock as NatsStorage +else: + from .nats.hub import KvTranslatorHub + from .nats.storage import NatsStorage + from .runner import TranslatorRunner from .transformers import DateTimeTransformer, MoneyTransformer from .translator import FluentTranslator @@ -6,7 +18,9 @@ __all__ = [ "DateTimeTransformer", "FluentTranslator", + "KvTranslatorHub", "MoneyTransformer", + "NatsStorage", "TranslatorHub", "TranslatorRunner", ] diff --git a/fluentogram/nats/hub.py b/fluentogram/nats/hub.py new file mode 100644 index 0000000..1c63da0 --- /dev/null +++ b/fluentogram/nats/hub.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import warnings +from typing import Any + +from fluentogram.nats.storage import NatsKvStorage +from fluentogram.translator_hub import TranslatorHub + + +class KvTranslatorHub(TranslatorHub): + def __init__(self, *args: Any, **kwargs: Any) -> None: + warnings.warn( + "KvTranslatorHub is deprecated and will be removed in 2.0.0 version. " + "Use TranslatorHub with NatsStorage instead.", + DeprecationWarning, + stacklevel=2, + ) + super().__init__(*args, **kwargs) + + async def from_storage(self, kv_storage: NatsKvStorage) -> None: + old_storage = self.storage + locales_map = old_storage.get_locales_map() + translators = old_storage.get_all_translators() + + self.storage = kv_storage + self.storage.add_translators(translators) + self.storage.set_locales_map(locales_map) + + async def put( + self, + locale: str, + key: str | None = None, + value: Any | None = None, + mapping_values: dict[str, Any] | None = None, + ) -> None: + if mapping_values is not None: + for k, v in mapping_values.items(): + await self.put(locale=locale, key=k, value=v) + + if key is not None and value is not None: + await self.storage.update_translation( + locale=locale, + key=key, + value=value, + ) + + async def create( + self, + locale: str, + key: str, + value: Any, + mapping_values: dict[str, Any] | None = None, + ) -> None: + await self.put(locale=locale, key=key, value=value, mapping_values=mapping_values) + + async def delete(self, _: str, *__: str) -> None: + pass diff --git a/fluentogram/nats/mock.py b/fluentogram/nats/mock.py new file mode 100644 index 0000000..b45bcb2 --- /dev/null +++ b/fluentogram/nats/mock.py @@ -0,0 +1,11 @@ +from typing import Any + + +class NatsStorageMock: + def __init__(self, *args: Any, **kwargs: Any) -> None: + raise NotImplementedError("NatsStorage is not implemented") + + +class KvTranslatorHubMock: + def __init__(self, *args: Any, **kwargs: Any) -> None: + raise NotImplementedError("KvTranslatorHub is not implemented") diff --git a/fluentogram/nats/storage.py b/fluentogram/nats/storage.py index 3959902..396af8e 100644 --- a/fluentogram/nats/storage.py +++ b/fluentogram/nats/storage.py @@ -8,7 +8,6 @@ from typing import Any, Callable from nats import connect -from nats.aio.client import Client from nats.aio.msg import Msg from nats.js import JetStreamContext from nats.js.api import KeyValueConfig @@ -25,15 +24,15 @@ class NatsKvStorage(BaseStorage): def __init__( # noqa: PLR0913 self, - nc: Client, kv: KeyValue, + js: JetStreamContext, separator: str = ".", serializer: _JsonDumps = lambda data: json.dumps(data).encode("utf-8"), deserializer: _JsonLoads = json.loads, consume_timeout: float = 1.0, ) -> None: - self._nc = nc - self._js = nc.jetstream() + self._js = js + self._nc = js._nc # noqa: SLF001 self._kv = kv self._stream_name = kv._stream # noqa: SLF001 self.separator = separator @@ -57,7 +56,7 @@ async def from_servers( nc = await connect(servers=servers) js = nc.jetstream() kv = await js.create_key_value(config=kv_config) - return cls(nc, kv, separator, serializer, deserializer) + return cls(kv, js, separator, serializer, deserializer) async def update_translation( self, @@ -81,6 +80,8 @@ async def _process_messages(self, consumer: JetStreamContext.PullSubscription) - """Process messages from the consumer.""" try: messages: list[Msg] = await consumer.fetch(50, timeout=self.consume_timeout) + if not messages: + return logger.debug("Received %d messages: %s", len(messages), messages) await self._update_compiled_messages(messages) except TimeoutError: @@ -131,3 +132,6 @@ async def close(self) -> None: await self._listen_for_changes_task await self._nc.close() + + +NatsStorage = NatsKvStorage # backward compatibility diff --git a/fluentogram/src/impl/transator_hubs/__init__.py b/fluentogram/src/impl/transator_hubs/__init__.py new file mode 100644 index 0000000..d5b60e9 --- /dev/null +++ b/fluentogram/src/impl/transator_hubs/__init__.py @@ -0,0 +1,3 @@ +from fluentogram import KvTranslatorHub + +__all__ = ["KvTranslatorHub"] diff --git a/fluentogram/storage/base.py b/fluentogram/storage/base.py index ccf84e9..d5af733 100644 --- a/fluentogram/storage/base.py +++ b/fluentogram/storage/base.py @@ -19,6 +19,10 @@ def __init__(self) -> None: self._locales_map: dict[str, Iterable[str]] = {} self._translators_map: dict[str, Iterable[FluentTranslator]] = {} + def get_locales_map(self) -> dict[str, Iterable[str]]: + """Get the locales mapping configuration.""" + return self._locales_map + def add_translator(self, translator: FluentTranslator) -> None: """Add a translator to storage.""" self._storage[translator.locale] = translator diff --git a/fluentogram/translator_hub.py b/fluentogram/translator_hub.py index d2052d5..e04133f 100644 --- a/fluentogram/translator_hub.py +++ b/fluentogram/translator_hub.py @@ -34,6 +34,10 @@ def __init__( if not self.storage.has_translator(root_locale): raise RootTranslatorNotFoundError(self.root_locale) + async def update_translation(self, locale: str, key: str, value: str) -> bool: + """Update translation for a given locale and key.""" + return await self.storage.update_translation(locale, key, value) + def get_translator_by_locale(self, locale: str) -> TranslatorRunner: """Here is a little tricky moment. There should be like a one-shot scheme. diff --git a/tests/test_deprecated_kv_translator.py b/tests/test_deprecated_kv_translator.py new file mode 100644 index 0000000..cdb3a7c --- /dev/null +++ b/tests/test_deprecated_kv_translator.py @@ -0,0 +1,25 @@ +import pytest +from fluent_compiler.bundle import FluentBundle + +from fluentogram import FluentTranslator +from fluentogram.src.impl.transator_hubs import KvTranslatorHub + + +def test_deprecated_msg_kv_translator() -> None: + with pytest.warns(DeprecationWarning): + KvTranslatorHub( + { + "ru": ("ru", "en"), + "en": ("en",), + }, + translators=[ + FluentTranslator( + locale="en", + translator=FluentBundle.from_string(locale="en-US", text="hello = Hello {$name}!"), + ), + FluentTranslator( + locale="ru", + translator=FluentBundle.from_string(locale="ru", text="hello = Привет {$name}!"), + ), + ], + ) From a8a0394549f736b475c4f5897bf17651ea227691 Mon Sep 17 00:00:00 2001 From: sheldy Date: Wed, 2 Jul 2025 15:14:29 +0200 Subject: [PATCH 33/36] Update README. --- README.md | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 63 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9cc3845..0a022e9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # fluentogram -Fluentogram is easy way to use i18n (Fluent) mechanism in any python app. +fluentogram is easy way to use i18n (Fluent) mechanism in any python app. ## Features @@ -96,14 +96,14 @@ translator = hub.get_translator_by_locale("en") print(translator.get("hello")) # Hello, world! ``` -## Fluentogram supports real-time translation updates using NATS KV storage: +## fluentogram supports real-time translation updates using NATS KV storage: Install: + ```sh pip install fluentogram[nats] ``` - ```python import asyncio @@ -180,3 +180,63 @@ except RootTranslatorNotFoundError as e: except FormatError as e: print(f"Formatting error for key {e.key}: {e.original_error}") ``` + +# Inside of fluentogram + +### TranslatorHub + +`TranslatorHub` is unit of distribution TranslatorRunner's. + +Init: + +```python +def __init__( + self, + locales_map: dict[str, str | Iterable[str]], + translators: list[FluentTranslator], + root_locale: str = "en", +) -> None: +``` + +### Locales map + +That's like a configuration map for "Rollback" feature. If you haven't configured translation for current locale - first in collection, "Rollback" will look to others locales' data and try to find a translation + +For example: + +```python +locales_map = { + "ua": ("ua", "de", "en"), + "de": ("de", "en"), + "en": ("en",) +} +``` + +Let's look at this example + +If translator does not find a translation for `ua` locale in `ua` data, next stop is `de` data. If it's failed too - it will look to `en` translations. + +### Translators + +You have 3 variants to use translators: + +- In memory + +```python +translators = [ + FluentTranslator( + "en", + translator=FluentBundle.from_string( + "en-US", + "welcome = Welcome, { $username }!\n" + "items-count = You have { $count } items", + ), + ), +] +hub = TranslatorHub(locales_map, translators) +``` + +- From files using file storage +- From NATS KV using storage + +[↑ Back to Storages](#storages) From cb4c9dc810bafec6b11422e3bbec4e1e2ee4704c Mon Sep 17 00:00:00 2001 From: sheldy Date: Wed, 2 Jul 2025 15:19:21 +0200 Subject: [PATCH 34/36] Run tests on supported python versions. --- .github/workflows/tests.yml | 8 ++++++-- pyproject.toml | 3 ++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 8cf9db9..92aa565 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,6 +26,10 @@ jobs: - uses: pre-commit/action@v3.0.1 test: + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12"] + runs-on: ubuntu-latest steps: @@ -35,7 +39,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: ${{ matrix.python-version }} - name: Install dependencies run: | @@ -44,4 +48,4 @@ jobs: uv pip install --system .[dev] - name: Run tests - run: pytest tests \ No newline at end of file + run: pytest tests diff --git a/pyproject.toml b/pyproject.toml index 334f4a9..1de27cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,8 @@ license = "MIT" dependencies = ["fluent-compiler~=1.1"] [project.optional-dependencies] -cli = ["jinja2>=3.1.0,<4"] +stubs = ["jinja2>=3.1.0,<4"] +cli = ["fluentogram[stubs]"] aiogram = ["aiogram~=3.20.0"] nats = ["nats-py~=2.10.0"] From a91a1b6f2317906286357541016583d997f6213a Mon Sep 17 00:00:00 2001 From: sheldy Date: Wed, 2 Jul 2025 15:20:34 +0200 Subject: [PATCH 35/36] Fix tests on python 3.9 --- tests/test_transformers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/test_transformers.py b/tests/test_transformers.py index c792efe..ba5c73b 100644 --- a/tests/test_transformers.py +++ b/tests/test_transformers.py @@ -1,4 +1,4 @@ -from datetime import UTC, datetime +from datetime import datetime, timezone from decimal import Decimal from fluent_compiler.bundle import FluentBundle @@ -22,7 +22,7 @@ def test_date_transformer() -> None: translator = hub.get_translator_by_locale("en") # Use DateTimeTransformer for proper date formatting - meeting_date = datetime(2024, 1, 15, 14, 30, tzinfo=UTC) + meeting_date = datetime(2024, 1, 15, 14, 30, tzinfo=timezone.utc) formatted_date = DateTimeTransformer(meeting_date, dateStyle="full", timeStyle="short") assert ( translator.get( From 586e1191d24169bdcdf864ca089dcca898ce53f6 Mon Sep 17 00:00:00 2001 From: sheldy Date: Wed, 2 Jul 2025 17:52:24 +0200 Subject: [PATCH 36/36] Add watchdog for cli and add backward compatibility. --- fluentogram/cli/main.py | 84 ++++++++++++++++++++++++++++++++++++++--- pyproject.toml | 3 +- 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/fluentogram/cli/main.py b/fluentogram/cli/main.py index 4e40a9a..35ab59e 100644 --- a/fluentogram/cli/main.py +++ b/fluentogram/cli/main.py @@ -1,16 +1,88 @@ +from __future__ import annotations + import argparse +import time +from pathlib import Path + +from watchdog.events import ( + FileModifiedEvent, + FileSystemEventHandler, +) +from watchdog.observers import Observer from fluentogram.stub_generator.generator import generate +class FluentogramFileHandler(FileSystemEventHandler): + """Handler for file system events that regenerates stubs when files change.""" + + def __init__(self, output_file: str, file_path: str | None, dir_path: str | None) -> None: + self.output_file = output_file + self.file_path = file_path + self.dir_path = dir_path + + def on_modified(self, event: FileModifiedEvent) -> None: + print(f"event type: {event.event_type}, path: {event.src_path}") + if not event.is_directory: + generate(self.output_file, self.file_path, self.dir_path) + print(f"Regenerated stubs for {event.src_path}") + + +def watch_files(output_file: str, file_path: str | None, dir_path: str | None) -> None: + """Start watching files/directories for changes.""" + event_handler = FluentogramFileHandler(output_file, file_path, dir_path) + observer = Observer() + + if file_path: + # Watch specific file + file_dir = str(Path(file_path).parent) + observer.schedule(event_handler, file_dir, recursive=False) + print(f"Watching file: {file_path}") + elif dir_path: + # Watch directory recursively + observer.schedule(event_handler, dir_path, recursive=True) + print(f"Watching directory: {dir_path}") + + observer.start() + print("File watcher started. Press Ctrl+C to stop.") + + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + observer.stop() + print("\nFile watcher stopped.") + + observer.join() + + def cli() -> None: parser = argparse.ArgumentParser(prog="fluentogram") - parser.add_argument("-o", dest="output_file", required=True) - parser.add_argument("-f", dest="file_path", required=False) - parser.add_argument("-d", dest="dir_path", required=False) + parser.add_argument("-o", "--output-file", dest="output_file", required=False, help="Path to the output file") + parser.add_argument("-f", "--file-path", dest="file_path", required=False, help="Path to the file to watch") + parser.add_argument( + "-w", + "--watch", + dest="watch", + required=False, + help="Watch for file changes and regenerate stubs automatically", + ) + parser.add_argument("-d", "--dir-path", dest="dir_path", required=False, help="Path to the directory to watch") + # back compatibility + + parser.add_argument("-ftl", dest="ftl_path", required=False, help="Path to the file to watch") + parser.add_argument("-track-ftl", dest="watch", required=False, help="Path to the file to watch") + parser.add_argument("-dir-ftl", dest="dir_path", required=False, help="Path to the directory to watch") + parser.add_argument("-stub", dest="output_file", required=False, help="Path to the output file") args = parser.parse_args() - if args.file_path is None and args.dir_path is None: - raise ValueError("You must provide either a file or a directory") - generate(args.output_file, args.file_path, args.dir_path) + if not args.output_file: + args.output_file = "fluentogram.pyi" + + if args.watch: + # Start watching for changes + watch_files(args.output_file, args.file_path, args.dir_path) + else: + # Generate stubs once + generate(args.output_file, args.file_path, args.dir_path) diff --git a/pyproject.toml b/pyproject.toml index 1de27cf..7817ccf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,7 +13,7 @@ dependencies = ["fluent-compiler~=1.1"] [project.optional-dependencies] stubs = ["jinja2>=3.1.0,<4"] -cli = ["fluentogram[stubs]"] +cli = ["fluentogram[stubs]", "watchdog>=3.0.0"] aiogram = ["aiogram~=3.20.0"] nats = ["nats-py~=2.10.0"] @@ -86,6 +86,7 @@ ignore = [ [tool.ruff.lint.per-file-ignores] "tests/**/*.py" = ["S101"] "fluentogram/stub_generator/**/*.py" = ["T201"] +"fluentogram/cli/**/*.py" = ["T201"] [tool.ruff.lint.mccabe] max-complexity = 8