diff --git a/SinaTools.egg-info/PKG-INFO b/SinaTools.egg-info/PKG-INFO index 5b1ebd4..31d9e34 100644 --- a/SinaTools.egg-info/PKG-INFO +++ b/SinaTools.egg-info/PKG-INFO @@ -1,6 +1,6 @@ -Metadata-Version: 2.2 +Metadata-Version: 2.4 Name: SinaTools -Version: 0.1.41 +Version: 1.0.7 Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow. Home-page: https://github.com/SinaLab/sinatools License: MIT license @@ -13,16 +13,19 @@ Requires-Dist: farasapy Requires-Dist: tqdm Requires-Dist: requests Requires-Dist: pathlib -Requires-Dist: transformers==4.47.1 -Requires-Dist: torchvision==0.20.1 -Requires-Dist: seqeval==1.2.2 -Requires-Dist: natsort==7.1.1 +Requires-Dist: torch>=2.0 +Requires-Dist: transformers<4.44,>=4.0 +Requires-Dist: seqeval==1.2.2 +Requires-Dist: natsort==7.1.1 +Requires-Dist: pandas +Requires-Dist: pyarabic Dynamic: description Dynamic: description-content-type Dynamic: home-page -Dynamic: keywords -Dynamic: license -Dynamic: requires-dist +Dynamic: keywords +Dynamic: license +Dynamic: license-file +Dynamic: requires-dist Dynamic: summary SinaTools diff --git a/SinaTools.egg-info/SOURCES.txt b/SinaTools.egg-info/SOURCES.txt index fe11659..c96e854 100644 --- a/SinaTools.egg-info/SOURCES.txt +++ b/SinaTools.egg-info/SOURCES.txt @@ -136,4 +136,6 @@ sinatools/utils/word_compare.py sinatools/wsd/__init__.py sinatools/wsd/disambiguator.py sinatools/wsd/settings.py -sinatools/wsd/wsd.py \ No newline at end of file +sinatools/wsd/wsd.py +tests/test_dependency_metadata.py +tests/test_sentence_tokenizer_cli.py \ No newline at end of file diff --git a/SinaTools.egg-info/requires.txt b/SinaTools.egg-info/requires.txt index 88cba91..7a9cf2b 100644 --- a/SinaTools.egg-info/requires.txt +++ b/SinaTools.egg-info/requires.txt @@ -3,7 +3,9 @@ farasapy tqdm requests pathlib -transformers==4.47.1 -torchvision==0.20.1 +torch>=2.0 +transformers<4.44,>=4.0 seqeval==1.2.2 natsort==7.1.1 +pandas +pyarabic diff --git a/build/lib/sinatools/CLI/utils/sentence_tokenizer.py b/build/lib/sinatools/CLI/utils/sentence_tokenizer.py index 12bd3d9..b265be4 100644 --- a/build/lib/sinatools/CLI/utils/sentence_tokenizer.py +++ b/build/lib/sinatools/CLI/utils/sentence_tokenizer.py @@ -2,22 +2,22 @@ About: ------ -The sentence_tokenizer command allows you to tokenize text into sentences using the SinaTools utility. It provides -flexibility in tokenizing at different punctuation marks, including dots, question marks, and exclamation marks. It also -allows tokenization at new lines. +The sentence_tokenizer command tokenizes text into sentences using the +SinaTools utility. It can split at dots, question marks, exclamation marks, +and new lines. Usage: ------ -Below is the usage information that can be generated by running sentence_tokenizer --help. +Run sentence_tokenizer --help to show this usage information. .. code-block:: none Usage: sentence_tokenizer --text=TEXT [options] sentence_tokenizer --file=FILE [options] - + .. code-block:: none - + Options: --text TEXT Text to be tokenized into sentences. @@ -37,24 +37,48 @@ .. code-block:: none - sentence_tokenizer --text "Your text here. Does it work? Yes! Try with new lines." --dot --question_mark --exclamation_mark - sentence_tokenizer --file "path/to/your/file.txt" --dot --question_mark --exclamation_mark + sentence_tokenizer --text "Your text here." --dot --question_mark + sentence_tokenizer --file "path/to/file.txt" --dot --question_mark """ import argparse + from sinatools.utils.tokenizer import sentence_tokenizer from sinatools.utils.readfile import read_file + def main(): - parser = argparse.ArgumentParser(description='Sentence Tokenization using SinaTools') - + parser = argparse.ArgumentParser( + description='Sentence Tokenization using SinaTools' + ) + # Adding arguments for the text, file, and tokenization options - parser.add_argument('--text', type=str, help='Text to be tokenized into sentences') - parser.add_argument('--file', type=str, help='File containing the text to be tokenized into sentences') + parser.add_argument( + '--text', + type=str, + help='Text to be tokenized into sentences', + ) + parser.add_argument( + '--file', + type=str, + help='File containing the text to be tokenized into sentences', + ) parser.add_argument('--dot', action='store_true', help='Tokenize at dots') - parser.add_argument('--new_line', action='store_true', help='Tokenize at new lines') - parser.add_argument('--question_mark', action='store_true', help='Tokenize at question marks') - parser.add_argument('--exclamation_mark', action='store_true', help='Tokenize at exclamation marks') + parser.add_argument( + '--new_line', + action='store_true', + help='Tokenize at new lines', + ) + parser.add_argument( + '--question_mark', + action='store_true', + help='Tokenize at question marks', + ) + parser.add_argument( + '--exclamation_mark', + action='store_true', + help='Tokenize at exclamation marks', + ) args = parser.parse_args() @@ -63,15 +87,21 @@ def main(): print("Either --text or --file argument must be provided.") return - text_content = args.text if args.text else read_file(args.file) + text_content = args.text if args.text else " ".join(read_file(args.file)) # Perform sentence tokenization - sentences = sentence_tokenizer(" ".join(text_content), dot=args.dot, new_line=args.new_line, - question_mark=args.question_mark, exclamation_mark=args.exclamation_mark) - + sentences = sentence_tokenizer( + text_content, + dot=args.dot, + new_line=args.new_line, + question_mark=args.question_mark, + exclamation_mark=args.exclamation_mark, + ) + # Print each sentence in a new line for sentence in sentences: print(sentence) + if __name__ == '__main__': main() diff --git a/build/lib/sinatools/environment.yml b/build/lib/sinatools/environment.yml index 02cd002..1462152 100644 --- a/build/lib/sinatools/environment.yml +++ b/build/lib/sinatools/environment.yml @@ -111,7 +111,6 @@ dependencies: - tk=8.6.14=h39e8969_0 - torchaudio=2.5.1=py311_cu124 - torchtriton=3.1.0=py311 - - torchvision=0.20.1=py311_cu124 - typing_extensions=4.12.2=py311h06a4308_0 - urllib3=2.2.3=py311h06a4308_0 - wheel=0.44.0=py311h06a4308_0 @@ -171,9 +170,9 @@ dependencies: - tensorboard==2.19.0 - tensorboard-data-server==0.7.2 - threadpoolctl==3.5.0 - - tokenizers==0.21.0 + - tokenizers==0.19.1 - tqdm==4.67.1 - - transformers==4.47.1 + - transformers==4.43.4 - trl==0.12.0 - tzdata==2024.2 - werkzeug==3.1.3 diff --git a/setup.cfg b/setup.cfg index c592034..a490d5f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,10 +25,7 @@ exclude = docs [aliases] test = pytest -[tool:pytest] -collect_ignore = ['setup.py'] - -[egg_info] -tag_build = -tag_date = 0 +[egg_info] +tag_build = +tag_date = 0 diff --git a/setup.py b/setup.py index 9930b3f..e7cceff 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -"""The setup script.""" -import os +"""The setup script.""" +import os from setuptools import setup, find_packages VERSION_FILE = os.path.join(os.path.dirname(__file__), 'sinatools', @@ -18,9 +18,8 @@ 'requests', # 'regex', 'pathlib', - # 'torch==2.5.1', - 'transformers==4.47.1', - 'torchvision==0.20.1', + 'torch>=2.0', + 'transformers>=4.0,<4.44', 'seqeval==1.2.2', 'natsort==7.1.1', 'pandas', @@ -39,7 +38,7 @@ setup( entry_points={ - 'console_scripts':[ + 'console_scripts': [ ('install_env=' 'sinatools.install_env:main'), ('arStrip=' @@ -72,12 +71,12 @@ 'sinatools.CLI.DataDownload.download_files:main'), ('corpus_entity_extractor=' 'sinatools.CLI.ner.corpus_entity_extractor:main'), - ('text_dublication_detector=' - 'sinatools.CLI.utils.text_dublication_detector:main'), - ('evaluate_synonyms=' - 'sinatools.CLI.synonyms.evaluate_synonyms:main'), - ('extend_synonyms=' - 'sinatools.CLI.synonyms.extend_synonyms:main'), + ('text_dublication_detector=' + 'sinatools.CLI.utils.text_dublication_detector:main'), + ('evaluate_synonyms=' + 'sinatools.CLI.synonyms.evaluate_synonyms:main'), + ('extend_synonyms=' + 'sinatools.CLI.synonyms.extend_synonyms:main'), ('semantic_relatedness=' 'sinatools.CLI.semantic_relatedness.compute_relatedness:main'), ('relation_extractor=' @@ -88,8 +87,11 @@ package_data={'sinatools': ['data/*.pickle', 'environment.yml']}, install_requires=requirements, license="MIT license", - description='Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.', - long_description = readme + "\n", + description=( + 'Open-source Python toolkit for Arabic Natural Understanding, ' + 'allowing people to integrate it in their system workflow.' + ), + long_description=readme + "\n", long_description_content_type='text/markdown', include_package_data=True, keywords='sinatools', diff --git a/sinatools/CLI/utils/sentence_tokenizer.py b/sinatools/CLI/utils/sentence_tokenizer.py index 12bd3d9..b265be4 100644 --- a/sinatools/CLI/utils/sentence_tokenizer.py +++ b/sinatools/CLI/utils/sentence_tokenizer.py @@ -2,22 +2,22 @@ About: ------ -The sentence_tokenizer command allows you to tokenize text into sentences using the SinaTools utility. It provides -flexibility in tokenizing at different punctuation marks, including dots, question marks, and exclamation marks. It also -allows tokenization at new lines. +The sentence_tokenizer command tokenizes text into sentences using the +SinaTools utility. It can split at dots, question marks, exclamation marks, +and new lines. Usage: ------ -Below is the usage information that can be generated by running sentence_tokenizer --help. +Run sentence_tokenizer --help to show this usage information. .. code-block:: none Usage: sentence_tokenizer --text=TEXT [options] sentence_tokenizer --file=FILE [options] - + .. code-block:: none - + Options: --text TEXT Text to be tokenized into sentences. @@ -37,24 +37,48 @@ .. code-block:: none - sentence_tokenizer --text "Your text here. Does it work? Yes! Try with new lines." --dot --question_mark --exclamation_mark - sentence_tokenizer --file "path/to/your/file.txt" --dot --question_mark --exclamation_mark + sentence_tokenizer --text "Your text here." --dot --question_mark + sentence_tokenizer --file "path/to/file.txt" --dot --question_mark """ import argparse + from sinatools.utils.tokenizer import sentence_tokenizer from sinatools.utils.readfile import read_file + def main(): - parser = argparse.ArgumentParser(description='Sentence Tokenization using SinaTools') - + parser = argparse.ArgumentParser( + description='Sentence Tokenization using SinaTools' + ) + # Adding arguments for the text, file, and tokenization options - parser.add_argument('--text', type=str, help='Text to be tokenized into sentences') - parser.add_argument('--file', type=str, help='File containing the text to be tokenized into sentences') + parser.add_argument( + '--text', + type=str, + help='Text to be tokenized into sentences', + ) + parser.add_argument( + '--file', + type=str, + help='File containing the text to be tokenized into sentences', + ) parser.add_argument('--dot', action='store_true', help='Tokenize at dots') - parser.add_argument('--new_line', action='store_true', help='Tokenize at new lines') - parser.add_argument('--question_mark', action='store_true', help='Tokenize at question marks') - parser.add_argument('--exclamation_mark', action='store_true', help='Tokenize at exclamation marks') + parser.add_argument( + '--new_line', + action='store_true', + help='Tokenize at new lines', + ) + parser.add_argument( + '--question_mark', + action='store_true', + help='Tokenize at question marks', + ) + parser.add_argument( + '--exclamation_mark', + action='store_true', + help='Tokenize at exclamation marks', + ) args = parser.parse_args() @@ -63,15 +87,21 @@ def main(): print("Either --text or --file argument must be provided.") return - text_content = args.text if args.text else read_file(args.file) + text_content = args.text if args.text else " ".join(read_file(args.file)) # Perform sentence tokenization - sentences = sentence_tokenizer(" ".join(text_content), dot=args.dot, new_line=args.new_line, - question_mark=args.question_mark, exclamation_mark=args.exclamation_mark) - + sentences = sentence_tokenizer( + text_content, + dot=args.dot, + new_line=args.new_line, + question_mark=args.question_mark, + exclamation_mark=args.exclamation_mark, + ) + # Print each sentence in a new line for sentence in sentences: print(sentence) + if __name__ == '__main__': main() diff --git a/sinatools/environment.yml b/sinatools/environment.yml index 02cd002..1462152 100644 --- a/sinatools/environment.yml +++ b/sinatools/environment.yml @@ -111,7 +111,6 @@ dependencies: - tk=8.6.14=h39e8969_0 - torchaudio=2.5.1=py311_cu124 - torchtriton=3.1.0=py311 - - torchvision=0.20.1=py311_cu124 - typing_extensions=4.12.2=py311h06a4308_0 - urllib3=2.2.3=py311h06a4308_0 - wheel=0.44.0=py311h06a4308_0 @@ -171,9 +170,9 @@ dependencies: - tensorboard==2.19.0 - tensorboard-data-server==0.7.2 - threadpoolctl==3.5.0 - - tokenizers==0.21.0 + - tokenizers==0.19.1 - tqdm==4.67.1 - - transformers==4.47.1 + - transformers==4.43.4 - trl==0.12.0 - tzdata==2024.2 - werkzeug==3.1.3 diff --git a/tests/test_dependency_metadata.py b/tests/test_dependency_metadata.py new file mode 100644 index 0000000..14f64d3 --- /dev/null +++ b/tests/test_dependency_metadata.py @@ -0,0 +1,47 @@ +import ast +import os +import unittest + + +ROOT = os.path.dirname(os.path.dirname(__file__)) + + +def load_setup_requirements(): + setup_path = os.path.join(ROOT, "setup.py") + with open(setup_path, encoding="utf-8") as handle: + tree = ast.parse(handle.read()) + + for node in tree.body: + if isinstance(node, ast.Assign): + for target in node.targets: + is_requirements = ( + isinstance(target, ast.Name) + and target.id == "requirements" + ) + if is_requirements: + return ast.literal_eval(node.value) + raise AssertionError("setup.py does not define requirements") + + +class DependencyMetadataTest(unittest.TestCase): + def test_torch_is_declared_directly(self): + requirements = load_setup_requirements() + + self.assertIn("torch>=2.0", requirements) + has_torchvision = any( + req.startswith("torchvision") for req in requirements + ) + self.assertFalse( + has_torchvision, + "SinaTools imports torch directly; do not rely on torchvision", + ) + + def test_transformers_range_matches_camel_tools(self): + requirements = load_setup_requirements() + + self.assertIn("transformers>=4.0,<4.44", requirements) + self.assertNotIn("transformers==4.47.1", requirements) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sentence_tokenizer_cli.py b/tests/test_sentence_tokenizer_cli.py new file mode 100644 index 0000000..2ddf8bd --- /dev/null +++ b/tests/test_sentence_tokenizer_cli.py @@ -0,0 +1,29 @@ +import io +import sys +import unittest +from contextlib import redirect_stdout +from unittest.mock import patch + +from sinatools.CLI.utils import sentence_tokenizer + + +class SentenceTokenizerCliTest(unittest.TestCase): + def test_text_argument_is_not_split_into_characters(self): + argv = [ + "sentence_tokenizer", + "--text", + "One. Two?", + "--dot", + "--question_mark", + ] + + stdout = io.StringIO() + with patch.object(sys, "argv", argv): + with redirect_stdout(stdout): + sentence_tokenizer.main() + + self.assertEqual(stdout.getvalue().splitlines(), ["One.", "Two?"]) + + +if __name__ == "__main__": + unittest.main()