Fix camel-tools dependency compatibility

SinaTools.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: SinaTools
-Version: 0.1.41
+Version: 1.0.7
 Summary: Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.
 Home-page: https://github.com/SinaLab/sinatools
 License: MIT license
@@ -13,16 +13,19 @@ Requires-Dist: farasapy
 Requires-Dist: tqdm
 Requires-Dist: requests
 Requires-Dist: pathlib
-Requires-Dist: transformers==4.47.1
-Requires-Dist: torchvision==0.20.1
-Requires-Dist: seqeval==1.2.2
-Requires-Dist: natsort==7.1.1
+Requires-Dist: torch>=2.0
+Requires-Dist: transformers<4.44,>=4.0
+Requires-Dist: seqeval==1.2.2
+Requires-Dist: natsort==7.1.1
+Requires-Dist: pandas
+Requires-Dist: pyarabic
 Dynamic: description
 Dynamic: description-content-type
 Dynamic: home-page
-Dynamic: keywords
-Dynamic: license
-Dynamic: requires-dist
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
 Dynamic: summary
 
 SinaTools

SinaTools.egg-info/SOURCES.txt

@@ -136,4 +136,6 @@ sinatools/utils/word_compare.py
 sinatools/wsd/__init__.py
 sinatools/wsd/disambiguator.py
 sinatools/wsd/settings.py
-sinatools/wsd/wsd.py
+sinatools/wsd/wsd.py
+tests/test_dependency_metadata.py
+tests/test_sentence_tokenizer_cli.py

SinaTools.egg-info/requires.txt

@@ -3,7 +3,9 @@ farasapy
 tqdm
 requests
 pathlib
-transformers==4.47.1
-torchvision==0.20.1
+torch>=2.0
+transformers<4.44,>=4.0
 seqeval==1.2.2
 natsort==7.1.1
+pandas
+pyarabic

build/lib/sinatools/CLI/utils/sentence_tokenizer.py

@@ -2,22 +2,22 @@
 About:
 ------
 
-The sentence_tokenizer command allows you to tokenize text into sentences using the SinaTools utility. It provides 
-flexibility in tokenizing at different punctuation marks, including dots, question marks, and exclamation marks. It also 
-allows tokenization at new lines.
+The sentence_tokenizer command tokenizes text into sentences using the
+SinaTools utility. It can split at dots, question marks, exclamation marks,
+and new lines.
 
 Usage:
 ------
-Below is the usage information that can be generated by running sentence_tokenizer --help.
+Run sentence_tokenizer --help to show this usage information.
 
 .. code-block:: none
 
     Usage:
         sentence_tokenizer --text=TEXT [options]
         sentence_tokenizer --file=FILE [options]
-        
+
 .. code-block:: none
-      
+
     Options:
       --text TEXT
             Text to be tokenized into sentences.
@@ -37,24 +37,48 @@
 
 .. code-block:: none
 
-  sentence_tokenizer --text "Your text here. Does it work? Yes! Try with new lines." --dot --question_mark --exclamation_mark
-  sentence_tokenizer --file "path/to/your/file.txt" --dot --question_mark --exclamation_mark
+  sentence_tokenizer --text "Your text here." --dot --question_mark
+  sentence_tokenizer --file "path/to/file.txt" --dot --question_mark
 
 """
 import argparse
+
 from sinatools.utils.tokenizer import sentence_tokenizer
 from sinatools.utils.readfile import read_file
 
+
 def main():
-    parser = argparse.ArgumentParser(description='Sentence Tokenization using SinaTools')
-
+    parser = argparse.ArgumentParser(
+        description='Sentence Tokenization using SinaTools'
+    )
+
     # Adding arguments for the text, file, and tokenization options
-    parser.add_argument('--text', type=str, help='Text to be tokenized into sentences')
-    parser.add_argument('--file', type=str, help='File containing the text to be tokenized into sentences')
+    parser.add_argument(
+        '--text',
+        type=str,
+        help='Text to be tokenized into sentences',
+    )
+    parser.add_argument(
+        '--file',
+        type=str,
+        help='File containing the text to be tokenized into sentences',
+    )
     parser.add_argument('--dot', action='store_true', help='Tokenize at dots')
-    parser.add_argument('--new_line', action='store_true', help='Tokenize at new lines')
-    parser.add_argument('--question_mark', action='store_true', help='Tokenize at question marks')
-    parser.add_argument('--exclamation_mark', action='store_true', help='Tokenize at exclamation marks')
+    parser.add_argument(
+        '--new_line',
+        action='store_true',
+        help='Tokenize at new lines',
+    )
+    parser.add_argument(
+        '--question_mark',
+        action='store_true',
+        help='Tokenize at question marks',
+    )
+    parser.add_argument(
+        '--exclamation_mark',
+        action='store_true',
+        help='Tokenize at exclamation marks',
+    )
 
     args = parser.parse_args()
 
@@ -63,15 +87,21 @@ def main():
         print("Either --text or --file argument must be provided.")
         return
 
-    text_content = args.text if args.text else read_file(args.file)
+    text_content = args.text if args.text else " ".join(read_file(args.file))
 
     # Perform sentence tokenization
-    sentences = sentence_tokenizer(" ".join(text_content), dot=args.dot, new_line=args.new_line, 
-                              question_mark=args.question_mark, exclamation_mark=args.exclamation_mark)
-
+    sentences = sentence_tokenizer(
+        text_content,
+        dot=args.dot,
+        new_line=args.new_line,
+        question_mark=args.question_mark,
+        exclamation_mark=args.exclamation_mark,
+    )
+
     # Print each sentence in a new line
     for sentence in sentences:
         print(sentence)
 
+
 if __name__ == '__main__':
     main()

build/lib/sinatools/environment.yml

@@ -111,7 +111,6 @@ dependencies:
   - tk=8.6.14=h39e8969_0
   - torchaudio=2.5.1=py311_cu124
   - torchtriton=3.1.0=py311
-  - torchvision=0.20.1=py311_cu124
   - typing_extensions=4.12.2=py311h06a4308_0
   - urllib3=2.2.3=py311h06a4308_0
   - wheel=0.44.0=py311h06a4308_0
@@ -171,9 +170,9 @@ dependencies:
       - tensorboard==2.19.0
       - tensorboard-data-server==0.7.2
       - threadpoolctl==3.5.0
-      - tokenizers==0.21.0
+      - tokenizers==0.19.1
       - tqdm==4.67.1
-      - transformers==4.47.1
+      - transformers==4.43.4
       - trl==0.12.0
       - tzdata==2024.2
       - werkzeug==3.1.3

setup.cfg

@@ -25,10 +25,7 @@ exclude = docs
 [aliases]
 test = pytest
 
-[tool:pytest]
-collect_ignore = ['setup.py']
-
-[egg_info]
-tag_build = 
-tag_date = 0
+[egg_info]
+tag_build =
+tag_date = 0
 

setup.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
-"""The setup script."""
-import os 
+"""The setup script."""
+import os
 from setuptools import setup, find_packages
 VERSION_FILE = os.path.join(os.path.dirname(__file__),
                             'sinatools',
@@ -18,9 +18,8 @@
     'requests',
     # 'regex',
     'pathlib',
-    # 'torch==2.5.1',
-    'transformers==4.47.1',
-    'torchvision==0.20.1',
+    'torch>=2.0',
+    'transformers>=4.0,<4.44',
     'seqeval==1.2.2',
     'natsort==7.1.1',
     'pandas',
@@ -39,7 +38,7 @@
 
 setup(
     entry_points={
-        'console_scripts':[
+        'console_scripts': [
             ('install_env='
                 'sinatools.install_env:main'),
             ('arStrip='
@@ -72,12 +71,12 @@
                 'sinatools.CLI.DataDownload.download_files:main'),
             ('corpus_entity_extractor='
                 'sinatools.CLI.ner.corpus_entity_extractor:main'),
-            ('text_dublication_detector='
-                'sinatools.CLI.utils.text_dublication_detector:main'),     
-            ('evaluate_synonyms='
-                'sinatools.CLI.synonyms.evaluate_synonyms:main'),  
-            ('extend_synonyms='
-                'sinatools.CLI.synonyms.extend_synonyms:main'),                    
+            ('text_dublication_detector='
+                'sinatools.CLI.utils.text_dublication_detector:main'),
+            ('evaluate_synonyms='
+                'sinatools.CLI.synonyms.evaluate_synonyms:main'),
+            ('extend_synonyms='
+                'sinatools.CLI.synonyms.extend_synonyms:main'),
             ('semantic_relatedness='
                 'sinatools.CLI.semantic_relatedness.compute_relatedness:main'),
             ('relation_extractor='
@@ -88,8 +87,11 @@
     package_data={'sinatools': ['data/*.pickle', 'environment.yml']},
     install_requires=requirements,
     license="MIT license",
-    description='Open-source Python toolkit for Arabic Natural Understanding, allowing people to integrate it in their system workflow.',
-    long_description = readme + "\n",
+    description=(
+        'Open-source Python toolkit for Arabic Natural Understanding, '
+        'allowing people to integrate it in their system workflow.'
+    ),
+    long_description=readme + "\n",
     long_description_content_type='text/markdown',
     include_package_data=True,
     keywords='sinatools',

sinatools/CLI/utils/sentence_tokenizer.py

@@ -2,22 +2,22 @@
 About:
 ------
 
-The sentence_tokenizer command allows you to tokenize text into sentences using the SinaTools utility. It provides 
-flexibility in tokenizing at different punctuation marks, including dots, question marks, and exclamation marks. It also 
-allows tokenization at new lines.
+The sentence_tokenizer command tokenizes text into sentences using the
+SinaTools utility. It can split at dots, question marks, exclamation marks,
+and new lines.
 
 Usage:
 ------
-Below is the usage information that can be generated by running sentence_tokenizer --help.
+Run sentence_tokenizer --help to show this usage information.
 
 .. code-block:: none
 
     Usage:
         sentence_tokenizer --text=TEXT [options]
         sentence_tokenizer --file=FILE [options]
-        
+
 .. code-block:: none
-      
+
     Options:
       --text TEXT
             Text to be tokenized into sentences.
@@ -37,24 +37,48 @@
 
 .. code-block:: none
 
-  sentence_tokenizer --text "Your text here. Does it work? Yes! Try with new lines." --dot --question_mark --exclamation_mark
-  sentence_tokenizer --file "path/to/your/file.txt" --dot --question_mark --exclamation_mark
+  sentence_tokenizer --text "Your text here." --dot --question_mark
+  sentence_tokenizer --file "path/to/file.txt" --dot --question_mark
 
 """
 import argparse
+
 from sinatools.utils.tokenizer import sentence_tokenizer
 from sinatools.utils.readfile import read_file
 
+
 def main():
-    parser = argparse.ArgumentParser(description='Sentence Tokenization using SinaTools')
-
+    parser = argparse.ArgumentParser(
+        description='Sentence Tokenization using SinaTools'
+    )
+
     # Adding arguments for the text, file, and tokenization options
-    parser.add_argument('--text', type=str, help='Text to be tokenized into sentences')
-    parser.add_argument('--file', type=str, help='File containing the text to be tokenized into sentences')
+    parser.add_argument(
+        '--text',
+        type=str,
+        help='Text to be tokenized into sentences',
+    )
+    parser.add_argument(
+        '--file',
+        type=str,
+        help='File containing the text to be tokenized into sentences',
+    )
     parser.add_argument('--dot', action='store_true', help='Tokenize at dots')
-    parser.add_argument('--new_line', action='store_true', help='Tokenize at new lines')
-    parser.add_argument('--question_mark', action='store_true', help='Tokenize at question marks')
-    parser.add_argument('--exclamation_mark', action='store_true', help='Tokenize at exclamation marks')
+    parser.add_argument(
+        '--new_line',
+        action='store_true',
+        help='Tokenize at new lines',
+    )
+    parser.add_argument(
+        '--question_mark',
+        action='store_true',
+        help='Tokenize at question marks',
+    )
+    parser.add_argument(
+        '--exclamation_mark',
+        action='store_true',
+        help='Tokenize at exclamation marks',
+    )
 
     args = parser.parse_args()
 
@@ -63,15 +87,21 @@ def main():
         print("Either --text or --file argument must be provided.")
         return
 
-    text_content = args.text if args.text else read_file(args.file)
+    text_content = args.text if args.text else " ".join(read_file(args.file))
 
     # Perform sentence tokenization
-    sentences = sentence_tokenizer(" ".join(text_content), dot=args.dot, new_line=args.new_line, 
-                              question_mark=args.question_mark, exclamation_mark=args.exclamation_mark)
-
+    sentences = sentence_tokenizer(
+        text_content,
+        dot=args.dot,
+        new_line=args.new_line,
+        question_mark=args.question_mark,
+        exclamation_mark=args.exclamation_mark,
+    )
+
     # Print each sentence in a new line
     for sentence in sentences:
         print(sentence)
 
+
 if __name__ == '__main__':
     main()