Merge Development into Main --> Release: BioProfileKit CLI v1.0

.dockerignore

@@ -64,10 +64,26 @@ compose.yaml
 # Data & Output files
 data/
 renders/
+*_renders/
 *.pdf
 *.csv
 *.xlsx
+benchmark/
+benchmark_data/
 
 # Project files (not needed in container)
 MANIFEST.in
-requirements.txt
+requirements.txt
+
+src/cython_wrapper/*.c
+src/cython_wrapper/*.so
+*.egg-info
+tests/
+.old_venv/
+.bioprofilekit
+iedb_test_report.html
+iedb_sub_renders.zip
+profile.out
+test.html
+tests_
+__pycache__

.gitignore

@@ -284,3 +284,6 @@ Temporary Items
 
 src/renders/
 /data/
+*_renders/
+*.zip
+renders/

Makefile

@@ -13,13 +13,14 @@ help:
 	@echo "$$help"
 
 install:
+	python -m pip install --upgrade pip
 	pip install -e .
 	python setup.py build_ext --inplace
 
 clean:
 	rm -rf build/ dist/ */*.egg-info/
-	find . -name ".so" -delete
-	find . -name ".c" -delete
+	find src/cython_wrapper -name "*.so" -delete
+	find src/cython_wrapper -name "*.c" -delete
 	find . -name "__pycache__" -type d -exec rm -rf {} +
 
 reinstall: clean install

README.md

@@ -1,27 +1,65 @@
 # BioProfileKit
-[![Python: 3.13](https://img.shields.io/badge/Python-3.13-green.svg)](https://www.python.org/downloads/release/python-3130/) [![License: MIT](https://img.shields.io/badge/License-MIT-darkred.svg)](https://opensource.org/licenses/MIT) 
+[![Python: 3.13](https://img.shields.io/badge/Python-3.13-green.svg)](https://www.python.org/downloads/release/python-3130/) [![License: MIT](https://img.shields.io/badge/License-MIT-darkred.svg)](https://opensource.org/licenses/MIT)
 [![pages-build-deployment](https://github.com/BioWarlock/BioProfileKit/actions/workflows/pages/pages-build-deployment/badge.svg?branch=main)](https://github.com/BioWarlock/BioProfileKit/actions/workflows/pages/pages-build-deployment)
 
 
 ## Overview
 
-BioProfileKit is a specialized bioinformatics tool that enables scientists to analyze large and diverse datasets. 
-Unlike traditional profilers, it offers customized analyzes for genomics, proteomics, transcriptomics, and metabolomics, providing sequence analysis and reports on nucleotide or amino acid distribution and abundance. 
-It includes advanced visualizations for pattern and anomaly detection, along with interactive dashboards showing key metrics. 
-Designed to be user-friendly, BioProfileKit is accessible to scientists without extensive data science skills.
-Example Results: [BioProfileKit Example](https://hansen-maria.github.io/BioProfileKit/)
+BioProfileKit is a domain-specific bioinformatics profiling tool for tabular biological data. It generates self-contained, interactive HTML reports covering exploratory data analysis, biological sequence profiling, and metadata validation — designed to be accessible to scientists without extensive data science skills.
+
+Example report: [BioProfileKit Example](https://hansen-maria.github.io/BioProfileKit/)
 
 ### Features
-- **EDA-Focused Analysis** - Offers a detailed overview of data structure, quality, and composition with automated detection of issues like missing values or correlations
-- **Specialized Sequence Profiling** - Examines DNA, RNA, and protein sequences using relevant metrics (such as GC content, k-mer frequencies, and amino acid composition)
-- **Biological Metadata Recognition** - Automatically identifies and verifies organism names, taxonomic identifiers, and biological annotations using controlled vocabularies from official databases
-- **Rich Visualizations** - Creates histograms and interactive charts to help quickly identify patterns
-- **Interactive HTML Reports** - Provides portable, user-friendly reports with dynamic filtering and cross-linked visualizations for seamless data exploration
-## WebApplication
 
+**Column-level EDA**
+- Automated column type inference (numeric, categorical, sequence, metadata)
+- Distributional metrics and quality flags per column type
+- Numeric quality metrics: IQR, outlier detection (adjusted IQR, modified Z-score, standard Z-score), zero/negative/infinity counts
+- Categorical imbalance metrics: class imbalance ratio, rare category detection, near-zero variance, top-1/top-5 coverage
+- Collapsible filter bar with text search and quality flag toggles (AND/OR logic)
+
+**Dataset-level EDA**
+- Correlation and pairwise structure analysis
+- Duplicate detection
+- Missing value pattern analysis: MCAR (Little's test), MAR (point-biserial correlation), MNAR (KS-test heuristic)
+- Interactive Plotly visualizations
+
+**Biological Sequence Profiling**
+- DNA/RNA: GC content, nucleotide distribution, k-mer frequencies, AT/GC skew, ambiguous base ratio, dinucleotide O/E ratios, codon completeness, low-complexity entropy, reverse complement duplicate detection
+- Protein: amino acid composition and distribution
+- Configurable k-mer size and top-N reporting
+
+**Biological Metadata Validation**
+- Organism name and taxonomic identifier verification (Cython-accelerated)
+- Functional annotation validation: GO terms and COG categories
+- Controlled vocabulary checks against official databases
+
+**Interactive HTML Reports**
+- Portable, single-file reports with no server dependency
+- Dynamic column filtering with quality flag toggles
+- Colorblind-safe theme (Okabe-Ito palette) with persistent toggle
+- WCAG AA accessible
+
+## Project Structure
+
+```
+BioProfileKit/
+├── src/
+│   ├── analysis/        # Column-level and dataset-level EDA
+│   ├── biological/      # Sequence profiling and metadata validation
+│   ├── cli/             # Entry point and report writer
+│   ├── cython_wrapper/  # Cython-accelerated components
+│   ├── data_utils/      # File I/O and remote data access
+│   ├── models/          # Data models per column type
+│   └── templates/       # Jinja templates for reports
+└── tests/               # Unit and integration tests
+```
+
+## Web Application
 
 https://bioprofilekit.computational.bio/
 
+
 ## Installation
 
 ```bash
@@ -34,30 +72,40 @@ pip install -e .
 python setup.py build_ext --inplace
 ```
 
-## Parameters
-Currently only supports .csv, .tsv, and .json as input files
-```bash 
- bioprofilekit -i input.csv
- Options:
-  -i,   --input PATH           Input file as .tsv, .csv or .json  [required]
-  -t,   --tax                  Enable taxonomy analysis
-  -f,   --func [cog|go]        Choose between COG or GO analysis, if validation is needed
-  -tc,  --target_column TEXT   Target column for further analysis
-  -k,   --kmer INTEGER         K-mer Size for sequence analysis. Default: 3
-  -n,   --top_n INTEGER        Top N entries analysis. Default: 20
-  -h,   --help                 Show this message and exit.
+
+## Usage
+
+Supports `.csv`, `.tsv`, and `.json` as input formats.
+
+```bash
+bioprofilekit -i input.csv
+
+Options:
+  -i,  --input PATH           Input file (.tsv, .csv, or .json)  [required]
+  -t,  --tax                  Enable taxonomy analysis
+  -f,  --func [cog|go]        Functional annotation validation (COG or GO)
+  -tc, --target_column TEXT   Target column for further analysis
+  -k,  --kmer INTEGER         K-mer size for sequence analysis  [default: 3]
+  -n,  --top_n INTEGER        Top N entries for reporting  [default: 20]
+  -h,  --help                 Show this message and exit.
 ```
 
+> [!WARNING]  
+> For high-dimensional datasets with many columns, report rendering times may increase significantly. In such cases, the hosted web application at https://bioprofilekit.computational.bio/ is recommended.
+
 ## Contributing
 
-Contributions to this project are welcome! Whether you find bugs, want to request features, or submit enhancements, please feel free to open an issue or submit a pull request. For major changes, it's recommended to discuss them first to ensure alignment with project goals.
-Please read the [`CODE OF CONDUCT`](CODE_OF_CONDUCT.md) to learn more about our guidelines and the contribution process.
+Contributions are welcome. For bug reports, feature requests, or enhancements, please open an issue or submit a pull request. For major changes, open an issue first to discuss alignment with project goals.
+
+Please read the [`CODE OF CONDUCT`](CODE_OF_CONDUCT.md) for contribution guidelines.
+
 
 ## License
 
-Licensed under MIT license ([LICENSE-MIT](LICENSE) or http://opensource.org/licenses/MIT). 
-Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in BioProfileKit by you, shall be licensed as above, without any additional terms or conditions.
+Licensed under the MIT license ([LICENSE](LICENSE) or https://opensource.org/licenses/MIT).
+Unless explicitly stated otherwise, any contribution intentionally submitted for inclusion in BioProfileKit shall be licensed under the same terms.
+
 
 ## Contact
 
-For inquiries or support regarding this project, you can reach out to the maintainers through GitHub issues or Discussion.
+For inquiries or support, reach out via GitHub Issues or Discussions.

__main__.py

@@ -1,9 +1,11 @@
-from app import cli
+from src.cli.app import cli
 
 if __name__ == '__main__':
-    cli(['--input', 'data/iedb_sub.tsv', '--tax', '-tc','bind_class'])
-    #cli(['-i', "data/Complete_nitrogenase.csv"])
-    #cli(['--input', 'data/upstream_sd.complete.tsv'])
-    #cli(['-i', "data/iedb_test.tsv"])
+    #cli(['--input', 'data/iedb.tsv', '--tax', '-tc','bind_class'])
+   # cli(['-i', "data/Complete_nitrogenase.csv"])
+    #cli(['--input', 'data/sorfdb_sub.tsv'])
+   # cli(['-i', "data/iedb_sub.tsv", "--tax"])
+    #cli(['-i', 'data/winequality-white.csv'])
 #    cli(['-i', "data/proteinGroups.tsv"])
-    #cli(['-i', 'data/sample_bakrep.tsv'])
+    cli(['-i', 'data/bakrep_sagalactiae.tsv'])
+    #cli(['-i', 'data/iedb_test.tsv', '-tc', 'bind_class'])

pyproject.toml

@@ -2,8 +2,8 @@
 requires = [
     "setuptools==82.0.0",
     "wheel>=0.44.0",
-    "cython>=3.2.1",
-    "numpy>=2.4.2"
+    "cython>=3.2.5",
+    "numpy>=2.4.6"
 ]
 build-backend = "setuptools.build_meta"
 
@@ -16,10 +16,9 @@ authors = [
     {name = "Sonja Diedrich", email = "sonja.diedrich@computational.bio.uni-giessen.de"},
     {name = "Maria Hansen", email = "maria.hansen@computational.bio.uni-giessen.de"}
 ]
-#license = "MIT"
-#license-files = ["LICENSE"]
+license = {file = "LICENSE"}
 requires-python = ">=3.13"
-keywords = ["bioinformatics", "data-analysis", "proteomics"]
+keywords = ["bioinformatics", "data-analysis", "proteomics", "EDA"]
 classifiers = [
     "Development Status :: 3 - Alpha",
     "Intended Audience :: Science/Research",
@@ -29,29 +28,34 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Bio-Informatics",
 ]
 dependencies = [
-    "cython==3.2.4",
-    "click>=8.3.1",
+    "cython==3.2.5",
+    "click>=8.3.3",
     "Jinja2==3.1.6",
-    "numpy==2.4.2",
-    "pandas==3.0.1",
-    "plotly==6.5.2",
+    "numpy==2.4.6",
+    "pandas==3.0.3",
+    "plotly==6.8.0",
     "scipy==1.17.1",
     "termcolor==3.3.0",
-    "pyarrow==23.0.0",
+    "pyarrow==24.0.0",
     "peptides==0.5.0",
     "iteration_utilities==0.13.0",
-    "requests==2.32.5",
-    "goatools==1.5.2",
-    "importlib_resources==6.5.2",
-    "biopython==1.86",
+    "requests==2.34.2",
+    "goatools==1.6.5",
+    "importlib_resources==7.1.0",
+    "biopython==1.87",
     "weblogo==3.9.0",
+    "scikit-learn==1.9.0"
 ]
 [project.urls]
 Homepage="https://github.com/BioWarlock/BioProfileKit/"
 Repository="https://github.com/BioWarlock/BioProfileKit/"
 
 [project.optional-dependencies]
 cli = ["click"]
+dev = [
+    "pytest>=9.1.0",
+    "pytest-cov>=6.0.0",
+]
 
 [tool.setuptools]
 package-dir = {"" = "src"}
@@ -60,7 +64,8 @@ include-package-data = true
 
 [tool.setuptools.package-data]
 "static" = ["static/**/*"]
-"templates"= ["*.jinja"]
+"templates" = ["*.jinja"]
+#"cython" = ["*.pyx"]
 
 [tool.setuptools.exclude-package-data]
 "*" = [
@@ -69,5 +74,26 @@ include-package-data = true
     "*.egg-info"
 ]
 
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+pythonpath = ["src"]
+addopts = "--tb=short -q"
+
+[tool.coverage.run]
+source = ["src"]
+omit = ["src/cython_wrapper/*"]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "if __name__ == .__main__.:",
+    "raise NotImplementedError",
+]
+show_missing = true
+skip_empty = true
+
 [project.scripts]
-bioprofilekit = "app:cli"
+bioprofilekit = "cli.app:cli"
+
+[pytest]
+strict = true

requirements.txt

@@ -1,6 +1,6 @@
 peptides~=0.5.0
 weblogo~=3.7.12
-requests~=2.32.5
+requests~=2.34.2
 goatools~=1.5.1
 pandas~=2.3.0
 jinja2~=3.1.6
@@ -11,4 +11,5 @@ termcolor~=3.1.0
 plotly~=6.3.0
 scipy~=1.16.1
 setuptools~=70.0.0
-matplotlib~=3.10.7
+matplotlib~=3.10.7
+pytest~=9.1.0

setup.py

@@ -1,5 +1,5 @@
 import sysconfig
-
+import numpy as np
 from Cython.Build import cythonize
 from setuptools import setup, Extension, find_packages
 
@@ -9,20 +9,23 @@ def get_python_include_dir():
 
 extensions = [
     Extension(
-        "qc_eda.basic.wrapper_utils",
-        ["src/qc_eda/basic/wrapper_utils.pyx"],
+        "cython_wrapper.wrapper_utils",
+        ["src/cython_wrapper/wrapper_utils.pyx"],
     ),
     Extension(
-            "qc_eda.basic.taxonomy_validator",
-            ["src/qc_eda/basic/taxonomy_validator.pyx"],
+        "cython_wrapper.taxonomy_validator",
+        ["src/cython_wrapper/taxonomy_validator.pyx"],
     ),
+    Extension("cython_wrapper.medcouple_fast",
+              ["src/cython_wrapper/medcouple_fast.pyx"],
+              include_dirs=[np.get_include()])
 ]
 
 setup(
     name="BioProfileKit",
     version="0.1",
     packages=find_packages(where="src"),
     package_dir={"": "src"},
-    ext_modules=cythonize(extensions, compiler_directives={'language_level': "3"}),
+    ext_modules=cythonize(extensions, compiler_directives={'language_level': "3"}, force=True),
     zip_safe=False,
 )

src/analysis/__init__.py

@@ -0,0 +1,6 @@
+from .overview import overview, column_overview
+from .numeric_analysis import numeric_columns
+from .categorical_analysis import categorical_columns
+from .multivariate_analysis import get_correlation, multivariate_analysis
+from .data_quality import check_mixed_types, check_suspect_values
+from .plot_utils import plot_overview