From c37003d6e04d5f70ad623f4ed1302f412af601c1 Mon Sep 17 00:00:00 2001 From: Sonja Diedrich Date: Tue, 7 Apr 2026 12:12:13 +0200 Subject: [PATCH 01/95] feat: added shape ratio and empty rows --- src/qc_eda/basic/numerical_data.py | 4 ++++ src/templates/numeric_overview.jinja | 8 ++++++++ src/utils/file_reader.py | 2 +- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/qc_eda/basic/numerical_data.py b/src/qc_eda/basic/numerical_data.py index 3f43512..15ce01f 100644 --- a/src/qc_eda/basic/numerical_data.py +++ b/src/qc_eda/basic/numerical_data.py @@ -34,8 +34,10 @@ class NumericalData: cols: int nulls: int nulls_percentage: float + empty_rows: int dup_row: int dup_col: int + ratio: float memory: float alerts: int @@ -101,8 +103,10 @@ def overview(df: pd.DataFrame, file) -> NumericalData: cols=df.shape[1], nulls=sum(df.isnull().sum()), nulls_percentage=100 if df.isnull().all().all() else round(sum(df.isnull().sum()) * 100 / df.size, 2), + empty_rows=df.replace("", np.nan).isna().all(axis=1).sum(), dup_row=int(df.duplicated().sum()), dup_col=int(df.columns.duplicated().sum()), + ratio=round(df.shape[0] / df.shape[1], 3), memory=int(df.memory_usage(deep=True).sum()), alerts=0 ) diff --git a/src/templates/numeric_overview.jinja b/src/templates/numeric_overview.jinja index c33389d..537c4b8 100644 --- a/src/templates/numeric_overview.jinja +++ b/src/templates/numeric_overview.jinja @@ -78,6 +78,10 @@ Missing values (%) {{ general.nulls_percentage }} + + Empty Rows + {{ general.empty_rows }} + Duplicate rows {{ general.dup_row }} @@ -86,6 +90,10 @@ Duplicate columns {{ general.dup_col }} + + Sample-Feature Ratio + {{ general.ratio }} + Memory usage {{ general.memory }} diff --git a/src/utils/file_reader.py b/src/utils/file_reader.py index 8ee2b9e..4f0a113 100644 --- a/src/utils/file_reader.py +++ b/src/utils/file_reader.py @@ -7,7 +7,7 @@ import click def read_file(file: click.Path) -> pd.DataFrame | None: - file = pathlib.Path(file).absolute() + file = pathlib.Path(file.__str__()).absolute() ext = pathlib.Path(file.__str__()).suffix seperator = _get_sep(file) From a193fbc2f03704f9113ea48a2737570c7169b47b Mon Sep 17 00:00:00 2001 From: Sonja Diedrich Date: Tue, 7 Apr 2026 12:34:03 +0200 Subject: [PATCH 02/95] fix: updated columns.jinja to keep negative high correlations --- __main__.py | 6 +++--- src/qc_eda/basic/numerical_data.py | 2 +- src/templates/columns.jinja | 10 +++++----- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/__main__.py b/__main__.py index 58d3e25..4003244 100644 --- a/__main__.py +++ b/__main__.py @@ -1,9 +1,9 @@ from app import cli if __name__ == '__main__': - cli(['--input', 'data/iedb_sub.tsv', '--tax', '-tc','bind_class']) + #cli(['--input', 'data/iedb_sub.tsv', '--tax', '-tc','bind_class']) #cli(['-i', "data/Complete_nitrogenase.csv"]) #cli(['--input', 'data/upstream_sd.complete.tsv']) - #cli(['-i', "data/iedb_test.tsv"]) + #cli(['-i', "data/iedb_sub.tsv"]) # cli(['-i', "data/proteinGroups.tsv"]) - #cli(['-i', 'data/sample_bakrep.tsv']) \ No newline at end of file + cli(['-i', 'data/sample_bakrep.tsv']) \ No newline at end of file diff --git a/src/qc_eda/basic/numerical_data.py b/src/qc_eda/basic/numerical_data.py index 15ce01f..67175e8 100644 --- a/src/qc_eda/basic/numerical_data.py +++ b/src/qc_eda/basic/numerical_data.py @@ -229,7 +229,7 @@ def get_correlation(df: pd.DataFrame, col) -> list | None: return None corr = df[ncols].corrwith(df[col], method='pearson') corr.drop(labels=col, inplace=True) - corr = corr.drop(corr[corr < .3].index) + corr = corr[corr.abs() >= 0.3] if corr.empty: return None return list(zip(corr.index, corr)) diff --git a/src/templates/columns.jinja b/src/templates/columns.jinja index 0ba21db..f495dd6 100644 --- a/src/templates/columns.jinja +++ b/src/templates/columns.jinja @@ -134,13 +134,13 @@ {% endif %} {% if col.correlation %} {% set ns = namespace(has_low=false, has_moderate=false, has_high=false) %} - {% for corr in col.correlation %} - {% if corr.1 < 0.5 %} - {% set ns.has_low = true %} - {% elif corr.1 < 0.7 %} + {% for corr in col.correlation %} + {% if corr.1 >= 0.7 or corr.1 <= -0.7 %} + {% set ns.has_high = true %} + {% elif corr.1 >= 0.5 or corr.1 <= -0.5 %} {% set ns.has_moderate = true %} {% else %} - {% set ns.has_high = true %} + {% set ns.has_low = true %} {% endif %} {% endfor %} {% if ns.has_low %} From 93681887f1b4f658d4ec189fd305436fc8193d84 Mon Sep 17 00:00:00 2001 From: Sonja Diedrich Date: Tue, 7 Apr 2026 14:21:46 +0200 Subject: [PATCH 03/95] feat: Added Cardinality ratio and cardinality dimension ratio. --- src/qc_eda/basic/numerical_data.py | 8 +++++--- src/templates/columns.jinja | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/qc_eda/basic/numerical_data.py b/src/qc_eda/basic/numerical_data.py index 67175e8..7cd460b 100644 --- a/src/qc_eda/basic/numerical_data.py +++ b/src/qc_eda/basic/numerical_data.py @@ -55,6 +55,7 @@ class ColumnOverview: describe_plot: str | None constant: bool | None correlation: list[str] | None + cardinality_dimension_ratio: float | None # taxonomy: bool @@ -92,8 +93,8 @@ class CategoricalColumns: value_counts: dict max_category_length: int min_category_length: int - memory: int cardinality_ratio: float + memory: int def overview(df: pd.DataFrame, file) -> NumericalData: @@ -128,6 +129,7 @@ def column_overview(df: pd.DataFrame, col) -> ColumnOverview: describe_plot=None if df[col].isnull().all() else plot_overview(df[col]), constant=True if (df[col].nunique() == 1) else False, correlation=get_correlation(df, col), + cardinality_dimension_ratio=round(df[col].nunique() / len(df), 3), ) # @@ -215,8 +217,8 @@ def categorical_columns(df: pd.DataFrame, col: str) -> CategoricalColumns: value_counts=df[col].value_counts().head(20).to_dict(), max_category_length=lengths.max(), min_category_length=lengths.min(), - memory=df[col].memory_usage(deep=True), - cardinality_ratio=round(cardinality_ratio, 3) + cardinality_ratio=round(cardinality_ratio, 3), + memory = df[col].memory_usage(deep=True), ) diff --git a/src/templates/columns.jinja b/src/templates/columns.jinja index f495dd6..134756d 100644 --- a/src/templates/columns.jinja +++ b/src/templates/columns.jinja @@ -172,9 +172,11 @@ Number of Values: {{ col.number }}
Number of Unique: {{ col.unique }}
Number of Missing: {{ col.missing }} ({{ col.missing_per }}%)
+ Cardinality Dimension Ratio: {{ col.cardinality_dimension_ratio }}
{% if col.type != 'object' or 'str' %} Number type: {{ col.type }}
{% endif %} + {% if col.sequence != 'None' %} Sequence type: {{ col.sequence }}
{% endif %} From 807ff1e2df68f008fb3404399c12cc14686f248e Mon Sep 17 00:00:00 2001 From: Julian Hahnfeld <84308019+jhahnfeld@users.noreply.github.com> Date: Tue, 14 Apr 2026 11:17:18 +0200 Subject: [PATCH 04/95] fix: unit column detection and badge --- src/qc_eda/biological/measurement_data.py | 2 +- src/templates/columns.jinja | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/qc_eda/biological/measurement_data.py b/src/qc_eda/biological/measurement_data.py index 65921e7..cfbde06 100644 --- a/src/qc_eda/biological/measurement_data.py +++ b/src/qc_eda/biological/measurement_data.py @@ -24,7 +24,7 @@ def measurement_columns(column_overview: ColumnOverview, df: pd.DataFrame) -> UN unit_counts=[{unit: 1}], with_measurement=False ) - if column_overview.type == 'object': + if column_overview.type == 'str': values = df[column_overview.name].dropna().unique().astype(str).tolist() if all(len(x) > 1 for x in values): measurement_and_unit = match_units(values, MEASUREMENTS.UNIT_COLUMN.value) diff --git a/src/templates/columns.jinja b/src/templates/columns.jinja index 134756d..8fee6f4 100644 --- a/src/templates/columns.jinja +++ b/src/templates/columns.jinja @@ -123,6 +123,8 @@ {% endif %} {% if col.sequence != 'None' %} Sequence + {% elif col.measurement_data %} + Unit {% elif col.type == 'str'%} Object {% elif col.type == 'bool'%} From 61b65c7d3dbfcd1768aa86233ec3c134706bc01d Mon Sep 17 00:00:00 2001 From: Sonja Diedrich Date: Thu, 16 Apr 2026 10:54:36 +0200 Subject: [PATCH 05/95] refactor: part 1 --- __main__.py | 6 +- pyproject.toml | 5 +- setup.py | 8 +- src/analysis/__init__.py | 6 + src/analysis/categorical_analysis.py | 38 +++ src/analysis/data_quality.py | 60 ++++ .../general.py => analysis/multivariate.py} | 149 ++++----- src/analysis/numeric_analysis.py | 64 ++++ src/analysis/overview.py | 46 +++ src/analysis/plot_utils.py | 21 ++ src/app.py | 115 ------- src/biological/__init__.py | 5 + .../biological/functional_annotation.py | 2 +- .../biological/measurement_data.py | 14 +- .../sequence_data.py} | 78 ++--- src/biological/sequence_detection.py | 54 +++ src/{qc_eda => }/biological/taxonomy.py | 0 src/cli/__init__.py | 1 + src/cli/app.py | 102 ++++++ src/cli/report_writer.py | 30 ++ .../basic => cython_wrapper}/__init__.py | 0 .../taxonomy_validator.pyx | 0 .../wrapper_utils.pyx | 0 src/data_utils/__init__.py | 2 + src/{utils => data_utils}/file_reader.py | 0 .../remote_data.py} | 0 src/enums/__init__.py | 2 + .../basic => enums}/measurement_enum.py | 0 src/{qc_eda/basic => enums}/sequence_enum.py | 0 src/models/__init__.py | 5 + src/models/categorical.py | 17 + src/models/measurement.py | 9 + src/models/numeric.py | 23 ++ src/models/overview.py | 34 ++ src/models/sequence.py | 32 ++ src/qc_eda/basic/numerical_data.py | 308 ------------------ src/qc_eda/biological/__init__.py | 0 src/templates/columns.jinja | 57 +++- src/utils/__init__.py | 0 39 files changed, 694 insertions(+), 599 deletions(-) create mode 100644 src/analysis/__init__.py create mode 100644 src/analysis/categorical_analysis.py create mode 100644 src/analysis/data_quality.py rename src/{qc_eda/basic/general.py => analysis/multivariate.py} (53%) create mode 100644 src/analysis/numeric_analysis.py create mode 100644 src/analysis/overview.py create mode 100644 src/analysis/plot_utils.py delete mode 100644 src/app.py create mode 100644 src/biological/__init__.py rename src/{qc_eda => }/biological/functional_annotation.py (95%) rename src/{qc_eda => }/biological/measurement_data.py (86%) rename src/{qc_eda/biological/biological_data.py => biological/sequence_data.py} (79%) create mode 100644 src/biological/sequence_detection.py rename src/{qc_eda => }/biological/taxonomy.py (100%) create mode 100644 src/cli/__init__.py create mode 100644 src/cli/app.py create mode 100644 src/cli/report_writer.py rename src/{qc_eda/basic => cython_wrapper}/__init__.py (100%) rename src/{qc_eda/basic => cython_wrapper}/taxonomy_validator.pyx (100%) rename src/{qc_eda/basic => cython_wrapper}/wrapper_utils.pyx (100%) create mode 100644 src/data_utils/__init__.py rename src/{utils => data_utils}/file_reader.py (100%) rename src/{utils/download_metadata.py => data_utils/remote_data.py} (100%) create mode 100644 src/enums/__init__.py rename src/{qc_eda/basic => enums}/measurement_enum.py (100%) rename src/{qc_eda/basic => enums}/sequence_enum.py (100%) create mode 100644 src/models/__init__.py create mode 100644 src/models/categorical.py create mode 100644 src/models/measurement.py create mode 100644 src/models/numeric.py create mode 100644 src/models/overview.py create mode 100644 src/models/sequence.py delete mode 100644 src/qc_eda/basic/numerical_data.py delete mode 100644 src/qc_eda/biological/__init__.py delete mode 100644 src/utils/__init__.py diff --git a/__main__.py b/__main__.py index 4003244..e5971b1 100644 --- a/__main__.py +++ b/__main__.py @@ -1,9 +1,9 @@ -from app import cli +from src.cli.app import cli if __name__ == '__main__': #cli(['--input', 'data/iedb_sub.tsv', '--tax', '-tc','bind_class']) #cli(['-i', "data/Complete_nitrogenase.csv"]) #cli(['--input', 'data/upstream_sd.complete.tsv']) - #cli(['-i', "data/iedb_sub.tsv"]) + cli(['-i', "data/iedb_test.tsv"]) # cli(['-i', "data/proteinGroups.tsv"]) - cli(['-i', 'data/sample_bakrep.tsv']) \ No newline at end of file + #cli(['-i', 'data/sample_bakrep.tsv']) \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 663ba06..6714213 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,7 +60,8 @@ include-package-data = true [tool.setuptools.package-data] "static" = ["static/**/*"] -"templates"= ["*.jinja"] +"templates" = ["*.jinja"] +#"cython" = ["*.pyx"] [tool.setuptools.exclude-package-data] "*" = [ @@ -70,4 +71,4 @@ include-package-data = true ] [project.scripts] -bioprofilekit = "app:cli" +bioprofilekit = "cli.app:cli" diff --git a/setup.py b/setup.py index c9c4176..4af58d4 100644 --- a/setup.py +++ b/setup.py @@ -9,12 +9,12 @@ def get_python_include_dir(): extensions = [ Extension( - "qc_eda.basic.wrapper_utils", - ["src/qc_eda/basic/wrapper_utils.pyx"], + "cython_wrapper.wrapper_utils", + ["src/cython_wrapper/wrapper_utils.pyx"], ), Extension( - "qc_eda.basic.taxonomy_validator", - ["src/qc_eda/basic/taxonomy_validator.pyx"], + "cython_wrapper.taxonomy_validator", + ["src/cython_wrapper/taxonomy_validator.pyx"], ), ] diff --git a/src/analysis/__init__.py b/src/analysis/__init__.py new file mode 100644 index 0000000..d1447cb --- /dev/null +++ b/src/analysis/__init__.py @@ -0,0 +1,6 @@ +from .overview import overview, column_overview +from .numeric_analysis import numeric_columns +from .categorical_analysis import categorical_columns +from .multivariate import get_correlation, general_plots +from .data_quality import check_mixed_types, check_suspect_values +from .plot_utils import plot_overview \ No newline at end of file diff --git a/src/analysis/categorical_analysis.py b/src/analysis/categorical_analysis.py new file mode 100644 index 0000000..c17bdf2 --- /dev/null +++ b/src/analysis/categorical_analysis.py @@ -0,0 +1,38 @@ +import numpy as np +import pandas as pd + +from models.categorical import CategoricalColumns + + +def categorical_columns(df: pd.DataFrame, col: str) -> CategoricalColumns: + value_counts = df[col].value_counts() + n = int(df[col].notna().sum()) + frequencies = value_counts / n if n > 0 else value_counts + if n > 0: + entropy = -(frequencies * np.log2(frequencies)).sum() + gini = 1 - (frequencies ** 2).sum() + simpson = 1 / (frequencies ** 2).sum() + cardinality_ratio = df[col].nunique() / n + mode_value = df[col].mode().iloc[0] + else: + entropy = np.nan + gini = np.nan + simpson = np.nan + cardinality_ratio = np.nan + mode_value = "" + lengths = df[col].astype(str).str.len() + + return CategoricalColumns( + name=col, + unique_categories=df[col].nunique(), + mode=mode_value, + entropy=round(entropy, 2), + frequencies=df[col].value_counts(normalize=True).head(20).to_dict(), + gini=round(gini, 2), + simpson_diversity=round(simpson, 2), + value_counts=df[col].value_counts().head(20).to_dict(), + max_category_length=lengths.max(), + min_category_length=lengths.min(), + cardinality_ratio=round(cardinality_ratio, 3), + memory = df[col].memory_usage(deep=True), + ) diff --git a/src/analysis/data_quality.py b/src/analysis/data_quality.py new file mode 100644 index 0000000..bc36646 --- /dev/null +++ b/src/analysis/data_quality.py @@ -0,0 +1,60 @@ +import numpy as np +import pandas as pd +from pandas.api.types import infer_dtype + + +def check_mixed_types(df, col): + series = df[col].dropna() + if series.empty: + return None + + inferred = infer_dtype(series) + + if 'mixed' in inferred: + is_numeric = series.apply(lambda x: isinstance(x, (int, float, complex))) + numeric_count = is_numeric.sum() + string_count = len(series) - numeric_count + if numeric_count == 0 or string_count == 0: + return None + if numeric_count >= string_count: + return series[~is_numeric].astype(str).tolist() + else: + return series[is_numeric].astype(str).tolist() + + if str(series.dtype) in ('str', 'string', 'object'): + numeric_mask = pd.to_numeric(series, errors='coerce').notna() & series.notna() + numeric_count = numeric_mask.sum() + string_count = len(series) - numeric_count + if numeric_count == 0 or string_count == 0: + return None + if numeric_count <= string_count: + return series[numeric_mask].tolist() + else: + return series[~numeric_mask].tolist() + + return None + + +def check_suspect_values(df, col): + series = df[col].dropna() + if series.empty: + return None + + dtype_str = str(series.dtype) + if dtype_str in ('str', 'string', 'object'): + numeric_mask = pd.to_numeric(series, errors='coerce').notna() & series.notna() + numeric_count = numeric_mask.sum() + if numeric_count == 0: + return None + if numeric_count / len(series) < 0.15: + return series[numeric_mask].tolist() + return None + + if pd.api.types.is_numeric_dtype(series): + sentinels = {np.inf, -np.inf, np.nan, float('inf'), float('-inf'), float('nan')} + found = [v for v in sentinels if v in series.values] + if found: + return [str(v) for v in found] + return None + + return None \ No newline at end of file diff --git a/src/qc_eda/basic/general.py b/src/analysis/multivariate.py similarity index 53% rename from src/qc_eda/basic/general.py rename to src/analysis/multivariate.py index 1bd13d0..de9b88d 100644 --- a/src/qc_eda/basic/general.py +++ b/src/analysis/multivariate.py @@ -1,7 +1,11 @@ -import plotly.express as px +from dataclasses import dataclass + import pandas as pd +import plotly.express as px import plotly.graph_objects as go -from dataclasses import dataclass + +from .plot_utils import apply_standard_axes + @dataclass class GeneralPlots: @@ -12,6 +16,7 @@ class GeneralPlots: boxplot: str scatter_matrix: str + def general_plots(df: pd.DataFrame, target: str) -> GeneralPlots: return GeneralPlots( correlation_heatmap=correlation_heatmap(df), @@ -19,143 +24,107 @@ def general_plots(df: pd.DataFrame, target: str) -> GeneralPlots: missing_values_barchart=missing_values_barchart(df), balance_plot=balance_plot(df, target) if target else None, boxplot=boxplot(df), - scatter_matrix=scatter_matrix(df) + scatter_matrix=scatter_matrix(df), ) + +def get_correlation(df: pd.DataFrame, col) -> list | None: + ncols = df.select_dtypes(include='number').dropna(axis=1, how='all').columns + if col in ncols: + std = df[ncols].std(ddof=0) + ncols = std[std > 0].index + if col not in ncols: + return None + corr = df[ncols].corrwith(df[col], method='pearson') + corr.drop(labels=col, inplace=True) + corr = corr[corr.abs() >= 0.3] + if corr.empty: + return None + return list(zip(corr.index, corr)) + return None + + def correlation_heatmap(df: pd.DataFrame): df_numeric = df.select_dtypes(include=['float64', 'int64']).dropna(axis=1, how='all') if not df_numeric.empty: std = df_numeric.std(ddof=0) df_numeric = df_numeric.loc[:, std[std > 0].index] - corr_matrix = df_numeric.corr() - corr_matrix = round(corr_matrix, 3) - fig = px.imshow(corr_matrix, text_auto=True, labels=dict(color="Correlation"), color_continuous_scale="RdBu_r", aspect="auto", height=700) + corr_matrix = round(df_numeric.corr(), 3) + fig = px.imshow(corr_matrix, text_auto=True, + labels=dict(color="Correlation"), + color_continuous_scale="RdBu_r", aspect="auto", height=700) fig.update_layout(title="Correlation Heatmap") return fig.to_html(full_html=False, include_plotlyjs=False) def missing_matrix(df: pd.DataFrame): missing_values = df.isnull().astype(int) - - fig = px.imshow( - missing_values, - labels=dict(color="Missing Values"), - aspect="auto", - color_continuous_scale="blues_r", - title="Missing Values Matrix" - ) + fig = px.imshow(missing_values, labels=dict(color="Missing Values"), + aspect="auto", color_continuous_scale="blues_r", + title="Missing Values Matrix") fig.update_yaxes(autorange='reversed') fig.update_layout(coloraxis_showscale=False) - - return fig.to_html(full_html=False, include_plotlyjs=False) + def missing_values_barchart(df: pd.DataFrame): missing_counts = df.isna().sum() - fig = px.bar(x=missing_counts.index, y=missing_counts.values, labels={'x': 'Columns', 'y': 'Missing Values'},color_discrete_sequence=['#0F65A0']) + fig = px.bar(x=missing_counts.index, y=missing_counts.values, + labels={'x': 'Columns', 'y': 'Missing Values'}, + color_discrete_sequence=['#0F65A0']) fig.update_layout(title="Missing Values per Column", bargap=0.2, plot_bgcolor='white') - fig.update_xaxes( - tickangle=-45, - mirror=True, - ticks='outside', - showline=True, - linecolor='black', - gridcolor='lightgrey' - ) - fig.update_yaxes( - mirror=True, - ticks='outside', - showline=True, - linecolor='black', - gridcolor='lightgrey' - ) - + apply_standard_axes(fig, tick_angle=-45) return fig.to_html(full_html=False, include_plotlyjs=False) + def balance_plot(df, target): fig = px.histogram(df, x=target, color_discrete_sequence=["#0F65A0"], text_auto=True) - fig.update_layout( - title="Class Balance (Target Distribution)", - bargap=0.2, - plot_bgcolor="white" - ) - fig.update_xaxes( - mirror=True, - ticks="outside", - showline=True, - linecolor="black", - gridcolor="lightgrey" - ) - fig.update_yaxes( - mirror=True, - ticks="outside", - showline=True, - linecolor="black", - gridcolor="lightgrey" - ) + fig.update_layout(title="Class Balance (Target Distribution)", bargap=0.2, plot_bgcolor="white") + apply_standard_axes(fig) return fig.to_html(full_html=False, include_plotlyjs=False) + def boxplot(df: pd.DataFrame): df = df.select_dtypes(include=['float64', 'int64']).dropna(axis=1, how='all') fig = go.Figure() - for col in df: fig.add_trace(go.Box(y=df[col].values, name=df[col].name)) - fig.update_yaxes(type="log", title="Logarithmic",showticklabels=False) + fig.update_yaxes(type="log", title="Logarithmic", showticklabels=False) fig.update_layout( xaxis=dict(rangeslider=dict(visible=True), type="linear"), - title="Boxplot", - xaxis_title="Columns", - yaxis_title="Values", - legend_title="Columns" + title="Boxplot", xaxis_title="Columns", + yaxis_title="Values", legend_title="Columns", ) - return fig.to_html(full_html=False, include_plotlyjs=False) + def scatter_matrix(df: pd.DataFrame): - df = df.select_dtypes(include=['float64', 'int64']).dropna(axis=1, how='all') + df = df.select_dtypes(include=['float64', 'int64']).dropna(axis=1, how='all') fig = px.scatter_matrix(df, color_discrete_sequence=["#0F65A0"], height=750) + fig.update_traces(diagonal_visible=False, + marker=dict(size=2, opacity=0.5, color="#0F65A0")) + fig.update_layout(title="Scatter Matrix", xaxis_title="Columns", + plot_bgcolor="white", bargap=0.2, dragmode="select") - fig.update_traces( - diagonal_visible=False, - marker=dict(size=2, opacity=0.5, color="#0F65A0") - ) - - # Layout anpassen - fig.update_layout( - title="Scatter Matrix", - xaxis_title="Columns", - plot_bgcolor="white", - bargap=0.2, - dragmode="select" - ) n_vars = len(df.columns) for i in range(1, n_vars + 1): for j in range(1, n_vars + 1): if i == 1: fig.update_layout({f'xaxis{j if j > 1 else ""}': dict( - mirror=True, - ticks="outside", - showline=True, - linecolor="black", - linewidth=1, - gridcolor="lightgrey", - title_standoff=25 + mirror=True, ticks="outside", showline=True, + linecolor="black", linewidth=1, gridcolor="lightgrey", + title_standoff=25, )}) if j == 1: fig.update_layout({f'yaxis{i if i > 1 else ""}': dict( - mirror=True, - ticks="outside", - showline=True, - linecolor="black", - linewidth=1, - gridcolor="lightgrey", - title_standoff=25 + mirror=True, ticks="outside", showline=True, + linecolor="black", linewidth=1, gridcolor="lightgrey", + title_standoff=25, )}) + fig.for_each_annotation(lambda a: a.update( textangle=0, x=-0.08 if a.textangle == 90 or 'y' in str(a.yref) else a.x, - y=-0.08 if a.textangle == 0 and 'x' in str(a.xref) else a.y + y=-0.08 if a.textangle == 0 and 'x' in str(a.xref) else a.y, )) - - return fig.to_html(full_html=False, include_plotlyjs=False) + return fig.to_html(full_html=False, include_plotlyjs=False) \ No newline at end of file diff --git a/src/analysis/numeric_analysis.py b/src/analysis/numeric_analysis.py new file mode 100644 index 0000000..e24aa04 --- /dev/null +++ b/src/analysis/numeric_analysis.py @@ -0,0 +1,64 @@ +import numpy as np +import pandas as pd +from scipy import stats + +from models.numeric import NumericColumns + + +def _safe_round(val, decimals=2): + return round(val, decimals) if np.isfinite(val) else np.nan + + +def numeric_columns(df: pd.DataFrame, col) -> NumericColumns: + series = df[col].dropna() + + if series.empty: + coefficient_of_variation = np.nan + quantiles = np.array([np.nan, np.nan, np.nan]) + mode_value = np.nan + value_counts = {} + frequencies = {} + mad_value = np.nan + min_value = np.nan + max_value = np.nan + mean_value = np.nan + median_value = np.nan + std_value = np.nan + sum_value = np.nan + kurtosis_value = np.nan + skewness_value = np.nan + else: + mean_value = float(series.mean()) + std_value = float(series.std()) + coefficient_of_variation = np.nan if mean_value == 0 else float(std_value / abs(mean_value)) + quantiles = np.array(series.quantile([0.25, 0.5, 0.75]).to_list(), dtype=float) + mode_series = series.mode() + mode_value = float(mode_series.iloc[0]) if not mode_series.empty else np.nan + value_counts = series.value_counts().head(20).to_dict() + frequencies = series.value_counts(normalize=True).head(20).to_dict() + mad_value = float(stats.median_abs_deviation(series, nan_policy='omit')) + min_value = float(series.min()) + max_value = float(series.max()) + median_value = float(series.median()) + sum_value = float(series.sum()) + kurtosis_value = float(series.kurtosis()) + skewness_value = float(series.skew()) + + return NumericColumns( + name=col, + min=round(min_value, 2) if np.isfinite(min_value) else np.nan, + max=round(max_value, 2) if np.isfinite(max_value) else np.nan, + mean=round(mean_value, 2) if np.isfinite(mean_value) else np.nan, + median=round(median_value, 2) if np.isfinite(median_value) else np.nan, + mode=round(mode_value, 2) if np.isfinite(mode_value) else np.nan, + std=round(std_value, 2) if np.isfinite(std_value) else np.nan, + sum=round(sum_value, 2) if np.isfinite(sum_value) else np.nan, + kurtosis=round(kurtosis_value, 2) if np.isfinite(kurtosis_value) else np.nan, + skewness=round(skewness_value, 2) if np.isfinite(skewness_value) else np.nan, + coefficient_of_variation=round(coefficient_of_variation, 2) if np.isfinite(coefficient_of_variation) else np.nan, + mad=mad_value if np.isfinite(mad_value) else np.nan, + quantiles=quantiles, + memory=df[col].memory_usage(deep=True), + value_counts=value_counts, + frequencies=frequencies + ) \ No newline at end of file diff --git a/src/analysis/overview.py b/src/analysis/overview.py new file mode 100644 index 0000000..3e2c251 --- /dev/null +++ b/src/analysis/overview.py @@ -0,0 +1,46 @@ +import numpy as np +import pandas as pd + +from biological.sequence_detection import check_sequence +from models.overview import DatasetSummary, ColumnOverview +from .data_quality import check_mixed_types, check_suspect_values +from .plot_utils import plot_overview +from .multivariate import get_correlation + + +def overview(df: pd.DataFrame, file) -> DatasetSummary: + return DatasetSummary( + filename=file, + rows=df.shape[0], + cols=df.shape[1], + nulls=sum(df.isnull().sum()), + nulls_percentage=100 if df.isnull().all().all() else round(sum(df.isnull().sum()) * 100 / df.size, 2), + empty_rows=df.replace("", np.nan).isna().all(axis=1).sum(), + dup_row=int(df.duplicated().sum()), + dup_col=int(df.columns.duplicated().sum()), + ratio=round(df.shape[0] / df.shape[1], 3), + memory=int(df.memory_usage(deep=True).sum()), + alerts=0, + ) + + +def column_overview(df: pd.DataFrame, col) -> ColumnOverview: + seq_type, invalid = check_sequence(df, col) + if seq_type != "None": + print(f"Column: {col:10s} is of type: {seq_type:10s} with invalid sequences: {invalid}.") + return ColumnOverview( + name=col, + number=int(df[col].notnull().sum()), + unique=df[col].nunique(), + missing=int(df[col].isnull().sum()), + missing_per=100 if df[col].isnull().all() else round(sum(df[col].isnull()) * 100 / df[col].size, 2), + type=str(df[col].dtype), + sequence=seq_type, + invalid_seqs=invalid, + mixed_types=check_mixed_types(df, col) if seq_type == "None" else None, + suspect_values=check_suspect_values(df, col) if seq_type == "None" else None, + describe_plot=None if df[col].isnull().all() else plot_overview(df[col]), + constant=(df[col].nunique() == 1), + correlation=get_correlation(df, col), + cardinality_dimension_ratio=round(df[col].nunique() / len(df), 3), + ) \ No newline at end of file diff --git a/src/analysis/plot_utils.py b/src/analysis/plot_utils.py new file mode 100644 index 0000000..eeebba5 --- /dev/null +++ b/src/analysis/plot_utils.py @@ -0,0 +1,21 @@ +import plotly.express as px + + +def apply_standard_axes(fig, tick_angle=None): + x_kwargs = dict(mirror=True, ticks='outside', showline=True, + linecolor='black', gridcolor='lightgrey') + if tick_angle is not None: + x_kwargs['tickangle'] = tick_angle + fig.update_xaxes(**x_kwargs) + fig.update_yaxes(mirror=True, ticks='outside', showline=True, + linecolor='black', gridcolor='lightgrey') + + +def plot_overview(col): + if col.dtype != 'object': + bins = None if col.nunique() < 10 else 10 + fig = px.histogram(x=col, color_discrete_sequence=['#0F65A0']) + fig.update_layout(bargap=0.2, plot_bgcolor='white') + apply_standard_axes(fig) + return fig.to_html(full_html=False, include_plotlyjs=False) + return None \ No newline at end of file diff --git a/src/app.py b/src/app.py deleted file mode 100644 index 6bf425f..0000000 --- a/src/app.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python3 - -import shutil -from pathlib import Path - -import click -import pandas as pd -from jinja2 import Environment, FileSystemLoader -from termcolor import colored -from importlib_resources import files -from qc_eda.basic.numerical_data import overview, column_overview, numeric_columns, categorical_columns -from qc_eda.biological.biological_data import dna_rna_columns, protein_columns -from qc_eda.biological.functional_annotation import annotation_flags -from qc_eda.biological.measurement_data import measurement_columns -from qc_eda.biological.taxonomy import taxonomy_flags -from utils.download_metadata import get_tax_ids -from utils.file_reader import read_file, parse_parquet -from qc_eda.basic.general import general_plots - -CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) -TEMPLATE_DIR = files("templates").joinpath() -STATIC_DIR = files("static").joinpath() -env = Environment(loader=FileSystemLoader(str(TEMPLATE_DIR)), autoescape=True) - - -@click.group(context_settings=CONTEXT_SETTINGS, invoke_without_command=True) -@click.option("-i", "--input", type=click.Path(exists=True, resolve_path=True), required=True, - help="Input file as .tsv, .csv or .json", ) -@click.option('-t','--tax', is_flag=True,help='Enable taxonomy analysis') -@click.option('-f', '--func', type=click.Choice(['cog','go']), help='Enable functional annotation analysis. Choose between cog or go') -@click.option('-tc', '--target_column', type=str, help='Target column for Analysis') -@click.option('-k', '--kmer', type=int, default=3, help="K-mer Size for sequence analysis") -@click.option('-n', '--top_n', type=int, default=20, help="Top N entries analysis") -def cli(input: str, tax: bool = False, func: str = None, target_column: str = None, kmer: int = None, top_n: int = None): - input_path = Path(input) - print(colored(f'Reading file {input_path.name}', 'green')) - df = read_file(input_path) if input_path.suffix != '.parquet' else parse_parquet(input_path) - general = overview(df, input_path.name) - plots = general_plots(df, target_column) - - dups = df[df.duplicated(keep=False)] - dups = dups.reset_index() - duplicates_table = dups.to_html(classes="table table-hover table-responsive nowrap", border="0", - table_id="dup_table", - index=False) - print(colored(f'Analyse {len(df.columns)} columns', 'blue')) - - column_overviews = [column_overview(df, col) for col in df.columns] - tax_df = None - if tax: - tax_df = get_tax_ids() - for col_overview in column_overviews: - if tax and tax_df is not None: - col_overview.taxonomy = taxonomy_flags(df, col_overview.name, tax_df) - if func and tax_df is not None: - col_overview.annotation = [annotation_flags(df, col_overview.name, func)] - if hasattr(col_overview, "top_10") and isinstance(col_overview.top_10, pd.Series): - col_overview.top_10_items = list(col_overview.top_10.items()) - if col_overview.sequence == 'dna': - print(colored(f'Analyzing DNA/RNA sequences in column: {col_overview.name}', 'cyan')) - bio_data = dna_rna_columns(df[col_overview.name], k=kmer, top_n=top_n) - col_overview.dna_rna_data = bio_data - elif col_overview.sequence == 'protein': - print(colored(f'Analyzing protein sequences in column: {col_overview.name}', 'cyan')) - bio_data = protein_columns(df[col_overview.name], k=kmer, top_n=top_n) - col_overview.protein_data = bio_data - else: - col_overview.dna_rna_data = None - col_overview.protein_data = None - - if col_overview.sequence == 'None': - measurement_data = measurement_columns(col_overview, df) - if measurement_data: - print(colored(f'Analyzing lab measurements in column: {col_overview.name}', 'cyan')) - col_overview.measurement_data = measurement_data - else: - col_overview.measurement_data = None - else: - col_overview.measurement_data = None - - print(colored(f'Analyse {len(df.select_dtypes(include="number").columns)} numeric columns ', 'blue')) - - #ToDo: check cat_columns - exclude_cols = list() - #numeric_overviews = [if df[col].isnull().all() numeric_columns(df, col) for col in df.select_dtypes(include='number').columns] - numeric_overviews = [numeric_columns(df, col) if not df[col].isnull().all() else exclude_cols.append(col) for col in - df.select_dtypes(include="number").columns] - cat_columns = [col for col in df.select_dtypes(include=['str','object', 'bool', 'int64', 'float64']).columns if - any(i.sequence == 'None' for i in column_overviews if i.name == col)] - print(colored(f'Analyse {len(cat_columns)} object columns ', 'blue')) - print(exclude_cols) - categorical_overviews = [categorical_columns(df, col) if col not in exclude_cols else None for col in cat_columns] - - output_path = Path(input_path.stem+"_renders") - output_path.mkdir(parents=True, exist_ok=True) - - shutil.copytree(str(STATIC_DIR), output_path.as_posix()+"/static/", dirs_exist_ok=True) - - landing_template = env.get_template('LandingPage.jinja') - numeric_template = env.get_template('numeric_overview.jinja') - columns = env.get_template('columns.jinja') - stats = env.get_template('general_statistics.jinja') - - print(colored('Writing report …', 'green')) - with open(output_path.as_posix()+"/index.html", 'w', encoding="utf-8") as output: - print(landing_template.render(), file=output) - - with open(output_path.as_posix()+"/numeric_data.html", "w",encoding="utf-8") as output: - print(numeric_template.render(general=general, dups=duplicates_table), file=output) - - with open(output_path.as_posix()+"/columns.html", "w",encoding="utf-8") as output: - print(columns.render(columns=column_overviews, overview=numeric_overviews, categorical=categorical_overviews, top_n=top_n), file=output) - - with open(output_path.as_posix()+"/general_statistics.html", "w",encoding="utf-8") as output: - print(stats.render(plots=plots), file=output) diff --git a/src/biological/__init__.py b/src/biological/__init__.py new file mode 100644 index 0000000..85080d1 --- /dev/null +++ b/src/biological/__init__.py @@ -0,0 +1,5 @@ +from .sequence_detection import check_sequence +from .sequence_data import dna_rna_columns, protein_columns +from .taxonomy import taxonomy_flags +from .functional_annotation import annotation_flags +from .measurement_data import measurement_columns \ No newline at end of file diff --git a/src/qc_eda/biological/functional_annotation.py b/src/biological/functional_annotation.py similarity index 95% rename from src/qc_eda/biological/functional_annotation.py rename to src/biological/functional_annotation.py index 3a726d4..07c5817 100644 --- a/src/qc_eda/biological/functional_annotation.py +++ b/src/biological/functional_annotation.py @@ -1,6 +1,6 @@ from dataclasses import dataclass import pandas as pd -from utils.download_metadata import get_clusters_of_orthologous_groups, get_gene_ontology +from data_utils.remote_data import get_clusters_of_orthologous_groups, get_gene_ontology @dataclass class AnnotationFlags: diff --git a/src/qc_eda/biological/measurement_data.py b/src/biological/measurement_data.py similarity index 86% rename from src/qc_eda/biological/measurement_data.py rename to src/biological/measurement_data.py index cfbde06..98fbeab 100644 --- a/src/qc_eda/biological/measurement_data.py +++ b/src/biological/measurement_data.py @@ -1,17 +1,11 @@ import re -from dataclasses import dataclass -from typing import List, Dict +from typing import List import pandas as pd -from qc_eda.basic.numerical_data import ColumnOverview -from ..basic.measurement_enum import MEASUREMENTS - -@dataclass -class UNITColumns: - units: List[str] - unit_counts: List[Dict[str | None, int]] - with_measurement: bool +from models.measurement import UNITColumns +from models.overview import ColumnOverview +from enums.measurement_enum import MEASUREMENTS def measurement_columns(column_overview: ColumnOverview, df: pd.DataFrame) -> UNITColumns | bool: diff --git a/src/qc_eda/biological/biological_data.py b/src/biological/sequence_data.py similarity index 79% rename from src/qc_eda/biological/biological_data.py rename to src/biological/sequence_data.py index 2fecb06..3253243 100644 --- a/src/qc_eda/biological/biological_data.py +++ b/src/biological/sequence_data.py @@ -1,10 +1,10 @@ import tempfile import warnings from collections import defaultdict, Counter -from dataclasses import dataclass from itertools import chain from pathlib import Path from typing import Tuple + from Bio import motifs from weblogo import * import numpy as np @@ -12,36 +12,14 @@ import peptides import plotly.express as px import ssl + +from analysis.plot_utils import apply_standard_axes +from models.sequence import DNARNAColumns, PROTEINColumns + ssl._create_default_https_context = ssl._create_stdlib_context -@dataclass -class DNARNAColumns: - sequence: List[str] - gc_content: List[float] - length: List[int] - count: List[int] - nucleotide_count: List[Dict[str, int]] - k_mers: List[List[Tuple[str, int]]] - plot: str - -#ToDo: Add Composition over all -@dataclass -class PROTEINColumns: - sequence: List[str] - length: List[int] - count: List[int] - composition: List[Dict[str, int]] - frequency: List[float] - hydrophobicity: List[float] - charge: List[float] - molecular_weight: List[float] - isoelectric_point: List[float] - aliphatic_index: List[float] - boman: List[float] - aromaticity: List[float] - instability: List[float] - k_mers: List[List[Tuple[str, int]]] - plot: str + +# ToDo: Add Composition over all def count_nmer(sequence, n) -> defaultdict: @@ -62,20 +40,21 @@ def top_mere(seq, n=3, top=5) -> List[Tuple[str, int]] | None: return sorted(counts.items(), key=lambda x: x[1], reverse=True)[:top] -def biological_data_top_entries(seqs: pd.Series, top_k: int = 20) -> Tuple[np.ndarray, np.ndarray, int, int, np.ndarray]: - seqs =seqs.str.upper() +def biological_data_top_entries(seqs: pd.Series, top_k: int = 20) -> Tuple[ + np.ndarray, np.ndarray, int, int, np.ndarray]: + seqs = seqs.str.upper() vc = seqs.value_counts() - uniq_tmp, counts_tmp =vc.index.to_numpy(), vc.values + uniq_tmp, counts_tmp = vc.index.to_numpy(), vc.values top_k = min(top_k, len(uniq_tmp)) top_idx = np.argsort(counts_tmp)[::-1][:top_k] - + uniques = uniq_tmp[top_idx].astype(str) counts = counts_tmp[top_idx] lengths = np.array([len(s) for s in uniques]) min_len, max_len = lengths.min(), lengths.max() - + return uniques, counts, min_len, max_len, lengths @@ -89,7 +68,7 @@ def dna_rna_columns(seqs: pd.Series, k: int = 3, top_n: int = 20, top: int = 5) k_mers = _kmer_check(k, top, uniques) if min_len == max_len: - plot = make_logo(uniques,'color_classic', seq_type="dna") + plot = make_logo(uniques, 'color_classic', seq_type="dna") else: flat_kmers = chain.from_iterable(k_mers) df_kmers = pd.DataFrame(flat_kmers, columns=['kmer', 'count']) @@ -106,6 +85,7 @@ def dna_rna_columns(seqs: pd.Series, k: int = 3, top_n: int = 20, top: int = 5) plot=plot ) + def _kmer_check(k: int, top: int, uniques: np.ndarray) -> list: check_length = any(len(i) <= k for i in uniques) if not check_length: @@ -136,14 +116,14 @@ def protein_descriptors(peptide: str) -> Dict[str, str | float | dict[str, float return descriptors -def protein_columns(seqs: pd.Series, k: int = 3, top_n: int = 20, top: int = 5) -> PROTEINColumns: +def protein_columns(seqs: pd.Series, k: int = 3, top_n: int = 20, top: int = 5) -> PROTEINColumns: uniques, counts, min_len, max_len, lengths = biological_data_top_entries(seqs, top_n) aa_composition = [dict(Counter(seq)) for seq in uniques] descriptors = [protein_descriptors(seq) for seq in uniques] k_mers = _kmer_check(k, top, uniques) - #ToDo Add Desclaimer + # ToDo Add Desclaimer if min_len == max_len: plot = make_logo(uniques, "chemistry", seq_type="protein") else: @@ -152,7 +132,6 @@ def protein_columns(seqs: pd.Series, k: int = 3, top_n: int = 20, top: int = 5) aggregated = df_kmers.groupby('kmer', as_index=False)['count'].sum() plot = plot_overview(aggregated['kmer'].tolist(), aggregated['count'].tolist()) - return PROTEINColumns( sequence=uniques.tolist(), length=lengths.tolist(), @@ -181,7 +160,8 @@ def make_logo(seqs, color, seq_type): tmp_path = tmp_file.name try: - m.weblogo(tmp_path, format="svg",sequence_type=seq_type, color=color,logo_font="Calibri",logo_margin=3, fontsize=12) + m.weblogo(tmp_path, format="svg", sequence_type=seq_type, color=color, logo_font="Calibri", logo_margin=3, + fontsize=12) with open(tmp_path, 'r', encoding='utf-8') as svg_file: svg_content = svg_file.read() @@ -201,22 +181,6 @@ def make_logo(seqs, color, seq_type): def plot_overview(kmer, count): fig = px.bar(x=kmer, y=count, color_discrete_sequence=['#0F65A0']) fig.update_layout(bargap=0.2, plot_bgcolor='white', xaxis_title='K-mers', - yaxis_title='Count') - fig.update_xaxes( - mirror=True, - ticks='outside', - showline=True, - linecolor='black', - gridcolor='lightgrey', - tickangle=-45 - - ) - fig.update_yaxes( - mirror=True, - ticks='outside', - showline=True, - linecolor='black', - gridcolor='lightgrey' - ) - # fig.write_image("test.png") + yaxis_title='Count') + apply_standard_axes(fig, tick_angle=-45) return fig.to_html(full_html=False, include_plotlyjs=False) \ No newline at end of file diff --git a/src/biological/sequence_detection.py b/src/biological/sequence_detection.py new file mode 100644 index 0000000..77b5455 --- /dev/null +++ b/src/biological/sequence_detection.py @@ -0,0 +1,54 @@ +import re +import math + +from pandas.api.types import infer_dtype + +from enums.sequence_enum import Sequence +from cython_wrapper.wrapper_utils import fast_check_sequence, char_entropy + + +def _alphabet_from_pattern(pattern): + match = re.search(r'\[([^]]+)]', pattern.pattern) + return set(match.group(1).upper()) if match else set() + + +DNA_ALPHABET = _alphabet_from_pattern(Sequence.DNA.value) +RNA_ALPHABET = _alphabet_from_pattern(Sequence.RNA.value) +PROTEIN_ALPHABET = _alphabet_from_pattern(Sequence.PROTEIN.value) + +ENTROPY_THRESHOLDS = { + "dna": 0.5 * math.log2(len(DNA_ALPHABET)), + "rna": 0.5 * math.log2(len(RNA_ALPHABET)), + "protein": 0.5 * math.log2(len(PROTEIN_ALPHABET)), +} + + +def check_sequence(df, col, threshold=0.92): + if df[col].name in df.select_dtypes(include=['number', 'bool']).columns or infer_dtype(df[col]).__contains__('mixed'): + return "None", [] + if df[col].astype(str).str.len().eq(1).all(): + return "None", [] + values = df[col].dropna().astype(str).tolist() + + unique_count = len(set(values)) + if unique_count < 10: + return "None", [] + + if all(len(x) > 2 for x in values): + match, invalid = fast_check_sequence(values, Sequence.DNA.value, threshold) + if match: + return "dna", _get_invalid(values, invalid) + match, invalid = fast_check_sequence(values, Sequence.RNA.value, threshold) + if match: + return "rna", _get_invalid(values, invalid) + match, invalid = fast_check_sequence(values, Sequence.PROTEIN.value, threshold) + if match: + if not invalid or char_entropy(values, PROTEIN_ALPHABET) >= ENTROPY_THRESHOLDS["protein"]: + return "protein", _get_invalid(values, invalid) + return "None", [] + + +def _get_invalid(values, invalid_indices): + if not invalid_indices: + return [] + return [values[i] for i in invalid_indices] \ No newline at end of file diff --git a/src/qc_eda/biological/taxonomy.py b/src/biological/taxonomy.py similarity index 100% rename from src/qc_eda/biological/taxonomy.py rename to src/biological/taxonomy.py diff --git a/src/cli/__init__.py b/src/cli/__init__.py new file mode 100644 index 0000000..247b1cb --- /dev/null +++ b/src/cli/__init__.py @@ -0,0 +1 @@ +from .app import cli diff --git a/src/cli/app.py b/src/cli/app.py new file mode 100644 index 0000000..d3c3fce --- /dev/null +++ b/src/cli/app.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +from pathlib import Path + +import click +import pandas as pd +from termcolor import colored + +from analysis.overview import overview, column_overview +from analysis.numeric_analysis import numeric_columns +from analysis.categorical_analysis import categorical_columns +from analysis.multivariate import general_plots +from biological.sequence_data import dna_rna_columns, protein_columns +from biological.functional_annotation import annotation_flags +from biological.measurement_data import measurement_columns +from biological.taxonomy import taxonomy_flags +from data_utils.remote_data import get_tax_ids +from data_utils.file_reader import read_file, parse_parquet +from cli.report_writer import write_report + +CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) + + +@click.group(context_settings=CONTEXT_SETTINGS, invoke_without_command=True) +@click.option("-i", "--input", type=click.Path(exists=True, resolve_path=True), required=True, + help="Input file as .tsv, .csv or .json") +@click.option('-t', '--tax', is_flag=True, help='Enable taxonomy analysis') +@click.option('-f', '--func', type=click.Choice(['cog', 'go']), + help='Enable functional annotation analysis. Choose between cog or go') +@click.option('-tc', '--target_column', type=str, help='Target column for Analysis') +@click.option('-k', '--kmer', type=int, default=3, help="K-mer Size for sequence analysis") +@click.option('-n', '--top_n', type=int, default=20, help="Top N entries analysis") +def cli(input: str, tax: bool = False, func: str = None, + target_column: str = None, kmer: int = None, top_n: int = None): + input_path = Path(input) + print(colored(f'Reading file {input_path.name}', 'green')) + df = read_file(input_path) if input_path.suffix != '.parquet' else parse_parquet(input_path) + general = overview(df, input_path.name) + plots = general_plots(df, target_column) + + dups = df[df.duplicated(keep=False)].reset_index() + duplicates_table = dups.to_html( + classes="table table-hover table-responsive nowrap", + border="0", table_id="dup_table", index=False, + ) + + print(colored(f'Analyse {len(df.columns)} columns', 'blue')) + column_overviews = [column_overview(df, col) for col in df.columns] + + tax_df = get_tax_ids() if tax else None + + for col_ov in column_overviews: + if tax and tax_df is not None: + col_ov.taxonomy = taxonomy_flags(df, col_ov.name, tax_df) + if func and tax_df is not None: + col_ov.annotation = [annotation_flags(df, col_ov.name, func)] + if hasattr(col_ov, "top_10") and isinstance(col_ov.top_10, pd.Series): + col_ov.top_10_items = list(col_ov.top_10.items()) + + # Biologische Sequenz-Analyse + if col_ov.sequence == 'dna': + print(colored(f'Analyzing DNA/RNA sequences in column: {col_ov.name}', 'cyan')) + col_ov.dna_rna_data = dna_rna_columns(df[col_ov.name], k=kmer, top_n=top_n) + elif col_ov.sequence == 'protein': + print(colored(f'Analyzing protein sequences in column: {col_ov.name}', 'cyan')) + col_ov.protein_data = protein_columns(df[col_ov.name], k=kmer, top_n=top_n) + else: + col_ov.dna_rna_data = None + col_ov.protein_data = None + + # Measurement-Analyse + if col_ov.sequence == 'None': + measurement_data = measurement_columns(col_ov, df) + if measurement_data: + print(colored(f'Analyzing lab measurements in column: {col_ov.name}', 'cyan')) + col_ov.measurement_data = measurement_data if measurement_data else None + else: + col_ov.measurement_data = None + + print(colored(f'Analyse {len(df.select_dtypes(include="number").columns)} numeric columns', 'blue')) + + exclude_cols = [] + numeric_overviews = [ + numeric_columns(df, col) if not df[col].isnull().all() else exclude_cols.append(col) + for col in df.select_dtypes(include="number").columns + ] + + cat_columns = [ + col for col in df.select_dtypes(include=['str', 'object', 'bool', 'int64', 'float64']).columns + if any(i.sequence == 'None' for i in column_overviews if i.name == col) + ] + print(colored(f'Analyse {len(cat_columns)} object columns', 'blue')) + + categorical_overviews = [ + categorical_columns(df, col) if col not in exclude_cols else None + for col in cat_columns + ] + + output_path = Path(input_path.stem + "_renders") + print(colored('Writing report …', 'green')) + write_report(output_path, general, plots, duplicates_table, + column_overviews, numeric_overviews, categorical_overviews, top_n) diff --git a/src/cli/report_writer.py b/src/cli/report_writer.py new file mode 100644 index 0000000..8407bac --- /dev/null +++ b/src/cli/report_writer.py @@ -0,0 +1,30 @@ +import shutil +from pathlib import Path + +from jinja2 import Environment, FileSystemLoader +from importlib_resources import files + +TEMPLATE_DIR = files("templates").joinpath() +STATIC_DIR = files("static").joinpath() +env = Environment(loader=FileSystemLoader(str(TEMPLATE_DIR)), autoescape=True) + + +def write_report(output_path: Path, general, plots, duplicates_table, + column_overviews, numeric_overviews, categorical_overviews, top_n): + output_path.mkdir(parents=True, exist_ok=True) + shutil.copytree(str(STATIC_DIR), str(output_path / "static"), dirs_exist_ok=True) + + _render_to_file(output_path / "index.html", 'LandingPage.jinja') + _render_to_file(output_path / "numeric_data.html", 'numeric_overview.jinja', + general=general, dups=duplicates_table) + _render_to_file(output_path / "columns.html", 'columns.jinja', + columns=column_overviews, overview=numeric_overviews, + categorical=categorical_overviews, top_n=top_n) + _render_to_file(output_path / "general_statistics.html", 'general_statistics.jinja', + plots=plots) + + +def _render_to_file(filepath: Path, template_name: str, **context): + template = env.get_template(template_name) + with open(filepath, 'w', encoding="utf-8") as f: + f.write(template.render(**context)) \ No newline at end of file diff --git a/src/qc_eda/basic/__init__.py b/src/cython_wrapper/__init__.py similarity index 100% rename from src/qc_eda/basic/__init__.py rename to src/cython_wrapper/__init__.py diff --git a/src/qc_eda/basic/taxonomy_validator.pyx b/src/cython_wrapper/taxonomy_validator.pyx similarity index 100% rename from src/qc_eda/basic/taxonomy_validator.pyx rename to src/cython_wrapper/taxonomy_validator.pyx diff --git a/src/qc_eda/basic/wrapper_utils.pyx b/src/cython_wrapper/wrapper_utils.pyx similarity index 100% rename from src/qc_eda/basic/wrapper_utils.pyx rename to src/cython_wrapper/wrapper_utils.pyx diff --git a/src/data_utils/__init__.py b/src/data_utils/__init__.py new file mode 100644 index 0000000..9d3f6e8 --- /dev/null +++ b/src/data_utils/__init__.py @@ -0,0 +1,2 @@ +from .file_reader import read_file, parse_parquet +from .remote_data import get_tax_ids, get_clusters_of_orthologous_groups, get_gene_ontology \ No newline at end of file diff --git a/src/utils/file_reader.py b/src/data_utils/file_reader.py similarity index 100% rename from src/utils/file_reader.py rename to src/data_utils/file_reader.py diff --git a/src/utils/download_metadata.py b/src/data_utils/remote_data.py similarity index 100% rename from src/utils/download_metadata.py rename to src/data_utils/remote_data.py diff --git a/src/enums/__init__.py b/src/enums/__init__.py new file mode 100644 index 0000000..03d1441 --- /dev/null +++ b/src/enums/__init__.py @@ -0,0 +1,2 @@ +from .sequence_enum import Sequence +from .measurement_enum import MEASUREMENTS diff --git a/src/qc_eda/basic/measurement_enum.py b/src/enums/measurement_enum.py similarity index 100% rename from src/qc_eda/basic/measurement_enum.py rename to src/enums/measurement_enum.py diff --git a/src/qc_eda/basic/sequence_enum.py b/src/enums/sequence_enum.py similarity index 100% rename from src/qc_eda/basic/sequence_enum.py rename to src/enums/sequence_enum.py diff --git a/src/models/__init__.py b/src/models/__init__.py new file mode 100644 index 0000000..fe93c54 --- /dev/null +++ b/src/models/__init__.py @@ -0,0 +1,5 @@ +from .overview import DatasetSummary, ColumnOverview +from .numeric import NumericColumns +from .categorical import CategoricalColumns +from .sequence import DNARNAColumns, PROTEINColumns +from .measurement import UNITColumns \ No newline at end of file diff --git a/src/models/categorical.py b/src/models/categorical.py new file mode 100644 index 0000000..d1d5621 --- /dev/null +++ b/src/models/categorical.py @@ -0,0 +1,17 @@ +from dataclasses import dataclass + + +@dataclass +class CategoricalColumns: + name: str + unique_categories: int + mode: str + entropy: float + frequencies: dict + gini: float + simpson_diversity: float + value_counts: dict + max_category_length: int + min_category_length: int + cardinality_ratio: float + memory: int diff --git a/src/models/measurement.py b/src/models/measurement.py new file mode 100644 index 0000000..97b14ce --- /dev/null +++ b/src/models/measurement.py @@ -0,0 +1,9 @@ +from dataclasses import dataclass +from typing import List, Dict + + +@dataclass +class UNITColumns: + units: List[str] + unit_counts: List[Dict[str | None, int]] + with_measurement: bool diff --git a/src/models/numeric.py b/src/models/numeric.py new file mode 100644 index 0000000..c1ce339 --- /dev/null +++ b/src/models/numeric.py @@ -0,0 +1,23 @@ +from dataclasses import dataclass + +from numpy import ndarray + + +@dataclass +class NumericColumns: + name: str + min: float + max: float + mean: float + median: float + mode: float + std: float + sum: float + kurtosis: float + skewness: float + coefficient_of_variation: float + mad: float + quantiles: ndarray + memory: int + value_counts: dict + frequencies: dict diff --git a/src/models/overview.py b/src/models/overview.py new file mode 100644 index 0000000..e49ccdf --- /dev/null +++ b/src/models/overview.py @@ -0,0 +1,34 @@ +from dataclasses import dataclass + + +@dataclass +class DatasetSummary: + filename: str + rows: int + cols: int + nulls: int + nulls_percentage: float + empty_rows: int + dup_row: int + dup_col: int + ratio: float + memory: float + alerts: int + + +@dataclass +class ColumnOverview: + name: str + number: int | None + unique: int | None + missing: int | None + missing_per: float | None + type: str + sequence: str | None + invalid_seqs: list[str] | None + mixed_types: list[str] | None + suspect_values: list[str] | None + describe_plot: str | None + constant: bool | None + correlation: list[str] | None + cardinality_dimension_ratio: float | None diff --git a/src/models/sequence.py b/src/models/sequence.py new file mode 100644 index 0000000..f47a1c9 --- /dev/null +++ b/src/models/sequence.py @@ -0,0 +1,32 @@ +from dataclasses import dataclass +from typing import List, Dict, Tuple + + +@dataclass +class DNARNAColumns: + sequence: List[str] + gc_content: List[float] + length: List[int] + count: List[int] + nucleotide_count: List[Dict[str, int]] + k_mers: List[List[Tuple[str, int]]] + plot: str + +# ToDo: Add Composition over all +@dataclass +class PROTEINColumns: + sequence: List[str] + length: List[int] + count: List[int] + composition: List[Dict[str, int]] + frequency: List[float] + hydrophobicity: List[float] + charge: List[float] + molecular_weight: List[float] + isoelectric_point: List[float] + aliphatic_index: List[float] + boman: List[float] + aromaticity: List[float] + instability: List[float] + k_mers: List[List[Tuple[str, int]]] + plot: str diff --git a/src/qc_eda/basic/numerical_data.py b/src/qc_eda/basic/numerical_data.py deleted file mode 100644 index 7cd460b..0000000 --- a/src/qc_eda/basic/numerical_data.py +++ /dev/null @@ -1,308 +0,0 @@ -import re -from dataclasses import dataclass - -import math -import numpy as np -import pandas as pd -import plotly.express as px -from numpy import ndarray -from pandas.api.types import infer_dtype -from scipy import stats - -from .sequence_enum import Sequence -from .wrapper_utils import fast_check_sequence, char_entropy - -def _alphabet_from_pattern(pattern): - match = re.search(r'\[([^]]+)]', pattern.pattern) - return set(match.group(1).upper()) if match else set() - -# ToDo test if DNA and RNA Entropy is needed -DNA_ALPHABET = _alphabet_from_pattern(Sequence.DNA.value) -RNA_ALPHABET = _alphabet_from_pattern(Sequence.RNA.value) -PROTEIN_ALPHABET = _alphabet_from_pattern(Sequence.PROTEIN.value) - -ENTROPY_THRESHOLDS = { - "dna": 0.5 * math.log2(len(DNA_ALPHABET)), - "rna": 0.5 * math.log2(len(RNA_ALPHABET)), - "protein": 0.5 * math.log2(len(PROTEIN_ALPHABET)), -} - -@dataclass -class NumericalData: - filename: str - rows: int - cols: int - nulls: int - nulls_percentage: float - empty_rows: int - dup_row: int - dup_col: int - ratio: float - memory: float - alerts: int - - -@dataclass -class ColumnOverview: - name: str - number: int | None - unique: int | None - missing: int | None - missing_per: float | None - type: str - sequence: str | None - invalid_seqs: list[str] | None - describe_plot: str | None - constant: bool | None - correlation: list[str] | None - cardinality_dimension_ratio: float | None - # taxonomy: bool - - -@dataclass -class NumericColumns: - name: str - min: float - max: float - mean: float - median: float - mode: float - std: float - sum: float - kurtosis: float - skewness: float - coefficient_of_variation: float - mad: float - quantiles: ndarray - memory: int - value_counts: dict - frequencies: dict - - # cardinalities: list[int] - - -@dataclass -class CategoricalColumns: - name: str - unique_categories: int - mode: str - entropy: float - frequencies: dict - gini: float - simpson_diversity: float - value_counts: dict - max_category_length: int - min_category_length: int - cardinality_ratio: float - memory: int - - -def overview(df: pd.DataFrame, file) -> NumericalData: - return NumericalData( - filename=file, - rows=df.shape[0], - cols=df.shape[1], - nulls=sum(df.isnull().sum()), - nulls_percentage=100 if df.isnull().all().all() else round(sum(df.isnull().sum()) * 100 / df.size, 2), - empty_rows=df.replace("", np.nan).isna().all(axis=1).sum(), - dup_row=int(df.duplicated().sum()), - dup_col=int(df.columns.duplicated().sum()), - ratio=round(df.shape[0] / df.shape[1], 3), - memory=int(df.memory_usage(deep=True).sum()), - alerts=0 - ) - -#ToDo check for empty Column and return -def column_overview(df: pd.DataFrame, col) -> ColumnOverview: - seq_type, invalid = check_sequence(df, col) - if seq_type != "None": - print(f"Column: {col:10s} is of type: {seq_type:10s} with invalid sequences: {invalid}.") - return ColumnOverview( - name=col, - number=int(df[col].notnull().sum()), - unique=df[col].nunique(), - missing=int(df[col].isnull().sum()), - missing_per=100 if df[col].isnull().all() else round(sum(df[col].isnull()) * 100 / df[col].size, 2), - type=str(df[col].dtype), - sequence=seq_type, - invalid_seqs=invalid, - describe_plot=None if df[col].isnull().all() else plot_overview(df[col]), - constant=True if (df[col].nunique() == 1) else False, - correlation=get_correlation(df, col), - cardinality_dimension_ratio=round(df[col].nunique() / len(df), 3), - ) - -# -def numeric_columns(df: pd.DataFrame, col) -> NumericColumns: - series = df[col].dropna() - - if series.empty: - coefficient_of_variation = np.nan - quantiles = np.array([np.nan, np.nan, np.nan]) - mode_value = np.nan - value_counts = {} - frequencies = {} - mad_value = np.nan - min_value = np.nan - max_value = np.nan - mean_value = np.nan - median_value = np.nan - std_value = np.nan - sum_value = np.nan - kurtosis_value = np.nan - skewness_value = np.nan - else: - mean_value = float(series.mean()) - std_value = float(series.std()) - coefficient_of_variation = np.nan if mean_value == 0 else float(std_value / abs(mean_value)) - quantiles = np.array(series.quantile([0.25, 0.5, 0.75]).to_list(), dtype=float) - mode_series = series.mode() - mode_value = float(mode_series.iloc[0]) if not mode_series.empty else np.nan - value_counts = series.value_counts().head(20).to_dict() - frequencies = series.value_counts(normalize=True).head(20).to_dict() - mad_value = float(stats.median_abs_deviation(series, nan_policy='omit')) - min_value = float(series.min()) - max_value = float(series.max()) - median_value = float(series.median()) - sum_value = float(series.sum()) - kurtosis_value = float(series.kurtosis()) - skewness_value = float(series.skew()) - - return NumericColumns( - name=col, - min=round(min_value, 2) if np.isfinite(min_value) else np.nan, - max=round(max_value, 2) if np.isfinite(max_value) else np.nan, - mean=round(mean_value, 2) if np.isfinite(mean_value) else np.nan, - median=round(median_value, 2) if np.isfinite(median_value) else np.nan, - mode=round(mode_value, 2) if np.isfinite(mode_value) else np.nan, - std=round(std_value, 2) if np.isfinite(std_value) else np.nan, - sum=round(sum_value, 2) if np.isfinite(sum_value) else np.nan, - kurtosis=round(kurtosis_value, 2) if np.isfinite(kurtosis_value) else np.nan, - skewness=round(skewness_value, 2) if np.isfinite(skewness_value) else np.nan, - coefficient_of_variation=round(coefficient_of_variation, 2) if np.isfinite(coefficient_of_variation) else np.nan, - mad=mad_value if np.isfinite(mad_value) else np.nan, - quantiles=quantiles, - memory=df[col].memory_usage(deep=True), - value_counts=value_counts, - frequencies=frequencies - ) - - -def categorical_columns(df: pd.DataFrame, col: str) -> CategoricalColumns: - value_counts = df[col].value_counts() - n = int(df[col].notna().sum()) - frequencies = value_counts / n if n > 0 else value_counts - if n > 0: - entropy = -(frequencies * np.log2(frequencies)).sum() - gini = 1 - (frequencies ** 2).sum() - simpson = 1 / (frequencies ** 2).sum() - cardinality_ratio = df[col].nunique() / n - mode_value = df[col].mode().iloc[0] - else: - entropy = np.nan - gini = np.nan - simpson = np.nan - cardinality_ratio = np.nan - mode_value = "" - lengths = df[col].astype(str).str.len() - - return CategoricalColumns( - name=col, - unique_categories=df[col].nunique(), - mode=mode_value, - entropy=round(entropy, 2), - frequencies=df[col].value_counts(normalize=True).head(20).to_dict(), - gini=round(gini, 2), - simpson_diversity=round(simpson, 2), - value_counts=df[col].value_counts().head(20).to_dict(), - max_category_length=lengths.max(), - min_category_length=lengths.min(), - cardinality_ratio=round(cardinality_ratio, 3), - memory = df[col].memory_usage(deep=True), - ) - - -def get_correlation(df: pd.DataFrame, col) -> list | None: - ncols = df.select_dtypes(include='number').dropna(axis=1, how='all').columns - if col in ncols: - std = df[ncols].std(ddof=0) - ncols = std[std > 0].index - if col not in ncols: - return None - corr = df[ncols].corrwith(df[col], method='pearson') - corr.drop(labels=col, inplace=True) - corr = corr[corr.abs() >= 0.3] - if corr.empty: - return None - return list(zip(corr.index, corr)) - return None - - -# ToDo: move to plot_utils -def plot_overview(col): - if col.dtype != 'object': - bins = None if col.nunique() < 10 else 10 - fig = px.histogram(col, nbins=bins, color_discrete_sequence=['#0F65A0']) - fig.update_layout(bargap=0.2, plot_bgcolor='white') - fig.update_xaxes( - mirror=True, - ticks='outside', - showline=True, - linecolor='black', - gridcolor='lightgrey' - ) - fig.update_yaxes( - mirror=True, - ticks='outside', - showline=True, - linecolor='black', - gridcolor='lightgrey' - ) - return fig.to_html(full_html=False, include_plotlyjs=False) - return None - - -# ToDo: move to sequence_utils -def check_sequence(df, col, threshold=0.95): - if df[col].name in df.select_dtypes(include=['number', 'bool']).columns or infer_dtype(df[col]).__contains__('mixed'): - return "None", [] - if df[col].astype(str).str.len().eq(1).all(): - return "None", [] - values = df[col].dropna().astype(str).tolist() - - #ToDo real cardinality - unique_count = len(set(values)) - if unique_count < 10: - return "None", [] - - if all(len(x) > 2 for x in values): - match, invalid = fast_check_sequence(values, Sequence.DNA.value, threshold) - if match: - return "dna", _get_invalid(values, invalid) - match, invalid = fast_check_sequence(values, Sequence.RNA.value, threshold) - if match: - return "rna", _get_invalid(values, invalid) - match, invalid = fast_check_sequence(values, Sequence.PROTEIN.value, threshold) - if match: - if not invalid or char_entropy(values, PROTEIN_ALPHABET) >= ENTROPY_THRESHOLDS["protein"]: - return "protein", _get_invalid(values, invalid) - return "None", [] - -def _get_invalid(values, invalid_indices): - if not invalid_indices: - return [] - return [values[i] for i in invalid_indices] - - -"""def rank_taxonomy(df, col): - if df[col].dtype != 'object': - return False - - results = df[col].astype(str).apply(lambda x: validate_taxonomy(x)) - results = results[~results.str.len().eq(0)] - - if not results.empty: - results = set(deepflatten(results.value_counts().index.tolist(), depth=1)) - print(results) - - return False""" diff --git a/src/qc_eda/biological/__init__.py b/src/qc_eda/biological/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/templates/columns.jinja b/src/templates/columns.jinja index 8fee6f4..e87bce5 100644 --- a/src/templates/columns.jinja +++ b/src/templates/columns.jinja @@ -162,6 +162,9 @@ Mixed Values {% endif %} {% endif %} + {% if col.suspect_values is not none %} + Suspect Values + {% endif %}
Number of Unique: {{ col.unique }}
Number of Missing: {{ col.missing }} ({{ col.missing_per }}%)
Cardinality Dimension Ratio: {{ col.cardinality_dimension_ratio }}
- {% if col.type != 'object' or 'str' %} + {% if col.type not in ('object', 'str') %} + Number type: {{ col.type }}
{% endif %} - + {% if col.mixed_types is not none %} + Mixed Types: {{ col.mixed_types }}
+ {% endif %} {% if col.sequence != 'None' %} Sequence type: {{ col.sequence }}
{% endif %} @@ -231,6 +237,13 @@ type="button" role="tab">Unit Statistics {% endif %} + {% if col.suspect_values is not none %} + + {% endif %} {% if col.type not in ['object', 'bool', 'str'] and col.type in ['int64', 'float64'] %} {% endif %} {% if col.type not in ['object', 'bool', 'str'] and col.type in ['int64', 'float64'] %} @@ -329,14 +331,14 @@ - + - {% for val in col.suspect_values %} + {% for idx, val in col.suspect_values.items() %} - + {% endfor %} @@ -456,6 +458,13 @@ data-bs-toggle="tab" data-bs-target="#sequences{{ col.name | replace(' ', '') | replace('.', '_') | replace('#', '') | capitalize }}" type="button" role="tab">Sequences + {% if col.invalid_seqs and col.invalid_seqs|length > 0 %} + + {% endif %}
#Index Value
{{ loop.index }}{{ idx }} {{ val }}
+ + + + + + + + {% for seq in col.invalid_seqs %} + + + + + {% endfor %} + +
#Value
{{ loop.index }} + {% if seq|length > 50 %} + {{ seq[:25] }}...{{ seq[-25:] }} + {% else %} + {{ seq }} + {% endif %} + +
+
+ + {% endif %}
@@ -622,6 +664,13 @@ data-bs-toggle="tab" data-bs-target="#proteinSequences{{ col.name | replace(' ', '') | replace('.', '_') | replace('#', '') | capitalize }}" type="button" role="tab">Sequences + {% if col.invalid_seqs and col.invalid_seqs|length > 0 %} + + {% endif %}
+ + {% if col.invalid_seqs and col.invalid_seqs|length > 0 %} +
+
+ Note: {{ col.invalid_seqs|length }} invalid sequence(s) detected in {{ col.sequence }} column. +
+
+ + + + + + + + + {% for seq in col.invalid_seqs %} + + + + + {% endfor %} + +
#Value
{{ loop.index }} + {% if seq|length > 50 %} + {{ seq[:25] }}...{{ seq[-25:] }} + {% else %} + {{ seq }} + {% endif %} +
+
+
+ {% endif %}
From 26aab3851b5e39c5eaa174db51df92d810122d2e Mon Sep 17 00:00:00 2001 From: Sonja Diedrich Date: Mon, 20 Apr 2026 16:04:53 +0200 Subject: [PATCH 08/95] fix: updated sequence detection --- src/analysis/plot_utils.py | 3 ++- src/biological/sequence_detection.py | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/analysis/plot_utils.py b/src/analysis/plot_utils.py index 12c6c4b..2bfd495 100644 --- a/src/analysis/plot_utils.py +++ b/src/analysis/plot_utils.py @@ -10,8 +10,9 @@ def apply_standard_axes(fig, tick_angle=None): fig.update_yaxes(mirror=True, ticks='outside', showline=True, linecolor='black', gridcolor='lightgrey') - +#ToDo: Check if we want to keep nan in count plot. def plot_overview(col): + col = col.dropna() if col.dtype in ('str', 'string'): truncated = col.apply(lambda x: str(x)[:20] + '…' if len(str(x)) > 20 else str(x)) if col.dtype != 'object': diff --git a/src/biological/sequence_detection.py b/src/biological/sequence_detection.py index 77b5455..54af5bf 100644 --- a/src/biological/sequence_detection.py +++ b/src/biological/sequence_detection.py @@ -34,6 +34,12 @@ def check_sequence(df, col, threshold=0.92): if unique_count < 10: return "None", [] + unique_values = list(set(values)) + non_alpha_pattern = re.compile(r'[^a-zA-Z]') + non_alpha_count = sum(1 for v in unique_values if non_alpha_pattern.search(v)) + if non_alpha_count / len(unique_values) > 0.3: + return "None", [] + if all(len(x) > 2 for x in values): match, invalid = fast_check_sequence(values, Sequence.DNA.value, threshold) if match: From 956f060ff20b0aa7e33f1af93f270eea6ea62ed8 Mon Sep 17 00:00:00 2001 From: Sonja Diedrich Date: Mon, 20 Apr 2026 16:29:44 +0200 Subject: [PATCH 09/95] fix: added threshold for suspect values --- src/analysis/data_quality.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/analysis/data_quality.py b/src/analysis/data_quality.py index 8ee7943..5e906c2 100644 --- a/src/analysis/data_quality.py +++ b/src/analysis/data_quality.py @@ -17,7 +17,7 @@ def _split_numeric_string(series): return None, None -def check_mixed_types(df, col): +def check_mixed_types(df, col, suspect_threshold=0.125): series = df[col].dropna() if series.empty: return None @@ -25,10 +25,21 @@ def check_mixed_types(df, col): numeric_part, string_part = _split_numeric_string(series) if numeric_part is None or len(numeric_part) == 0 or len(string_part) == 0: return None + + minority_count = min(len(numeric_part), len(string_part)) + minority_ratio = minority_count / len(series) + print(col, minority_ratio) if len(numeric_part) >= len(string_part): - return "String", string_part + majority_type = "Numeric" + minority = string_part + else: + majority_type = "String" + minority = numeric_part + + if minority_ratio < suspect_threshold: + return majority_type, minority else: - return "Numeric", numeric_part + return majority_type, None def check_suspect_values(df, col): From c91e5925f828ea1d41d4c53c342bcd86b12398f8 Mon Sep 17 00:00:00 2001 From: Julian Hahnfeld <84308019+jhahnfeld@users.noreply.github.com> Date: Mon, 4 May 2026 14:26:32 +0200 Subject: [PATCH 10/95] fix: too broad unit regex --- src/biological/measurement_data.py | 4 ++-- src/enums/measurement_enum.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/biological/measurement_data.py b/src/biological/measurement_data.py index 98fbeab..5ca2e40 100644 --- a/src/biological/measurement_data.py +++ b/src/biological/measurement_data.py @@ -32,8 +32,8 @@ def measurement_columns(column_overview: ColumnOverview, df: pd.DataFrame) -> UN def match_units(entries: List[str], regex: re.Pattern) -> List[str]: units: List[str] = [] - for entrie in entries: - measurement_and_unit = regex.fullmatch(entrie) + for entry in entries: + measurement_and_unit = regex.fullmatch(entry) if measurement_and_unit: units.append(measurement_and_unit.group(0)) return units diff --git a/src/enums/measurement_enum.py b/src/enums/measurement_enum.py index 6d719f4..c711c87 100644 --- a/src/enums/measurement_enum.py +++ b/src/enums/measurement_enum.py @@ -4,5 +4,5 @@ class MEASUREMENTS(Enum): UNIT_IN_COL_TITLE = re.compile(r'\[[a-zA-Zµμ°/]+]', re.I) UNIT_PREFIXES = r'(Q|R|Y|Z|E|P|T|G|M|k|h|da|d|c|m|µ|μ|micro|n|p|f|a|z|y|r|q)' - UNITS = r'(g|mol|M|m|L|l|s|min|unit|units|°C|C|K|A|cd|h|24 h|d|g creatinine|Eq|%|U|kU|IU|L (%)|% of total Hb|Fraction of total Hb|Osm|AU|angstroms|\d)' - UNIT_COLUMN = re.compile(rf'^((\d\.,)+\s*)?{UNIT_PREFIXES}?{UNITS}(/{UNIT_PREFIXES}?{UNITS})*$', re.I) + UNITS = r'(g|mol|M|m|L|l|s|min|unit|units|°C|C|K|A|cd|h|24 h|d|g creatinine|Eq|%|U|kU|IU|L (%)|% of total Hb|Fraction of total Hb|Osm|AU|angstroms)' + UNIT_COLUMN = re.compile(rf'^((\d\.,)+\s+)?{UNIT_PREFIXES}?{UNITS}(/{UNIT_PREFIXES}?{UNITS})*$', re.I) From 52a9cbeb562803111bc50b066e05a4d3369bce87 Mon Sep 17 00:00:00 2001 From: Sonja Diedrich Date: Tue, 5 May 2026 12:02:23 +0200 Subject: [PATCH 11/95] fix: minor fixes --- Makefile | 1 + __main__.py | 4 +- pyproject.toml | 6 +-- src/analysis/data_quality.py | 1 - src/cli/app.py | 2 +- src/templates/columns.jinja | 6 +-- src/templates/general_statistics.jinja | 60 ++++++++++++++++++++++++++ 7 files changed, 69 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index d16f694..f0bfaa2 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,7 @@ help: @echo "$$help" install: + python -m pip install --upgrade pip pip install -e . python setup.py build_ext --inplace diff --git a/__main__.py b/__main__.py index 4dbda9b..c9bdc18 100644 --- a/__main__.py +++ b/__main__.py @@ -4,6 +4,6 @@ #cli(['--input', 'data/iedb_sub.tsv', '--tax', '-tc','bind_class']) # cli(['-i', "data/Complete_nitrogenase.csv"]) #cli(['--input', 'data/upstream_sd.complete.tsv']) - cli(['-i', "data/iedb_test.tsv"]) + cli(['-i', "data/iedb_sub.tsv"]) # cli(['-i', "data/proteinGroups.tsv"]) - #cli(['-i', 'data/sample_bakrep.tsv']) \ No newline at end of file + #cli(['-i', 'data/bakrep_sagalactiae.tsv']) \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 6714213..c1989dd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,10 +16,10 @@ authors = [ {name = "Sonja Diedrich", email = "sonja.diedrich@computational.bio.uni-giessen.de"}, {name = "Maria Hansen", email = "maria.hansen@computational.bio.uni-giessen.de"} ] -#license = "MIT" -#license-files = ["LICENSE"] +license = "MIT" +license-files = ["LICENSE"] requires-python = ">=3.13" -keywords = ["bioinformatics", "data-analysis", "proteomics"] +keywords = ["bioinformatics", "data-analysis", "proteomics", "EDA"] classifiers = [ "Development Status :: 3 - Alpha", "Intended Audience :: Science/Research", diff --git a/src/analysis/data_quality.py b/src/analysis/data_quality.py index 5e906c2..4258b34 100644 --- a/src/analysis/data_quality.py +++ b/src/analysis/data_quality.py @@ -28,7 +28,6 @@ def check_mixed_types(df, col, suspect_threshold=0.125): minority_count = min(len(numeric_part), len(string_part)) minority_ratio = minority_count / len(series) - print(col, minority_ratio) if len(numeric_part) >= len(string_part): majority_type = "Numeric" minority = string_part diff --git a/src/cli/app.py b/src/cli/app.py index d3c3fce..f3d5522 100644 --- a/src/cli/app.py +++ b/src/cli/app.py @@ -84,7 +84,7 @@ def cli(input: str, tax: bool = False, func: str = None, numeric_columns(df, col) if not df[col].isnull().all() else exclude_cols.append(col) for col in df.select_dtypes(include="number").columns ] - + #Todo Identify numeric cat columnd cat_columns = [ col for col in df.select_dtypes(include=['str', 'object', 'bool', 'int64', 'float64']).columns if any(i.sequence == 'None' for i in column_overviews if i.name == col) diff --git a/src/templates/columns.jinja b/src/templates/columns.jinja index bab399d..1e1a5e2 100644 --- a/src/templates/columns.jinja +++ b/src/templates/columns.jinja @@ -14,7 +14,7 @@ - + @@ -976,8 +976,7 @@
{% if col.protein_data.plot %}
-
- {{ col.protein_data.plot | safe }} +
{{ col.protein_data.plot | safe }}
{% endif %} @@ -1072,7 +1071,6 @@ - + +
\ No newline at end of file diff --git a/src/templates/columns.jinja b/src/templates/columns.jinja index cfae6f0..40b346c 100644 --- a/src/templates/columns.jinja +++ b/src/templates/columns.jinja @@ -5,7 +5,7 @@ - Columns + Column Analysis – BioProfileKit @@ -31,10 +31,17 @@
Overview of Analysis - Column Analysis + Column Analysis General Statistics
+ +
  • + +
    • @@ -82,7 +89,7 @@ @@ -247,7 +254,7 @@ id="multiCollapse{{ col.name | replace(' ', '') | replace('.', '_') | replace('#', '') | capitalize }}">
    - +
    Column: {{ col.name }}
    Number of Values: {{ col.number }}
    @@ -255,7 +262,7 @@ Number of Missing: {{ col.missing }} ({{ col.missing_per }}%)
    Cardinality Dimension Ratio: {{ col.cardinality_dimension_ratio }}
    {% if col.type not in ('object', 'str') %} - + Number type: {{ col.type }}
    {% endif %} {% if col.mixed_types is not none %} @@ -294,7 +301,9 @@
    {% if col.describe_plot %}
    - {{ col.describe_plot | safe }} +
    + {{ col.describe_plot | safe }} +
    {% endif %}
    @@ -346,7 +355,8 @@
    -
    +
    @@ -374,7 +384,8 @@ {% if col.measurement_data %} -
    +
    @@ -397,7 +408,8 @@ {% endif %} {% if col.suspect_values is not none %} -
    +
    Note: {{ col.suspect_values|length }} suspect value(s) detected. These values appear to be of a different type than the majority of this column.
    @@ -423,7 +435,8 @@ {% endif %} {% if col.taxonomy is defined and col.taxonomy is not none and col.taxonomy.is_taxonomy and col.taxonomy.taxonomy is not string %} -
    +
    @@ -444,7 +457,8 @@ {% endif %} {% if col.type not in ['object', 'bool', 'str'] and col.type in ['int64', 'float64'] %} -
    +
    {% for ov in overview %} {% if ov.name == col.name %} @@ -485,7 +499,8 @@ {% endif %} {% if col.type in ['object', 'bool', 'str'] and col.type not in ['int64', 'float64'] %} -
    +
    {% for cat in categorical %} {% if col.name == cat.name %}
    @@ -622,7 +637,7 @@ Note: This column is empty.
    {% endif %} - + {% if col.sequence != 'None' and col.dna_rna_data and col.missing_per < 100 %}
    @@ -677,7 +692,8 @@
    -
    +
    @@ -732,7 +748,8 @@ {% if col.invalid_seqs and col.invalid_seqs|length > 0 %} -
    +
    Note: {{ col.invalid_seqs|length }} invalid sequence(s) detected in {{ col.sequence }} column.
    @@ -761,7 +778,8 @@ {% endif %} -
    +
    {% set d = col.dna_rna_data %}
    @@ -1011,7 +1029,8 @@ -
    +
    {% set d = col.dna_rna_data %}
    {# GC Content Distribution #} @@ -1019,7 +1038,9 @@
    GC Content Distribution

    Violin plot of per-sequence GC content across all sequences. Bimodal distributions may indicate contamination from a second organism.

    - {{ d.gc_distribution | safe }} +
    + {{ d.gc_distribution | safe }} +
    {% endif %} @@ -1029,7 +1050,9 @@
    AT / GC Skewness

    Scatter plot of AT skew vs. GC skew per sequence. Deviation from (0, 0) indicates strand asymmetry.

    - {{ d.at_gc_skewness | safe }} +
    + {{ d.at_gc_skewness | safe }} +
    {% endif %} @@ -1039,7 +1062,9 @@
    Sequence Length Distribution

    Histogram of sequence lengths across all sequences. High variance may indicate mixed amplicons or assembly artefacts.

    - {{ d.length_distribution | safe }} +
    + {{ d.length_distribution | safe }} +
    {% else %}
    All sequences have equal length — length distribution plot not generated.
    @@ -1051,7 +1076,9 @@
    Ambiguous Base (N) Distribution

    N-count per sequence in chronological order. Spikes may indicate specific flowcell tiles or regions with poor sequencing quality.

    - {{ d.ambiguous_distribution | safe }} +
    + {{ d.ambiguous_distribution | safe }} +
    {% elif d.ambiguous_base_ratio is defined and d.ambiguous_base_ratio.mean == 0 %}
    No ambiguous bases (N) detected — distribution plot not generated.
    @@ -1060,11 +1087,14 @@
    -
    +
    {% if col.dna_rna_data.plot %}
    - {{ col.dna_rna_data.plot | safe }} +
    + {{ col.dna_rna_data.plot | safe }} +
    {% endif %} @@ -1077,7 +1107,8 @@

    @@ -1220,7 +1251,8 @@
    {% if col.invalid_seqs and col.invalid_seqs|length > 0 %} -
    +
    Note: {{ col.invalid_seqs|length }} invalid sequence(s) detected in {{ col.sequence }} column.
    @@ -1555,14 +1587,16 @@ // First Button const firstLi = document.createElement('li'); - firstLi.className = `page-item ${currentPage === 1 ? 'disabled' : ''}`; - firstLi.innerHTML = `≪`; + const firstDisabled = currentPage === 1; + firstLi.className = `page-item ${firstDisabled ? 'disabled' : ''}`; + firstLi.innerHTML = `≪`; pagination.appendChild(firstLi); // Previous Button const prevLi = document.createElement('li'); - prevLi.className = `page-item ${currentPage === 1 ? 'disabled' : ''}`; - prevLi.innerHTML = `<`; + const prevDisabled = currentPage === 1; + prevLi.className = `page-item ${prevDisabled ? 'disabled' : ''}`; + prevLi.innerHTML = `<`; pagination.appendChild(prevLi); // Page Numbers @@ -1603,14 +1637,16 @@ // Next Button const nextLi = document.createElement('li'); - nextLi.className = `page-item ${currentPage === totalPages ? 'disabled' : ''}`; - nextLi.innerHTML = `>`; + const nextDisabled = currentPage === totalPages; + nextLi.className = `page-item ${nextDisabled ? 'disabled' : ''}`; + nextLi.innerHTML = `>`; pagination.appendChild(nextLi); // Last Button const lastLi = document.createElement('li'); - lastLi.className = `page-item ${currentPage === totalPages ? 'disabled' : ''}`; - lastLi.innerHTML = `≫`; + const lastDisabled = currentPage === totalPages; + lastLi.className = `page-item ${lastDisabled ? 'disabled' : ''}`; + lastLi.innerHTML = `≫`; pagination.appendChild(lastLi); }); @@ -1738,5 +1774,28 @@ }); }); + + + \ No newline at end of file diff --git a/src/templates/general_statistics.jinja b/src/templates/general_statistics.jinja index 3af5819..b2a23ca 100644 --- a/src/templates/general_statistics.jinja +++ b/src/templates/general_statistics.jinja @@ -5,7 +5,7 @@ - Overview + General Statistics – BioProfileKit @@ -15,7 +15,7 @@ - +
    @@ -74,27 +81,32 @@ {% endif %}
    -
    +
    - {{ plots.missing_matrix | safe }} +
    + {{ plots.missing_matrix | safe }} +
    -
    +
    - {{ plots.missing_values_barchart | safe }} +
    + {{ plots.missing_values_barchart | safe }} +
    - + {% if plots.missing_mechanisms %} -
    +
    + @@ -149,7 +161,9 @@
    - {{ plots.balance_plot | safe }} +
    + {{ plots.balance_plot | safe }} +
    @@ -158,7 +172,9 @@
    - {{ plots.boxplot | safe }} +
    + {{ plots.boxplot | safe }} +
    @@ -166,7 +182,9 @@
    - {{ plots.scatter_matrix | safe }} +
    + {{ plots.scatter_matrix | safe }} +
    @@ -174,7 +192,9 @@
    - {{ plots.correlation_heatmap | safe }} +
    + {{ plots.correlation_heatmap | safe }} +
    @@ -185,7 +205,7 @@
    - Github + BioProfileKit on GitHub
    @@ -198,6 +218,28 @@ + + diff --git a/src/templates/numeric_overview.jinja b/src/templates/numeric_overview.jinja index 537c4b8..af42d15 100644 --- a/src/templates/numeric_overview.jinja +++ b/src/templates/numeric_overview.jinja @@ -5,7 +5,7 @@ - Overview + Analysis Overview – BioProfileKit @@ -15,7 +15,7 @@ - +
    @@ -55,6 +62,7 @@
    Statistical test results for missing data mechanisms per column
    Column
    + @@ -116,7 +124,7 @@
    - Github + BioProfileKit on GitHub
    @@ -128,6 +136,28 @@ + + From 5dd94017f1757c673510ead06309f098fe136d1d Mon Sep 17 00:00:00 2001 From: Sonja Diedrich Date: Tue, 2 Jun 2026 15:04:16 +0200 Subject: [PATCH 25/95] fix: minor fixes --- __main__.py | 4 ++-- src/biological/plot_utils.py | 22 +++++++++++++++++----- src/biological/sequence_data.py | 5 +++-- src/biological/sequence_detection.py | 2 +- src/enums/sequence_enum.py | 4 ++-- src/templates/columns.jinja | 2 +- 6 files changed, 26 insertions(+), 13 deletions(-) diff --git a/__main__.py b/__main__.py index cfed85e..a477c16 100644 --- a/__main__.py +++ b/__main__.py @@ -3,9 +3,9 @@ if __name__ == '__main__': #cli(['--input', 'data/iedb_sub.tsv', '--tax', '-tc','bind_class']) # cli(['-i', "data/Complete_nitrogenase.csv"]) - #cli(['--input', 'data/upstream_sd.complete.tsv']) + cli(['--input', 'data/upstream_sd.complete.tsv']) #cli(['-i', "data/iedb_sub.tsv"]) #cli(['-i', 'data/winequality-white.csv']) # cli(['-i', "data/proteinGroups.tsv"]) #cli(['-i', 'data/bakrep_sagalactiae.tsv']) - cli(['-i', 'data/upstream_sd.complete.tsv']) \ No newline at end of file + #cli(['-i', 'data/iedb.tsv']) \ No newline at end of file diff --git a/src/biological/plot_utils.py b/src/biological/plot_utils.py index fb95d24..7b4c98b 100644 --- a/src/biological/plot_utils.py +++ b/src/biological/plot_utils.py @@ -2,6 +2,17 @@ from plotly import express as px, graph_objs as go, graph_objects as go from plotly.subplots import make_subplots +config = { + 'toImageButtonOptions': { + 'format': 'png', # one of png, svg, jpeg, webp + 'filename': None, + 'height': 1200, + 'width': 1600, + 'scale': 4 + } +} + + def length_distribution(all_overview: DataFrame, unit: str = "bp"): bins = 300 if all_overview.shape[0] > 10000 else all_overview.shape[0] @@ -24,7 +35,8 @@ def length_distribution(all_overview: DataFrame, unit: str = "bp"): yaxis_title="Sequence Count", bargap=0.5 ) - return fig.to_html(full_html=False, include_plotlyjs=False) + config['toImageButtonOptions']['filename'] = "length_distribution" + return fig.to_html(full_html=False, include_plotlyjs=False, config=config) def gc_distribution(all_overview: DataFrame): fig = px.violin( @@ -54,8 +66,8 @@ def gc_distribution(all_overview: DataFrame): ticksuffix="%" ) ) - - return fig.to_html(full_html=False, include_plotlyjs=False) + config['toImageButtonOptions']['filename'] = "gc_distribution" + return fig.to_html(full_html=False, include_plotlyjs=False, config=config) def ambiguous_distribution(all_overview: DataFrame, col: str = "N", label: str = "N-Count"): fig = go.Figure() @@ -159,5 +171,5 @@ def at_gc_skewness(all_overview: DataFrame): fig.add_hline(y=0, line_dash="dash", line_color="rgba(0,0,0,0.2)", row=2, col=1) fig.add_vline(x=0, line_dash="dash", line_color="rgba(0,0,0,0.2)", row=2, col=1) - - return fig.to_html(full_html=False, include_plotlyjs=False) \ No newline at end of file + config['toImageButtonOptions']['filename'] = "at_gc_skewness" + return fig.to_html(full_html=False, include_plotlyjs=False, config=config) \ No newline at end of file diff --git a/src/biological/sequence_data.py b/src/biological/sequence_data.py index c48f2ce..e0621c3 100644 --- a/src/biological/sequence_data.py +++ b/src/biological/sequence_data.py @@ -63,6 +63,7 @@ def biological_data_top_entries(seqs: pd.Series, top_k: int = 20) -> Tuple[ def dna_rna_columns(seqs: pd.Series, k: int = 3, top_n: int = 20, top: int = 5, invalid: list=None) -> DNARNAColumns: if invalid: seqs = seqs[~seqs.isin(invalid)] + seqs.dropna(inplace=True) #Over Top N uniques, counts, min_len, max_len, lengths = biological_data_top_entries(seqs, top_n) nucleotide_count = [dict(Counter(s)) for s in uniques] @@ -354,8 +355,8 @@ def make_logo(seqs, color, seq_type): tmp_path = tmp_file.name try: - m.weblogo(tmp_path, format="svg", sequence_type=seq_type, color=color, logo_font="Calibri", logo_margin=3, - fontsize=12) + m.weblogo(tmp_path, format="svg", sequence_type=seq_type, color_scheme=color, logo_font="Calibri", logo_margin=3, + fontsize=12, scale_width=True) with open(tmp_path, 'r', encoding='utf-8') as svg_file: svg_content = svg_file.read() diff --git a/src/biological/sequence_detection.py b/src/biological/sequence_detection.py index 54af5bf..58de1f0 100644 --- a/src/biological/sequence_detection.py +++ b/src/biological/sequence_detection.py @@ -35,7 +35,7 @@ def check_sequence(df, col, threshold=0.92): return "None", [] unique_values = list(set(values)) - non_alpha_pattern = re.compile(r'[^a-zA-Z]') + non_alpha_pattern = re.compile(r'[^a-zA-Z-]') non_alpha_count = sum(1 for v in unique_values if non_alpha_pattern.search(v)) if non_alpha_count / len(unique_values) > 0.3: return "None", [] diff --git a/src/enums/sequence_enum.py b/src/enums/sequence_enum.py index 86e98cb..9bf68a8 100644 --- a/src/enums/sequence_enum.py +++ b/src/enums/sequence_enum.py @@ -2,6 +2,6 @@ from enum import Enum class Sequence(Enum): - DNA = re.compile('^[acgtn]*$', re.I) - RNA = re.compile('^[acgun]*$', re.I) + DNA = re.compile('^[-acgtn]+$', re.I) + RNA = re.compile('^[-acgun]*$', re.I) PROTEIN = re.compile('^[acdefghiklmnpqrstvwyxju]*$', re.I) \ No newline at end of file diff --git a/src/templates/columns.jinja b/src/templates/columns.jinja index 40b346c..bf3b142 100644 --- a/src/templates/columns.jinja +++ b/src/templates/columns.jinja @@ -809,7 +809,7 @@
    = 5 %}class="table-warning"{% endif %}> + + + + {% else %} + + + + + {% endif %} + + + + +
    General dataset statistics
    - Length Outliers (Adj. Boxplot): + Length Outliers (Adj. IQR): {{ d.length_outliers.n_lower_iqr }} below {{ d.length_outliers.lower_bound }} bp  |  {{ d.length_outliers.n_upper_iqr }} above {{ d.length_outliers.upper_bound }} bp  |  Medcouple: {{ d.length_outliers.medcouple }} From 8e9d58a7de45ae814c33616b89372ec19b2fa0c9 Mon Sep 17 00:00:00 2001 From: Maria Hansen Date: Wed, 3 Jun 2026 16:49:29 +0200 Subject: [PATCH 26/95] [feat] display numeric quality metrics and outlier detection results --- src/templates/columns.jinja | 241 ++++++++++++++++++++++++++++++++---- 1 file changed, 215 insertions(+), 26 deletions(-) diff --git a/src/templates/columns.jinja b/src/templates/columns.jinja index bf3b142..5449fd8 100644 --- a/src/templates/columns.jinja +++ b/src/templates/columns.jinja @@ -238,6 +238,50 @@ {% endif %} {% endif %}{% endfor %} {% endif %} + {# ── Numeric quality badges ── #} + {% if col.type in ['int64', 'float64'] %} + {% for ov in overview %}{% if ov.name == col.name %} + {% set n_total = col.number if col.number and col.number > 0 else 1 %} + {# Infinity — pipeline-critical #} + {% if ov.infinity is defined and ov.infinity > 0 %} + + ∞ Infinity ({{ ov.infinity }}) + + {% endif %} + {# Sparse feature #} + {% if ov.zero_count is defined %} + {% set zero_pct = (ov.zero_count / n_total * 100) | round(1) %} + {% if zero_pct >= 80 %} + + Sparse ({{ zero_pct }}%) + + {% elif zero_pct >= 50 %} + + Sparse ({{ zero_pct }}%) + + {% endif %} + {% endif %} + {# Outliers (Adj. IQR) #} + {% if ov.outliers %} + {% set n_out = ov.outliers.n_lower_iqr + ov.outliers.n_upper_iqr %} + {% set out_pct = (n_out / n_total * 100) | round(1) %} + {% if out_pct >= 10 %} + + Outliers ({{ out_pct }}%) + + {% elif out_pct >= 5 %} + + Outliers ({{ out_pct }}%) + + {% endif %} + {% endif %} + {% endif %}{% endfor %} + {% endif %} {% if col.measurement_data %} {% if col.measurement_data.with_measurement %} Mixed Values @@ -462,36 +506,181 @@
    {% for ov in overview %} {% if ov.name == col.name %} - - - - - + {% set n_total = col.number if col.number and col.number > 0 else 1 %} +
    StatisticValue
    + + + + + + + + + {# ── Central Tendency & Dispersion ── #} + + + + + + + + + + + + + + + + + + + + {# ── Distribution Shape ── #} + + + + + + + + + + + {# ── Data Quality ── #} + + + {# Zero count #} + {% set zero_pct = (ov.zero_count / n_total * 100) | round(1) %} + + + + + + {# Negative count #} + {% set neg_pct = (ov.negative_count / n_total * 100) | round(1) %} + + + + + + {# Infinity count #} + 0 %}class="table-danger"{% endif %}> + + + + + {# Outliers — only if detect_outliers returned a result #} + {% if ov.outliers %} + {% set out = ov.outliers %} + {% set n_iqr = out.n_lower_iqr + out.n_upper_iqr %} + {% set n_mz = out.n_lower_mzscore + out.n_upper_mzscore %} + {% set n_z = out.n_lower_zscore + out.n_upper_zscore %} + {% set iqr_pct = (n_iqr / n_total * 100) | round(1) %} + {% set mz_pct = (n_mz / n_total * 100) | round(1) %} + {% set z_pct = (n_z / n_total * 100) | round(1) %} + + {# Adjusted IQR (Hubert & Vandervieren, 2008) #} + = 5 %}class="table-warning"{% endif %}> + + - - - - - - - - - - - - - - - + + {# Modified Z-Score (Iglewicz & Hoaglin, 1993) #} + = 5 %}class="table-warning"{% endif %}> + - - -
    StatisticValue
    Central Tendency & Dispersion
    Minimum{{ ov.min }}
    Maximum{{ ov.max }}
    Mean{{ ov.mean }}
    Median{{ ov.median }}
    Mode{{ ov.mode }}
    Standard Deviation{{ ov.std }}
    Sum{{ ov.sum }}
    QuantilesQ1: {{ ov.quantiles[0] }} | Q2: {{ ov.quantiles[1] }} | Q3: {{ ov.quantiles[2] }}
    IQR (Q3 − Q1, linear interpolation){{ (ov.quantiles[2] - ov.quantiles[0]) | round(4) }}
    Median Absolute Deviation{{ ov.mad }}
    Coefficient of Variation{{ ov.coefficient_of_variation }}
    Distribution Shape
    Skewness + {{ ov.skewness }} + {% if ov.skewness is not none %} + {% set sk = ov.skewness | float %} + {% if sk | abs > 1 %} + Highly skewed + {% elif sk | abs > 0.5 %} + Moderate skew + {% endif %} + {% endif %} +
    Kurtosis + {{ ov.kurtosis }} + {% if ov.kurtosis is not none %} + {% set kt = ov.kurtosis | float %} + {% if kt > 3 %} + Leptokurtic + {% elif kt < -1 %} + Platykurtic + {% endif %} + {% endif %} +
    Data Quality
    Zero Values (sparse feature indicator) + {{ ov.zero_count }} ({{ zero_pct }}%) + {% if zero_pct >= 80 %}Highly sparse + {% elif zero_pct >= 50 %}Sparse + {% endif %} +
    Negative Values{{ ov.negative_count }} ({{ neg_pct }}%)
    + Infinity Values (Inf / -Inf) + {% if ov.infinity > 0 %} + Critical + {% endif %} + + {{ ov.infinity }} + {% if ov.infinity > 0 %} + replace with NaN or clip before use + {% endif %} +
    + Outliers — Adjusted IQR + Hubert & Vandervieren (2008); bounds adjusted via Medcouple + + {{ n_iqr }} ({{ iqr_pct }}%) + {% if iqr_pct >= 10 %}High + {% elif iqr_pct >= 5 %}Moderate + {% endif %} + + ↓ {{ out.n_lower_iqr }} below {{ out.lower_bound }} +  ↑ {{ out.n_upper_iqr }} above {{ out.upper_bound }} +  MC: {{ out.medcouple }} + +
    Minimum{{ ov.min }}
    Maximum{{ ov.max }}
    Mean{{ ov.mean }}
    Median{{ ov.median }}
    Mode{{ ov.mode }}
    Standard Deviation{{ ov.std }}
    Sum{{ ov.sum }}
    Kurtosis{{ ov.kurtosis }}
    Skewness{{ ov.skewness }}
    Median Abs Deviation{{ ov.mad }}
    Coefficient of Variation{{ ov.coefficient_of_variation }}
    Quantiles
    + Outliers — Modified Z-Score + Iglewicz & Hoaglin (1993); |M_i| > 3.5, MAD-based + - 25 %: {{ ov.quantiles[0] }}
    - 50 %: {{ ov.quantiles[1] }}
    - 75 %: {{ ov.quantiles[2] }} + {{ n_mz }} ({{ mz_pct }}%) + {% if mz_pct >= 10 %}High + {% elif mz_pct >= 5 %}Moderate + {% endif %} + + ↓ {{ out.n_lower_mzscore }}  ↑ {{ out.n_upper_mzscore }} +
    Memory{{ ov.memory }}
    + + {# Standard Z-Score #} +
    + Outliers — Z-Score + |z| > 3.0; sensitive to non-normal distributions + + {{ n_z }} ({{ z_pct }}%) + {% if z_pct >= 10 %}High + {% elif z_pct >= 5 %}Moderate + {% endif %} + + ↓ {{ out.n_lower_zscore }}  ↑ {{ out.n_upper_zscore }} + +
    Outlier Detection + Not performed + Requires ≥ 20 values with > 10 distinct values and IQR > 0 +
    Memory{{ ov.memory }}
    {% endif %} {% endfor %}
    From c7735639ab6e6856ee6a2e7fa44d84b916a13c3f Mon Sep 17 00:00:00 2001 From: Maria Hansen Date: Wed, 3 Jun 2026 17:17:01 +0200 Subject: [PATCH 27/95] [feat] add collapsible filter bar with name search, type and quality flag filters --- src/templates/columns.jinja | 292 +++++++++++++++++++++++++++++++++++- 1 file changed, 288 insertions(+), 4 deletions(-) diff --git a/src/templates/columns.jinja b/src/templates/columns.jinja index 5449fd8..5f567f6 100644 --- a/src/templates/columns.jinja +++ b/src/templates/columns.jinja @@ -15,6 +15,39 @@ + @@ -74,6 +107,93 @@
    + + +
    + +
    + + + + Showing {{ columns|length }} + of {{ columns|length }} columns + + +
    + + +
    +
    +
    + + +
    + +
    + + + + + +
    +
    + + +
    +
    + +
    + Match: +
    + + +
    +
    +
    +
    + + + + + + + + + + +
    +
    + +
    +
    +
    +
    + {% set items_per_page = 20 %} {% set total_pages = (columns|length / items_per_page)|round(0, 'ceil')|int %} @@ -100,7 +220,33 @@
    {% for col in columns %} + {# fns = filter namespace, separate from badge ns #} + {% set fns = namespace(has_inf='no', is_sparse='no', has_outliers='no', imbalanced='no', nzv='no', rare='no') %} + {% for ov in overview %}{% if ov.name == col.name %} + {% set _n = col.number if col.number and col.number > 0 else 1 %} + {% if ov.infinity is defined and ov.infinity > 0 %}{% set fns.has_inf = 'yes' %}{% endif %} + {% if ov.zero_count is defined and (ov.zero_count / _n) >= 0.5 %}{% set fns.is_sparse = 'yes' %}{% endif %} + {% if ov.outliers and ((ov.outliers.n_lower_iqr + ov.outliers.n_upper_iqr) / _n) >= 0.05 %}{% set fns.has_outliers = 'yes' %}{% endif %} + {% endif %}{% endfor %} + {% for cat in categorical %}{% if cat.name == col.name %} + {% if cat.cib_ratio is defined and cat.cib_ratio > 3 %}{% set fns.imbalanced = 'yes' %}{% endif %} + {% if cat.top_1_coverage is defined and cat.top_1_coverage >= 0.95 %}{% set fns.nzv = 'yes' %}{% endif %} + {% if cat.rare_categories is defined and cat.rare_categories > 0 %}{% set fns.rare = 'yes' %}{% endif %} + {% endif %}{% endfor %}
    Showing {{ columns|length }} @@ -133,13 +129,9 @@
    - -
    - -
    @@ -150,8 +142,6 @@
    - -
    @@ -159,36 +149,25 @@ Match:
    + title="Show only columns matching ALL selected flags">ALL + title="Show columns matching ANY selected flag">ANY
    - - - - - - - - - - + + + + + + + + + +
    -
    @@ -220,8 +199,8 @@
    {% for col in columns %} - {# fns = filter namespace, separate from badge ns #} - {% set fns = namespace(has_inf='no', is_sparse='no', has_outliers='no', imbalanced='no', nzv='no', rare='no') %} + {# fns = filter namespace — separate from badge ns #} + {% set fns = namespace(has_inf='no',is_sparse='no',has_outliers='no',imbalanced='no',nzv='no',rare='no') %} {% for ov in overview %}{% if ov.name == col.name %} {% set _n = col.number if col.number and col.number > 0 else 1 %} {% if ov.infinity is defined and ov.infinity > 0 %}{% set fns.has_inf = 'yes' %}{% endif %} @@ -1893,12 +1872,12 @@ - + - - - - + + + + ", + ) + content = (output / "numeric_data.html").read_text() + assert " + - + - +
    -
    -
    - -
    -
    -

    General Statistics

    -

    -

    -
    +
    +
    + + +
    +
    +

    Multivariate Statistics

    +
    +
    + +
    +
    + + {# ================================================================ + 1) MISSINGNESS + ================================================================ #} +

    Missingness

    + + + +
    +
    +
    +
    + {{ plots.missing_matrix | safe }} +
    +
    - -
    -
    -
    - -
    -
    -
    -
    -
    - {{ plots.missing_matrix | safe }} -
    -
    -
    -
    -
    -
    -
    -
    - {{ plots.missing_values_barchart | safe }} -
    -
    -
    -
    - - {% if plots.missing_mechanisms %} -
    -
    - -
    - - - - - - - - - - - - - - {% for col_name, info in plots.missing_mechanisms.items() %} - - - - - - - - - {% endfor %} - -
    Statistical test results for missing data mechanisms per column
    ColumnMissing CountMechanismp-ValueTestRelated Column
    {{ col_name }}{{ info.count }} - {% if info.mechanism == 'MAR' %} - {{ info.mechanism }} - {% else %} - {{ info.mechanism }} - {% endif %} - {{ info.p_value }}{{ info.test if info.test else '—' }}{{ info.related_col if info.related_col else '—' }}
    -
    -
    -
    - MCAR: Missing Completely At Random — no pattern detected; safe to drop or impute simply.
    - MAR: Missing At Random — missingness depends on other observed columns (p < 0.05).
    -
    - Mann-Whitney U:Tests whether a numeric column has a different distribution when the target column is missing versus present. A significant result (p < 0.05) suggests the missingness is related to that numeric column.
    - Chi-Squared: Tests whether the missing patterns of two columns are statistically associated. A significant result suggests the columns tend to be missing together. -
    - These results are statistical estimates derived from hypothesis testing and should be interpreted cautiously. - MNAR (Missing Not At Random) cannot be detected from the observed data alone and may require domain knowledge. - A single significant test does not confirm a specific mechanism — consider the overall context of your dataset. -
    -
    -
    -
    - {% endif %} - {% if plots.mcar_result %} -
    -
    - {% if plots.mcar_result is mapping and plots.mcar_result.keys() | list == ['Hello'] %} - {# Placeholder result from littles_mcar_test stub #} -
    - Little’s MCAR Test is not yet fully implemented. - Results will appear here once the statistical test is complete. -
    - {% else %} -
    - - - - - - - - - - {% for key, value in plots.mcar_result.items() %} - - - - - {% endfor %} - -
    Little’s MCAR test result for the full dataset
    MetricValue
    {{ key }}{{ value }}
    -
    -
    -
    - Little’s MCAR Test: Tests whether data is Missing Completely At Random. - A significant p-value (p < 0.05) suggests the data is not MCAR — - missingness may depend on observed or unobserved values. -
    - MNAR (Missing Not At Random) cannot be detected from the observed data alone - and may require domain knowledge. -
    -
    - {% endif %} -
    -
    - {% endif %} -
    -
    -
    -
    -
    -
    -
    Strong Associations - threshold ≥ 0.7 -
    -

    - Column pairs with an association strength ≥ 0.7 across all applicable methods. - Method is determined automatically by column type (Pearson / Cramér’s V / Eta²). - A 0.7 Pearson and a 0.7 Cramér’s V are not directly comparable. -

    - {% if plots.top_associations %} -
    - - - - - - - - - - - - - {% for pair in plots.top_associations %} - {% set pct = (pair.value * 100) | round(1) %} - - - - - - - - - {% endfor %} - -
    #Variable 1Variable 2StrengthDistributionMethod
    {{ loop.index }}{{ pair.var1 }}{{ pair.var2 }} - {{ "%.3f"|format(pair.value) }} - -
    -
    -
    -
    -     		- - {{ pair.method }} - -
    -
    +
    +
    +
    + {{ plots.missing_values_barchart | safe }} +
    +
    +
    + {% if plots.missing_mechanisms %} +
    +
    +
    + + + + + + + + + + + + + + {% for col_name, info in plots.missing_mechanisms.items() %} + + + + + + + + + {% endfor %} + +
    ColumnMissing CountMechanismp-ValueTestRelated Column
    {{ col_name }}{{ info.count }} + {% if info.mechanism == 'MAR' %} + {{ info.mechanism }} {% else %} -
    - No column pairs exceed the association threshold of 0.7 — no strong associations detected. -
    + {{ info.mechanism }} {% endif %} - +     		{{ info.p_value }}{{ info.test if info.test else '—' }}{{ info.related_col if info.related_col else '—' }}
    - - {% if plots.feature_target_plot or plots.feature_target_correlation %} -
    -
    -
    Feature–Target Association
    -
    - {% if plots.feature_target_plot %} -
    -
    -
    - {{ plots.feature_target_plot | safe }} -
    -
    -
    - {% endif %} - {% if plots.feature_target_correlation %} -
    -
    - - - - - - - - - - {% for feature, info in plots.feature_target_correlation.items() | sort(attribute='1.value', reverse=true) %} - - - - - - {% endfor %} - -
    FeatureStrengthMethod
    {{ feature }} - {{ "%.3f"|format(info.value) }} -
    -
    -
    -
    -     		- - {{ info.method }} - -
    -
    -
    - {% endif %} -
    -
    +
    + MCAR: Missing Completely At Random — safe to drop or impute simply.
    + MAR: Missing At Random — missingness depends on other observed columns (p < 0.05).
    +
    + Mann-Whitney U: Tests numeric distribution difference when target is missing vs. present.
    + Chi-Squared: Tests whether missing patterns of two columns are associated.
    +
    + Results are statistical estimates — interpret cautiously. MNAR cannot be detected from observed data alone. +
    +
    +
    + {% endif %} + {% if plots.mcar_result %} +
    +
    + {% if plots.mcar_result is mapping and plots.mcar_result.keys() | list == ['Hello'] %} +
    + Little’s MCAR Test is not yet fully implemented. + Results will appear here once the statistical test is complete. +
    + {% else %} +
    + + + + + + + {% for key, value in plots.mcar_result.items() %} + + {% endfor %} + +
    MetricValue
    {{ key }}{{ value }}
    +
    +
    + Little’s MCAR Test: p < 0.05 suggests data is not MCAR. + MNAR cannot be detected from observed data alone. +
    {% endif %} +
    +
    + {% endif %} +
    - {% if plots.mutual_information or plots.mutual_information_plot or plots.mi_relationship_plots %} -
    -
    -
    Mutual Information - feature → target, non-linear -
    -

    - Mutual information measures non-linear statistical dependency between each feature and the target. - Unlike correlation, it captures any kind of relationship — not just linear ones. - Values are non-negative and unbounded; higher means more information shared. -

    - - - -
    - - {# ── Ranking plot ── #} - {% if plots.mutual_information_plot %} -
    -
    -
    - {{ plots.mutual_information_plot | safe }} -
    -
    -
    - {% endif %} - - {# ── Table ── #} - {% if plots.mutual_information %} -
    -
    - - - - - - - - - - - {% set max_mi = plots.mutual_information.values() | map(attribute='value') | max %} - {% for feature, info in plots.mutual_information.items() | sort(attribute='1.value', reverse=true) %} - {% set pct = ((info.value / max_mi) * 100) | round(1) if max_mi > 0 else 0 %} - - - - - - - {% endfor %} - -
    #FeatureMI ScoreRelative Strength
    {{ loop.index }}{{ feature }}{{ "%.4f"|format(info.value) }} -
    -
    -
    -
    -
    -
    -
    - {% endif %} - - {# ── Top Relationship plots ── #} - {% if plots.mi_relationship_plots %} -
    -

    - Pairwise relationship plots for the top {{ plots.mi_relationship_plots | length }} features - by mutual information score. Plot type adapts automatically to column types. -

    -
    - {% for item in plots.mi_relationship_plots %} -
    -
    - {{ item.feature }} - - MI = {{ "%.4f"|format(item.value) }} - -
    -
    -
    - {{ item.plot | safe }} -
    -
    -
    - {% endfor %} -
    -
    - {% endif %} - -
    {# /tab-content #} -
    - {% endif %} +
    - {% if plots.balance_plot %} -
    -
    -
    -
    -
    - {{ plots.balance_plot | safe }} -
    + {# ================================================================ + 2) CORRELATIONS + ================================================================ #} +

    Correlations

    + + {# ── Correlation heatmap tabs: Association first, then individual ── #} + + +
    + + {# Association Heatmap (combined) #} +
    +

    + Absolute association strengths in [0, 1]. Methods differ by variable type (hover for detail): + Pearson for numeric pairs, Cramér’s V for categorical pairs, Eta² for mixed. + A 0.7 Pearson and a 0.7 Cramér’s V are not directly comparable. +

    + {% if plots.correlation_heatmap %} +
    +
    + {{ plots.correlation_heatmap | safe }} +
    +
    + {% else %} +
    Association heatmap not available.
    + {% endif %} +
    + + {# Strong Associations table #} +
    +
    +

    + Column pairs with association ≥ 0.7. Method is chosen automatically by column type. + Values from different methods are not directly comparable. +

    + {% if plots.top_associations %} +
    + + + + + + + + + + + {% for pair in plots.top_associations %} + {% set pct = (pair.value * 100) | round(1) %} + + + + + + + {% endfor %} + +
    Variable 1Variable 2StrengthMethod
    {{ pair.var1 }}{{ pair.var2 }} + {{ "%.3f"|format(pair.value) }} +
    +
    - - +     		+ {{ pair.method }} +
    +
    + {% else %} +
    No column pairs exceed the association threshold of 0.7.
    {% endif %} -
    -
    -
    -
    -
    - {{ plots.boxplot | safe }} -
    -
    -
    +
    +
    + + {# Pearson #} +
    + {% if plots.pearson_heatmap %} +
    +
    {{ plots.pearson_heatmap | safe }}
    +
    + {% else %} +
    No numeric column pairs — Pearson heatmap not available.
    + {% endif %} +
    + + {# Cramér's V #} +
    + {% if plots.cramers_heatmap %} +
    +
    {{ plots.cramers_heatmap | safe }}
    +
    + {% else %} +
    At least two categorical columns required — Cramér’s V not available.
    + {% endif %} +
    + + {# Eta² #} +
    + {% if plots.eta_heatmap %} +
    +
    {{ plots.eta_heatmap | safe }}
    -
    -
    -
    -
    -
    - {{ plots.scatter_matrix | safe }} -
    + {% else %} +
    At least one categorical and one numeric column required — Eta² not available.
    + {% endif %} +
    + +
    {# /corrPlotContent #} + +
    + + {# ================================================================ + 3) TARGET (only rendered when a target column was specified) + ================================================================ #} + {% if plots.feature_target_plot or plots.feature_target_correlation + or plots.mutual_information or plots.mutual_information_plot + or plots.mi_relationship_plots or plots.balance_plot %} + +

    Target

    + + {# ── Feature–Target Association ── #} + {% if plots.feature_target_plot or plots.feature_target_correlation %} +
    Feature–Target Association
    +
    + {% if plots.feature_target_plot %} +
    +
    +
    + {{ plots.feature_target_plot | safe }} +
    +
    +
    + {% endif %} + {% if plots.feature_target_correlation %} +
    +
    + + + + + + + + + + {% for feature, info in plots.feature_target_correlation.items() | sort(attribute='1.value', reverse=true) %} + + + + + + {% endfor %} + +
    FeatureStrengthMethod
    {{ feature }} + {{ "%.3f"|format(info.value) }} +
    +
    - +     		+ {{ info.method }} +
    -
    -
    -
    -
    Correlation Plots
    - - - -
    - - {# ── Association Heatmap (combined) ── #} -
    -

    - Absolute association strengths in [0, 1]. Methods differ by variable type (hover for detail): - Pearson for numeric pairs, Cramér’s V for categorical pairs, Eta² for mixed. - A 0.7 Pearson and a 0.7 Cramér’s V are not directly comparable. -

    - {% if plots.correlation_heatmap %} -
    -
    - {{ plots.correlation_heatmap | safe }} -
    -
    - {% else %} -
    - Association heatmap not available. -
    - {% endif %} -
    +
    + {% endif %} +
    + {% endif %} - {# ── Pearson ── #} -
    - {% if plots.pearson_heatmap %} -
    -
    - {{ plots.pearson_heatmap | safe }} -
    -
    - {% else %} -
    - No numeric column pairs with non-zero variance found — Pearson heatmap not available. -
    - {% endif %} -
    + {# ── Mutual Information ── #} + {% if plots.mutual_information or plots.mutual_information_plot or plots.mi_relationship_plots %} +
    + Mutual Information + feature → target, non-linear +
    +

    + Mutual information measures non-linear statistical dependency between each feature and the target. + Unlike correlation, it captures any kind of relationship. Values are non-negative and unbounded — + higher means more information shared. +

    - {# ── Cramér's V ── #} -
    - {% if plots.cramers_heatmap %} -
    -
    - {{ plots.cramers_heatmap | safe }} -
    -
    - {% else %} -
    - At least two categorical columns are required — Cramér’s V heatmap not available. -
    - {% endif %} -
    + - {# ── Eta² ── #} -
    - {% if plots.eta_heatmap %} -
    -
    - {{ plots.eta_heatmap | safe }} -
    -
    - {% else %} -
    - At least one categorical and one numeric column are required — Eta² heatmap not available. -
    - {% endif %} -
    +
    + {% if plots.mutual_information_plot %} +
    +
    +
    + {{ plots.mutual_information_plot | safe }} +
    +
    +
    + {% endif %} + + {% if plots.mutual_information %} +
    +
    + + + + + + + + + + {% set max_mi = plots.mutual_information.values() | map(attribute='value') | max %} + {% for feature, info in plots.mutual_information.items() | sort(attribute='1.value', reverse=true) %} + {% set pct = ((info.value / max_mi) * 100) | round(1) if max_mi > 0 else 0 %} + + + + + + {% endfor %} + +
    FeatureMI ScoreRelative Strength
    {{ feature }} + {{ "%.4f"|format(info.value) }} +
    +
    +
    +     		{{ pct }}%
    +
    +
    + {% endif %} -
    {# /tab-content #} + {% if plots.mi_relationship_plots %} +
    +

    + Pairwise plots for top {{ plots.mi_relationship_plots | length }} features by MI score. + Plot type adapts to column types automatically. +

    +
    + {% for item in plots.mi_relationship_plots %} +
    +
    + {{ item.feature }} + MI = {{ "%.4f"|format(item.value) }} +
    +
    +
    + {{ item.plot | safe }} +
    +
    + {% endfor %}
    +
    + {% endif %} +
    {# /miTabContent #} + {% endif %}{# /MI block #} + + {# ── Class Balance ── #} + {% if plots.balance_plot %} +
    Class Balance
    +
    +
    + {{ plots.balance_plot | safe }} +
    -
    + {% endif %} + +
    + + {% endif %}{# /Target section #} + + {# ================================================================ + 4) GENERAL STATISTICS + ================================================================ #} +

    General Statistics

    + +
    Boxplot
    +
    +
    + {{ plots.boxplot | safe }} +
    +
    + +
    Scatter Matrix
    +
    +
    + {{ plots.scatter_matrix | safe }} +
    +
    + +
    {# /col-12 #} +
    {# /row #} +
    {# /container #} +
    {# /content-wrap #} + +
    +
    + + BioProfileKit on GitHub +
    - -
    -
    - - BioProfileKit on GitHub - -
    -
    -

    Copyright © JLU 2025

    -
    -
    +

    Copyright © JLU 2025

    +
    - - - - + - + + diff --git a/src/templates/numeric_overview.jinja b/src/templates/numeric_overview.jinja index af42d15..99be0a8 100644 --- a/src/templates/numeric_overview.jinja +++ b/src/templates/numeric_overview.jinja @@ -31,7 +31,7 @@
    Overview of Analysis Column Analysis - General Statistics + Multivariate Statistics
    From edb04609263a814633a1f7e485c475a18643e84e Mon Sep 17 00:00:00 2001 From: Maria Hansen Date: Wed, 17 Jun 2026 13:39:50 +0200 Subject: [PATCH 69/95] [refactor] restructure layout, add section filter, fix table typography and simplify MI table --- src/templates/general_statistics.jinja | 662 ++++++++++++++----------- 1 file changed, 362 insertions(+), 300 deletions(-) diff --git a/src/templates/general_statistics.jinja b/src/templates/general_statistics.jinja index 118eb85..88259ab 100644 --- a/src/templates/general_statistics.jinja +++ b/src/templates/general_statistics.jinja @@ -13,25 +13,53 @@ - + - +
    -
    -
    - -
    -
    -

    Overview of Analysis

    -

    - Get a compact, numerical summary of all analyses carried out. Visualisations and key figures provide a quick overview of key results and developments - ideal for evaluation at a glance. -

    -
    +
    +
    + + +
    +
    +

    Overview of Analysis

    +

    + Get a compact, numerical summary of all analyses carried out. Visualisations and key figures + provide a quick overview of key results and developments — ideal for evaluation at a glance. +

    +
    +
    + +
    +
    + + {# ================================================================ + 1) GENERAL DATASET STATISTICS + ================================================================ #} +

    General Dataset Statistics

    + + + + + + + + + + + + + + + + + + + + +
    Filename {{ general.filename }}
    Rows {{ general.rows }}
    Columns {{ general.cols }}
    Missing values {{ general.nulls }}
    Missing values (%) {{ general.nulls_percentage }}
    Empty rows {{ general.empty_rows }}
    Duplicate rows + {% if general.dup_row > 0 %} + {{ general.dup_row }} + {% else %} + {{ general.dup_row }} + {% endif %} +
    Duplicate columns + {% if general.dup_col > 0 %} + {{ general.dup_col }} + {% else %} + {{ general.dup_col }} + {% endif %} +
    Sample-Feature Ratio {{ general.ratio }}
    Memory usage {{ general.memory }}
    Alerts {{ general.alerts }}
    + +
    + + {# ================================================================ + 2) DUPLICATES + ================================================================ #} +

    + Duplicates + {% if general.dup_row > 0 %} + + {{ general.dup_row }} duplicate row{{ 's' if general.dup_row != 1 else '' }} found + + {% endif %} +

    + + {% if general.dup_row > 0 %} +
    + {{ dups | safe }}
    - -
    -
    - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    General dataset statistics
    Filename{{ general.filename }}
    Row number{{ general.rows }}
    Column number{{ general.cols }}
    Missing values{{ general.nulls }}
    Missing values (%){{ general.nulls_percentage }}
    Empty Rows{{ general.empty_rows }}
    Duplicate rows{{ general.dup_row }}
    Duplicate columns{{ general.dup_col }}
    Sample-Feature Ratio{{ general.ratio }}
    Memory usage{{ general.memory }}
    Alerts{{ general.alters }}
    -

    Duplicates

    -
    - {{ dups | safe }} -
    -
    + {% else %} +
    + + + + No duplicate rows detected in this dataset.
    + {% endif %} +
    +
    - -
    -
    - - BioProfileKit on GitHub - -
    -
    -

    Copyright © JLU 2025

    -
    -
    +
    + +
    +
    + + BioProfileKit on GitHub + +
    +

    Copyright © JLU 2025

    +
    - + - - + - - \ No newline at end of file + From a071d807c6f385fa92d95a435b15249900bb4372 Mon Sep 17 00:00:00 2001 From: Sonja Diedrich Date: Wed, 17 Jun 2026 15:12:55 +0200 Subject: [PATCH 72/95] feat: added quality assessment --- __main__.py | 2 +- src/analysis/multivariate_analysis.py | 1 - src/cli/app.py | 15 ++- src/models/__init__.py | 3 +- src/models/overview.py | 3 + src/models/quality.py | 26 ++++ src/quality_assessment/__init__.py | 1 + src/quality_assessment/biological_quality.py | 104 ++++++++++++++++ src/quality_assessment/column_quality.py | 124 +++++++++++++++++++ src/quality_assessment/dataset_quality.py | 66 ++++++++++ src/quality_assessment/quality_assessment.py | 79 ++++++++++++ src/quality_assessment/relationships.py | 37 ++++++ src/quality_assessment/utils.py | 10 ++ 13 files changed, 462 insertions(+), 9 deletions(-) create mode 100644 src/models/quality.py create mode 100644 src/quality_assessment/__init__.py create mode 100644 src/quality_assessment/biological_quality.py create mode 100644 src/quality_assessment/column_quality.py create mode 100644 src/quality_assessment/dataset_quality.py create mode 100644 src/quality_assessment/quality_assessment.py create mode 100644 src/quality_assessment/relationships.py create mode 100644 src/quality_assessment/utils.py diff --git a/__main__.py b/__main__.py index d1c354b..a93fb3d 100644 --- a/__main__.py +++ b/__main__.py @@ -8,4 +8,4 @@ #cli(['-i', 'data/winequality-white.csv']) # cli(['-i', "data/proteinGroups.tsv"]) #cli(['-i', 'data/bakrep_sagalactiae.tsv']) - cli(['-i', 'data/iedb_sub.tsv', '-tc', 'bind_class']) \ No newline at end of file + cli(['-i', 'data/iedb_test.tsv', '-tc', 'bind_class']) \ No newline at end of file diff --git a/src/analysis/multivariate_analysis.py b/src/analysis/multivariate_analysis.py index a954a67..b4e69ac 100644 --- a/src/analysis/multivariate_analysis.py +++ b/src/analysis/multivariate_analysis.py @@ -417,7 +417,6 @@ def mutual_information_plot(mi: dict | None): bargap=0.3, height=max(400, 40 * len(features) + 150), ) - fig.show() return fig.to_html(full_html=False, include_plotlyjs=False) diff --git a/src/cli/app.py b/src/cli/app.py index f1c21af..9139a8f 100644 --- a/src/cli/app.py +++ b/src/cli/app.py @@ -6,17 +6,18 @@ import pandas as pd from termcolor import colored -from analysis.overview import overview, column_overview -from analysis.numeric_analysis import numeric_columns from analysis.categorical_analysis import categorical_columns from analysis.multivariate_analysis import multivariate_analysis -from biological.sequence_data import dna_rna_columns, protein_columns +from analysis.numeric_analysis import numeric_columns +from analysis.overview import overview, column_overview from biological.functional_annotation import annotation_flags from biological.measurement_data import measurement_columns +from biological.sequence_data import dna_rna_columns, protein_columns from biological.taxonomy import taxonomy_flags, build_lookups -from data_utils.remote_data import get_tax_ids -from data_utils.file_reader import read_file, parse_parquet from cli.report_writer import write_report +from data_utils.file_reader import read_file, parse_parquet +from data_utils.remote_data import get_tax_ids +from quality_assessment.quality_assessment import quality_assessment, print_quality_report CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) @@ -99,7 +100,9 @@ def cli(input: str, tax: bool = False, func: str = None, categorical_columns(df, col) if col not in exclude_cols else None for col in cat_columns ] - + readiness = quality_assessment(general, column_overviews, numeric_overviews, + categorical_overviews, plots) + print_quality_report(readiness) output_path = Path(input_path.stem + "_renders") print(colored('Writing report …', 'green')) write_report(output_path, general, plots, duplicates_table, diff --git a/src/models/__init__.py b/src/models/__init__.py index 4d6adaa..68d9847 100644 --- a/src/models/__init__.py +++ b/src/models/__init__.py @@ -4,4 +4,5 @@ from .sequence import DNARNAColumns, PROTEINColumns from .measurement import UNITColumns from .outliers import Outliers -from .multivariate import MultivariateAnalysis \ No newline at end of file +from .multivariate import MultivariateAnalysis +from .quality import QualityCategory, QualityCheck, QualityAssessment \ No newline at end of file diff --git a/src/models/overview.py b/src/models/overview.py index 04569dd..c6e3823 100644 --- a/src/models/overview.py +++ b/src/models/overview.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from typing import Optional @dataclass @@ -34,3 +35,5 @@ class ColumnOverview: correlation: list[str] | None cardinality_dimension_ratio: float | None monotonicity: bool | None + taxonomy: Optional = None + measurement_data: Optional = None \ No newline at end of file diff --git a/src/models/quality.py b/src/models/quality.py new file mode 100644 index 0000000..c0cb03d --- /dev/null +++ b/src/models/quality.py @@ -0,0 +1,26 @@ +from dataclasses import dataclass + + +@dataclass +class QualityCheck: + name: str + status: str # pass | warn | fail + message: str + detail_link: str | None + + +@dataclass +class QualityCategory: + name: str + status: str + checks: list # List[QualityCheck] + + +@dataclass +class QualityAssessment: + categories: list # List[QualityCategory] + passed: int + warnings: int + failed: int + total: int + overall: str diff --git a/src/quality_assessment/__init__.py b/src/quality_assessment/__init__.py new file mode 100644 index 0000000..350b1bd --- /dev/null +++ b/src/quality_assessment/__init__.py @@ -0,0 +1 @@ +from .quality_assessment import quality_assessment, print_quality_report \ No newline at end of file diff --git a/src/quality_assessment/biological_quality.py b/src/quality_assessment/biological_quality.py new file mode 100644 index 0000000..a628ca3 --- /dev/null +++ b/src/quality_assessment/biological_quality.py @@ -0,0 +1,104 @@ +from models import QualityCheck +from .utils import _worst + +AMBIGUOUS_SEQ_WARN = 5.0 +AMBIGUOUS_SEQ_FAIL = 15.0 +STOP_CODON_WARN = 1.0 +RC_REDUNDANCY_WARN = 10.0 +RC_REDUNDANCY_FAIL = 30.0 + + +def _check_sequence_validity(column_overviews) -> QualityCheck: + seq_cols = [c for c in column_overviews if c.sequence in ("dna", "protein")] + if not seq_cols: + return QualityCheck(name="Sequence Validity", status="pass", + message="No sequence columns present", detail_link=None) + statuses, notes = [], [] + for c in seq_cols: + n_invalid = len(c.invalid_seqs) if c.invalid_seqs else 0 + if n_invalid > 0: + statuses.append("warn") + notes.append(f"{c.name}: {n_invalid} invalid") + data = getattr(c, "dna_rna_data", None) or getattr(c, "protein_data", None) + if data is None: + continue + amb = getattr(data, "ambiguous_base_ratio", None) or getattr(data, "ambiguous_residue_ratio", None) + if amb is not None: + if amb.mean > AMBIGUOUS_SEQ_FAIL: + statuses.append("fail") + notes.append(f"{c.name}: {amb.mean:.1f}% mean ambiguous") + elif amb.mean > AMBIGUOUS_SEQ_WARN: + statuses.append("warn") + notes.append(f"{c.name}: {amb.mean:.1f}% mean ambiguous") + stop = getattr(data, "stop_codon_ratio", None) + if stop is not None and stop > STOP_CODON_WARN: + statuses.append("warn") + notes.append(f"{c.name}: {stop:.1f}% with stop codons") + status = _worst(statuses) if statuses else "pass" + return QualityCheck(name="Sequence Validity", status=status, + message="; ".join(notes) if notes else "All sequences valid", + detail_link="#sequences") + + +def _check_sequence_redundancy(column_overviews) -> QualityCheck: + dna_cols = [c for c in column_overviews if c.sequence == "dna"] + if not dna_cols: + return QualityCheck(name="Sequence Redundancy (Reverse Complement)", status="pass", + message="No DNA/RNA columns present", detail_link=None) + statuses, notes = [], [] + for c in dna_cols: + data = getattr(c, "dna_rna_data", None) + if data is None: + continue + rc = getattr(data, "reverse_complement_ratio", None) + if rc is None: + continue + if rc > RC_REDUNDANCY_FAIL: + statuses.append("fail") + notes.append(f"{c.name}: {rc:.1f}% RC duplicates") + elif rc > RC_REDUNDANCY_WARN: + statuses.append("warn") + notes.append(f"{c.name}: {rc:.1f}% RC duplicates") + status = _worst(statuses) if statuses else "pass" + return QualityCheck(name="Sequence Redundancy (Reverse Complement)", status=status, + message="; ".join(notes) if notes else "No significant reverse-complement redundancy", + detail_link="#sequences") + + +def _check_taxonomy_validity(column_overviews) -> QualityCheck: + tax_cols = [c for c in column_overviews if c.taxonomy is not None] + if not tax_cols: + return QualityCheck(name="Taxonomy Validity", status="pass", + message="No taxonomy columns present", detail_link=None) + statuses, notes = [], [] + for c in tax_cols: + n_invalid = len(c.taxonomy.invalid_names) if c.taxonomy.invalid_names else 0 + if n_invalid > 0: + statuses.append("warn") + notes.append(f"{c.name}: {n_invalid} invalid") + + status = _worst(statuses) if statuses else "pass" + return QualityCheck(name="Taxonomy Validity", status=status, + message="; ".join(notes) if notes else "Taxonomy valid", + detail_link="#taxonomy") + + +def _check_unit_validity(column_overviews) -> QualityCheck: + unit_cols = [c for c in column_overviews if c.measurement_data is not None] + if not unit_cols: + return QualityCheck(name="Unit Validity", status="pass", + message="No unit columns present", detail_link=None) + statuses, notes = [], [] + for c in unit_cols: + ucount = len(c.measurement_data.unit_counts) if c.measurement_data.unit_counts else 0 + if ucount > 1: + statuses.append("warn") + notes.append(f"{c.name}: {ucount} units found.") + if c.measurement_data.with_measurement: + statuses.append("warn") + notes.append(f"{c.name}: measurement with unit found.") + + status = _worst(statuses) if statuses else "pass" + return QualityCheck(name="Unit Validity", status=status, + message="; ".join(notes) if notes else "Unit valid", + detail_link="#unit") diff --git a/src/quality_assessment/column_quality.py b/src/quality_assessment/column_quality.py new file mode 100644 index 0000000..c212457 --- /dev/null +++ b/src/quality_assessment/column_quality.py @@ -0,0 +1,124 @@ +from quality_assessment.utils import _worst, _rate +from models import QualityCheck + +MISSING_COL_WARN = 0.30 +MISSING_COL_FAIL = 0.50 + +QUASI_CONSTANT_WARN = 0.95 +HIGH_CARDINALITY_WARN = 0.90 + +SKEW_WARN = 1.0 +SKEW_FAIL = 2.0 + +OUTLIER_WARN = 0.05 +OUTLIER_FAIL = 0.15 + +def _missing_rate(col): + mp = col.missing_per or 0.0 + return mp / 100.0 if mp > 1 else mp + + +def _check_missing(column_overviews) -> QualityCheck: + flagged = [] + worst = "pass" + for c in column_overviews: + rate = _missing_rate(c) + if rate > MISSING_COL_FAIL: + flagged.append((c.name, rate)); worst = "fail" + elif rate > MISSING_COL_WARN: + flagged.append((c.name, rate)) + worst = "fail" if worst == "fail" else "warn" + if not flagged: + return QualityCheck(name="Missing Values", status="pass", + message="No column above 30% missing", detail_link="#multivariate") + flagged.sort(key=lambda x: x[1], reverse=True) + listed = ", ".join(f"{n} ({r * 100:.0f}%)" for n, r in flagged) + return QualityCheck(name="Missing Values", status=worst, + message=f"Columns above 30% missing: {listed}", detail_link="#multivariate") + + +def _check_variance(column_overviews, categorical_overviews) -> QualityCheck: + constant_cols = [c.name for c in column_overviews if c.constant] + quasi = [c.name for c in categorical_overviews + if c is not None and c.top_1_coverage is not None and c.top_1_coverage >= QUASI_CONSTANT_WARN] + if constant_cols: + return QualityCheck(name="Feature Variance", status="fail", + message=f"Constant column(s): {', '.join(constant_cols)}", detail_link="#columns") + if quasi: + return QualityCheck(name="Feature Variance", status="warn", + message=f"Quasi-constant column(s) (top category ≥{QUASI_CONSTANT_WARN:.0%}): {', '.join(quasi)}", + detail_link="#columns") + return QualityCheck(name="Feature Variance", status="pass", + message="No constant or quasi-constant columns", detail_link="#columns") + + +def _check_high_cardinality(column_overviews) -> QualityCheck: + flagged = [c.name for c in column_overviews + if c.cardinality_dimension_ratio is not None and c.cardinality_dimension_ratio >= HIGH_CARDINALITY_WARN] + status = "warn" if flagged else "pass" + return QualityCheck( + name="High Cardinality", status=status, + message=(f"High-cardinality column(s) (≥{HIGH_CARDINALITY_WARN:.0%} unique): {', '.join(flagged)}" + if flagged else "No high-cardinality columns"), + detail_link="#columns", + ) + + +def _check_mixed_types(column_overviews) -> QualityCheck: + flagged = [c.name for c in column_overviews if c.mixed_types] + status = "fail" if flagged else "pass" + return QualityCheck( + name="Mixed Types", status=status, + message=(f"Mixed-type column(s): {', '.join(flagged)}" if flagged else "No mixed-type columns"), + detail_link="#columns", + ) + + +def _check_infinity(numeric_overviews) -> QualityCheck: + total_inf = sum(int(n.infinity) for n in numeric_overviews if n.infinity) + status = "fail" if total_inf > 0 else "pass" + return QualityCheck(name="Infinity Check", status=status, + message=(f"{total_inf} infinite value(s)" if total_inf else "No infinite values"), + detail_link="#columns") + + +def _check_skewness(numeric_overviews) -> QualityCheck: + statuses, notes = [], [] + for n in numeric_overviews: + s = abs(n.skewness) if n.skewness == n.skewness else 0.0 # guard NaN + if s > SKEW_FAIL: + statuses.append("fail") + notes.append(f"{n.name} ({n.skewness:.1f})") + elif s > SKEW_WARN: + statuses.append("warn") + notes.append(f"{n.name} ({n.skewness:.1f})") + status = _worst(statuses) if statuses else "pass" + return QualityCheck( + name="Skewness", status=status, + message=(f"Strongly skewed (transform recommended): {', '.join(notes)}" + if notes else "No strongly skewed numeric columns"), + detail_link="#columns", + ) + + +def _check_outliers(numeric_overviews, n_rows) -> QualityCheck: + statuses, notes = [], [] + for n in numeric_overviews: + o = n.outliers + if o is None: + continue + flagged = (o.n_lower_iqr or 0) + (o.n_upper_iqr or 0) # adjusted IQR + rate = _rate(flagged, n_rows) + if rate > OUTLIER_FAIL: + statuses.append("fail") + notes.append(f"{n.name} ({rate * 100:.1f}%)") + elif rate > OUTLIER_WARN: + statuses.append("warn") + notes.append(f"{n.name} ({rate * 100:.1f}%)") + status = _worst(statuses) if statuses else "pass" + return QualityCheck( + name="Outliers (Adjusted IQR)", status=status, + message=(f"High outlier share: {', '.join(notes)}" if notes + else "No columns with excessive outliers"), + detail_link="#columns", + ) diff --git a/src/quality_assessment/dataset_quality.py b/src/quality_assessment/dataset_quality.py new file mode 100644 index 0000000..e09432e --- /dev/null +++ b/src/quality_assessment/dataset_quality.py @@ -0,0 +1,66 @@ +from models import QualityCheck +from quality_assessment.utils import _rate + +MIN_ROWS_PER_FEATURE_WARN = 10 +MIN_ROWS_PER_FEATURE_FAIL = 3 + +DUP_ROW_WARN = 0.05 +DUP_ROW_FAIL = 0.20 +EMPTY_ROW_WARN = 0.01 + +MISSING_COL_WARN = 0.30 +MISSING_COL_FAIL = 0.50 + + +def _check_sample_size(general) -> QualityCheck: + ratio = general.ratio + if ratio < MIN_ROWS_PER_FEATURE_FAIL: + status = "fail" + elif ratio < MIN_ROWS_PER_FEATURE_WARN: + status = "warn" + else: + status = "pass" + return QualityCheck( + name="Sample Size vs Feature Count", status=status, + message=f"{ratio:.1f} rows per feature " + f"({general.rows} rows / {general.cols} columns; ≥{MIN_ROWS_PER_FEATURE_WARN} recommended)", + detail_link="#overview", + ) + + +def _check_duplicate_rows(general) -> QualityCheck: + dup = general.dup_row + rate = _rate(dup, general.rows) + if rate > DUP_ROW_FAIL: + status = "fail" + elif rate > DUP_ROW_WARN: + status = "warn" + else: + status = "pass" + unique_rows = general.rows - dup + return QualityCheck( + name="Duplicate Rows", status=status, + message=(f"{dup} duplicate rows ({rate * 100:.1f}%); " + f"{unique_rows} unique of {general.rows}" if dup else "No duplicate rows"), + detail_link="#overview", + ) + + +def _check_duplicate_columns(general) -> QualityCheck: + dup = general.dup_col + status = "warn" if dup > 0 else "pass" + return QualityCheck( + name="Duplicate Columns", status=status, + message=(f"{dup} duplicate column(s)" if dup else "No duplicate columns"), + detail_link="#overview", + ) + + +def _check_empty_rows(general) -> QualityCheck: + empty = general.empty_rows + status = "warn" if empty > 0 else "pass" + return QualityCheck( + name="Empty Rows", status=status, + message=(f"{empty} completely empty row(s)" if empty else "No empty rows"), + detail_link="#overview", + ) diff --git a/src/quality_assessment/quality_assessment.py b/src/quality_assessment/quality_assessment.py new file mode 100644 index 0000000..ffeb092 --- /dev/null +++ b/src/quality_assessment/quality_assessment.py @@ -0,0 +1,79 @@ +from models.quality import QualityCategory, QualityAssessment +from quality_assessment.biological_quality import _check_sequence_validity, _check_sequence_redundancy, \ + _check_taxonomy_validity, _check_unit_validity +from quality_assessment.column_quality import _check_missing, _check_variance, _check_high_cardinality, \ + _check_mixed_types, _check_infinity, _check_skewness, _check_outliers +from quality_assessment.dataset_quality import _check_sample_size, _check_duplicate_rows, _check_duplicate_columns, \ + _check_empty_rows +from quality_assessment.relationships import _check_leakage, _check_multicollinearity +from quality_assessment.utils import _worst + + +def quality_assessment(general, column_overviews, numeric_overviews, categorical_overviews, multivariate) -> QualityAssessment: + numeric_overviews = [n for n in numeric_overviews if n is not None] + categorical_overviews = [c for c in categorical_overviews if c is not None] + + structure = [ + _check_sample_size(general), + _check_duplicate_rows(general), + _check_duplicate_columns(general), + _check_empty_rows(general), + ] + column_quality = [ + _check_missing(column_overviews), + _check_variance(column_overviews, categorical_overviews), + _check_high_cardinality(column_overviews), + _check_mixed_types(column_overviews), + _check_infinity(numeric_overviews), + _check_skewness(numeric_overviews), + _check_outliers(numeric_overviews, general.rows), + ] + biological = [ + _check_sequence_validity(column_overviews), + _check_sequence_redundancy(column_overviews), + _check_taxonomy_validity(column_overviews), + _check_unit_validity(column_overviews), + ] + relationships = [ + _check_leakage(multivariate), + _check_multicollinearity(multivariate), + ] + + categories = [ + QualityCategory("Dataset Structure", _worst([c.status for c in structure]), structure), + QualityCategory("Column Quality", _worst([c.status for c in column_quality]), column_quality), + QualityCategory("Biological", _worst([c.status for c in biological]), biological), + QualityCategory("Relationships", _worst([c.status for c in relationships]), relationships), + ] + + all_checks = [c for cat in categories for c in cat.checks] + passed = sum(1 for c in all_checks if c.status == "pass") + warnings = sum(1 for c in all_checks if c.status == "warn") + failed = sum(1 for c in all_checks if c.status == "fail") + + overall = "not_ready" if failed else ("caution" if warnings else "ready") + + return QualityAssessment( + categories=categories, passed=passed, warnings=warnings, + failed=failed, total=len(all_checks), overall=overall, + ) + + +def print_quality_report(quality: QualityAssessment) -> None: + symbols = {"pass": "[PASS]", "warn": "[WARN]", "fail": "[FAIL]"} + overall_label = {"ready": "READY", "caution": "READY WITH CAUTION", "not_ready": "NOT READY"} + + print("\n" + "=" * 64) + print("QUALITY ASSESSMENT".center(64)) + print("=" * 64) + print(f" Overall: {overall_label.get(quality.overall, quality.overall)}") + print(f" {quality.passed} passed | {quality.warnings} warnings | " + f"{quality.failed} failed (of {quality.total})") + + for cat in quality.categories: + print("-" * 64) + print(f" {symbols.get(cat.status)} {cat.name.upper()}") + for c in cat.checks: + print(f" {symbols.get(c.status)} {c.name}") + print(f" {c.message}") + print("=" * 64) diff --git a/src/quality_assessment/relationships.py b/src/quality_assessment/relationships.py new file mode 100644 index 0000000..0a6fae2 --- /dev/null +++ b/src/quality_assessment/relationships.py @@ -0,0 +1,37 @@ +from models import QualityCheck + +LEAKAGE_WARN = 0.90 +LEAKAGE_FAIL = 0.98 +MULTICOLLINEAR_WARN = 0.90 + + +def _check_leakage(multivariate) -> QualityCheck: + ftc = getattr(multivariate, "feature_target_correlation", None) + if not ftc: + return QualityCheck(name="Target Leakage", status="pass", + message="No target set; leakage check skipped", detail_link=None) + suspects = [(f, info["value"]) for f, info in ftc.items() if info["value"] >= LEAKAGE_WARN] + if not suspects: + return QualityCheck(name="Target Leakage", status="pass", + message="No feature strongly associated with the target", + detail_link="#multivariate") + suspects.sort(key=lambda x: x[1], reverse=True) + status = "fail" if suspects[0][1] >= LEAKAGE_FAIL else "warn" + listed = ", ".join(f"{f} ({v:.2f})" for f, v in suspects) + return QualityCheck(name="Target Leakage", status=status, + message=f"Suspiciously high feature-target association: {listed}", + detail_link="#multivariate") + + +def _check_multicollinearity(multivariate) -> QualityCheck: + pairs = getattr(multivariate, "top_associations", None) + if not pairs: + return QualityCheck(name="Multicollinearity", status="pass", + message="No strongly associated feature pairs", detail_link="#multivariate") + high = [p for p in pairs if p["value"] >= MULTICOLLINEAR_WARN] + if not high: + return QualityCheck(name="Multicollinearity", status="pass", + message="No feature pair above 0.90 association", detail_link="#multivariate") + listed = ", ".join(f"{p['var1']}↔{p['var2']} ({p['value']:.2f})" for p in high) + return QualityCheck(name="Multicollinearity", status="warn", + message=f"Highly associated feature pairs: {listed}", detail_link="#multivariate") diff --git a/src/quality_assessment/utils.py b/src/quality_assessment/utils.py new file mode 100644 index 0000000..ede1ef4 --- /dev/null +++ b/src/quality_assessment/utils.py @@ -0,0 +1,10 @@ +def _worst(statuses: list) -> str: + if "fail" in statuses: + return "fail" + if "warn" in statuses: + return "warn" + return "pass" + + +def _rate(value, total): + return value / total if total else 0.0 From 8c39bc32de34b07fa8bbb1847e77dd03ff41627d Mon Sep 17 00:00:00 2001 From: Sonja Diedrich Date: Wed, 17 Jun 2026 15:14:39 +0200 Subject: [PATCH 73/95] fix: updated print --- src/quality_assessment/dataset_quality.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/quality_assessment/dataset_quality.py b/src/quality_assessment/dataset_quality.py index e09432e..c7ecd30 100644 --- a/src/quality_assessment/dataset_quality.py +++ b/src/quality_assessment/dataset_quality.py @@ -23,7 +23,7 @@ def _check_sample_size(general) -> QualityCheck: return QualityCheck( name="Sample Size vs Feature Count", status=status, message=f"{ratio:.1f} rows per feature " - f"({general.rows} rows / {general.cols} columns; ≥{MIN_ROWS_PER_FEATURE_WARN} recommended)", + f"({general.rows} rows / {general.cols} columns", detail_link="#overview", ) From 2350b6679bbdee45693b2ee8b0b987aafc404d1b Mon Sep 17 00:00:00 2001 From: Maria Hansen Date: Wed, 17 Jun 2026 15:25:45 +0200 Subject: [PATCH 74/95] [feat] add quality assessment report section with category cards and overall banner --- src/cli/app.py | 6 +- src/cli/report_writer.py | 4 +- src/templates/numeric_overview.jinja | 127 ++++++++++++++++++++++++++- 3 files changed, 131 insertions(+), 6 deletions(-) diff --git a/src/cli/app.py b/src/cli/app.py index 9139a8f..4f2b4e9 100644 --- a/src/cli/app.py +++ b/src/cli/app.py @@ -100,10 +100,10 @@ def cli(input: str, tax: bool = False, func: str = None, categorical_columns(df, col) if col not in exclude_cols else None for col in cat_columns ] - readiness = quality_assessment(general, column_overviews, numeric_overviews, + quality = quality_assessment(general, column_overviews, numeric_overviews, categorical_overviews, plots) - print_quality_report(readiness) + print_quality_report(quality) output_path = Path(input_path.stem + "_renders") print(colored('Writing report …', 'green')) write_report(output_path, general, plots, duplicates_table, - column_overviews, numeric_overviews, categorical_overviews, top_n) + column_overviews, numeric_overviews, categorical_overviews, top_n, quality) diff --git a/src/cli/report_writer.py b/src/cli/report_writer.py index 8407bac..60f2c51 100644 --- a/src/cli/report_writer.py +++ b/src/cli/report_writer.py @@ -10,13 +10,13 @@ def write_report(output_path: Path, general, plots, duplicates_table, - column_overviews, numeric_overviews, categorical_overviews, top_n): + column_overviews, numeric_overviews, categorical_overviews, top_n, quality): output_path.mkdir(parents=True, exist_ok=True) shutil.copytree(str(STATIC_DIR), str(output_path / "static"), dirs_exist_ok=True) _render_to_file(output_path / "index.html", 'LandingPage.jinja') _render_to_file(output_path / "numeric_data.html", 'numeric_overview.jinja', - general=general, dups=duplicates_table) + general=general, dups=duplicates_table, quality=quality) _render_to_file(output_path / "columns.html", 'columns.jinja', columns=column_overviews, overview=numeric_overviews, categorical=categorical_overviews, top_n=top_n) diff --git a/src/templates/numeric_overview.jinja b/src/templates/numeric_overview.jinja index 3b10081..a32829e 100644 --- a/src/templates/numeric_overview.jinja +++ b/src/templates/numeric_overview.jinja @@ -127,8 +127,133 @@
    {# ================================================================ - 2) DUPLICATES + 2) QUALITY ASSESSMENT REPORT ================================================================ #} + {% if quality %} +

    Quality Assessment Report

    + + {# ── Overall banner ── #} + {% if quality.overall == 'ready' %} + {% set banner_class = 'success' %} + {% set banner_icon = 'M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zm-3.97-3.03a.75.75 0 0 0-1.08.022L7.477 9.417 5.384 7.323a.75.75 0 0 0-1.06 1.06L6.97 11.03a.75.75 0 0 0 1.079-.02l3.992-4.99a.75.75 0 0 0-.01-1.05z' %} + {% set banner_label = 'Ready' %} + {% elif quality.overall == 'caution' %} + {% set banner_class = 'warning' %} + {% set banner_icon = 'M8.982 1.566a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566zM8 5c.535 0 .954.462.9.995l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995A.905.905 0 0 1 8 5zm.002 6a1 1 0 1 1 0 2 1 1 0 0 1 0-2z' %} + {% set banner_label = 'Ready with Caution' %} + {% else %} + {% set banner_class = 'danger' %} + {% set banner_icon = 'M8.982 1.566a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566zM8 5c.535 0 .954.462.9.995l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995A.905.905 0 0 1 8 5zm.002 6a1 1 0 1 1 0 2 1 1 0 0 1 0-2z' %} + {% set banner_label = 'Not Ready' %} + {% endif %} + + + + {# ── Category cards ── #} +
    + {% for cat in quality.categories %} + {# category header colour #} + {% if cat.status == 'fail' %} + {% set cat_color = 'danger' %} + {% set cat_icon = 'M8.982 1.566a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566zM8 5c.535 0 .954.462.9.995l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995A.905.905 0 0 1 8 5zm.002 6a1 1 0 1 1 0 2 1 1 0 0 1 0-2z' %} + {% elif cat.status == 'warn' %} + {% set cat_color = 'warning' %} + {% set cat_icon = 'M8.982 1.566a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566zM8 5c.535 0 .954.462.9.995l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995A.905.905 0 0 1 8 5zm.002 6a1 1 0 1 1 0 2 1 1 0 0 1 0-2z' %} + {% else %} + {% set cat_color = 'success' %} + {% set cat_icon = 'M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zm-3.97-3.03a.75.75 0 0 0-1.08.022L7.477 9.417 5.384 7.323a.75.75 0 0 0-1.06 1.06L6.97 11.03a.75.75 0 0 0 1.079-.02l3.992-4.99a.75.75 0 0 0-.01-1.05z' %} + {% endif %} + +
    +
    + + {# card header #} +
    + + {{ cat.name }} + {# mini pass/warn/fail summary #} + {% set n_pass = cat.checks | selectattr('status','equalto','pass') | list | length %} + {% set n_warn = cat.checks | selectattr('status','equalto','warn') | list | length %} + {% set n_fail = cat.checks | selectattr('status','equalto','fail') | list | length %} + + {% if n_pass %}{{ n_pass }} ✓{% endif %} + {% if n_warn %}{{ n_warn }} ⚠{% endif %} + {% if n_fail %}{{ n_fail }} ✗{% endif %} + +
    + + {# check rows #} +
    + + + {% for check in cat.checks %} + {# row background #} + {% if check.status == 'fail' %} + {% set row_class = 'table-danger' %} + {% set dot_color = 'text-danger' %} + {% set dot = '✗' %} + {% elif check.status == 'warn' %} + {% set row_class = 'table-warning' %} + {% set dot_color = 'text-warning-emphasis' %} + {% set dot = '⚠' %} + {% else %} + {% set row_class = '' %} + {% set dot_color = 'text-success' %} + {% set dot = '✓' %} + {% endif %} + + + {# status icon #} + + {# check name #} + + {# message – truncate long ones with a title tooltip #} + + {# optional deep-link #} + + + {% endfor %} + +
    + + {{ dot }} + + + {{ check.name }} + + {{ check.message | truncate(140, True, '…') }} + + {% if check.detail_link %} + ↗ + {% endif %} +
    +
    + +
    +
    + {% endfor %} +
    + {% endif %}{# /quality #} + +

    Duplicates {% if general.dup_row > 0 %} From bbf99ac908a6bd47bed46af0d0c9edb2da9276d6 Mon Sep 17 00:00:00 2001 From: Sonja Diedrich Date: Thu, 18 Jun 2026 11:49:08 +0200 Subject: [PATCH 75/95] chore/fix: updating methods for speed up and memory efficiency --- __main__.py | 4 +- src/analysis/multivariate_analysis.py | 69 ++++++++++---------- src/biological/sequence_data.py | 15 +++-- src/biological/taxonomy.py | 18 +++-- src/cli/app.py | 1 + src/quality_assessment/column_quality.py | 29 ++++---- src/quality_assessment/quality_assessment.py | 4 +- 7 files changed, 79 insertions(+), 61 deletions(-) diff --git a/__main__.py b/__main__.py index a93fb3d..3eb087a 100644 --- a/__main__.py +++ b/__main__.py @@ -7,5 +7,5 @@ # cli(['-i', "data/iedb_sub.tsv", "--tax"]) #cli(['-i', 'data/winequality-white.csv']) # cli(['-i', "data/proteinGroups.tsv"]) - #cli(['-i', 'data/bakrep_sagalactiae.tsv']) - cli(['-i', 'data/iedb_test.tsv', '-tc', 'bind_class']) \ No newline at end of file + cli(['-i', 'data/bakrep_sagalactiae.tsv']) + #cli(['-i', 'data/iedb_test.tsv', '-tc', 'bind_class']) \ No newline at end of file diff --git a/src/analysis/multivariate_analysis.py b/src/analysis/multivariate_analysis.py index b4e69ac..f238ed5 100644 --- a/src/analysis/multivariate_analysis.py +++ b/src/analysis/multivariate_analysis.py @@ -2,6 +2,7 @@ import pandas as pd import plotly.express as px import plotly.graph_objects as go +from scipy.sparse import coo_matrix from scipy.stats import chi2_contingency from models.multivariate import MultivariateAnalysis @@ -9,6 +10,7 @@ def multivariate_analysis(df: pd.DataFrame, target: str) -> MultivariateAnalysis: + types = _classify_columns(df) values, methods = compute_correlation_matrix(df) feat_target_corr = feature_target_correlation(df, target) if target else None mi = mutual_information(df, target) if target else None @@ -16,8 +18,8 @@ def multivariate_analysis(df: pd.DataFrame, target: str) -> MultivariateAnalysis return MultivariateAnalysis( correlation_heatmap=correlation_heatmap(values, methods), pearson_heatmap=pearson_correlation_heatmap(df), - cramers_heatmap=cramers_heatmap(df), - eta_heatmap=eta_heatmap(df), + cramers_heatmap=cramers_heatmap(values, types), + eta_heatmap=eta_heatmap(values, types), missing_matrix=missing_matrix(df), missing_values_barchart=missing_values_barchart(df), balance_plot=balance_plot(df, target) if target else None, @@ -36,11 +38,6 @@ def multivariate_analysis(df: pd.DataFrame, target: str) -> MultivariateAnalysis ) def _classify_columns(df: pd.DataFrame) -> dict: - """Return {column: 'numeric' | 'categorical'} for usable columns. - - Drops constant numeric columns and single-value categoricals, since they - carry no association signal. - """ num = df.select_dtypes(include='number') num_std = num.std(ddof=0) numeric = num_std[num_std > 0].index @@ -50,6 +47,7 @@ def _classify_columns(df: pd.DataFrame) -> dict: categorical = cat_nunique[cat_nunique > 1].index return {**{c: 'numeric' for c in numeric}, **{c: 'categorical' for c in categorical}} + """ Correlation """ @@ -67,14 +65,26 @@ def _cramers_v(df: pd.DataFrame, a: str, b: str) -> float: if pair.empty: return np.nan - confusion = pd.crosstab(pair[a], pair[b]) - if confusion.shape[0] < 2 or confusion.shape[1] < 2: + a_codes, _ = pair[a].factorize() + b_codes, _ = pair[b].factorize() + if len(a_codes) == 0 or len(b_codes) == 0: return np.nan + r = int(a_codes.max()) + 1 + k = int(b_codes.max()) + 1 + if r < 2 or k < 2: + return np.nan + + n = len(pair) + obs = coo_matrix((np.ones(n), (a_codes, b_codes)), shape=(r, k)).tocsr() + row_sum = np.asarray(obs.sum(axis=1)).ravel() + col_sum = np.asarray(obs.sum(axis=0)).ravel() + + coo = obs.tocoo() + expected = row_sum[coo.row] * col_sum[coo.col] / n + chi2 = np.sum(coo.data ** 2 / expected) - n - chi2 = chi2_contingency(confusion)[0] - n = confusion.to_numpy().sum() phi2 = chi2 / n - r, k = confusion.shape + r, k = obs.shape # Bergsma-Bias Correction phi2corr = max(0.0, phi2 - ((k - 1) * (r - 1)) / (n - 1)) @@ -241,19 +251,13 @@ def correlation_heatmap(df: pd.DataFrame, methods: pd.DataFrame): return fig.to_html(full_html=False, include_plotlyjs=False) -def cramers_heatmap(df: pd.DataFrame) -> str | None: +def cramers_heatmap(df: pd.DataFrame, types: dict) -> str | None: """Cramér's V matrix over categorical columns only (0..1).""" - cols = [c for c, t in _classify_columns(df).items() if t == 'categorical'] + cols = [c for c, t in types.items() if t == 'categorical' and c in df.index] if len(cols) < 2: return None - mat = pd.DataFrame(np.eye(len(cols)), index=cols, columns=cols) - for i in range(len(cols)): - for j in range(i + 1, len(cols)): - val = _cramers_v(df, cols[i], cols[j]) - val = round(float(val), 3) if val == val else np.nan - mat.iat[i, j] = val - mat.iat[j, i] = val + mat = df.loc[cols, cols] # light -> primary blue (#0F65A0) colorscale = [ @@ -282,28 +286,24 @@ def cramers_heatmap(df: pd.DataFrame) -> str | None: )) fig.update_layout( title="Cramér's V (categorical pairs)", + autosize=True, height=max(450, 55 * len(cols) + 200), template="plotly_white", xaxis=dict(tickangle=-45), annotations=annotations, - plot_bgcolor="#A1ACBD", # NaN cells appear neutral grey + plot_bgcolor="#A1ACBD", ) fig.update_yaxes(autorange="reversed") - return fig.to_html(full_html=False, include_plotlyjs=False) + return fig.to_html(full_html=False, include_plotlyjs=False, config={"responsive": True}) -def eta_heatmap(df: pd.DataFrame) -> str | None: - types = _classify_columns(df) - cats = [c for c, t in types.items() if t == 'categorical'] - nums = [c for c, t in types.items() if t == 'numeric'] +def eta_heatmap(df: pd.DataFrame, types: dict) -> str | None: + cats = [c for c, t in types.items() if t == 'categorical' and c in df.index] + nums = [c for c, t in types.items() if t == 'numeric' and c in df.index] if not cats or not nums: return None - mat = pd.DataFrame(np.zeros((len(cats), len(nums))), index=cats, columns=nums) - for c in cats: - for nu in nums: - val = _eta_squared(df, c, nu) - mat.at[c, nu] = round(float(val), 3) if val == val else np.nan + mat = df.loc[cats, nums] # light -> magenta (#994564) colorscale = [ @@ -332,6 +332,7 @@ def eta_heatmap(df: pd.DataFrame) -> str | None: )) fig.update_layout( title="Eta² — variance in numeric explained by categorical (directional)", + autosize=True, height=max(450, 55 * len(cats) + 200), template="plotly_white", xaxis_title="Numeric", yaxis_title="Categorical", @@ -340,7 +341,7 @@ def eta_heatmap(df: pd.DataFrame) -> str | None: plot_bgcolor="#A1ACBD", ) fig.update_yaxes(autorange="reversed") - return fig.to_html(full_html=False, include_plotlyjs=False) + return fig.to_html(full_html=False, include_plotlyjs=False, config={"responsive": True}) def top_associations(values, methods, threshold=0.7): pairs = [] @@ -379,7 +380,7 @@ def mutual_information(df: pd.DataFrame, target: str) -> dict | None: discrete_features = [] for col in features: if types[col] == 'categorical': - X[col], _ = X[col].factorize() # your preferred encoding + X[col], _ = X[col].factorize() discrete_features.append(True) else: X[col] = X[col].astype(float) diff --git a/src/biological/sequence_data.py b/src/biological/sequence_data.py index 6482a56..b38ee58 100644 --- a/src/biological/sequence_data.py +++ b/src/biological/sequence_data.py @@ -3,12 +3,12 @@ from collections import defaultdict, Counter from itertools import chain from pathlib import Path -from typing import Tuple +from typing import Tuple, List, Dict from urllib.error import URLError, HTTPError from Bio import motifs from Bio.SeqUtils.ProtParam import ProteinAnalysis -from weblogo import * +#from weblogo import * import numpy as np import pandas as pd import peptides @@ -207,13 +207,16 @@ def _reverse_complement_duplicates(all_seqs: pd.Series) -> Tuple[float,set]: return (round(redundant / len(seq_set) * 100, 2) if len(seq_set) > 0 else 0.0), redundant_seqs def _normalized_shanon_entropy(seq: str) -> np.float64: - if len(seq) == 0: + n = len(seq) + if n == 0: return np.float64(0.0) - counts = np.array(list(Counter(seq).values())) - max_entropy = np.log2(min(len(seq), len(counts))) # ToDo or 4? + counts = np.fromiter(Counter(seq).values(), dtype=np.float64) + max_entropy = np.log2(min(n, len(counts))) # ToDo or 4? if max_entropy == 0: return np.float64(0.0) - return np.round(entropy(counts, base=2) / max_entropy * 100, 2).astype(np.float64) + p = counts / n + h = -np.sum(p * np.log2(p)) + return np.float64(round(float(h / max_entropy * 100), 2)) def _dinucleotide_oe(df: pd.DataFrame) -> Tuple[SequenceMetricSummary, SequenceMetricSummary]: seqs = df["sequence"] diff --git a/src/biological/taxonomy.py b/src/biological/taxonomy.py index ed18f41..7388bd6 100644 --- a/src/biological/taxonomy.py +++ b/src/biological/taxonomy.py @@ -1,7 +1,6 @@ from dataclasses import dataclass from typing import Tuple -import numpy as np import pandas as pd @dataclass @@ -93,13 +92,19 @@ def is_taxid(col: pd.Series, valid_tax_ids: set, threshold: float = 0.9) -> set def is_taxonomy(col: pd.Series, valid_names: set, name_to_rank: dict, name_to_scientific: dict, threshold: float = 0.8) -> dict | None: - is_valid = col.isin(valid_names) + col_obj = col.astype(object) + uniques = pd.unique(col_obj.dropna()) + valid_unique = {u for u in uniques if u in valid_names} + + is_valid = col_obj.isin(valid_unique) validity_rate = is_valid.sum() / len(col) - cleaned_names = col + cleaned_names = col_obj if validity_rate < threshold: - cleaned_names = col.astype(str).str.extract(r'^([^(]+)')[0].str.strip() - is_valid_cleaned = cleaned_names.isin(valid_names) + cleaned_names = col_obj.astype(str).str.extract(r'^([^(]+)')[0].str.strip() + cleaned_uniques = pd.unique(cleaned_names.dropna()) + valid_cleaned_unique = {u for u in cleaned_uniques if u in valid_names} + is_valid_cleaned = cleaned_names.isin(valid_cleaned_unique) validity_rate_cleaned = is_valid_cleaned.sum() / len(col) if validity_rate_cleaned > validity_rate: @@ -144,7 +149,8 @@ def rank_distribution(col: pd.Series, name_to_rank: dict, threshold: float = 0.0 def find_outdated_names(col: pd.Series, valid_names: set, name_to_scientific: dict) -> dict: cleaned = col.astype(str).str.strip() - valid_used = cleaned[cleaned.isin(valid_names)].unique() + uniques = pd.unique(cleaned.dropna()) + valid_used = [u for u in uniques if u in valid_names] outdated = {} for name in valid_used: diff --git a/src/cli/app.py b/src/cli/app.py index 4f2b4e9..63845b5 100644 --- a/src/cli/app.py +++ b/src/cli/app.py @@ -37,6 +37,7 @@ def cli(input: str, tax: bool = False, func: str = None, print(colored(f'Reading file {input_path.name}', 'green')) df = read_file(input_path) if input_path.suffix != '.parquet' else parse_parquet(input_path) general = overview(df, input_path.name) + print("Starting Multivariate Analysis") plots = multivariate_analysis(df, target_column) # ToDo Rename Plots to Multivariate dups = df[df.duplicated(keep=False)].reset_index() diff --git a/src/quality_assessment/column_quality.py b/src/quality_assessment/column_quality.py index c212457..1300a73 100644 --- a/src/quality_assessment/column_quality.py +++ b/src/quality_assessment/column_quality.py @@ -18,24 +18,31 @@ def _missing_rate(col): return mp / 100.0 if mp > 1 else mp -def _check_missing(column_overviews) -> QualityCheck: - flagged = [] - worst = "pass" +def _check_missing(column_overviews) -> list[QualityCheck]: + flagged, empty, worst = [], [], "pass" for c in column_overviews: rate = _missing_rate(c) - if rate > MISSING_COL_FAIL: + if rate >= 1.0: + empty.append(c.name) + elif rate > MISSING_COL_FAIL: flagged.append((c.name, rate)); worst = "fail" elif rate > MISSING_COL_WARN: flagged.append((c.name, rate)) worst = "fail" if worst == "fail" else "warn" - if not flagged: - return QualityCheck(name="Missing Values", status="pass", - message="No column above 30% missing", detail_link="#multivariate") - flagged.sort(key=lambda x: x[1], reverse=True) - listed = ", ".join(f"{n} ({r * 100:.0f}%)" for n, r in flagged) - return QualityCheck(name="Missing Values", status=worst, - message=f"Columns above 30% missing: {listed}", detail_link="#multivariate") + checks = [] + if flagged: + flagged.sort(key=lambda x: x[1], reverse=True) + listed = ", ".join(f"{n} ({r*100:.0f}%)" for n, r in flagged) + checks.append(QualityCheck("Missing Values", worst, + f"Columns above 30% missing: {listed}", "#multivariate")) + if empty: + checks.append(QualityCheck("Empty Column", "fail", + f"Empty column(s): {', '.join(empty)}", "#multivariate")) + if not checks: + checks.append(QualityCheck("Missing Values", "pass", + "No column above 30% missing", "#multivariate")) + return checks def _check_variance(column_overviews, categorical_overviews) -> QualityCheck: constant_cols = [c.name for c in column_overviews if c.constant] diff --git a/src/quality_assessment/quality_assessment.py b/src/quality_assessment/quality_assessment.py index ffeb092..985485c 100644 --- a/src/quality_assessment/quality_assessment.py +++ b/src/quality_assessment/quality_assessment.py @@ -20,7 +20,7 @@ def quality_assessment(general, column_overviews, numeric_overviews, categorical _check_empty_rows(general), ] column_quality = [ - _check_missing(column_overviews), + *_check_missing(column_overviews), _check_variance(column_overviews, categorical_overviews), _check_high_cardinality(column_overviews), _check_mixed_types(column_overviews), @@ -61,7 +61,7 @@ def quality_assessment(general, column_overviews, numeric_overviews, categorical def print_quality_report(quality: QualityAssessment) -> None: symbols = {"pass": "[PASS]", "warn": "[WARN]", "fail": "[FAIL]"} - overall_label = {"ready": "READY", "caution": "READY WITH CAUTION", "not_ready": "NOT READY"} + overall_label = {"ready": "NO QUALITY ISSUES", "caution": "MINOR QUALITY ISSUES", "not_ready": "MAJOR QUALITY ISSUES"} print("\n" + "=" * 64) print("QUALITY ASSESSMENT".center(64)) From 2a332086264edf13102e6bdd6355df4e17a4957d Mon Sep 17 00:00:00 2001 From: Sonja Diedrich Date: Thu, 18 Jun 2026 14:17:41 +0200 Subject: [PATCH 76/95] chore: updates tool prints --- src/analysis/multivariate_analysis.py | 6 +- src/analysis/numeric_analysis.py | 2 - src/analysis/outlier_detection.py | 3 +- src/analysis/overview.py | 4 +- src/biological/sequence_data.py | 1 - src/cli/app.py | 94 +++++++++++++------- src/cli/utils.py | 27 ++++++ src/quality_assessment/quality_assessment.py | 6 +- 8 files changed, 99 insertions(+), 44 deletions(-) create mode 100644 src/cli/utils.py diff --git a/src/analysis/multivariate_analysis.py b/src/analysis/multivariate_analysis.py index f238ed5..1517813 100644 --- a/src/analysis/multivariate_analysis.py +++ b/src/analysis/multivariate_analysis.py @@ -103,10 +103,12 @@ def _eta_squared(df: pd.DataFrame, cat: str, num: str) -> float: if len(pair) < 2: return np.nan - groups = pair.groupby(cat)[num] grand_mean = pair[num].mean() + agg = pair.groupby(cat)[num].agg(['count', 'mean']) + counts = agg['count'].to_numpy() + means = agg['mean'].to_numpy() - ss_between = sum(len(g) * (g.mean() - grand_mean) ** 2 for _, g in groups) + ss_between = np.sum(counts * (means - grand_mean) ** 2) ss_total = ((pair[num] - grand_mean) ** 2).sum() if ss_total == 0: diff --git a/src/analysis/numeric_analysis.py b/src/analysis/numeric_analysis.py index 45f0f00..2696322 100644 --- a/src/analysis/numeric_analysis.py +++ b/src/analysis/numeric_analysis.py @@ -12,8 +12,6 @@ def _safe_round(val, decimals=2): def numeric_columns(df: pd.DataFrame, col) -> NumericColumns: series = df[col].dropna() - print("Column: ", col) - res = detect_outliers(series.to_numpy(dtype=np.double)) if series.empty: coefficient_of_variation = np.nan diff --git a/src/analysis/outlier_detection.py b/src/analysis/outlier_detection.py index a57609a..d857b45 100644 --- a/src/analysis/outlier_detection.py +++ b/src/analysis/outlier_detection.py @@ -19,12 +19,11 @@ def detect_outliers(values: np.ndarray) -> Optional[Outliers]: if iqr <= 0: return None - # Adjusted Boxplot (Hubert & Vandervieren, 2008) + # Adjusted IQR (Hubert & Vandervieren, 2008) """ Adjusted implementation adapted from robpy (v.0.0.6) Source: https://robpy.readthedocs.io/en/latest/_modules/robpy/univariate/adjusted_boxplot.html """ - #print("Adjusted IQR") mc = fast_medcouple(values) if mc >= 0: diff --git a/src/analysis/overview.py b/src/analysis/overview.py index c6235ae..7df9332 100644 --- a/src/analysis/overview.py +++ b/src/analysis/overview.py @@ -26,8 +26,8 @@ def overview(df: pd.DataFrame, file) -> DatasetSummary: def column_overview(df: pd.DataFrame, col) -> ColumnOverview: seq_type, invalid = check_sequence(df, col) - if seq_type != "None": - print(f"Column: {col:10s} is of type: {seq_type:10s} with invalid sequences: {invalid}.") + #if seq_type != "None": + # print(f"Column: {col:10s} is of type: {seq_type:10s} with invalid sequences: {invalid}.") mixed = check_mixed_types(df, col) if seq_type == "None" else None if mixed is not None: suspect = mixed[1] diff --git a/src/biological/sequence_data.py b/src/biological/sequence_data.py index b38ee58..5ec4324 100644 --- a/src/biological/sequence_data.py +++ b/src/biological/sequence_data.py @@ -8,7 +8,6 @@ from Bio import motifs from Bio.SeqUtils.ProtParam import ProteinAnalysis -#from weblogo import * import numpy as np import pandas as pd import peptides diff --git a/src/cli/app.py b/src/cli/app.py index 63845b5..12f4b99 100644 --- a/src/cli/app.py +++ b/src/cli/app.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 +import time from pathlib import Path import click @@ -15,6 +16,7 @@ from biological.sequence_data import dna_rna_columns, protein_columns from biological.taxonomy import taxonomy_flags, build_lookups from cli.report_writer import write_report +from cli.utils import _fmt_duration, print_step, info from data_utils.file_reader import read_file, parse_parquet from data_utils.remote_data import get_tax_ids from quality_assessment.quality_assessment import quality_assessment, print_quality_report @@ -33,12 +35,17 @@ @click.option('-n', '--top_n', type=int, default=20, help="Top N entries analysis") def cli(input: str, tax: bool = False, func: str = None, target_column: str = None, kmer: int = None, top_n: int = None): + run_start = time.perf_counter() input_path = Path(input) - print(colored(f'Reading file {input_path.name}', 'green')) + + print(f"\n{'=' * 64}") + print(colored(f" BioProfileKit — {input_path.name}", "magenta", attrs=["bold"])) + print(f"{'=' * 64}") + + done = print_step(f"Reading {input_path.name}") df = read_file(input_path) if input_path.suffix != '.parquet' else parse_parquet(input_path) general = overview(df, input_path.name) - print("Starting Multivariate Analysis") - plots = multivariate_analysis(df, target_column) # ToDo Rename Plots to Multivariate + done(f"{df.shape[0]:,} rows × {df.shape[1]} columns") dups = df[df.duplicated(keep=False)].reset_index() duplicates_table = dups.to_html( @@ -46,65 +53,86 @@ def cli(input: str, tax: bool = False, func: str = None, border="0", table_id="dup_table", index=False, ) - print(colored(f'Analyse {len(df.columns)} columns', 'blue')) + done = print_step(f"Column overview ({len(df.columns)} columns)") column_overviews = [column_overview(df, col) for col in df.columns] + done() tax_df = get_tax_ids() if tax else None - valid_names, valid_tax_ids, name_to_rank, taxid_to_rank, name_to_scientific = None, None, None, None, None if tax and tax_df is not None: + done = print_step("Building taxonomy lookups") valid_names, valid_tax_ids, name_to_rank, taxid_to_rank, name_to_scientific = build_lookups(tax_df) + done(f"{len(valid_names):,} scientific names") + done = print_step("Per-column analysis") + seq_count = 0 for col_ov in column_overviews: if tax and tax_df is not None: - col_ov.taxonomy = taxonomy_flags(df, col_ov.name, valid_names, valid_tax_ids, name_to_rank, taxid_to_rank, name_to_scientific) - if func and tax_df is not None: + col_ov.taxonomy = taxonomy_flags(df, col_ov.name, valid_names, valid_tax_ids, name_to_rank, taxid_to_rank, name_to_scientific) + + if func and col_ov.taxonomy is not None: col_ov.annotation = [annotation_flags(df, col_ov.name, func)] + if hasattr(col_ov, "top_10") and isinstance(col_ov.top_10, pd.Series): col_ov.top_10_items = list(col_ov.top_10.items()) - # Biologische Sequenz-Analyse if col_ov.sequence == 'dna': - print(colored(f'Analyzing DNA/RNA sequences in column: {col_ov.name}', 'cyan')) + info(f"DNA sequences: {col_ov.name}") + col_ov.dna_rna_data = dna_rna_columns(df[col_ov.name], k=kmer, top_n=top_n, invalid=col_ov.invalid_seqs) + seq_count += 1 + elif col_ov.sequence == 'rna': + info(f"RNA sequences: {col_ov.name}") col_ov.dna_rna_data = dna_rna_columns(df[col_ov.name], k=kmer, top_n=top_n, invalid=col_ov.invalid_seqs) + seq_count += 1 elif col_ov.sequence == 'protein': - print(colored(f'Analyzing protein sequences in column: {col_ov.name}', 'cyan')) - col_ov.protein_data = protein_columns(df[col_ov.name], k=kmer, top_n=top_n, invalid=col_ov.invalid_seqs) + info(f"protein sequences: {col_ov.name}") + col_ov.protein_data = protein_columns(df[col_ov.name], k=kmer, top_n=top_n, invalid=col_ov.invalid_seqs) + seq_count += 1 else: col_ov.dna_rna_data = None col_ov.protein_data = None - # Measurement-Analyse if col_ov.sequence == 'None': measurement_data = measurement_columns(df[col_ov.name], col_ov.name, col_ov.type) if measurement_data: - print(colored(f'Analyzing lab measurements in column: {col_ov.name}', 'cyan')) + info(f"lab measurements: {col_ov.name}") col_ov.measurement_data = measurement_data if measurement_data else None else: col_ov.measurement_data = None + done(f"{seq_count} sequence column(s)") - print(colored(f'Analyse {len(df.select_dtypes(include="number").columns)} numeric columns', 'blue')) - - exclude_cols = [] - numeric_overviews = [ - numeric_columns(df, col) if not df[col].isnull().all() else exclude_cols.append(col) - for col in df.select_dtypes(include="number").columns - ] - #Todo Identify numeric cat columns - cat_columns = [ - col for col in df.select_dtypes(include=['str', 'object', 'bool', 'int64', 'float64']).columns - if any(i.sequence == 'None' for i in column_overviews if i.name == col) - ] - print(colored(f'Analyse {len(cat_columns)} object columns', 'blue')) - - categorical_overviews = [ - categorical_columns(df, col) if col not in exclude_cols else None - for col in cat_columns - ] + empty_cols = [col for col in df.columns if df[col].isnull().all()] + sequence_cols = {col.name for col in column_overviews if col.sequence in ('dna', 'rna', 'protein')} + + numeric_cols = [col for col in df.select_dtypes(include="number").columns + if col not in empty_cols and col not in sequence_cols] + done = print_step(f"Numeric analysis ({len(numeric_cols)} columns)") + numeric_overviews = [numeric_columns(df, col) for col in numeric_cols] + done() + + cat_columns = [col for col in df.select_dtypes(include=['str', 'object', 'bool']).columns + if col not in empty_cols and col not in sequence_cols] + done = print_step(f"Categorical analysis ({len(cat_columns)} columns)") + categorical_overviews = [categorical_columns(df, col) for col in cat_columns] + done() + + done = print_step("Multivariate analysis") + plots = multivariate_analysis(df, target_column) + done() + + done = print_step("Quality assessment") quality = quality_assessment(general, column_overviews, numeric_overviews, - categorical_overviews, plots) + categorical_overviews, plots) + done() print_quality_report(quality) + output_path = Path(input_path.stem + "_renders") - print(colored('Writing report …', 'green')) + done = print_step("Writing report") write_report(output_path, general, plots, duplicates_table, column_overviews, numeric_overviews, categorical_overviews, top_n, quality) + done(f"→ {output_path}/") + + total = _fmt_duration(time.perf_counter() - run_start) + print(f"\n{'=' * 64}") + print(colored(f" Done in {total}", "green", attrs=["bold"])) + print(f"{'=' * 64}\n") diff --git a/src/cli/utils.py b/src/cli/utils.py new file mode 100644 index 0000000..e498f79 --- /dev/null +++ b/src/cli/utils.py @@ -0,0 +1,27 @@ +import time + +from termcolor import colored + + +def _fmt_duration(seconds: float) -> str: + if seconds < 60: + return f"{seconds:.1f}s" + m, s = divmod(seconds, 60) + return f"{int(m)}m {s:.0f}s" + + +def print_step(label: str): + """Print a phase header and return a callable that closes it with timing.""" + print(f"▶ {label} …") + start = time.perf_counter() + + def done(detail: str = ""): + elapsed = _fmt_duration(time.perf_counter() - start) + suffix = f" — {detail}" if detail else "" + print(colored(f" ✓", "green"), f"{label} ({elapsed}){suffix}") + + return done + + +def info(message: str): + print(colored(f" {message}", "blue")) diff --git a/src/quality_assessment/quality_assessment.py b/src/quality_assessment/quality_assessment.py index 985485c..26180ec 100644 --- a/src/quality_assessment/quality_assessment.py +++ b/src/quality_assessment/quality_assessment.py @@ -1,3 +1,5 @@ +from termcolor import colored + from models.quality import QualityCategory, QualityAssessment from quality_assessment.biological_quality import _check_sequence_validity, _check_sequence_redundancy, \ _check_taxonomy_validity, _check_unit_validity @@ -60,8 +62,8 @@ def quality_assessment(general, column_overviews, numeric_overviews, categorical def print_quality_report(quality: QualityAssessment) -> None: - symbols = {"pass": "[PASS]", "warn": "[WARN]", "fail": "[FAIL]"} - overall_label = {"ready": "NO QUALITY ISSUES", "caution": "MINOR QUALITY ISSUES", "not_ready": "MAJOR QUALITY ISSUES"} + symbols = {"pass": colored("[PASS]", 'green'), "warn": colored("[WARN]", 'yellow'), "fail": colored("[FAIL]", 'red')} + overall_label = {"ready": colored("NO QUALITY ISSUES", 'green'), "caution": colored("MINOR QUALITY ISSUES", 'yellow'), "not_ready": colored("MAJOR QUALITY ISSUES", 'red')} print("\n" + "=" * 64) print("QUALITY ASSESSMENT".center(64)) From ed9b238729e7a136122e4e93ed61af3fa98ea553 Mon Sep 17 00:00:00 2001 From: Sonja Diedrich Date: Thu, 18 Jun 2026 15:39:04 +0200 Subject: [PATCH 77/95] feat: added values for General Dataset Statistics --- src/cli/app.py | 11 +++++++++++ src/models/overview.py | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/src/cli/app.py b/src/cli/app.py index 12f4b99..8431335 100644 --- a/src/cli/app.py +++ b/src/cli/app.py @@ -126,6 +126,17 @@ def cli(input: str, tax: bool = False, func: str = None, done() print_quality_report(quality) + general.n_number = len(numeric_cols) + general.n_categorical = len(cat_columns) + general.n_empty = len(empty_cols) + general.n_dna = sum(1 for col in column_overviews if col.sequence == 'dna') + general.n_rna = sum(1 for col in column_overviews if col.sequence == 'rna') + general.n_protein = sum(1 for col in column_overviews if col.sequence == 'protein') + general.n_taxonomy = sum(1 for col in column_overviews + if getattr(col, 'taxonomy', None) and col.taxonomy.is_taxonomy) + general.n_unit = sum(1 for col in column_overviews if col.measurement_data is not None) + general.n_functional = sum(1 for col in column_overviews if getattr(col, 'annotation', None)) + output_path = Path(input_path.stem + "_renders") done = print_step("Writing report") write_report(output_path, general, plots, duplicates_table, diff --git a/src/models/overview.py b/src/models/overview.py index c6e3823..f01b7e6 100644 --- a/src/models/overview.py +++ b/src/models/overview.py @@ -15,6 +15,16 @@ class DatasetSummary: ratio: float memory: float alerts: int + #ToDo Added for General Dataset Statistics Table + n_number: int = 0 + n_dna: int = 0 + n_rna: int = 0 + n_protein: int = 0 + n_taxonomy: int = 0 + n_unit: int = 0 + n_functional: int = 0 + n_categorical: int = 0 + n_empty: int = 0 @dataclass From 5b5e1eb98ce996e6060d08d06fa36b755db35b1f Mon Sep 17 00:00:00 2001 From: Maria Hansen Date: Thu, 18 Jun 2026 15:44:45 +0200 Subject: [PATCH 78/95] [refactor] flat navbar, fixed colorblind pill, remove message truncation --- src/templates/LandingPage.jinja | 73 ++++++++++----------- src/templates/columns.jinja | 91 +++++++++++++++----------- src/templates/general_statistics.jinja | 63 +++++++++--------- src/templates/numeric_overview.jinja | 59 +++++++++-------- 4 files changed, 152 insertions(+), 134 deletions(-) diff --git a/src/templates/LandingPage.jinja b/src/templates/LandingPage.jinja index e40a88a..2407e18 100644 --- a/src/templates/LandingPage.jinja +++ b/src/templates/LandingPage.jinja @@ -24,23 +24,10 @@

    @@ -111,27 +98,39 @@
    + + +
    + + + Colorblind Mode +
    \ No newline at end of file diff --git a/src/templates/columns.jinja b/src/templates/columns.jinja index f85dae3..e0f1298 100644 --- a/src/templates/columns.jinja +++ b/src/templates/columns.jinja @@ -55,23 +55,10 @@
    @@ -502,6 +489,11 @@ {% if col.taxonomy.outdated_names %} Outdated names: {{ col.taxonomy.outdated_names | length }}
    {% endif %} + {% elif col.taxonomy is defined and col.taxonomy is none %} + Taxonomy: + Not checked + (run with --tax to enable) +
    {% endif %} {% if col.annotation and col.annotation.is_annotation %} {% if col.annotation.valid_annotation is string %} @@ -529,8 +521,8 @@
    {% if col.describe_plot %} -
    -
    +
    +
    {{ col.describe_plot | safe }}
    @@ -586,6 +578,14 @@ {% endif %} + {% elif col.taxonomy is defined and col.taxonomy is none %} + {% endif %} @@ -850,7 +850,7 @@ {% for tid in tax.taxid | sort %} {{ loop.index }} - {{ tid }} + {{ tid }} {% endfor %} @@ -2672,25 +2672,38 @@ + + +
    + + + Colorblind Mode +
    \ No newline at end of file diff --git a/src/templates/general_statistics.jinja b/src/templates/general_statistics.jinja index 0fbf033..ac501f2 100644 --- a/src/templates/general_statistics.jinja +++ b/src/templates/general_statistics.jinja @@ -78,23 +78,10 @@
    @@ -281,7 +268,7 @@ @@ -392,8 +379,8 @@ {# Cramér's V #}
    {% if plots.cramers_heatmap %} -
    -
    {{ plots.cramers_heatmap | safe }}
    +
    +
    {{ plots.cramers_heatmap | safe }}
    {% else %}
    At least two categorical columns required — Cramér’s V not available.
    @@ -403,8 +390,8 @@ {# Eta² #}
    {% if plots.eta_heatmap %} -
    -
    {{ plots.eta_heatmap | safe }}
    +
    +
    {{ plots.eta_heatmap | safe }}
    {% else %}
    At least one categorical and one numeric column required — Eta² not available.
    @@ -631,22 +618,22 @@ @@ -707,5 +694,21 @@ $(document).ready(function () { }); + + +
    + + + Colorblind Mode +
    diff --git a/src/templates/numeric_overview.jinja b/src/templates/numeric_overview.jinja index a32829e..0cd2c9a 100644 --- a/src/templates/numeric_overview.jinja +++ b/src/templates/numeric_overview.jinja @@ -46,23 +46,10 @@
    @@ -229,10 +216,9 @@ {{ check.name }} - {# message – truncate long ones with a title tooltip #} - - {{ check.message | truncate(140, True, '…') }} + {# message – full text, no truncation #} + + {{ check.message }} {# optional deep-link #} @@ -298,21 +284,22 @@ @@ -343,5 +330,21 @@ {% endif %} }); + + +
    + + + Colorblind Mode +
    From 6b213c4833cf5dc57089ed10466581cd716d5b38 Mon Sep 17 00:00:00 2001 From: Maria Hansen Date: Thu, 18 Jun 2026 15:53:05 +0200 Subject: [PATCH 79/95] [feat] move taxonomy not-checked to overview, add column type breakdown, render quality messages as lists --- src/templates/columns.jinja | 13 ------ src/templates/numeric_overview.jinja | 62 +++++++++++++++++++++++++++- 2 files changed, 60 insertions(+), 15 deletions(-) diff --git a/src/templates/columns.jinja b/src/templates/columns.jinja index e0f1298..ee75833 100644 --- a/src/templates/columns.jinja +++ b/src/templates/columns.jinja @@ -489,11 +489,6 @@ {% if col.taxonomy.outdated_names %} Outdated names: {{ col.taxonomy.outdated_names | length }}
    {% endif %} - {% elif col.taxonomy is defined and col.taxonomy is none %} - Taxonomy: - Not checked - (run with --tax to enable) -
    {% endif %} {% if col.annotation and col.annotation.is_annotation %} {% if col.annotation.valid_annotation is string %} @@ -578,14 +573,6 @@ {% endif %} - {% elif col.taxonomy is defined and col.taxonomy is none %} - {% endif %} diff --git a/src/templates/numeric_overview.jinja b/src/templates/numeric_overview.jinja index 0cd2c9a..f3b38fb 100644 --- a/src/templates/numeric_overview.jinja +++ b/src/templates/numeric_overview.jinja @@ -111,6 +111,53 @@ + {# ── Column type breakdown ── #} +
    Column Types
    + + + {% if general.n_number %} + + {% endif %} + {% if general.n_categorical %} + + {% endif %} + {% if general.n_dna %} + + {% endif %} + {% if general.n_rna %} + + {% endif %} + {% if general.n_protein %} + + {% endif %} + {% if general.n_taxonomy %} + + {% endif %} + {% if general.n_unit %} + + {% endif %} + {% if general.n_functional %} + + {% endif %} + {% if general.n_empty %} + + + + + {% endif %} + +
    Numeric {{ general.n_number }}
    Categorical {{ general.n_categorical }}
    DNA sequences {{ general.n_dna }}
    RNA sequences {{ general.n_rna }}
    Protein sequences {{ general.n_protein }}
    Taxonomy {{ general.n_taxonomy }}
    Lab measurements {{ general.n_unit }}
    Functional annotation {{ general.n_functional }}
    Empty (all-null){{ general.n_empty }}
    + + {# ── Taxonomy not checked notice ── #} + {% if general.n_taxonomy == 0 %} +
    + Taxonomy: + Not checked — + run with --tax to enable taxonomic validation. + +
    + {% endif %} +
    {# ================================================================ @@ -216,9 +263,20 @@ {{ check.name }} - {# message – full text, no truncation #} + {# message – render comma-separated items as a list #} - {{ check.message }} + {%- if ': ' in check.message and ', ' in check.message.split(': ', 1)[1] %} + {%- set prefix = check.message.split(': ', 1)[0] %} + {%- set items = check.message.split(': ', 1)[1].split(', ') %} + {{ prefix }}: +
      + {%- for item in items %} +
    • {{ item }}
      • + {%- endfor %} +
    + {%- else %} + {{ check.message }} + {%- endif %} {# optional deep-link #} From 194afa4a05100788759e3a52ee4f96f611993025 Mon Sep 17 00:00:00 2001 From: Sonja Diedrich Date: Thu, 18 Jun 2026 16:01:39 +0200 Subject: [PATCH 80/95] fix: minor adjustments --- src/analysis/plot_utils.py | 4 +++- src/quality_assessment/dataset_quality.py | 2 +- src/templates/numeric_overview.jinja | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/analysis/plot_utils.py b/src/analysis/plot_utils.py index 2bfd495..7c77209 100644 --- a/src/analysis/plot_utils.py +++ b/src/analysis/plot_utils.py @@ -13,6 +13,8 @@ def apply_standard_axes(fig, tick_angle=None): #ToDo: Check if we want to keep nan in count plot. def plot_overview(col): col = col.dropna() + if col.nunique() == 1: + return " " #ToDo find better solution if col.dtype in ('str', 'string'): truncated = col.apply(lambda x: str(x)[:20] + '…' if len(str(x)) > 20 else str(x)) if col.dtype != 'object': @@ -23,7 +25,7 @@ def plot_overview(col): else: fig = px.histogram(x=col, color_discrete_sequence=['#0F65A0'], nbins=bins) apply_standard_axes(fig) - fig.update_layout(bargap=0.4, plot_bgcolor='white', xaxis_title=col.name,yaxis_title='Count') + fig.update_layout(bargap=0.4, plot_bgcolor='white', xaxis_title=col.name,yaxis_title='Count', autosize=True) fig.layout.xaxis.automargin = True return fig.to_html(full_html=False, include_plotlyjs=False) return None \ No newline at end of file diff --git a/src/quality_assessment/dataset_quality.py b/src/quality_assessment/dataset_quality.py index c7ecd30..a2d826e 100644 --- a/src/quality_assessment/dataset_quality.py +++ b/src/quality_assessment/dataset_quality.py @@ -23,7 +23,7 @@ def _check_sample_size(general) -> QualityCheck: return QualityCheck( name="Sample Size vs Feature Count", status=status, message=f"{ratio:.1f} rows per feature " - f"({general.rows} rows / {general.cols} columns", + f"({general.rows} rows / {general.cols} columns)", detail_link="#overview", ) diff --git a/src/templates/numeric_overview.jinja b/src/templates/numeric_overview.jinja index f3b38fb..fa882c7 100644 --- a/src/templates/numeric_overview.jinja +++ b/src/templates/numeric_overview.jinja @@ -170,15 +170,15 @@ {% if quality.overall == 'ready' %} {% set banner_class = 'success' %} {% set banner_icon = 'M16 8A8 8 0 1 1 0 8a8 8 0 0 1 16 0zm-3.97-3.03a.75.75 0 0 0-1.08.022L7.477 9.417 5.384 7.323a.75.75 0 0 0-1.06 1.06L6.97 11.03a.75.75 0 0 0 1.079-.02l3.992-4.99a.75.75 0 0 0-.01-1.05z' %} - {% set banner_label = 'Ready' %} + {% set banner_label = 'No Quality Issues' %} {% elif quality.overall == 'caution' %} {% set banner_class = 'warning' %} {% set banner_icon = 'M8.982 1.566a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566zM8 5c.535 0 .954.462.9.995l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995A.905.905 0 0 1 8 5zm.002 6a1 1 0 1 1 0 2 1 1 0 0 1 0-2z' %} - {% set banner_label = 'Ready with Caution' %} + {% set banner_label = 'Minor Quality Issues' %} {% else %} {% set banner_class = 'danger' %} {% set banner_icon = 'M8.982 1.566a1.13 1.13 0 0 0-1.96 0L.165 13.233c-.457.778.091 1.767.98 1.767h13.713c.889 0 1.438-.99.98-1.767L8.982 1.566zM8 5c.535 0 .954.462.9.995l-.35 3.507a.552.552 0 0 1-1.1 0L7.1 5.995A.905.905 0 0 1 8 5zm.002 6a1 1 0 1 1 0 2 1 1 0 0 1 0-2z' %} - {% set banner_label = 'Not Ready' %} + {% set banner_label = 'Major Quality Issues' %} {% endif %} @@ -206,7 +209,7 @@
    {% for col in columns %} {# fns = filter namespace — separate from badge ns #} - {% set fns = namespace(has_inf='no',is_sparse='no',has_outliers='no',imbalanced='no',nzv='no',rare='no',ambiguous_bases='no',rc_duplicates='no',low_complexity='no',mixed_values='no',data_anomalies='no',taxonomy_invalid='no',annotation_invalid='no') %} + {% set fns = namespace(has_inf='no',is_sparse='no',has_outliers='no',imbalanced='no',nzv='no',rare='no',ambiguous_bases='no',rc_duplicates='no',low_complexity='no',mixed_values='no',data_anomalies='no',taxonomy_invalid='no',taxonomy_valid='no',taxonomy_not_checked='no',annotation_invalid='no') %} {% for ov in overview %}{% if ov.name == col.name %} {% set _n = col.number if col.number and col.number > 0 else 1 %} {% if ov.infinity is defined and ov.infinity > 0 %}{% set fns.has_inf = 'yes' %}{% endif %} @@ -230,6 +233,8 @@ {% if col.measurement_data and col.measurement_data.with_measurement %}{% set fns.mixed_values = 'yes' %}{% endif %} {% if col.suspect_values is not none and not (col.invalid_seqs and col.invalid_seqs|length > 0) %}{% set fns.data_anomalies = 'yes' %}{% endif %} {% if col.taxonomy and col.taxonomy.is_taxonomy and col.taxonomy.taxonomy is not string %}{% set fns.taxonomy_invalid = 'yes' %}{% endif %} + {% if col.taxonomy and col.taxonomy.is_taxonomy and col.taxonomy.taxonomy is string %}{% set fns.taxonomy_valid = 'yes' %}{% endif %} + {% if col.taxonomy is defined and col.taxonomy is none and col.taxonomy_candidate %}{% set fns.taxonomy_not_checked = 'yes' %}{% endif %} {% if col.annotation and col.annotation.is_annotation and col.taxonomy.valid_annotation is not string %}{% set fns.annotation_invalid = 'yes' %}{% endif %}
    @@ -262,7 +269,11 @@
    {% if col.taxonomy and col.taxonomy.is_taxonomy and col.taxonomy.taxonomy is not string %} - Taxonomy invalid + Taxonomy Invalid + {% elif col.taxonomy and col.taxonomy.is_taxonomy and col.taxonomy.taxonomy is string %} + Taxonomy Valid + {% elif col.taxonomy is defined and col.taxonomy is none and col.taxonomy_candidate %} + Taxonomy Not Checked {% endif %} {% if col.annotation and col.annotation.is_annotation and col.taxonomy.valid_annotation is not string %} Annotation invalid @@ -489,6 +500,11 @@ {% if col.taxonomy.outdated_names %} Outdated names: {{ col.taxonomy.outdated_names | length }}
    {% endif %} + {% elif col.taxonomy is defined and col.taxonomy is none %} + Taxonomy: + Not checked + (run with --tax to enable) +
    {% endif %} {% if col.annotation and col.annotation.is_annotation %} {% if col.annotation.valid_annotation is string %} @@ -573,6 +589,14 @@ {% endif %} + {% elif col.taxonomy is defined and col.taxonomy is none %} + {% endif %} @@ -2543,6 +2567,18 @@ 'taxonomyinvalid','annotationinvalid','dataanomalies' ]; + // Composite filters: one button matches if ANY of the listed flags is 'yes' + const COMPOSITE_QUALITY = { + 'taxonomy': ['taxonomyinvalid', 'taxonomyvalid', 'taxonomynotchecked'], + }; + + function matchesQuality(card, q) { + if (COMPOSITE_QUALITY[q]) { + return COMPOSITE_QUALITY[q].some(f => card.dataset[f] === 'yes'); + } + return card.dataset[q] === 'yes'; + } + let activeType = 'all'; let activeQualities = new Set(); let filterLogic = 'or'; @@ -2581,10 +2617,10 @@ } if (show && activeQualities.size > 0) { if (filterLogic === 'and') { - for (const q of activeQualities) { if (card.dataset[q] !== 'yes') { show = false; break; } } + for (const q of activeQualities) { if (!matchesQuality(card, q)) { show = false; break; } } } else { let any = false; - for (const q of activeQualities) { if (card.dataset[q] === 'yes') { any = true; break; } } + for (const q of activeQualities) { if (matchesQuality(card, q)) { any = true; break; } } if (!any) show = false; } } From 5d46c157cc9a04b92dadec9c10a99acbf399b978 Mon Sep 17 00:00:00 2001 From: Maria Hansen Date: Thu, 18 Jun 2026 16:17:32 +0200 Subject: [PATCH 82/95] [fix] make quality optional (for older tests to still work) --- src/cli/report_writer.py | 3 ++- tests/integration/test_report_writer.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/cli/report_writer.py b/src/cli/report_writer.py index 60f2c51..9f1d9be 100644 --- a/src/cli/report_writer.py +++ b/src/cli/report_writer.py @@ -10,7 +10,8 @@ def write_report(output_path: Path, general, plots, duplicates_table, - column_overviews, numeric_overviews, categorical_overviews, top_n, quality): + column_overviews, numeric_overviews, categorical_overviews, + top_n, quality=None): output_path.mkdir(parents=True, exist_ok=True) shutil.copytree(str(STATIC_DIR), str(output_path / "static"), dirs_exist_ok=True) diff --git a/tests/integration/test_report_writer.py b/tests/integration/test_report_writer.py index 921038a..f9a3b1a 100644 --- a/tests/integration/test_report_writer.py +++ b/tests/integration/test_report_writer.py @@ -38,7 +38,7 @@ def invoke_write_report(tmp_path, general="general", plots="plots", dups="
    ", column_overviews=None, numeric_overviews=None, categorical_overviews=None, - top_n=20): + top_n=20, quality=None): """Call write_report with mocked env and static dir.""" output = tmp_path / "report" with patch(ENV_PATCH, TEST_ENV), \ @@ -55,6 +55,7 @@ def invoke_write_report(tmp_path, general="general", plots="plots", numeric_overviews=numeric_overviews or [], categorical_overviews=categorical_overviews or [], top_n=top_n, + quality=quality, ) return output From f2e9483ec97c4f32aebaeabbece1c346c8382f4e Mon Sep 17 00:00:00 2001 From: Maria Hansen Date: Thu, 18 Jun 2026 16:31:05 +0200 Subject: [PATCH 83/95] [refactor] place dataset overview and column types tables side by side --- src/cli/app.py | 4 + src/templates/numeric_overview.jinja | 167 +++++++++++++++------------ 2 files changed, 95 insertions(+), 76 deletions(-) diff --git a/src/cli/app.py b/src/cli/app.py index 8431335..bafcd61 100644 --- a/src/cli/app.py +++ b/src/cli/app.py @@ -136,6 +136,10 @@ def cli(input: str, tax: bool = False, func: str = None, if getattr(col, 'taxonomy', None) and col.taxonomy.is_taxonomy) general.n_unit = sum(1 for col in column_overviews if col.measurement_data is not None) general.n_functional = sum(1 for col in column_overviews if getattr(col, 'annotation', None)) + general.n_taxonomy_candidates = sum( + 1 for col in column_overviews + if getattr(col, 'taxonomy_candidate', False) + ) output_path = Path(input_path.stem + "_renders") done = print_step("Writing report") diff --git a/src/templates/numeric_overview.jinja b/src/templates/numeric_overview.jinja index fa882c7..ea3e77c 100644 --- a/src/templates/numeric_overview.jinja +++ b/src/templates/numeric_overview.jinja @@ -78,85 +78,100 @@ ================================================================ #}

    General Dataset Statistics

    - - - - - - - - - - - - - - - - - - - -
    Filename {{ general.filename }}
    Rows {{ general.rows }}
    Columns {{ general.cols }}
    Missing values {{ general.nulls }}
    Missing values (%) {{ general.nulls_percentage }}
    Empty rows {{ general.empty_rows }}
    Duplicate rows - {% if general.dup_row > 0 %} - {{ general.dup_row }} - {% else %} - {{ general.dup_row }} - {% endif %} -
    Duplicate columns - {% if general.dup_col > 0 %} - {{ general.dup_col }} - {% else %} - {{ general.dup_col }} - {% endif %} -
    Sample-Feature Ratio {{ general.ratio }}
    Memory usage {{ general.memory }}
    Alerts {{ general.alerts }}
    +
    - {# ── Column type breakdown ── #} -
    Column Types
    - - - {% if general.n_number %} - - {% endif %} - {% if general.n_categorical %} - - {% endif %} - {% if general.n_dna %} - - {% endif %} - {% if general.n_rna %} - - {% endif %} - {% if general.n_protein %} - - {% endif %} - {% if general.n_taxonomy %} - - {% endif %} - {% if general.n_unit %} - - {% endif %} - {% if general.n_functional %} - - {% endif %} - {% if general.n_empty %} - - - - + {# ── Left: Dataset overview ── #} +
    +
    + Dataset Overview +
    +
    Numeric {{ general.n_number }}
    Categorical {{ general.n_categorical }}
    DNA sequences {{ general.n_dna }}
    RNA sequences {{ general.n_rna }}
    Protein sequences {{ general.n_protein }}
    Taxonomy {{ general.n_taxonomy }}
    Lab measurements {{ general.n_unit }}
    Functional annotation {{ general.n_functional }}
    Empty (all-null){{ general.n_empty }}
    + + + + + + + + + + + + + + + + + + +
    Filename {{ general.filename }}
    Rows {{ general.rows }}
    Columns {{ general.cols }}
    Missing values {{ general.nulls }}
    Missing values (%) {{ general.nulls_percentage }}
    Empty rows {{ general.empty_rows }}
    Duplicate rows + {% if general.dup_row > 0 %} + {{ general.dup_row }} + {% else %} + {{ general.dup_row }} + {% endif %} +
    Duplicate columns + {% if general.dup_col > 0 %} + {{ general.dup_col }} + {% else %} + {{ general.dup_col }} + {% endif %} +
    Sample-Feature Ratio {{ general.ratio }}
    Memory usage {{ general.memory }}
    Alerts {{ general.alerts }}
    +
    + + {# ── Right: Column type breakdown ── #} +
    +
    + Column Types +
    + + + + {% if general.n_number %} + + {% endif %} + {% if general.n_categorical %} + + {% endif %} + {% if general.n_dna %} + + {% endif %} + {% if general.n_rna %} + + {% endif %} + {% if general.n_protein %} + + {% endif %} + {% if general.n_taxonomy %} + + {% endif %} + {% if general.n_unit %} + + {% endif %} + {% if general.n_functional %} + + {% endif %} + {% if general.n_empty %} + + + + + {% endif %} + +
    Numeric {{ general.n_number }}
    Categorical {{ general.n_categorical }}
    DNA sequences {{ general.n_dna }}
    RNA sequences {{ general.n_rna }}
    Protein sequences {{ general.n_protein }}
    Taxonomy {{ general.n_taxonomy }}
    Lab measurements {{ general.n_unit }}
    Functional annotation {{ general.n_functional }}
    Empty (all-null){{ general.n_empty }}
    + + {# Only shown when taxonomy candidates exist but --tax was not used #} + {% if general.n_taxonomy == 0 and general.n_taxonomy_candidates is defined and general.n_taxonomy_candidates > 0 %} +
    + Taxonomy: + Not checked — + run with --tax to enable taxonomic validation. + +
    {% endif %} - - +
    - {# ── Taxonomy not checked notice ── #} - {% if general.n_taxonomy == 0 %} -
    - Taxonomy: - Not checked — - run with --tax to enable taxonomic validation. - -
    - {% endif %} +
    {# /row #}
    From a1bc8af555a2c5a9b9ef3545d5450cb75e0ef313 Mon Sep 17 00:00:00 2001 From: Sonja Diedrich Date: Thu, 18 Jun 2026 16:29:25 +0200 Subject: [PATCH 84/95] fix: updated links --- src/cli/report_writer.py | 17 +++++++++++++++++ src/quality_assessment/biological_quality.py | 8 ++++---- 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/src/cli/report_writer.py b/src/cli/report_writer.py index 9f1d9be..820d89f 100644 --- a/src/cli/report_writer.py +++ b/src/cli/report_writer.py @@ -8,6 +8,21 @@ STATIC_DIR = files("static").joinpath() env = Environment(loader=FileSystemLoader(str(TEMPLATE_DIR)), autoescape=True) +PAGE_MAP = { + "overview": "numeric_data.html", + "columns": "columns.html", + "multivariate": "general_statistics.html", +} + +def _resolve_detail_links(quality): + if quality is None: + return + for category in quality.categories: + for check in category.checks: + if not check.detail_link: + continue + anchor = check.detail_link.lstrip("#") + check.detail_link = PAGE_MAP.get(anchor) def write_report(output_path: Path, general, plots, duplicates_table, column_overviews, numeric_overviews, categorical_overviews, @@ -15,6 +30,8 @@ def write_report(output_path: Path, general, plots, duplicates_table, output_path.mkdir(parents=True, exist_ok=True) shutil.copytree(str(STATIC_DIR), str(output_path / "static"), dirs_exist_ok=True) + _resolve_detail_links(quality) + _render_to_file(output_path / "index.html", 'LandingPage.jinja') _render_to_file(output_path / "numeric_data.html", 'numeric_overview.jinja', general=general, dups=duplicates_table, quality=quality) diff --git a/src/quality_assessment/biological_quality.py b/src/quality_assessment/biological_quality.py index a628ca3..67532db 100644 --- a/src/quality_assessment/biological_quality.py +++ b/src/quality_assessment/biological_quality.py @@ -37,7 +37,7 @@ def _check_sequence_validity(column_overviews) -> QualityCheck: status = _worst(statuses) if statuses else "pass" return QualityCheck(name="Sequence Validity", status=status, message="; ".join(notes) if notes else "All sequences valid", - detail_link="#sequences") + detail_link="#columns") def _check_sequence_redundancy(column_overviews) -> QualityCheck: @@ -62,7 +62,7 @@ def _check_sequence_redundancy(column_overviews) -> QualityCheck: status = _worst(statuses) if statuses else "pass" return QualityCheck(name="Sequence Redundancy (Reverse Complement)", status=status, message="; ".join(notes) if notes else "No significant reverse-complement redundancy", - detail_link="#sequences") + detail_link="#columns") def _check_taxonomy_validity(column_overviews) -> QualityCheck: @@ -80,7 +80,7 @@ def _check_taxonomy_validity(column_overviews) -> QualityCheck: status = _worst(statuses) if statuses else "pass" return QualityCheck(name="Taxonomy Validity", status=status, message="; ".join(notes) if notes else "Taxonomy valid", - detail_link="#taxonomy") + detail_link="#columns") def _check_unit_validity(column_overviews) -> QualityCheck: @@ -101,4 +101,4 @@ def _check_unit_validity(column_overviews) -> QualityCheck: status = _worst(statuses) if statuses else "pass" return QualityCheck(name="Unit Validity", status=status, message="; ".join(notes) if notes else "Unit valid", - detail_link="#unit") + detail_link="#columns") From 7fd1763b288d328471f385bdfbba2e1a24fe0e68 Mon Sep 17 00:00:00 2001 From: Maria Hansen Date: Thu, 18 Jun 2026 17:12:29 +0200 Subject: [PATCH 85/95] [refactor] add autosize to correlation plot to ensure responsive layout --- src/templates/general_statistics.jinja | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/templates/general_statistics.jinja b/src/templates/general_statistics.jinja index ac501f2..f463682 100644 --- a/src/templates/general_statistics.jinja +++ b/src/templates/general_statistics.jinja @@ -643,6 +643,19 @@