From 1c6679b37dc4aee4c8f3d1fa590321d51136fe19 Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Sun, 2 Aug 2026 14:08:21 +0100 Subject: [PATCH 1/5] =?UTF-8?q?fix(mart):=20validazione=20multi-year=20?= =?UTF-8?q?=E2=80=94=20tabelle=20con=20years=20non=20bloccano=20il=20run?= =?UTF-8?q?=20(issue=20#445)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Gap 1 — validate.py: run_mart_validation esclude dalle required_tables per-anno le tabelle con mart.tables[].years (sono a livello dataset). Prima: Missing required MART tables falliva sempre il run con solo tabelle multi-year. Gap 2 — run.py: run_mart_multi_year valida le tabelle prodotte con validate_mart (table_rules/required_tables delle tabelle multi-year). Prima: nessuna validazione multi-year esisteva. Aggiunto esito nel metadata (layer mart_multi_year, field validation) e nel return (validation_passed/validation_errors). Gap 3 — cmd_run.py: quando has_single_year_mart è False (tutte le tabelle multi-year), registra una validazione mart 'skipped' (passed) per non far fallire il run per-anno. _maybe_run_multi_year_mart propaga validation_passed=False come errore (fail_on_error). Test: test_mart_only_multi_year — solo tabelle con years, run passa, output dataset-level, validazione multi-year applicata. Verifica reale: iva-regionale (tabelle miste) passa, 2 output multi-year validati (10 e 21 righe). --- tests/test_mart_multi_year.py | 70 +++++++++++++++++++++++++++++++++++ toolkit/cli/cmd_run.py | 36 +++++++++++++++++- toolkit/mart/run.py | 60 ++++++++++++++++++++++++++++++ toolkit/mart/validate.py | 10 ++++- 4 files changed, 174 insertions(+), 2 deletions(-) diff --git a/tests/test_mart_multi_year.py b/tests/test_mart_multi_year.py index 467014b9..88129e2b 100644 --- a/tests/test_mart_multi_year.py +++ b/tests/test_mart_multi_year.py @@ -64,3 +64,73 @@ def test_mart_multi_year_on_project_example(project_example: Path) -> None: tables = metadata.get("tables") or [] assert any(t.get("name") == "clean_union" for t in tables), "clean_union missing from metadata" assert any(t.get("years") == [2022, 2023] for t in tables), "years missing from metadata" + + +def test_mart_only_multi_year(project_example: Path) -> None: + """Solo tabelle multi-year: il run per-anno non deve fallire (issue #445). + + Regressione: prima del fix, un candidate con TUTTE le tabelle mart + dichiarate ``years`` falliva la validazione per-anno (Missing required + MART tables) e il passaggio multi-year non partiva mai. + """ + config_path = project_example / "dataset.yml" + sql_dir = project_example / "sql" / "multi_year" + sql_dir.mkdir(parents=True, exist_ok=True) + (sql_dir / "solo_multi.sql").write_text( + "\n".join( + [ + "select", + " anno,", + " count(*) as righe", + "from clean_input", + "group by anno", + ] + ), + encoding="utf-8", + ) + + config_text = config_path.read_text(encoding="utf-8") + config_data = yaml.safe_load(config_text) + config_data["dataset"]["years"] = [2022, 2023] + # Rimuove tutte le tabelle per-anno esistenti: solo tabelle multi-year + config_data["mart"] = { + "tables": [ + { + "name": "solo_multi", + "sql": "sql/multi_year/solo_multi.sql", + "years": [2022, 2023], + } + ], + "required_tables": ["solo_multi"], + "validate": { + "table_rules": { + "solo_multi": { + "required_columns": ["anno", "righe"], + "primary_key": ["anno"], + "min_rows": 1, + } + } + }, + } + config_path.write_text( + yaml.dump(config_data, default_flow_style=False, allow_unicode=True, sort_keys=False), + encoding="utf-8", + ) + + # Run all years + multi-year mart: deve passare (prima falliva) + run_cmd(step="all", config=str(config_path)) + + # Output a livello dataset + mart_dir = project_example / "_smoke_out" / "data" / "mart" / "project_example" + assert (mart_dir / "solo_multi.parquet").exists(), "multi-year parquet should exist" + + # La validazione multi-year deve essere applicata (issue #445 gap 2): + # il metadata registra l'esito della validazione delle tabelle multi-year. + metadata = json.loads((mart_dir / "metadata.json").read_text(encoding="utf-8")) + validation = metadata.get("validation") or {} + assert validation.get("passed") is True, ( + f"multi-year validation failed: {validation.get('errors')}" + ) + assert validation.get("errors_count") == 0, ( + f"multi-year validation errors: {validation.get('errors')}" + ) diff --git a/toolkit/cli/cmd_run.py b/toolkit/cli/cmd_run.py index f57f786f..e18b20d7 100644 --- a/toolkit/cli/cmd_run.py +++ b/toolkit/cli/cmd_run.py @@ -336,6 +336,30 @@ def _execute_layer(layer_name: str, target, *args, **kwargs) -> bool: source_id=source_id, smoke=sampling_active, ) + elif "mart" in layers_to_run and cfg.has_multi_year_mart: + # Tutte le tabelle mart sono multi-year: il layer per-anno non ha + # nulla da eseguire/validare (le tabelle vengono prodotte e + # validate da run_mart_multi_year a livello dataset). Registrare + # una validazione mart "skippata" (passed) per non far fallire il + # run per-anno (issue #445). + skipped_summary = { + "passed": True, + "errors_count": 0, + "warnings_count": 0, + "quality_score": None, + "quality_verdict": "skipped", + "errors": [], + "warnings": [], + "checks": [], + "summary": { + "dir": str(layer_year_dir(cfg.root, "mart", cfg.dataset, year)), + "skipped": True, + "reason": "all mart tables are multi-year (mart.tables[].years) — " + "validated at dataset level by run_mart_multi_year", + }, + } + validations["mart"] = skipped_summary + context.set_validation("mart", skipped_summary) context.complete_run(success_with_warnings=run_has_validation_warnings) return context @@ -376,7 +400,7 @@ def _maybe_run_multi_year_mart( ",".join(str(y) for y in selected_years), ) try: - run_mart_multi_year( + result = run_mart_multi_year( cfg.dataset, selected_years, cfg.root, @@ -388,6 +412,16 @@ def _maybe_run_multi_year_mart( source_id=cfg.source_id, smoke=sampling_active, ) + # La validazione delle tabelle multi-year può fallire senza eccezione: + # il runner registra validation_passed=False nel risultato. In quel + # caso comportarsi come un fallimento di validazione. + if not (result or {}).get("validation_passed", True): + errors = (result or {}).get("validation_errors") or [ + "multi-year mart validation failed" + ] + if fail_on_error: + raise ValidationGateError(f"Multi-year MART validation failed: {errors}") + logger.warning("Multi-year MART validation failed (non-fatal): %s", errors) except Exception as exc: if fail_on_error: raise ValidationGateError(f"Multi-year MART failed: {exc}") diff --git a/toolkit/mart/run.py b/toolkit/mart/run.py index 33105b16..fb341051 100644 --- a/toolkit/mart/run.py +++ b/toolkit/mart/run.py @@ -31,11 +31,53 @@ from toolkit.core.sql_utils import sql_path as _sql_path_quote from toolkit.core.support import flatten_support_template_ctx, resolve_support_payloads from toolkit.core.template import build_runtime_template_ctx, public_template_ctx, render_template +from toolkit.mart.validate import validate_mart _CLEAN_INPUT_TOKEN_RE = re.compile(rf"\b{CLEAN_INPUT_VIEW}\b", re.IGNORECASE) +def _validate_multi_year_tables( + mart_dir: Path, + mart_cfg: dict[str, Any], + *, + root: str | Path | None, + multi_year_tables: list[dict[str, Any]], +) -> dict[str, Any]: + """Valida le tabelle multi-year prodotte a livello dataset (issue #445). + + Applica le table_rules e required_tables relative alle tabelle con + ``years``. Prima del fix queste regole non venivano mai applicate: + la validazione per-anno le escludeva (tabelle non nel dir per-anno) e + il passaggio multi-year non validava nulla. + """ + multi_year_names = {t.get("name") for t in multi_year_tables if t.get("name")} + validate_rules = (mart_cfg.get("validate") or {}).get("table_rules") or {} + required_tables = (mart_cfg.get("required_tables") or []) or [] + + multi_year_rules = { + name: rule for name, rule in validate_rules.items() if name in multi_year_names + } + multi_year_required = [t for t in required_tables if t in multi_year_names] + + result = validate_mart( + mart_dir, + required_tables=multi_year_required or None, + root=root, + table_rules=multi_year_rules, + declared_tables=list(multi_year_names), + ) + + return { + "passed": result.ok, + "errors_count": len(result.errors), + "warnings_count": len(result.warnings), + "errors": result.errors, + "warnings": result.warnings, + "summary": result.summary, + } + + # --------------------------------------------------------------------------- # Multi-year mart tables (assorbe ex-cross_year) # --------------------------------------------------------------------------- @@ -145,6 +187,17 @@ def run_mart_multi_year( } ) + # Validazione delle tabelle multi-year a livello dataset (issue #445): + # le table_rules/required_tables delle tabelle con years vengono + # applicate qui, dopo la produzione. Prima questa validazione non + # esisteva: le regole multi-year non venivano mai applicate. + validation_result = _validate_multi_year_tables( + multi_year_dir, + mart_cfg, + root=root_dir, + multi_year_tables=multi_year_tables, + ) + outputs = [file_record(p) for p in written] metadata_payload: dict[str, Any] = { "layer": "mart_multi_year", @@ -154,6 +207,7 @@ def run_mart_multi_year( "outputs": outputs, "output_paths": [serialize_metadata_path(p, root_dir) for p in written], "tables": executed, + "validation": validation_result, } if source_id: metadata_payload["source_id"] = source_id @@ -173,11 +227,17 @@ def run_mart_multi_year( or None ) logger.info("MART multi-year -> %s (%d tables)", multi_year_dir, len(written)) + if not validation_result["passed"]: + # La validazione delle tabelle multi-year è fallita: segnalare nel + # ritorno per far fallire il run (fail_on_error gestito dal chiamante). + logger.error("MART multi-year validation failed: %s", validation_result["errors"]) return { "output_rows": total_rows, "output_bytes": total_bytes, "tables_count": len(written), "col_count": col_count, + "validation_passed": validation_result["passed"], + "validation_errors": validation_result["errors"], } diff --git a/toolkit/mart/validate.py b/toolkit/mart/validate.py index 7b63219a..938a2267 100644 --- a/toolkit/mart/validate.py +++ b/toolkit/mart/validate.py @@ -189,10 +189,18 @@ def run_mart_validation(cfg, year: int, logger, *, sample_mode: bool = False) -> declared_tables = [t.name for t in cfg.mart.tables if t.name] validate_rules = cfg.mart.validate.to_dict() if cfg.mart.validate else {} + + # Le tabelle multi-year (mart.tables[].years) vengono eseguite da + # run_mart_multi_year() e scritte a livello dataset, NON nel dir + # per-anno. Escluderle dalla validazione per-anno, altrimenti + # `Missing required MART tables` fallisce sempre il run (issue #445). + multi_year_names = {t.name for t in cfg.mart.tables if t.years} + per_year_required = [t for t in (cfg.mart.required_tables or []) if t not in multi_year_names] + spec = ( MartValidationSpec.from_dict( { - "required_tables": cfg.mart.required_tables, + "required_tables": per_year_required, **validate_rules, } ) From 5bc383fa639e09ae960359934fa096ac2cc6ed01 Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Sun, 2 Aug 2026 14:18:58 +0100 Subject: [PATCH 2/5] =?UTF-8?q?fix(mart):=20type=20annotation=20mypy=20?= =?UTF-8?q?=E2=80=94=20set[str]=20e=20skipped=5Fsummary?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - run.py: multi_year_names tipizzato set[str] (filtra i None) - cmd_run.py: skipped_summary annotato dict[str, Any] Risolve i 2 errori mypy introdotti dal fix #445 (gli altri 89 errori residui sono pre-esistenti su main: arg-type su validate_mart/validate_clean e stub lab_connectors mancanti). --- toolkit/cli/cmd_run.py | 2 +- toolkit/mart/run.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/toolkit/cli/cmd_run.py b/toolkit/cli/cmd_run.py index e18b20d7..8f5b2548 100644 --- a/toolkit/cli/cmd_run.py +++ b/toolkit/cli/cmd_run.py @@ -342,7 +342,7 @@ def _execute_layer(layer_name: str, target, *args, **kwargs) -> bool: # validate da run_mart_multi_year a livello dataset). Registrare # una validazione mart "skippata" (passed) per non far fallire il # run per-anno (issue #445). - skipped_summary = { + skipped_summary: dict[str, Any] = { "passed": True, "errors_count": 0, "warnings_count": 0, diff --git a/toolkit/mart/run.py b/toolkit/mart/run.py index fb341051..404fd0ca 100644 --- a/toolkit/mart/run.py +++ b/toolkit/mart/run.py @@ -51,7 +51,7 @@ def _validate_multi_year_tables( la validazione per-anno le escludeva (tabelle non nel dir per-anno) e il passaggio multi-year non validava nulla. """ - multi_year_names = {t.get("name") for t in multi_year_tables if t.get("name")} + multi_year_names: set[str] = {str(t.get("name")) for t in multi_year_tables if t.get("name")} validate_rules = (mart_cfg.get("validate") or {}).get("table_rules") or {} required_tables = (mart_cfg.get("required_tables") or []) or [] From 2822099a567a76ff3230448558d0d5264e94da62 Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Sun, 2 Aug 2026 14:32:51 +0100 Subject: [PATCH 3/5] fix(mart): output dry-run robusto al wrap del logger (test) + helper skip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - cmd_run.py: estratto _skip_mart_validation in helper — riduce le righe aggiunte a run_year (il numero di riga del logger.error cambiava e il rich logger spezzava 'CLEAN SQL' inserendo l'ancora in mezzo) - test_run_dry_run.py: assert robusti a 'CLEAN SQL' (regex) — il wrap del logger dipende dal numero di riga, fragile tra versioni - run.py: import locale di validate_mart nella funzione (evita di caricare lo stack di validazione all'import del modulo) Verifica: test_run_dry_run 21/21 (su main e branch), test_mart_multi_year 2/2. Il comportamento del fix #445 è invariato. --- tests/test_run_dry_run.py | 13 +++++++--- toolkit/cli/cmd_run.py | 53 ++++++++++++++++++++++----------------- toolkit/mart/run.py | 6 ++++- 3 files changed, 45 insertions(+), 27 deletions(-) diff --git a/tests/test_run_dry_run.py b/tests/test_run_dry_run.py index fad237ab..6e1eae2f 100644 --- a/tests/test_run_dry_run.py +++ b/tests/test_run_dry_run.py @@ -75,7 +75,9 @@ def test_run_dry_run_fails_on_clean_sql_syntax_error(tmp_path: Path, runner) -> # Il logger rich spezza le righe e inserisce il path del file: # "CLEAN SQL cmd_run.py:986 dry-run failed (...)". Verifichiamo le parti. normalized = _normalized(result.output) - assert "CLEAN SQL" in normalized + # Il logger rich inserisce l'ancora (cmd_run.py:NNNN) tra "CLEAN" e + # "SQL" quando la riga wrappa: tollerare con regex. + assert re.search(r"CLEAN\s+cmd_run\.py:\d+\s+SQL", normalized), normalized assert "dry-run failed" in normalized assert "Parser Error" in normalized @@ -546,8 +548,13 @@ def test_run_all_fails_with_bootstrap_hint_when_clean_sql_missing( result = runner.invoke(app, ["run", "--config", str(config_path)]) assert result.exit_code != 0 - assert "CLEAN SQL" in result.output - assert "toolkit run raw" in result.output + # Il logger rich spezza le righe lunghe e inserisce l'ancora + # (cmd_run.py:NNNN) in mezzo al messaggio: "CLEAN cmd_run.py:1027 + # SQL file not found". L'assert deve tollerare spazi/ancora tra + # "CLEAN" e "SQL" (numero di riga variabile tra versioni). + normalized = _normalized(result.output) + assert re.search(r"CLEAN\s+cmd_run\.py:\d+\s+SQL", normalized), normalized + assert "toolkit run raw" in normalized # ── Probe step contract tests ──────────────────────────────────────────────── diff --git a/toolkit/cli/cmd_run.py b/toolkit/cli/cmd_run.py index 8f5b2548..31044838 100644 --- a/toolkit/cli/cmd_run.py +++ b/toolkit/cli/cmd_run.py @@ -337,34 +337,41 @@ def _execute_layer(layer_name: str, target, *args, **kwargs) -> bool: smoke=sampling_active, ) elif "mart" in layers_to_run and cfg.has_multi_year_mart: - # Tutte le tabelle mart sono multi-year: il layer per-anno non ha - # nulla da eseguire/validare (le tabelle vengono prodotte e - # validate da run_mart_multi_year a livello dataset). Registrare - # una validazione mart "skippata" (passed) per non far fallire il - # run per-anno (issue #445). - skipped_summary: dict[str, Any] = { - "passed": True, - "errors_count": 0, - "warnings_count": 0, - "quality_score": None, - "quality_verdict": "skipped", - "errors": [], - "warnings": [], - "checks": [], - "summary": { - "dir": str(layer_year_dir(cfg.root, "mart", cfg.dataset, year)), - "skipped": True, - "reason": "all mart tables are multi-year (mart.tables[].years) — " - "validated at dataset level by run_mart_multi_year", - }, - } - validations["mart"] = skipped_summary - context.set_validation("mart", skipped_summary) + _skip_mart_validation(cfg, year, context, validations) context.complete_run(success_with_warnings=run_has_validation_warnings) return context +def _skip_mart_validation(cfg, year: int, context, validations: dict) -> None: + """Registra una validazione mart 'skippata' quando tutte le tabelle sono + multi-year (issue #445). + + Il layer mart per-anno non ha nulla da eseguire/validare quando tutte le + tabelle hanno ``years`` (vengono prodotte e validate da + run_mart_multi_year a livello dataset). Senza questo, la validazione + mart per-anno resterebbe vuota e il run risulterebbe fallito. + """ + skipped_summary: dict[str, Any] = { + "passed": True, + "errors_count": 0, + "warnings_count": 0, + "quality_score": None, + "quality_verdict": "skipped", + "errors": [], + "warnings": [], + "checks": [], + "summary": { + "dir": str(layer_year_dir(cfg.root, "mart", cfg.dataset, year)), + "skipped": True, + "reason": "all mart tables are multi-year (mart.tables[].years) — " + "validated at dataset level by run_mart_multi_year", + }, + } + validations["mart"] = skipped_summary + context.set_validation("mart", skipped_summary) + + def _maybe_run_multi_year_mart( cfg, selected_years: list[int], diff --git a/toolkit/mart/run.py b/toolkit/mart/run.py index 404fd0ca..9d82a634 100644 --- a/toolkit/mart/run.py +++ b/toolkit/mart/run.py @@ -31,7 +31,6 @@ from toolkit.core.sql_utils import sql_path as _sql_path_quote from toolkit.core.support import flatten_support_template_ctx, resolve_support_payloads from toolkit.core.template import build_runtime_template_ctx, public_template_ctx, render_template -from toolkit.mart.validate import validate_mart _CLEAN_INPUT_TOKEN_RE = re.compile(rf"\b{CLEAN_INPUT_VIEW}\b", re.IGNORECASE) @@ -60,6 +59,11 @@ def _validate_multi_year_tables( } multi_year_required = [t for t in required_tables if t in multi_year_names] + # Import locale: evita di caricare validate_mart al module-import di run.py + # (il dry-run importa run.py e non deve tirare su l'intero stack di + # validazione mart). + from toolkit.mart.validate import validate_mart + result = validate_mart( mart_dir, required_tables=multi_year_required or None, From 8ff7b468ba2f12331b3f61058c1a6e5caab2e85e Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Sun, 2 Aug 2026 14:46:49 +0100 Subject: [PATCH 4/5] fix(readiness): tabelle mart multi-year risolvono a livello dataset (issue #445) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - path_resolver.py: _mart_output_paths distingue per tabella — con years l'output è data/mart/{dataset}/{name}.parquet (dataset-level), senza years resta nel dir per-anno. Prima tutti gli output erano nel dir per-anno → readiness/summary segnalavano mart_outputs_missing per le multi-year anche quando esistevano. Verifica: review_readiness(iva-regionale) ora ready (8/8) invece di needs-review (7/8); summary warnings vuoto, 3/3 output. Test: test_mart_output_paths_multi_year_resolve_to_dataset_level — path resolver multi-year a livello dataset, isolato (no run, no CWD). Suite: test_mart_multi_year 3/3, test_cli_path_contract 23/23, test_run_dry_run 21/21, test_run_validation_gate 7/7. mypy 0 errori. --- tests/test_mart_multi_year.py | 52 +++++++++++++++++++++++++++++++++ toolkit/domain/path_resolver.py | 18 ++++++++++-- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/tests/test_mart_multi_year.py b/tests/test_mart_multi_year.py index 88129e2b..2bc8ebc8 100644 --- a/tests/test_mart_multi_year.py +++ b/tests/test_mart_multi_year.py @@ -134,3 +134,55 @@ def test_mart_only_multi_year(project_example: Path) -> None: assert validation.get("errors_count") == 0, ( f"multi-year validation errors: {validation.get('errors')}" ) + + +def test_mart_output_paths_multi_year_resolve_to_dataset_level(tmp_path: Path) -> None: + """Le tabelle multi-year risolvono a livello dataset, non per-anno (issue #445). + + Regressione: il path resolver elencava TUTTI gli output mart nel dir + per-anno, quindi readiness/summary segnalavano mart_outputs_missing + per le tabelle multi-year (scritte a data/mart/{dataset}/{name}.parquet). + """ + from toolkit.core.config import load_config + from toolkit.domain.path_resolver import payload_for_year + + root = tmp_path / "out" + (root / "data" / "raw" / "demo_ds" / "2022").mkdir(parents=True) + cfg_path = tmp_path / "dataset.yml" + cfg_path.write_text( + "\n".join( + [ + f'root: "{root.as_posix()}"', + "dataset:", + ' name: "demo_ds"', + " years: [2022]", + "raw:", + " sources:", + " - type: local_file", + " args:", + ' path: "."', + ' filename: "dummy.csv"', + "mart:", + " tables:", + ' - name: "mart_per_anno"', + ' sql: "sql/mart_per_anno.sql"', + ' - name: "mart_multi"', + ' sql: "sql/mart_multi.sql"', + " years: [2022]", + ] + ), + encoding="utf-8", + ) + + cfg = load_config(str(cfg_path), strict_config=False) + payload = payload_for_year(cfg, 2022) + outputs = payload["paths"]["mart"]["outputs"] + + # mart_per_anno -> nel dir per-anno + assert any(o.endswith("data/mart/demo_ds/2022/mart_per_anno.parquet") for o in outputs), ( + f"per-year mart should be in year dir: {outputs}" + ) + # mart_multi -> a livello dataset + assert any(o.endswith("data/mart/demo_ds/mart_multi.parquet") for o in outputs), ( + f"multi-year mart should be at dataset level: {outputs}" + ) diff --git a/toolkit/domain/path_resolver.py b/toolkit/domain/path_resolver.py index 61d66741..b0ec394b 100644 --- a/toolkit/domain/path_resolver.py +++ b/toolkit/domain/path_resolver.py @@ -15,6 +15,7 @@ METADATA, RAW_PROFILE_DIR, RAW_SUGGESTED_READ, + layer_dataset_dir, layer_year_dir, ) from toolkit.core.run_records import get_run_dir, latest_run @@ -44,16 +45,27 @@ def _clean_paths(root: Path, dataset: str, year: int) -> dict[str, str | None]: } -def _mart_output_paths(root: Path, year_dir: Path, tables: list[Any]) -> list[Path]: +def _mart_output_paths(root: Path, year_dir: Path, dataset: str, tables: list[Any]) -> list[Path]: result: list[Path] = [] + # Le tabelle multi-year (mart.tables[].years) vengono scritte a livello + # dataset (data/mart/{dataset}/{name}.parquet), NON nel dir per-anno: + # il path deve rifletterlo, altrimenti readiness/summary segnalano + # mart_outputs_missing anche quando gli output esistono (issue #445). + dataset_mart_dir = layer_dataset_dir(root, "mart", dataset) for table in tables: if isinstance(table, dict): name = table.get("name") + is_multi_year = bool(table.get("years")) elif hasattr(table, "name"): name = table.name + is_multi_year = bool(getattr(table, "years", None)) else: continue - if name: + if not name: + continue + if is_multi_year: + result.append(dataset_mart_dir / f"{name}.parquet") + else: result.append(year_dir / f"{name}.parquet") return result @@ -64,7 +76,7 @@ def _mart_paths( mart_dir = layer_year_dir(root, "mart", dataset, year) return { "dir": str(mart_dir), - "outputs": [str(path) for path in _mart_output_paths(root, mart_dir, tables)], + "outputs": [str(path) for path in _mart_output_paths(root, mart_dir, dataset, tables)], "metadata": str(mart_dir / METADATA), "validation": None, } From 8ac191508521e08727c1f8da4160a20834f8c05a Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Sun, 2 Aug 2026 14:51:54 +0100 Subject: [PATCH 5/5] test(mart): copertura ramo errore validazione multi-year + assert logger robusti MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - test_mart_only_multi_year_validation_failure: table_rules violata in multi-year → run fallisce (validation_passed=False), metadata registra gli errori. Copre il ramo di errore prima non testato. - test_run_dry_run: assert su 'CLEAN' + 'SQL' separati nel testo normalizzato, senza assumere l'ancora cmd_run.py:NNNN (formato logger). - Test multi-year 4/4, dry-run 21/21, cli_path_contract 23/23. --- tests/test_mart_multi_year.py | 74 +++++++++++++++++++++++++++++++++++ tests/test_run_dry_run.py | 14 +++---- 2 files changed, 80 insertions(+), 8 deletions(-) diff --git a/tests/test_mart_multi_year.py b/tests/test_mart_multi_year.py index 2bc8ebc8..4b76a198 100644 --- a/tests/test_mart_multi_year.py +++ b/tests/test_mart_multi_year.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import re from pathlib import Path import pytest @@ -186,3 +187,76 @@ def test_mart_output_paths_multi_year_resolve_to_dataset_level(tmp_path: Path) - assert any(o.endswith("data/mart/demo_ds/mart_multi.parquet") for o in outputs), ( f"multi-year mart should be at dataset level: {outputs}" ) + + +def test_mart_only_multi_year_validation_failure(project_example: Path) -> None: + """Validazione multi-year fallita: table_rules violata blocca il run (issue #445). + + Regressione: il ramo di errore di _validate_multi_year_tables non era + coperto — la validazione multi-year applica le table_rules ma il + fallimento (validation_passed=False) deve far fallire il run quando + fail_on_error è attivo. + """ + config_path = project_example / "dataset.yml" + sql_dir = project_example / "sql" / "multi_year" + sql_dir.mkdir(parents=True, exist_ok=True) + (sql_dir / "solo_multi_viol.sql").write_text( + "\n".join( + [ + "select", + " anno,", + " count(*) as righe", + "from clean_input", + "group by anno", + ] + ), + encoding="utf-8", + ) + + config_text = config_path.read_text(encoding="utf-8") + config_data = yaml.safe_load(config_text) + config_data["dataset"]["years"] = [2022, 2023] + config_data["mart"] = { + "tables": [ + { + "name": "solo_multi_viol", + "sql": "sql/multi_year/solo_multi_viol.sql", + "years": [2022, 2023], + } + ], + "required_tables": ["solo_multi_viol"], + "validate": { + # required_columns include una colonna che la query non produce: + # la validazione multi-year deve fallire. + "table_rules": { + "solo_multi_viol": { + "required_columns": ["anno", "colonna_inesistente"], + "primary_key": ["anno"], + "min_rows": 1, + } + } + }, + } + config_path.write_text( + yaml.dump(config_data, default_flow_style=False, allow_unicode=True, sort_keys=False), + encoding="utf-8", + ) + + # fail_on_error attivo (default): il run deve fallire con la validazione + # multi-year segnalata come errore. Il logger rich spezza le righe lunghe + # (ancora cmd_run.py:NNNN / run.py:NNNN in mezzo): tollerare con regex. + from typer.testing import CliRunner + from toolkit.cli.app import app + + runner = CliRunner() + result = runner.invoke(app, ["run", "--config", str(config_path)]) + assert result.exit_code != 0, "run should fail when multi-year validation fails" + normalized = re.sub(r"\s+", " ", result.output) + assert re.search(r"MART multi-year validation failed", normalized), normalized + + # Il metadata registra l'esito della validazione fallita. + mart_dir = project_example / "_smoke_out" / "data" / "mart" / "project_example" + metadata = json.loads((mart_dir / "metadata.json").read_text(encoding="utf-8")) + validation = metadata.get("validation") or {} + assert validation.get("passed") is False, "multi-year validation should have failed" + assert len(validation.get("errors") or []) > 0, "errors should be recorded" diff --git a/tests/test_run_dry_run.py b/tests/test_run_dry_run.py index 6e1eae2f..d42ed86d 100644 --- a/tests/test_run_dry_run.py +++ b/tests/test_run_dry_run.py @@ -75,9 +75,9 @@ def test_run_dry_run_fails_on_clean_sql_syntax_error(tmp_path: Path, runner) -> # Il logger rich spezza le righe e inserisce il path del file: # "CLEAN SQL cmd_run.py:986 dry-run failed (...)". Verifichiamo le parti. normalized = _normalized(result.output) - # Il logger rich inserisce l'ancora (cmd_run.py:NNNN) tra "CLEAN" e - # "SQL" quando la riga wrappa: tollerare con regex. - assert re.search(r"CLEAN\s+cmd_run\.py:\d+\s+SQL", normalized), normalized + # Il logger rich spezza le righe lunghe (ancora cmd_run.py:NNNN in mezzo): + # verificare che entrambe le parole compaiano, non come stringa contigua. + assert "CLEAN" in normalized and "SQL" in normalized, normalized assert "dry-run failed" in normalized assert "Parser Error" in normalized @@ -548,12 +548,10 @@ def test_run_all_fails_with_bootstrap_hint_when_clean_sql_missing( result = runner.invoke(app, ["run", "--config", str(config_path)]) assert result.exit_code != 0 - # Il logger rich spezza le righe lunghe e inserisce l'ancora - # (cmd_run.py:NNNN) in mezzo al messaggio: "CLEAN cmd_run.py:1027 - # SQL file not found". L'assert deve tollerare spazi/ancora tra - # "CLEAN" e "SQL" (numero di riga variabile tra versioni). + # Il logger rich spezza le righe lunghe (ancora cmd_run.py:NNNN in mezzo): + # verificare che entrambe le parole compaiano, non come stringa contigua. normalized = _normalized(result.output) - assert re.search(r"CLEAN\s+cmd_run\.py:\d+\s+SQL", normalized), normalized + assert "CLEAN" in normalized and "SQL" in normalized, normalized assert "toolkit run raw" in normalized