From 83a41c6165a469a439c950ca6df43b26ec3aeebb Mon Sep 17 00:00:00 2001 From: Gabriel Bressan Date: Thu, 9 Jul 2026 10:41:53 -0400 Subject: [PATCH] Flatten JSON with underscore separator to keep columns DuckDB-safe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit json_normalize used a dot separator, producing columns like `day.c`. MindsDB runs projections through DuckDB, where a dotted name is parsed as a struct/table qualifier (column `c` of `day`) instead of a column literally named "day.c" — and no quoting form (backticks/double quotes) survives that rewrite, so the column becomes unselectable. Switch to sep='_' (matching the XML path) so nested fields flatten to day_c / prevDay_c, which are plain, selectable identifiers. Co-Authored-By: Claude Opus 4.8 --- .../handlers/multi_format_api_handler/format_parsers.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mindsdb/integrations/handlers/multi_format_api_handler/format_parsers.py b/mindsdb/integrations/handlers/multi_format_api_handler/format_parsers.py index 9b46ab61da9..98eed928600 100644 --- a/mindsdb/integrations/handlers/multi_format_api_handler/format_parsers.py +++ b/mindsdb/integrations/handlers/multi_format_api_handler/format_parsers.py @@ -175,7 +175,7 @@ def parse_json(content: str, record_path: Optional[str] = None, auto_explode: bo # Legacy single-row behavior for object payloads when explosion is disabled. if not auto_explode and isinstance(data, dict): - return _ensure_scalar_columns(pd.json_normalize(data)) + return _ensure_scalar_columns(pd.json_normalize(data, sep='_')) records, chosen_path = _resolve_records(data, record_path) if records is None: @@ -183,7 +183,11 @@ def parse_json(content: str, record_path: Optional[str] = None, auto_explode: bo if len(records) == 0: return pd.DataFrame() - df = pd.json_normalize(records, sep='.') + # Flatten nested dicts with an underscore separator (e.g. day_c), NOT a dot. + # MindsDB executes projections through DuckDB, where a dotted name like `day.c` + # is parsed as a struct/table qualifier (column `c` of `day`) instead of a + # column literally named "day.c" — and no quoting form survives that rewrite. + df = pd.json_normalize(records, sep='_') # Reattach top-level scalar siblings (status, count, request_id, ...) as # constant columns so envelope metadata is not lost. Skip the array key;