Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions DECISIONS.md

Large diffs are not rendered by default.

8 changes: 8 additions & 0 deletions docs/concepts/queries.md
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,14 @@ Query results are returned as a `SlayerResponse`:
}
```

Result keys are always this canonical dotted form, on every backend. Where a
database's own rules force a different spelling in the emitted SQL — BigQuery
and SQL Server reject dotted column aliases, and most engines cap identifier
length — the alias is rewritten on the way out and restored on the way back, so
`data` and `columns` do not vary by dialect. Only `sql` shows the rewritten
form, since that is what actually ran. See
[Database support](../database-support.md#identifier-length-limits).

Comment thread
coderabbitai[bot] marked this conversation as resolved.
---

## Filters
Expand Down
10 changes: 10 additions & 0 deletions docs/database-support.md
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,16 @@ Unit tests for SQL generation; no live-instance verification.
Redshift, Trino/Presto (Athena uses the Presto dialect), Databricks/Spark,
Oracle.

## Identifier length limits

Some databases cap identifier length (Postgres is the tightest at 63 bytes, and
truncates over-limit names *silently*). SLayer's join-path aliases like
`orders.customers.regions.name` can exceed that on a deep chain, so any
over-limit alias is trimmed at emission to `<head>_<hash>_<tail>`. This does
**not** affect output column names — `response.data` / `response.columns` keep
the canonical dotted alias — but `response.sql` (and `dry_run` / `EXPLAIN`)
shows the trimmed form, so account for it if you introspect the SQL.

## Aggregation support

Most aggregations (`sum`, `avg`, `min`, `max`, `count`, `count_distinct`,
Expand Down
32 changes: 32 additions & 0 deletions slayer/core/errors.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,38 @@ def __init__(
)


class IdentifierCollisionError(SlayerError, ValueError):
"""Two distinct SLayer-generated names collapse onto one identifier after
the dialect's length fitting (DEV-1756).

Raised loudly rather than left to corrupt a result set. Also covers an
already-short name equal to another name's fitted form.
"""

def __init__(
self,
*,
first: str,
second: str,
emitted: str,
dialect: str,
limit: int | None,
namespace: str = "identifier",
) -> None:
self.first = first
self.second = second
self.emitted = emitted
self.dialect = dialect
self.limit = limit
self.namespace = namespace
super().__init__(
f"{namespace} collision on dialect '{dialect}' "
f"(max_identifier_bytes={limit}): {first!r} and {second!r} both "
f"emit as {emitted!r}. Rename one of the underlying models or "
f"columns to break the tie."
)


class ForcedFilterError(SlayerError):
"""Raised when the session policy's ruleset cannot be safely applied to a query.

Expand Down
17 changes: 17 additions & 0 deletions slayer/engine/enriched.py
Original file line number Diff line number Diff line change
Expand Up @@ -274,6 +274,23 @@ class CrossModelMeasure(BaseModel):
CrossModelMeasure.model_rebuild()


def all_projection_aliases(enriched: EnrichedQuery) -> list[str]:
"""Every alias ``enriched`` can put into an emitted SELECT, in bucket order.

Superset of :func:`public_projection_aliases` WITHOUT internal-prefix
filtering: hidden entries (``_inner_*``/``_ft*``/``_ts*`` hoists, ORDER-BY
aggregates) are still projected and must be length-fitted with the same map.
Deduplicated, first-seen order, so the derived rewrite map is deterministic.
"""
out: list[str] = [d.alias for d in enriched.dimensions]
out.extend(td.alias for td in enriched.time_dimensions)
out.extend(m.alias for m in enriched.measures)
out.extend(e.alias for e in enriched.expressions)
out.extend(t.alias for t in enriched.transforms)
out.extend(cm.alias for cm in enriched.cross_model_measures)
return list(dict.fromkeys(out))


def public_projection_aliases(enriched: EnrichedQuery) -> list[str]:
"""Return the ordered list of public-projection aliases for ``enriched``.

Expand Down
112 changes: 72 additions & 40 deletions slayer/engine/query_engine.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,11 @@
from sqlglot import exp

from slayer.core.enums import DEFAULT_AGGREGATIONS_BY_TYPE, DataType
from slayer.core.errors import AmbiguousModelError, ForcedFilterError
from slayer.core.errors import (
AmbiguousModelError,
ForcedFilterError,
IdentifierCollisionError,
)
from slayer.core.policy import JoinFilterRuleset, SessionPolicy
from slayer.core.format import NumberFormat, NumberFormatType, format_number
from slayer.core.models import (
Expand Down Expand Up @@ -61,6 +65,7 @@
CrossModelMeasure,
EnrichedMeasure,
EnrichedQuery,
all_projection_aliases,
public_projection_aliases,
)
from slayer.engine.enrichment import enrich_query
Expand All @@ -72,11 +77,11 @@
)
from slayer.sql.client import SlayerSQLClient
from slayer.sql.dialects import SqlDialect, dialect_for_ds_type, get_dialect
from slayer.sql.dialects._identifier_fit import fit_identifier
from slayer.sql import engine_factory
from slayer.sql.engine_factory import EngineCacheKey
from slayer.sql.engine_factory import _cache_key as _engine_cache_key
from slayer.sql.generator import SQLGenerator
from slayer.sql.reserved_keywords import SLAYER_RESERVED_KEYWORDS
from slayer.sql.session_policy import ScopedTable, apply_session_policy
from slayer.storage.base import StorageBackend

Expand Down Expand Up @@ -1386,10 +1391,11 @@ async def _run_and_build(
)
raise
timing.record("execute", _t)
# Dialect-driven read-side decode: BigQuery reverses its alias
# mangling here so the response keys match SLayer's universal
# dotted shape. Default hook is identity for every other dialect.
rows = get_dialect(prepared.dialect).decode_result_keys(rows)
# Reverse dialect alias fitting/mangling so response keys are canonical.
# Pass the unfiltered alias set: a row may carry a hidden ORDER-BY hoist.
rows = get_dialect(prepared.dialect).decode_result_keys(
rows, aliases=all_projection_aliases(prepared.enriched),
)
Comment on lines +1394 to +1398

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

📐 Maintainability & Code Quality | 🟡 Minor | ⚡ Quick win

Use keyword arguments for the decode and fitting calls.

Use keyword arguments for each supported parameter in these calls.

  • slayer/engine/query_engine.py#L1391-L1395: pass rows to decode_result_keys by keyword.
  • slayer/engine/query_engine.py#L2049-L2053: pass the type-probe row list to decode_result_keys by keyword.
  • slayer/engine/query_engine.py#L3231-L3239: pass name to fit_identifier by keyword.
Proposed fix
-            rows, aliases=all_projection_aliases(prepared.enriched),
+            rows=rows, aliases=all_projection_aliases(prepared.enriched),

-            [raw_types], aliases=all_projection_aliases(enriched),
+            rows=[raw_types], aliases=all_projection_aliases(enriched),

-                name, limit=get_dialect(dialect).max_identifier_bytes,
+                name=name, limit=get_dialect(dialect).max_identifier_bytes,

As per coding guidelines, “Use keyword arguments for functions with more than one parameter.”

📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
# Reverse dialect alias fitting/mangling so response keys are canonical.
# Pass the unfiltered alias set: a row may carry a hidden ORDER-BY hoist.
rows = get_dialect(prepared.dialect).decode_result_keys(
rows, aliases=all_projection_aliases(prepared.enriched),
)
# Reverse dialect alias fitting/mangling so response keys are canonical.
# Pass the unfiltered alias set: a row may carry a hidden ORDER-BY hoist.
rows = get_dialect(prepared.dialect).decode_result_keys(
rows=rows, aliases=all_projection_aliases(prepared.enriched),
)
raw_types = get_dialect(dialect).decode_result_keys(
rows=[raw_types], aliases=all_projection_aliases(enriched),
)[0]
def _fit_short(name: str) -> str:
"""Fit a virtual-model short to the dialect's budget; identity under the limit.
Every short is both an output alias and a downstream ``Column.name``,
and caller-supplied names can be arbitrarily long.
"""
return fit_identifier(
name=name, limit=get_dialect(dialect).max_identifier_bytes,
)
📍 Affects 1 file
  • slayer/engine/query_engine.py#L1391-L1395 (this comment)
  • slayer/engine/query_engine.py#L2049-L2053
  • slayer/engine/query_engine.py#L3231-L3239
🤖 Prompt for AI Agents
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@slayer/engine/query_engine.py` around lines 1391 - 1395, Use keyword
arguments for all supported parameters in the three calls: in
slayer/engine/query_engine.py lines 1391-1395, pass rows by keyword to
decode_result_keys; in lines 2049-2053, pass the type-probe row list by keyword
to decode_result_keys; and in lines 3231-3239, pass name by keyword to
fit_identifier.

Source: Coding guidelines

columns = prepared.expected_columns if not rows else [] # [] auto-derives
return SlayerResponse(
data=rows,
Expand Down Expand Up @@ -2043,6 +2049,12 @@ async def get_column_types(
logger.warning("get_column_types probe failed for model '%s'", model_name)
return {}

# The probe's metadata keys are the emitted (fitted) names; decode them
# to canonical or an over-limit measure drops out of the type map.
raw_types = get_dialect(dialect).decode_result_keys(
[raw_types], aliases=all_projection_aliases(enriched),
)[0]

# Map qualified aliases (e.g., "orders.revenue_max") back to bare measure names
result: dict[str, str] = {}
for em in enriched.measures:
Expand Down Expand Up @@ -3211,19 +3223,23 @@ async def _query_as_model( # NOSONAR S3776 — variable-precedence + enrich + S
# from the alias by stripping the source model prefix and replacing
# dots with underscores.
def _alias_to_short(alias: str) -> str:
"""Convert result alias to a flat column name for the virtual model.
"""Flatten a result alias to the virtual model's column name, then fit it.

The query result is a self-contained table without the joins the
source model may have had, so dot syntax (join paths) is not
applicable. We use ``__`` to preserve the path information:

'orders.customers.regions.name' → 'customers__regions__name'
'orders.count' → 'count'
The result is a self-contained table without joins, so join-path
dots become ``__``: 'orders.customers.regions.name' → 'customers__regions__name'.
"""
# Strip source model prefix
stripped = alias.split(".", 1)[-1] if "." in alias else alias
# Replace remaining dots with __ to encode the original join path
return stripped.replace(".", "__")
return _fit_short(stripped.replace(".", "__"))

def _fit_short(name: str) -> str:
"""Fit a virtual-model short to the dialect's budget; identity under the limit.

Every short is both an output alias and a downstream ``Column.name``,
and caller-supplied names can be arbitrarily long.
"""
return fit_identifier(
name, limit=get_dialect(dialect).max_identifier_bytes,
)

# (inner_alias, short_name, data_type, label, description, format)
column_map = []
Expand All @@ -3246,14 +3262,14 @@ def _alias_to_short(alias: str) -> str:
measure_name=src_name,
aggregation=m.aggregation,
)
column_map.append((m.alias, m.name, DataType.DOUBLE, label, desc, fmt))
column_map.append((m.alias, _fit_short(m.name), DataType.DOUBLE, label, desc, fmt))
for t in enriched.transforms:
column_map.append(
(t.alias, t.name, DataType.DOUBLE, t.label, None, NumberFormat(type=NumberFormatType.FLOAT))
(t.alias, _fit_short(t.name), DataType.DOUBLE, t.label, None, NumberFormat(type=NumberFormatType.FLOAT))
)
for e in enriched.expressions:
column_map.append(
(e.alias, e.name, DataType.DOUBLE, e.label, None, NumberFormat(type=NumberFormatType.FLOAT))
(e.alias, _fit_short(e.name), DataType.DOUBLE, e.label, None, NumberFormat(type=NumberFormatType.FLOAT))
)
for cm in enriched.cross_model_measures:
# DEV-1448: when the user supplied an explicit ``name``, cm.name is
Expand All @@ -3272,37 +3288,51 @@ def _alias_to_short(alias: str) -> str:
# leak into the virtual model's column set. Only user-declared
# renames qualify for the bare-name short.
if cm.user_declared and cm.name and "." not in cm.name:
short = cm.name
short = _fit_short(cm.name) # still fitted: deterministic + collision-checked
else:
short = _alias_to_short(cm.alias)
column_map.append((cm.alias, short, DataType.DOUBLE, cm.label, None, cm.format))

# Wrap inner SQL: SELECT <inner_alias> AS <short>, ... FROM (inner) AS _inner
# DEV-1571 Bug 3 follow-up: identifier quoting must match the
# dialect ``inner_sql`` was generated for. On MySQL the inner CTEs
# use backticks; on T-SQL the inner CTEs use brackets AND have
# their dotted aliases mangled. Hardcoded ANSI double quotes
# would either fail to parse (MySQL) or reference an alias the
# mangled inner subquery doesn't expose (T-SQL).
# DEV-1686: the inner ``alias`` is always dialect-quoted; the ``short``
# output alias must also be quoted when it is a reserved word (a
# user-declared cross-model rename like ``order``, or an
# ``_alias_to_short`` that yields one), else ``AS order`` is bare and
# the wrapped SQL fails to parse/execute.
# Wrap inner SQL: SELECT <inner_alias> AS <short>, ... FROM (inner) AS _inner.
# The ``short`` output alias must be spelled EXACTLY as the downstream
# reference will be, so both resolve to the same column. Run it through the
# two mechanisms the reference side uses: ``_maybe_quote_ident`` (DEV-1645:
# quote iff mixed-case) and ``Identifier.sql`` (dialect-reserved words /
# unsafe shapes). Both dialect-driven — unconditional quoting breaks bare
# refs on upper-folding backends; a fixed keyword set misses native words
# like MySQL's ``index``.
def _short_sql(short: str) -> str:
if short.lower() in SLAYER_RESERVED_KEYWORDS:
return exp.Identifier(this=short, quoted=True).sql(dialect=dialect)
return short
ident = exp.Identifier(this=short, quoted=False)
SQLGenerator._maybe_quote_ident(ident)
return ident.sql(dialect=dialect)

# Validate the whole short allocation: two shorts colliding on one output
# name silently drop a column. Keyed CASEFOLDED — conservative, since a
# mixed-case short is quoted (case-preserved) and would not really collide
# with its folded twin, but SLayer carries no per-dialect fold direction, so
# a loud error beats a silent duplicate on the backends this protects.
short_owner: dict[str, str] = {}
for alias, short, _, _, _, _ in column_map:
prior_alias = short_owner.setdefault(short.casefold(), alias)
if prior_alias != alias:
raise IdentifierCollisionError(
first=prior_alias, second=alias, emitted=short,
dialect=dialect,
limit=get_dialect(dialect).max_identifier_bytes,
namespace="query-backed model column",
)

rename_parts = [
f'{exp.Identifier(this=alias, quoted=True).sql(dialect=dialect)} AS {_short_sql(short)}'
for alias, short, _, _, _, _ in column_map
]
wrapped_sql = f"SELECT {', '.join(rename_parts)} FROM ({inner_sql}) AS _inner"
# DEV-1571 Bug 2: apply the dialect's emitted-SQL rewrite (e.g.
# T-SQL bracket-mangling) so the rename clause's inner-alias
# references match what the inner subquery actually projects.
wrapped_sql = get_dialect(dialect).rewrite_emitted_sql(wrapped_sql)
# Apply the dialect's emitted-SQL rewrite (T-SQL bracket-mangling, DEV-1756
# length-fitting) so the wrapper's inner-alias references match the fitted
# names the inner subquery projects. Same alias set the inner SQL used.
wrapped_sql = get_dialect(dialect).rewrite_emitted_sql(
sql=wrapped_sql, aliases=all_projection_aliases(enriched),
)

# One Column per result column — each is potentially both a dimension
# (group-by) or measure (with colon-aggregation) at query time.
Expand All @@ -3329,7 +3359,9 @@ def _short_sql(short: str) -> str:
agg_shorts.add(_alias_to_short(cm.alias))
for m in enriched.measures:
if m.from_cross_model_intercept:
agg_shorts.add(m.name)
# Same fitting the column_map entry got, else the breadcrumb
# names a column the virtual model never has.
agg_shorts.add(_fit_short(m.name))

# DEV-1449: record the lineage breadcrumb so outer-stage dotted-ref
# lookup can strip the right ancestor prefix and find the flat
Expand Down
Loading
Loading