From bb0d57aae1cfd4495e8d814fb6bfe737053ce4eb Mon Sep 17 00:00:00 2001 From: waralexrom <108349432+waralexrom@users.noreply.github.com> Date: Tue, 11 Aug 2026 17:58:19 +0200 Subject: [PATCH 1/6] fix(schema-compiler): promote DATE columns in Trino/Presto convertTz (#11516) * fix(schema-compiler): promote DATE columns in Trino/Presto convertTz Scaffolding maps DATE columns to `type: time` without a cast, and both dialects then apply `AT TIME ZONE` straight to the field, which Trino and Presto reject with TYPE_MISMATCH: Type of value must be a time or timestamp with/without time zone (actual date) Lift the field to a timestamp first. The promotion uses `COALESCE(field, CAST(NULL AS TIMESTAMP))` rather than a plain `CAST(... AS TIMESTAMP)`: resolving the common supertype triggers the implicit DATE -> TIMESTAMP coercion while leaving both timestamp types intact, whereas an explicit cast strips the zone off a `timestamp with time zone` column so the following `AT TIME ZONE` reinterprets its wall clock in the session timezone and shifts the result. Verified against Trino and PrestoDB: the DATE case now runs, and the converted value is unchanged for `timestamp` and `timestamp with time zone` inputs under a non-UTC session timezone. Note for operators: `convertTz` feeds pre-aggregation `loadSql`, which is hashed into the structure version, so existing Presto/Trino/Athena pre-aggregations with a time dimension are rebuilt once on upgrade. Co-Authored-By: Claude Opus 5 (1M context) * fix(schema-compiler): document the DATE day shift, cover Athena Rename `coerceToTimestamp` to `promoteDateToTimestamp`: the old name described the very operation the body avoids, inviting a "simplification" back to `CAST(... AS TIMESTAMP)` that would strip the zone off a `timestamp with time zone` column. Spell out in the doc comment that a promoted DATE lands on midnight and is then converted like any other naive timestamp, so its calendar date moves under a negative offset. That matches what the other dialects do with a date column, verified against a live Postgres and Trino: both bucket `DATE '2024-01-15'` under `2024-01-14` for `America/Los_Angeles`. Treating DATE specially here would diverge instead. Extend the test matrix with `AthenaQuery`, pinning that Athena keeps inheriting the Presto form, and cover the no-timezone early return. Co-Authored-By: Claude Opus 5 (1M context) --------- Co-authored-by: Claude Opus 5 (1M context) --- .../src/adapter/PrestodbQuery.ts | 31 ++++- .../src/adapter/TrinoQuery.ts | 2 +- .../trino-presto-date-time-dimension.test.ts | 109 ++++++++++++++++++ 3 files changed, 137 insertions(+), 5 deletions(-) create mode 100644 packages/cubejs-schema-compiler/test/unit/trino-presto-date-time-dimension.test.ts diff --git a/packages/cubejs-schema-compiler/src/adapter/PrestodbQuery.ts b/packages/cubejs-schema-compiler/src/adapter/PrestodbQuery.ts index e889beaa80438..c8cac69163058 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PrestodbQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PrestodbQuery.ts @@ -49,11 +49,34 @@ export class PrestodbQuery extends BaseQuery { return `from_iso8601_timestamp(${value})`; } + /** + * Lifts a DATE expression to a timestamp so that timezone arithmetic accepts + * it. `COALESCE` with a NULL timestamp resolves to the common supertype, + * which triggers the implicit DATE -> TIMESTAMP coercion while leaving both + * timestamp types intact. Deliberately not `CAST(... AS TIMESTAMP)`: that + * strips the zone off a `timestamp with time zone` expression, so the + * subsequent `AT TIME ZONE` reinterprets its wall clock in the session + * timezone and shifts the result. + * + * A promoted DATE lands on midnight and is then converted like any other + * naive timestamp, which moves its calendar date under a negative offset: + * `DATE '2024-01-15'` day-truncates to `2024-01-14` for `America/Los_Angeles`. + * That is what every other dialect does with a date column — `PostgresQuery` + * reaches the same bucket via `::timestamptz AT TIME ZONE` — so treating DATE + * specially here would diverge instead. + */ + protected promoteDateToTimestamp(field: string): string { + return `COALESCE(${field}, CAST(NULL AS TIMESTAMP))`; + } + public override convertTz(field) { - const atTimezone = `${field} AT TIME ZONE '${this.timezone}'`; - return this.timezone ? - `CAST(date_add('minute', timezone_minute(${atTimezone}), date_add('hour', timezone_hour(${atTimezone}), ${field})) AS TIMESTAMP)` : - field; + if (!this.timezone) { + return field; + } + + const timestampField = this.promoteDateToTimestamp(field); + const atTimezone = `${timestampField} AT TIME ZONE '${this.timezone}'`; + return `CAST(date_add('minute', timezone_minute(${atTimezone}), date_add('hour', timezone_hour(${atTimezone}), ${timestampField})) AS TIMESTAMP)`; } /** diff --git a/packages/cubejs-schema-compiler/src/adapter/TrinoQuery.ts b/packages/cubejs-schema-compiler/src/adapter/TrinoQuery.ts index 9939280bc9c62..a3ddde089414c 100644 --- a/packages/cubejs-schema-compiler/src/adapter/TrinoQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/TrinoQuery.ts @@ -4,6 +4,6 @@ export class TrinoQuery extends PrestodbQuery { // Trino doesn't require odd prestodb manual datetime offset calculations // as it uses mature timestamps models public override convertTz(field) { - return this.timezone ? `CAST((${field} AT TIME ZONE '${this.timezone}') AS TIMESTAMP)` : field; + return this.timezone ? `CAST((${this.promoteDateToTimestamp(field)} AT TIME ZONE '${this.timezone}') AS TIMESTAMP)` : field; } } diff --git a/packages/cubejs-schema-compiler/test/unit/trino-presto-date-time-dimension.test.ts b/packages/cubejs-schema-compiler/test/unit/trino-presto-date-time-dimension.test.ts new file mode 100644 index 0000000000000..6b11976fa0225 --- /dev/null +++ b/packages/cubejs-schema-compiler/test/unit/trino-presto-date-time-dimension.test.ts @@ -0,0 +1,109 @@ +/* eslint-disable no-restricted-syntax, quotes */ +import { AthenaQuery } from '../../src/adapter/AthenaQuery'; +import { PrestodbQuery } from '../../src/adapter/PrestodbQuery'; +import { TrinoQuery } from '../../src/adapter/TrinoQuery'; +import { prepareJsCompiler } from './PrepareCompiler'; + +// Trino/Presto reject timezone arithmetic over DATE: +// "Type of value must be a time or timestamp with/without time zone (actual date)". +// Scaffolding maps DATE columns to `type: time` without a cast, so `convertTz` +// has to promote the field to a timestamp itself. The promotion must not be a +// plain `CAST(... AS TIMESTAMP)`: that strips the zone off a +// `timestamp with time zone` column and shifts the converted value. +describe('Trino/Presto time dimensions over DATE columns', () => { + const { compiler, joinGraph, cubeEvaluator } = prepareJsCompiler(` + cube('events', { + sql: \` + SELECT + 1 AS id, + CAST('2024-01-15' AS DATE) AS d, + CAST('2024-01-15 10:20:30' AS TIMESTAMP) AS ts, + CAST('2024-01-15 10:20:30 UTC' AS TIMESTAMP WITH TIME ZONE) AS tstz + \`, + dimensions: { + id: { + sql: 'id', + type: 'number', + primaryKey: true + }, + d: { + sql: 'd', + type: 'time' + }, + ts: { + sql: 'ts', + type: 'time' + }, + tstz: { + sql: 'tstz', + type: 'time' + } + }, + measures: { + count: { + type: 'count' + } + } + }); + `); + + const timezone = 'America/Los_Angeles'; + + const buildSql = (QueryClass: any, column: string) => { + const query = new QueryClass({ joinGraph, cubeEvaluator, compiler }, { + measures: ['events.count'], + timeDimensions: [{ + dimension: `events.${column}`, + granularity: 'day' + }], + timezone + }); + + return query.buildSqlAndParams()[0]; + }; + + const promoted = (column: string) => `COALESCE("events".${column}, CAST(NULL AS TIMESTAMP))`; + + const trinoConvertTz = (column: string) => `CAST((${promoted(column)} AT TIME ZONE '${timezone}') AS TIMESTAMP)`; + + const prestoConvertTz = (column: string) => { + const atTimezone = `${promoted(column)} AT TIME ZONE '${timezone}'`; + return `CAST(date_add('minute', timezone_minute(${atTimezone}), ` + + `date_add('hour', timezone_hour(${atTimezone}), ${promoted(column)})) AS TIMESTAMP)`; + }; + + const dialects = [ + { name: 'TrinoQuery', QueryClass: TrinoQuery, convertTz: trinoConvertTz }, + { name: 'PrestodbQuery', QueryClass: PrestodbQuery, convertTz: prestoConvertTz }, + // Athena has no convertTz of its own; pin that it keeps inheriting the fix. + { name: 'AthenaQuery', QueryClass: AthenaQuery, convertTz: prestoConvertTz } + ] as const; + + for (const { name, QueryClass, convertTz } of dialects) { + describe(name, () => { + // `d` is the column the engine rejects; `ts`/`tstz` pin that the + // promotion leaves the types that already work alone. + for (const column of ['d', 'ts', 'tstz']) { + it(`promotes the ${column} column instead of feeding it to AT TIME ZONE`, async () => { + await compiler.compile(); + + const sql = buildSql(QueryClass, column); + + expect(sql).not.toMatch(new RegExp(`"events"\\.${column} AT TIME ZONE`)); + expect(sql).toContain(convertTz(column)); + }); + } + + it('leaves the field untouched without a timezone', async () => { + await compiler.compile(); + + const query = new QueryClass({ joinGraph, cubeEvaluator, compiler }, { + measures: ['events.count'], + timezone: null + }); + + expect(query.convertTz('"events".d')).toBe('"events".d'); + }); + }); + } +}); From b951c004038acf4e7640867bf3702d3144ad80a9 Mon Sep 17 00:00:00 2001 From: Alex Qyoun-ae <4062971+MazterQyou@users.noreply.github.com> Date: Tue, 11 Aug 2026 20:08:45 +0400 Subject: [PATCH 2/6] feat(cubesql): Support `WIDTH_BUCKET` SQL pushdown (#11500) --- .../core-data-apis/sql-api/reference.mdx | 10 ++++ .../driver/DremioQuery.js | 1 + .../cubejs-druid-driver/src/DruidQuery.ts | 1 + .../cubejs-duckdb-driver/src/DuckDBQuery.ts | 1 + .../src/FireboltQuery.ts | 1 + packages/cubejs-ksql-driver/src/KsqlQuery.ts | 1 + .../cubejs-pinot-driver/src/PinotQuery.ts | 1 + .../cubejs-questdb-driver/src/QuestQuery.ts | 2 + .../src/adapter/BaseQuery.js | 1 + .../src/adapter/BigqueryQuery.ts | 1 + .../src/adapter/CrateQuery.ts | 6 ++ .../src/adapter/CubeStoreQuery.ts | 1 + .../src/adapter/HiveQuery.ts | 6 ++ .../src/adapter/MssqlQuery.ts | 1 + .../src/adapter/MysqlQuery.ts | 1 + .../src/adapter/SqliteQuery.ts | 6 ++ .../cubesql/src/compile/engine/udf/common.rs | 8 ++- rust/cubesql/cubesql/src/compile/mod.rs | 58 +++++++++++++++++++ rust/cubesql/cubesql/src/compile/test/mod.rs | 1 + 19 files changed, 107 insertions(+), 1 deletion(-) diff --git a/docs-mintlify/reference/core-data-apis/sql-api/reference.mdx b/docs-mintlify/reference/core-data-apis/sql-api/reference.mdx index 3923924ffcf24..7def4e0c65d2c 100644 --- a/docs-mintlify/reference/core-data-apis/sql-api/reference.mdx +++ b/docs-mintlify/reference/core-data-apis/sql-api/reference.mdx @@ -254,6 +254,16 @@ of the PostgreSQL documentation. | `SIGN` | Sign of the argument (`-1`, `0`, or `+1`) | ✅ Yes | ✅ Outer
❌ Inner (selections)
✅ Inner (projections) | | `SQRT` | Square root | ✅ Yes | ✅ Outer
❌ Inner (selections)
✅ Inner (projections) | | `TRUNC` | Truncates to integer (towards zero) | ✅ Yes | ✅ Outer
✅ Inner (selections)
❌ Inner (projections) | +| `WIDTH_BUCKET` | Assigns a value to a bucket in an equal-width histogram | ✅ Yes | ❌ No | + + + +`WIDTH_BUCKET` pushdown is only available on data sources whose SQL dialect +supports it. It is not supported with Apache Pinot, BigQuery, CrateDB, Cube Store, +Dremio, Druid, DuckDB, Firebolt, Hive, ksqlDB, Microsoft SQL Server, MySQL, +QuestDB, or SQLite. + + ### Trigonometric functions diff --git a/packages/cubejs-dremio-driver/driver/DremioQuery.js b/packages/cubejs-dremio-driver/driver/DremioQuery.js index eb62241e44a1c..237e53c5c162e 100644 --- a/packages/cubejs-dremio-driver/driver/DremioQuery.js +++ b/packages/cubejs-dremio-driver/driver/DremioQuery.js @@ -167,6 +167,7 @@ class DremioQuery extends BaseQuery { templates.expressions.interval_single_date_part = 'CAST({{ num }} as INTERVAL {{ date_part }})'; templates.expressions.like = '{{ expr }} {% if negated %}NOT {% endif %}LIKE {{ pattern }}{% if default_escape %} ESCAPE \'\\\'{% endif %}'; delete templates.expressions.ilike; + delete templates.functions.WIDTH_BUCKET; templates.quotes.identifiers = '"'; return templates; } diff --git a/packages/cubejs-druid-driver/src/DruidQuery.ts b/packages/cubejs-druid-driver/src/DruidQuery.ts index 69c1d6d289d0c..c4e2d212b1e61 100644 --- a/packages/cubejs-druid-driver/src/DruidQuery.ts +++ b/packages/cubejs-druid-driver/src/DruidQuery.ts @@ -68,6 +68,7 @@ export class DruidQuery extends BaseQuery { // Druid evaluates CURRENT_TIMESTAMP in the sqlTimeZone query context, which // defaults to UTC — assumes the connection does not override sqlTimeZone templates.functions.UTCTIMESTAMP = 'CURRENT_TIMESTAMP'; + delete templates.functions.WIDTH_BUCKET; return templates; } diff --git a/packages/cubejs-duckdb-driver/src/DuckDBQuery.ts b/packages/cubejs-duckdb-driver/src/DuckDBQuery.ts index d9c0061a890df..11a0d4626ed0d 100644 --- a/packages/cubejs-duckdb-driver/src/DuckDBQuery.ts +++ b/packages/cubejs-duckdb-driver/src/DuckDBQuery.ts @@ -67,6 +67,7 @@ export class DuckDBQuery extends BaseQuery { templates.functions.LEAST = 'LEAST({{ args_concat }})'; templates.functions.GREATEST = 'GREATEST({{ args_concat }})'; templates.functions.STRING_AGG = 'STRING_AGG({% if distinct %}DISTINCT {% endif %}{{ args[0] }}, COALESCE({{ args[1] }}, \'\'))'; + delete templates.functions.WIDTH_BUCKET; templates.expressions.like = '{{ expr }} {% if negated %}NOT {% endif %}LIKE {{ pattern }}{% if default_escape %} ESCAPE \'\\\'{% endif %}'; templates.expressions.ilike = '{{ expr }} {% if negated %}NOT {% endif %}ILIKE {{ pattern }}{% if default_escape %} ESCAPE \'\\\'{% endif %}'; // DuckDB `/` performs float division even for integer operands (since v0.8); diff --git a/packages/cubejs-firebolt-driver/src/FireboltQuery.ts b/packages/cubejs-firebolt-driver/src/FireboltQuery.ts index 2b42e699afa64..62b1bd3b968de 100644 --- a/packages/cubejs-firebolt-driver/src/FireboltQuery.ts +++ b/packages/cubejs-firebolt-driver/src/FireboltQuery.ts @@ -57,6 +57,7 @@ export class FireboltQuery extends BaseQuery { // value bare, which is invalid syntax templates.expressions.timestamp_literal = 'TIMESTAMPTZ \'{{ value }}\''; templates.tesseract.bool_param_cast = 'CAST({{ expr }} AS BOOLEAN)'; + delete templates.functions.WIDTH_BUCKET; return templates; } diff --git a/packages/cubejs-ksql-driver/src/KsqlQuery.ts b/packages/cubejs-ksql-driver/src/KsqlQuery.ts index c67fca19847d9..3852e647e1ff4 100644 --- a/packages/cubejs-ksql-driver/src/KsqlQuery.ts +++ b/packages/cubejs-ksql-driver/src/KsqlQuery.ts @@ -78,6 +78,7 @@ export class KsqlQuery extends BaseQuery { // ksqlDB does not support positional GROUP BY — group by the full // expressions instead of column ordinals. templates.statements.group_by_exprs = '{{ group_by | map(attribute=\'expr\') | join(\', \') }}'; + delete templates.functions.WIDTH_BUCKET; return templates; } diff --git a/packages/cubejs-pinot-driver/src/PinotQuery.ts b/packages/cubejs-pinot-driver/src/PinotQuery.ts index e1b3a1ef44296..7d739317bcfd3 100644 --- a/packages/cubejs-pinot-driver/src/PinotQuery.ts +++ b/packages/cubejs-pinot-driver/src/PinotQuery.ts @@ -216,6 +216,7 @@ export class PinotQuery extends BaseQuery { // epoch-millis representation produced by the timestamp_literal template templates.functions.UTCTIMESTAMP = 'NOW()'; templates.functions.STRING_AGG = 'LISTAGG({% if distinct %}DISTINCT {% endif %}{{ args_concat }})'; + delete templates.functions.WIDTH_BUCKET; templates.statements.select = '{% if ctes %} WITH \n' + '{{ ctes | join(\',\n\') }}\n' + '{% endif %}' + diff --git a/packages/cubejs-questdb-driver/src/QuestQuery.ts b/packages/cubejs-questdb-driver/src/QuestQuery.ts index 61a5942b8718c..777c1184660cd 100644 --- a/packages/cubejs-questdb-driver/src/QuestQuery.ts +++ b/packages/cubejs-questdb-driver/src/QuestQuery.ts @@ -309,6 +309,8 @@ export class QuestQuery extends BaseQuery { '{% elif offset is not none %}\nLIMIT {{ offset }}, 2147483647' + '{% elif limit is not none %}\nLIMIT {{ limit }}{% endif %}'; + delete templates.functions.WIDTH_BUCKET; + return templates; } } diff --git a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js index c0fb818999e92..7d601c015365b 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js +++ b/packages/cubejs-schema-compiler/src/adapter/BaseQuery.js @@ -4565,6 +4565,7 @@ export class BaseQuery { DATE: 'DATE({{ args_concat }})', PERCENTILECONT: 'PERCENTILE_CONT({{ args_concat }})', + WIDTH_BUCKET: 'WIDTH_BUCKET({{ args_concat }})', }, statements: { select: '{% if ctes %} WITH {% if recursive %}RECURSIVE {% endif %}\n' + diff --git a/packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts b/packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts index d9b45a063e8f8..eb773b66b7598 100644 --- a/packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/BigqueryQuery.ts @@ -335,6 +335,7 @@ export class BigqueryQuery extends BaseQuery { templates.functions.UTCTIMESTAMP = 'CURRENT_TIMESTAMP()'; delete templates.functions.TO_CHAR; delete templates.functions.PERCENTILECONT; + delete templates.functions.WIDTH_BUCKET; templates.expressions.binary = '{% if op == \'%\' %}MOD({{ left }}, {{ right }}){% else %}({{ left }} {{ op }} {{ right }}){% endif %}'; templates.expressions.interval = 'INTERVAL {{ interval }}'; // BigQuery `/` on INT64 operands returns FLOAT64; DIV() is integer division diff --git a/packages/cubejs-schema-compiler/src/adapter/CrateQuery.ts b/packages/cubejs-schema-compiler/src/adapter/CrateQuery.ts index 5c7d13895656b..ccc1e1f444d2b 100644 --- a/packages/cubejs-schema-compiler/src/adapter/CrateQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/CrateQuery.ts @@ -13,4 +13,10 @@ export class CrateQuery extends PostgresQuery { public countDistinctApprox(sql: string): string { return `hyperloglog_distinct(${sql})`; } + + public sqlTemplates() { + const templates = super.sqlTemplates(); + delete templates.functions.WIDTH_BUCKET; + return templates; + } } diff --git a/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts b/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts index 9cb0d58968b0a..d8c16d9c84b7a 100644 --- a/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/CubeStoreQuery.ts @@ -363,6 +363,7 @@ export class CubeStoreQuery extends BaseQuery { // across partitioned tables is unsafe. Don't push those join types down to CubeStore. delete templates.join_types.full; delete templates.join_types.right; + delete templates.functions.WIDTH_BUCKET; return templates; } } diff --git a/packages/cubejs-schema-compiler/src/adapter/HiveQuery.ts b/packages/cubejs-schema-compiler/src/adapter/HiveQuery.ts index 479039c82da18..4ed72e2ba4746 100644 --- a/packages/cubejs-schema-compiler/src/adapter/HiveQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/HiveQuery.ts @@ -109,4 +109,10 @@ export class HiveQuery extends BaseQuery { public defaultRefreshKeyRenewalThreshold() { return 120; } + + public sqlTemplates() { + const templates = super.sqlTemplates(); + delete templates.functions.WIDTH_BUCKET; + return templates; + } } diff --git a/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts b/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts index e91da7efa9ce4..d462acc766136 100644 --- a/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/MssqlQuery.ts @@ -275,6 +275,7 @@ export class MssqlQuery extends BaseQuery { delete templates.functions.STRING_AGG; // PERCENTILE_CONT works but requires PARTITION BY delete templates.functions.PERCENTILECONT; + delete templates.functions.WIDTH_BUCKET; templates.expressions.like = '{{ expr }} {% if negated %}NOT {% endif %}LIKE {{ pattern }}{% if default_escape %} ESCAPE \'\\\'{% endif %}'; delete templates.expressions.ilike; // MSSQL uses + for string concatenation instead of || diff --git a/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts b/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts index a19744df4b80b..243e47b7a9f83 100644 --- a/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/MysqlQuery.ts @@ -189,6 +189,7 @@ export class MysqlQuery extends BaseQuery { templates.functions.UTCTIMESTAMP = 'UTC_TIMESTAMP()'; // PERCENTILE_CONT works but requires PARTITION BY delete templates.functions.PERCENTILECONT; + delete templates.functions.WIDTH_BUCKET; templates.quotes.identifiers = '`'; templates.quotes.escape = '\\`'; // NOTE: this template contains a comma; two order expressions are being generated diff --git a/packages/cubejs-schema-compiler/src/adapter/SqliteQuery.ts b/packages/cubejs-schema-compiler/src/adapter/SqliteQuery.ts index 952d77b63ffdb..e28c488605c57 100644 --- a/packages/cubejs-schema-compiler/src/adapter/SqliteQuery.ts +++ b/packages/cubejs-schema-compiler/src/adapter/SqliteQuery.ts @@ -81,4 +81,10 @@ export class SqliteQuery extends BaseQuery { // eslint-disable-next-line quotes return `strftime('%s','now')`; } + + public sqlTemplates() { + const templates = super.sqlTemplates(); + delete templates.functions.WIDTH_BUCKET; + return templates; + } } diff --git a/rust/cubesql/cubesql/src/compile/engine/udf/common.rs b/rust/cubesql/cubesql/src/compile/engine/udf/common.rs index 0be4ef4231e42..e951e23666ae1 100644 --- a/rust/cubesql/cubesql/src/compile/engine/udf/common.rs +++ b/rust/cubesql/cubesql/src/compile/engine/udf/common.rs @@ -5382,10 +5382,16 @@ pub fn register_fun_stubs(mut ctx: SessionContext) -> SessionContext { vol = Stable ); register_fun_stub!(udf, "unistr", tsig = [Utf8], rettyp = Utf8); + // In Postgres the bucket count is int4, but integer literals are parsed as Int64 here, + // and Int64 is never coerced down to Int32. Accept both so that a plain literal count + // like "width_bucket(x, 0, 100, 10)" plans. register_fun_stub!( udf, "width_bucket", - tsig = [Float64, Float64, Float64, Int32], + tsigs = [ + [Float64, Float64, Float64, Int32], + [Float64, Float64, Float64, Int64], + ], rettyp = Int32 ); // TODO: "width_bucket" also has a two-arg variant with anyarray args diff --git a/rust/cubesql/cubesql/src/compile/mod.rs b/rust/cubesql/cubesql/src/compile/mod.rs index 669e2f4565e33..2e3ffa4b13696 100644 --- a/rust/cubesql/cubesql/src/compile/mod.rs +++ b/rust/cubesql/cubesql/src/compile/mod.rs @@ -15126,6 +15126,64 @@ ORDER BY "source"."str0" ASC .contains("NOT (")); } + #[tokio::test] + async fn test_width_bucket_push_down() { + if !Rewriter::sql_push_down_enabled() { + return; + } + init_testing_logger(); + + // The bucket count is an Int64 literal here, while Postgres types it as int4: + // the stub signature has to accept both, otherwise planning fails on coercion. + let query_plan = convert_select_to_query_plan( + " + SELECT WIDTH_BUCKET(k.taxful_total_price, -301, 2200, 36) AS b, COUNT(1) + FROM KibanaSampleDataEcommerce AS k + GROUP BY 1 + " + .to_string(), + DatabaseProtocol::PostgreSQL, + ) + .await; + + let physical_plan = query_plan.as_physical_plan().await.unwrap(); + println!( + "Physical plan: {}", + displayable(physical_plan.as_ref()).indent() + ); + + let logical_plan = query_plan.as_logical_plan(); + assert!(logical_plan + .find_cube_scan_wrapped_sql() + .wrapped_sql + .sql + .contains( + "WIDTH_BUCKET(${KibanaSampleDataEcommerce.taxful_total_price}, -301, 2200, 36)" + )); + + // Same call with an Int32 bucket count, which matches the other arm of the + // stub signature. It has to render identically. + let query_plan = convert_select_to_query_plan( + " + SELECT WIDTH_BUCKET(k.taxful_total_price, -301, 2200, CAST(36 AS INT)) AS b, COUNT(1) + FROM KibanaSampleDataEcommerce AS k + GROUP BY 1 + " + .to_string(), + DatabaseProtocol::PostgreSQL, + ) + .await; + + assert!(query_plan + .as_logical_plan() + .find_cube_scan_wrapped_sql() + .wrapped_sql + .sql + .contains( + "WIDTH_BUCKET(${KibanaSampleDataEcommerce.taxful_total_price}, -301, 2200, 36)" + )); + } + #[tokio::test] async fn test_datetrunc_push_down() { if !Rewriter::sql_push_down_enabled() { diff --git a/rust/cubesql/cubesql/src/compile/test/mod.rs b/rust/cubesql/cubesql/src/compile/test/mod.rs index 637ad40a05cbd..283d0e00190b8 100644 --- a/rust/cubesql/cubesql/src/compile/test/mod.rs +++ b/rust/cubesql/cubesql/src/compile/test/mod.rs @@ -704,6 +704,7 @@ pub fn sql_generator( ("functions/LOWER".to_string(), "LOWER({{ args_concat }})".to_string()), ("functions/UPPER".to_string(), "UPPER({{ args_concat }})".to_string()), ("functions/PERCENTILECONT".to_string(), "PERCENTILE_CONT({{ args_concat }})".to_string()), + ("functions/WIDTH_BUCKET".to_string(), "WIDTH_BUCKET({{ args_concat }})".to_string()), ("expressions/query_aliased".to_string(), "{{ query }} AS {{ quoted_alias }}".to_string()), ("expressions/extract".to_string(), "EXTRACT({{ date_part }} FROM {{ expr }})".to_string()), ( From 9587d107e26775a9ec1caefd5bd1ec41abc5ec7f Mon Sep 17 00:00:00 2001 From: Artyom Keydunov Date: Tue, 11 Aug 2026 11:00:01 -0700 Subject: [PATCH 3/6] docs(mcp): document the single MCP endpoint and the four undocumented tools (#11530) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The centralized MCP endpoint shipped, so https://cubecloud.dev/mcp is the same URL for every account, deployment and region. The page still told users to find a per-tenant host first, which is no longer the shortest path to a working client — every connect snippet now uses the one URL, with the region-specific endpoint kept as the explicit alternative and self-hosted installations pointed at their own console domain. Also documents the four tools added since the last pass (getBranchDiff, getDeploymentEnv, getPreAggregationStatus, buildPreAggregation), taking the count from 16 to 20, and states that the permission gate covers all ten schema-gated tools rather than the six model-editing ones. Adds a pre-aggregations section and a verification workflow, notes that getDeploymentEnv redacts secrets, and that buildPreAggregation consumes warehouse resources. Fixes a misnumbered step list in the Claude Code flow along the way. --- .../docs/integrations/mcp-server.mdx | 112 +++++++++++++++--- 1 file changed, 93 insertions(+), 19 deletions(-) diff --git a/docs-mintlify/docs/integrations/mcp-server.mdx b/docs-mintlify/docs/integrations/mcp-server.mdx index 1108c262e0a00..aa01896cce835 100644 --- a/docs-mintlify/docs/integrations/mcp-server.mdx +++ b/docs-mintlify/docs/integrations/mcp-server.mdx @@ -19,13 +19,30 @@ Model Context Protocol (MCP) is an open standard that enables AI assistants to s ## Overview -Cube hosts an MCP server endpoint for your tenant. MCP clients connect over HTTPS and authenticate via OAuth. +Cube hosts an MCP server endpoint. MCP clients connect over HTTPS and authenticate via OAuth. -- **Endpoint:** `https:///api/mcp` -- **OAuth discovery:** `https:///.well-known/oauth` -- **OAuth flow:** Authorization Code + PKCE, `client_id` = `cube-mcp-client`, scope = `mcp-agent-access` +- **Endpoint:** `https://cubecloud.dev/mcp` +- **OAuth flow:** Authorization Code + PKCE, `client_id` = `cube-mcp-client`, scope = `mcp-agent-access`. Clients discover it automatically from the endpoint — there is nothing to configure by hand. +- **Account selection:** You sign in to Cube as part of the OAuth flow and, if you belong to more than one account, choose which one to connect. - **Deployment selection:** On connect, the client lands on the tenant **default deployment** set by your admin (or the first deployment you can access). Clients can also target a specific deployment and agent per request — see [Select a deployment and agent](#select-a-deployment-and-agent). +### One endpoint for everyone + +`https://cubecloud.dev/mcp` is the same URL for every account, deployment and region. Cube +identifies your account from the OAuth token and routes each request to whichever +deployment serves it, so there is no per-tenant host to look up before you can connect. + + + +If Cube runs in your own cloud account or on your own domain, use that console domain +instead — `https:///mcp`. + + + +Each deployment also keeps a **region-specific endpoint**, shown under +**Admin → MCP Server**. Both work. Prefer the single endpoint above unless you need a +client to reach a deployment's region directly without passing through the control plane. + ## Admin setup ### Prerequisites @@ -34,11 +51,12 @@ Before enabling MCP, make sure you have: - **Admin privileges** in your Cube instance - An active Cube tenant -- MCP server URL configured -### 1) Confirm MCP server URL +### 1) Check the MCP page -MCP uses your Cube MCP server host. If the URL isn’t configured, the MCP page will show “MCP configuration is unavailable.” +Go to **Admin → MCP Server**. The page shows the endpoint to hand to clients along with +ready-made setup snippets for each supported client. If it reads “MCP configuration is +unavailable,” the MCP server host isn’t configured for the account yet. ### 2) Configure deployment access @@ -70,7 +88,7 @@ authenticated user is allowed to see. ### Claude Code ```bash -claude mcp add --transport http cube-mcp-server https:///api/mcp +claude mcp add --transport http cube-mcp-server https://cubecloud.dev/mcp ``` #### Authentication and usage flow: @@ -78,9 +96,9 @@ claude mcp add --transport http cube-mcp-server https:///a 1. Run the command copied from **Admin → MCP Server → AI Clients → Claude Code**. 2. Then run Claude and use `/mcp` to list available servers. 3. Select `cube-mcp-server` and choose `Authenticate`. -2. A browser window opens for authentication. -3. Log into Cube and choose your tenant. -4. Return to Claude Code and start asking questions. +4. A browser window opens for authentication. +5. Log into Cube and choose your tenant. +6. Return to Claude Code and start asking questions. @@ -92,7 +110,7 @@ claude mcp add --transport http cube-mcp-server https:///a 2. Scroll to **Integrations** and click **Add more**. 3. Use: - **Integration name:** Cube MCP - - **Integration URL:** `https:///api/mcp` + - **Integration URL:** `https://cubecloud.dev/mcp` 4. Complete the OAuth flow to grant access. 5. Enable tools in any new chats. @@ -109,7 +127,7 @@ claude mcp add --transport http cube-mcp-server https:///a "mcpServers": { "cube-mcp-server": { "command": "npx", - "args": ["-y", "mcp-remote", "--transport", "http", "https:///api/mcp"] + "args": ["-y", "mcp-remote", "--transport", "http", "https://cubecloud.dev/mcp"] } } } @@ -124,7 +142,7 @@ Add the MCP endpoint under Tools & MCP Settings, then complete the OAuth flow. "mcpServers": { "cube-mcp-server": { "command": "npx", - "args": ["-y", "mcp-remote", "--transport", "http", "https:///api/mcp"] + "args": ["-y", "mcp-remote", "--transport", "http", "https://cubecloud.dev/mcp"] } } } @@ -135,7 +153,7 @@ Add the MCP endpoint under Tools & MCP Settings, then complete the OAuth flow. Preferred (CLI): ```bash -codex mcp add cube-mcp-server --url https:///api/mcp +codex mcp add cube-mcp-server --url https://cubecloud.dev/mcp ``` If this is your first time using MCP in Codex, enable the feature in `~/.codex/config.toml`: @@ -152,7 +170,7 @@ Manual setup: rmcp_client = true [mcp_servers."cube-mcp-server"] -url = "https:///api/mcp" +url = "https://cubecloud.dev/mcp" ``` Then run `codex mcp login cube-mcp-server` to authenticate. @@ -211,7 +229,7 @@ neither `listDeployments` nor the `chat` selection can reach an excluded deploym ## Available actions -The MCP server exposes 16 tools, grouped below. +The MCP server exposes 20 tools, grouped below. Every tool runs as the authenticated user. Queries respect the same [permissions][ref-roles] as the rest of Cube, including row-level security — MCP is a new @@ -277,6 +295,21 @@ default. Users without it never see them. | `writeDataModelFile` | Creates or overwrites a model source file on the dev branch (whole-file replacement). Recompiles the model and reports `valid` plus any `validationError`. | Destructive — prompts | | `deleteDataModelFile` | Deletes a model source file on the dev branch. | Destructive — prompts | | `getDataModelChanges` | Shows the diff of the dev branch against its parent — the pending changes, for review before committing. | Read-only | +| `getBranchDiff` | Shows what **any** branch changed against the deploy branch: the changed-file list with per-file line counts, plus the unified diff. | Read-only | +| `getDeploymentEnv` | Lists the deployment's environment variables, with every secret-looking value redacted to `[ENCRYPTED]`. Useful for confirming configuration is present and shaped as expected. | Read-only | + +`getDataModelChanges` answers "what have I changed?" for your own dev branch against its +immediate parent. `getBranchDiff` answers "what did this branch change?" for any branch, +including feature branches — reach for it when a change edits existing cubes, where the +file list looks identical on both branches and `listDataModelFiles` reveals nothing. + + + +`getDeploymentEnv` never returns secret values. Anything that looks like a credential is +replaced with `[ENCRYPTED]` before it leaves Cube, so an AI client can verify that a +variable is set without ever seeing what it is set to. + + #### How model edits stay safe @@ -293,11 +326,37 @@ into the MCP server: from the Cube UI, as described in [Development mode][ref-dev-mode]. The MCP server deliberately exposes no commit tool — an AI client can prepare changes, but only a person can ship them. -- **Registration is permission-gated.** The six tools above are only offered to users - whose role allows editing the semantic model. +- **Registration is permission-gated.** Every tool in the two sections above — the six + model-editing tools, `getBranchDiff`, `getDeploymentEnv`, and both pre-aggregation + tools — is offered only to users whose role allows editing the semantic model. A Viewer + never sees them at all. Review pending work with `getDataModelChanges` before you commit. +### Pre-aggregations + +These tools inspect and trigger [pre-aggregation][ref-pre-aggregations] builds. They are +registered under the same semantic-model permission as the data model tools above. + +| Tool | Description | Access | +| --- | --- | --- | +| `getPreAggregationStatus` | Lists the data model's pre-aggregations with their definitions and, for each, how many partitions exist, how many were built, when the newest build landed, and the exact error if a build failed. | Read-only | +| `buildPreAggregation` | Queues an on-demand build of one pre-aggregation and returns once it is accepted. The build runs asynchronously. | Write | + +Together these close the loop on pre-aggregation work: a query alone cannot prove a rollup +was used, and external pre-aggregations fail in configuration-specific ways — a missing +export bucket, denied bucket permissions — that only an actual build surfaces. Call +`buildPreAggregation`, then poll `getPreAggregationStatus` to see whether the partitions +built or to read the failure. + + + +`buildPreAggregation` runs real queries against your data warehouse and, for external +pre-aggregations, writes through the export bucket. It consumes warehouse resources, so +expect a cost per build. + + + ## Example workflows ### Ask a data question @@ -350,6 +409,20 @@ recompiles the model and reports validation errors, so you can iterate until it Review the result with `getDataModelChanges`, then commit the branch from the Cube UI to publish it. +To review a branch you didn't author — a colleague's feature branch, say — call +`getBranchDiff` with its name instead. It compares against the deploy branch and returns +the full changed-file list even when the patch itself is trimmed. + +### Verify a pre-aggregation + +After adding or changing a pre-aggregation, call `getPreAggregationStatus` to see whether +its partitions exist and when they were last built. If nothing has been built yet — or you +want to confirm an external pre-aggregation's export bucket actually works — call +`buildPreAggregation`, then poll `getPreAggregationStatus` again. A failed build reports +the exact error, which is usually a configuration problem rather than a modeling one; +`getDeploymentEnv` will tell you whether the expected variables (`CUBEJS_DB_EXPORT_BUCKET` +and friends) are set. + ## Troubleshooting - **MCP configuration is unavailable**: Configure the MCP server URL. @@ -358,6 +431,7 @@ publish it. - **`Deployment is not available via MCP for this account` (403)**: The requested deployment is excluded by the deployment-access allow-list or by the user's permissions. Call `listDeployments` to see which deployments are reachable, or adjust the allow-list in **Admin → MCP Server → Deployment Access**. [ref-roles]: /admin/users-and-permissions/roles-and-permissions +[ref-pre-aggregations]: /docs/pre-aggregations/using-pre-aggregations [ref-sql-api]: /reference/core-data-apis/sql-api [ref-workbooks]: /docs/explore-analyze/workbooks [ref-dashboards]: /docs/explore-analyze/dashboards From 9715bf189db8b1e725db5bc7c3241e5e793be1fe Mon Sep 17 00:00:00 2001 From: Alex Qyoun-ae <4062971+MazterQyou@users.noreply.github.com> Date: Tue, 11 Aug 2026 22:20:31 +0400 Subject: [PATCH 4/6] fix(tesseract): Resolve join for hint-less member expressions on views (#11501) --- .../cubesqlplanner/src/planner/join_hints.rs | 51 ++- .../src/planner/multi_fact_join_groups.rs | 195 ++++++++-- .../test_fixtures/cube_bridge/mock_schema.rs | 29 ++ .../yaml_files/common/integration_views.yaml | 50 +++ .../tests/integration/member_expressions.rs | 350 ++++++++++++++++++ ...r_no_hints_beside_multi_stage_measure.snap | 8 + ...ount_star_no_hints_on_multi_fact_view.snap | 7 + ...r_measure_count_star_no_hints_on_view.snap | 7 + ...tar_only_member_on_view_with_join_map.snap | 7 + .../src/tests/join_hints_collector.rs | 12 +- 10 files changed, 674 insertions(+), 42 deletions(-) create mode 100644 rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__member_expressions__expr_measure_count_star_no_hints_beside_multi_stage_measure.snap create mode 100644 rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__member_expressions__expr_measure_count_star_no_hints_on_multi_fact_view.snap create mode 100644 rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__member_expressions__expr_measure_count_star_no_hints_on_view.snap create mode 100644 rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__member_expressions__expr_measure_count_star_only_member_on_view_with_join_map.snap diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/join_hints.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/join_hints.rs index f7efa2650b8d5..d2b741e017f03 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/join_hints.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/join_hints.rs @@ -1,9 +1,17 @@ use crate::cube_bridge::join_hints::JoinHintItem; -/// Ordered list of cube-join hints. Adjacent redundant entries are -/// silently dropped on `push` / `extend` — a `Single` is skipped when -/// it duplicates either the previous `Single` or the tail of the -/// previous `Vector`. +/// Ordered list of cube-join hints. `push` / `extend` drop an entry that +/// is redundant against the one before it — an item repeating the previous +/// one verbatim, or a `Single` duplicating either the previous `Single` or +/// the tail of the previous `Vector`. +/// +/// That is a local rule, not a normal form. `from_items` stores what it is +/// given as-is, and nothing collapses a `Vector` that is a strict prefix of +/// another (`[V[customers], V[customers, orders]]`) or a repeat that is not +/// adjacent. So two hint lists that resolve to the same join tree can still +/// differ — which matters, since `JoinHints` is a join tree cache key: +/// equal hints hit the same entry, but unequal ones are not proof of +/// different trees. #[derive(Debug, Clone, Eq, PartialEq, Hash)] pub struct JoinHints { items: Vec, @@ -19,13 +27,12 @@ impl JoinHints { } pub fn push(&mut self, item: JoinHintItem) { - if let JoinHintItem::Single(ref name) = item { - if let Some(last) = self.items.last() { - let redundant = match last { - JoinHintItem::Single(s) => s == name, - JoinHintItem::Vector(v) => v.last() == Some(name), - }; - if redundant { + if let Some(last) = self.items.last() { + if last == &item { + return; + } + if let (JoinHintItem::Single(name), JoinHintItem::Vector(v)) = (&item, last) { + if v.last() == Some(name) { return; } } @@ -161,6 +168,28 @@ mod tests { assert_eq!(hints.len(), 3, "Different Single is added"); } + #[test] + fn test_push_skips_repeated_vector() { + let mut hints = JoinHints::new(); + hints.push(v(&["customers", "orders"])); + hints.push(v(&["customers", "orders"])); + assert_eq!( + hints.len(), + 1, + "Vector repeating the previous one is skipped" + ); + + hints.push(v(&["customers", "returns"])); + assert_eq!(hints.len(), 2, "Different Vector is added"); + + hints.push(v(&["customers", "orders"])); + assert_eq!( + hints.len(), + 3, + "Only adjacent repeats are dropped, not every earlier occurrence" + ); + } + #[test] fn test_into_items_and_into_iter() { let hints = JoinHints::from_items(vec![s("b"), s("a"), v(&["x", "y"])]); diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/multi_fact_join_groups.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/multi_fact_join_groups.rs index 425f6eca3743d..47f0568410018 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/multi_fact_join_groups.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/planner/multi_fact_join_groups.rs @@ -57,7 +57,7 @@ impl MeasuresJoinHintsBuilder { base_hints.extend(&collect_join_hints(sym)?); } - MeasuresJoinHints::from_base_hints(base_hints, measures) + MeasuresJoinHints::from_base_hints(base_hints, measures, None) } } @@ -68,10 +68,25 @@ impl MeasuresJoinHintsBuilder { /// - `measure_hints` — per-measure incremental hints, one entry per /// non-multi-stage measure. Multi-stage measures plan their joins /// separately and are skipped here. +/// - `hints_by_cube` — the hints the measures of the whole query collected, +/// grouped by the cube the measure itself belongs to, which for a measure of a +/// view is that view. Multi-stage measures are included. Only used to resolve a +/// measure that carries no hints of its own and sits on a view (see +/// `MultiFactJoinGroups::fallback_hints_for_measure`), which is why the +/// grouping matters: such a measure may only borrow from members of its own +/// view. It is inherited as-is when regrouping over a measure subset, so the +/// measure stays in the join tree of the query it came from. +/// +/// Dimensions, filters and query-level join hints are deliberately absent: they +/// land in `base_hints`, so a measure of a query that has any of them never +/// reaches the fallback in the first place. That also means the view grouping +/// only guards the case where `base_hints` is empty - a dimension of an +/// unrelated view still pulls a hint-less member expression into its join. #[derive(Clone, Debug)] pub struct MeasuresJoinHints { base_hints: JoinHints, measure_hints: Vec, + hints_by_cube: HashMap, } impl MeasuresJoinHints { @@ -85,37 +100,59 @@ impl MeasuresJoinHints { } /// Reuse the existing `base_hints` to produce a new - /// `MeasuresJoinHints` over a different measure subset. + /// `MeasuresJoinHints` over a different measure subset. `hints_by_cube` + /// keeps describing the whole query, not the subset. pub fn for_measures(&self, measures: &[Rc]) -> Result { - Self::from_base_hints(self.base_hints.clone(), measures) + Self::from_base_hints( + self.base_hints.clone(), + measures, + Some(self.hints_by_cube.clone()), + ) } + /// `inherited_hints_by_cube` describes the whole query these measures were + /// taken from, so the measures add nothing to it; without it they are grouped + /// from scratch. fn from_base_hints( base_hints: JoinHints, measures: &[Rc], + inherited_hints_by_cube: Option>, ) -> Result { - let mut filtered_measures = Vec::new(); + let inherited = inherited_hints_by_cube.is_some(); + let mut hints_by_cube = inherited_hints_by_cube.unwrap_or_default(); + + let mut measure_hints: Vec = Vec::new(); for m in measures { - if !has_multi_stage_members(m, true)? { - filtered_measures.push(m.clone()); + // Multi-stage measures plan their joins separately, so they get no + // entry of their own - but their hints still count towards their + // cube's. With inherited hints there is nothing left to collect + // them for. + let is_multi_stage = has_multi_stage_members(m, true)?; + if is_multi_stage && inherited { + continue; + } + let own_hints = collect_join_hints(m)?; + if !inherited { + hints_by_cube + .entry(m.cube_name()) + .or_insert_with(JoinHints::new) + .extend(&own_hints); } + if is_multi_stage { + continue; + } + let mut hints = base_hints.clone(); + hints.extend(&own_hints); + measure_hints.push(MeasureJoinHints { + measure: m.clone(), + hints, + }); } - let measure_hints: Vec = filtered_measures - .iter() - .map(|m| -> Result<_, CubeError> { - let mut hints = base_hints.clone(); - hints.extend(&collect_join_hints(m)?); - Ok(MeasureJoinHints { - measure: m.clone(), - hints, - }) - }) - .collect::, _>>()?; - Ok(Self { base_hints, measure_hints, + hints_by_cube, }) } @@ -209,10 +246,20 @@ impl MultiFactJoinGroups { .iter() .map(|mh| -> Result<_, CubeError> { let measure_hints = if mh.hints.is_empty() { - Self::fallback_hints_for_measure(query_tools, &mh.measure)? + Self::fallback_hints_for_measure(query_tools, &mh.measure, hints)? } else { mh.hints.clone() }; + if measure_hints.is_empty() { + return Err(CubeError::user(format!( + "Can't resolve the cube to query for '{}': the member references no \ + members of '{}', and neither the rest of the query nor the join map \ + of '{}' gives a cube to join from", + mh.measure.full_name(), + mh.measure.cube_name(), + mh.measure.cube_name() + ))); + } let (key, join_tree) = resolve(&measure_hints)?; Ok((vec![mh.measure.clone()], key, join_tree)) }) @@ -237,23 +284,117 @@ impl MultiFactJoinGroups { } /// Hints to use for a measure whose own hint set resolved to empty. - /// Seeds the measure's owning cube when it is a real, joinable cube; - /// returns empty for views (resolved via the query's other members). + /// Seeds the measure's owning cube when it is a real, joinable cube. + /// + /// A view is not a joinable cube, so it can't seed anything. Such a measure + /// borrows the hints of the other members **of that same view** instead, and + /// lands in the same join group as the members it borrowed from. Members of + /// another view or of a bare cube are not borrowed from: their cubes need not + /// appear in this view at all, and counting rows of a join tree the view is + /// not built on would answer a different question than the one asked. + /// + /// Borrowing at all is what the legacy planner does, but it borrows wider: it + /// unions the join hints of every query member into one join tree, with no + /// notion of which view a member came from. Narrowing that union to the + /// measure's own view is the difference here. + /// + /// When there is nothing to borrow from either, the view's own join map is + /// the last resort: its paths start at the cube the view is rooted at, so + /// that cube is the one to query. This is what makes a query built only from + /// such member expressions, like `COUNT(*)` over a view, resolvable. It only + /// covers views that have a join map at all: a view over a single directly + /// joinable cube records no path, and such a query is rejected - see + /// `test_expr_measure_count_star_only_member_on_view`. + /// + /// Note that borrowing makes the meaning of such a measure depend on the rest + /// of the query: `COUNT(*)` over a view with two facts counts the rows of the + /// cube the view is rooted at when selected alone, and the rows of the fanned + /// out join tree when selected together with measures from both facts. The + /// legacy planner behaves the same way, since it pools the hints of all query + /// members into one join tree. + /// + /// Known hole, kept for legacy parity: the same-view rule only reaches + /// measures. Dimensions, filters and query-level hints land in `base_hints`, + /// which is not view-scoped, and a measure whose `base_hints` are non-empty + /// never gets here at all. So a dimension of an *unrelated* view still drags a + /// hint-less member expression into that view's join and yields a number for a + /// join tree its own view is not built on - see + /// `test_expr_measure_count_star_no_hints_beside_other_view_dimension`, which + /// pins that behaviour. Closing it means resolving from the view bucket + /// whenever the measure's *own* hints are empty, which would also make the + /// ordinary shape - a view dimension next to `COUNT(*)` on the same view - + /// depend on that bucket carrying dimensions, so it is a larger change than + /// this fix. fn fallback_hints_for_measure( query_tools: &Rc, measure: &Rc, + all_hints: &MeasuresJoinHints, ) -> Result { let cube_name = measure.cube_name(); - let is_view = query_tools + let cube_definition = query_tools .cube_evaluator() .cube_from_path(cube_name.clone()) - .ok() + .ok(); + let is_view = cube_definition + .as_ref() .and_then(|cube| cube.static_data().is_view) .unwrap_or(false); - if is_view { - Ok(JoinHints::new()) - } else { - Ok(JoinHints::from_items(vec![JoinHintItem::Single(cube_name)])) + if !is_view { + return Ok(JoinHints::from_items(vec![JoinHintItem::Single(cube_name)])); + } + + match all_hints.hints_by_cube.get(&cube_name) { + Some(hints) if !hints.is_empty() => return Ok(hints.clone()), + _ => {} + } + + let join_map = cube_definition + .and_then(|cube| cube.static_data().join_map.clone()) + .unwrap_or_default(); + if join_map.is_empty() { + return Ok(JoinHints::new()); + } + // A cube that heads one path but is reached from another one is not a + // root of the view - the path it heads is just the tail of a longer walk. + // Only the heads that nothing else reaches are candidates. + let reached = join_map + .iter() + .flat_map(|path| path.iter().skip(1)) + .collect::>(); + let roots = join_map + .iter() + .filter_map(|path| path.first()) + .filter(|head| !reached.contains(*head)) + .unique() + .collect_vec(); + + let no_single_root = |detail: String| { + CubeError::user(format!( + "Can't resolve the cube to query for '{}': the member references no members of \ + '{}', and {detail}", + measure.full_name(), + cube_name, + )) + }; + + match roots.as_slice() { + [root_cube] => Ok(JoinHints::from_items(vec![JoinHintItem::Single( + (*root_cube).clone(), + )])), + // Every path of the join map is headed by a cube some other path + // reaches, so the paths lead in a circle and none of them starts at + // the view's root. + [] => Err(no_single_root(format!( + "the join paths of that view are cyclic: {}", + join_map.iter().map(|path| path.join(".")).join(", ") + ))), + // The join map is ordered by the order the view lists its cubes, so + // picking one root out of several would make the answer depend on + // that order with nothing to hint at it. + _ => Err(no_single_root(format!( + "that view is built on cubes that don't share a single root: {}", + roots.iter().join(", ") + ))), } } diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/mock_schema.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/mock_schema.rs index 4b31bf0d2988b..706a15664c0c2 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/mock_schema.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/cube_bridge/mock_schema.rs @@ -645,9 +645,38 @@ impl MockViewBuilder { } } + // Like the schema compiler, only multi-hop join paths land in the join + // map: a direct cube needs no path to be reached. Note this is what makes + // a root cube member of a view carry `Vector([cube])` rather than + // `Single(cube)` - `collect_join_hints` enriches a hint into the prefix of + // the path it sits on - and both forms are distinct join tree cache keys. + // + // The schema compiler fills the map in `CubeSymbols.prepareIncludes`, + // inside the pass over `dimensions`, but it pushes the entry for an + // included cube before looking at that cube's includes - so a cube + // contributing no dimension still gets one. `customer_overview` includes + // only measures from `customers.orders` and is mapped all the same. + // Emitting one entry per view cube here matches that. What the compiler + // does differently is evaluate the join path as a reference instead of + // splitting the raw string, so a fixture would only diverge with a join + // path that is not a literal. + let join_map = self + .view_cubes + .iter() + .map(|view_cube| { + view_cube + .join_path + .split('.') + .map(|part| part.to_string()) + .collect::>() + }) + .filter(|path| path.len() > 1) + .collect::>(); + let view_def = MockCubeDefinition::builder() .name(self.view_name.clone()) .is_view(Some(true)) + .join_map(Some(join_map)) .default_filters(self.default_filters) .build(); diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_views.yaml b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_views.yaml index 49ab7e13b7bbe..10c468544028c 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_views.yaml +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/test_fixtures/schemas/yaml_files/common/integration_views.yaml @@ -52,6 +52,18 @@ cubes: - name: total_amount type: sum sql: amount + - name: total_amount_by_status + type: number + sql: "{CUBE.total_amount}" + multi_stage: true + add_group_by: + - orders.status + # Not multi-stage itself, but depends on a multi-stage measure, so it + # is skipped when building per-measure hints while still counting + # towards the hints of the view it is included in. + - name: amount_share + type: number + sql: "{CUBE.total_amount} / NULLIF({CUBE.total_amount_by_status}, 0)" segments: - name: completed_orders sql: "{CUBE}.status = 'completed'" @@ -89,6 +101,7 @@ views: - status - count - total_amount + - amount_share - name: orders_with_customers cubes: @@ -105,6 +118,43 @@ views: - city - ny_customers + # Join paths whose heads differ, but where one head is reached from the + # other, so `returns` is the single root. Real view YAML anchors every path + # at the view root and cannot produce this, but the root rule should not + # depend on that. + # The reached cube heads the *first* path on purpose, so that taking the first + # head instead of the unreached one picks the wrong cube. + - name: nested_root_view + cubes: + - join_path: customers.orders + includes: + - status + - join_path: returns.customers + includes: + - city + + # Join paths that lead in a circle, so every head is reached from another + # path and none of them is the view's root. + - name: cyclic_paths_view + cubes: + - join_path: orders.customers + includes: + - city + - join_path: customers.orders + includes: + - status + + # Join paths under two different roots, so the join map alone does not say + # which cube a member expression with no hints of its own should query. + - name: two_roots_view + cubes: + - join_path: orders.customers + includes: + - city + - join_path: returns.customers + includes: + - name + - name: customer_overview cubes: - join_path: customers diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/member_expressions.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/member_expressions.rs index 8c1d2db39683b..b58f7465efe00 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/member_expressions.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/member_expressions.rs @@ -40,6 +40,12 @@ fn make_measure_expression(name: &str, cube: &str, sql: &str) -> OptionsMember { OptionsMember::MemberExpression(Rc::new(expr)) } +// Collapses whitespace so an assertion on the shape of a query does not pin the +// renderer's spacing. +fn normalize_sql(sql: &str) -> String { + sql.split_whitespace().collect::>().join(" ") +} + // Mirrors a SQL-API `subqueryJoins` entry: opaque sub-query `sql`, a join type // and alias, and an `on` condition expressed as a member expression (the alias // arrives pre-quoted and is referenced verbatim inside `on`). @@ -203,6 +209,350 @@ async fn test_expr_measure_count_star_no_hints() { } } +// Same hint-less `COUNT(*)` case, but the member expression belongs to a view. +// The view is not a joinable cube, so the fallback cannot seed it as a hint; the +// join must be taken from the other measures of the query (here the +// `COUNT(DISTINCT {orders_view.status})` expression, which pulls in `orders`). +#[tokio::test(flavor = "multi_thread")] +async fn test_expr_measure_count_star_no_hints_on_view() { + let schema = MockSchema::from_yaml_file("common/integration_views.yaml"); + let ctx = TestContext::new(schema).unwrap(); + + let distinct_status = make_measure_expression( + "distinct_status", + "orders_view", + "COUNT(DISTINCT {orders_view.status})", + ); + let total_count = make_measure_expression("total_count", "orders_view", "COUNT(*)"); + + let options = Rc::new( + MockBaseQueryOptions::builder() + .cube_evaluator(ctx.query_tools().cube_evaluator().clone()) + .base_tools(ctx.query_tools().base_tools().clone()) + .join_graph(ctx.query_tools().join_graph().clone()) + .security_context(ctx.security_context().clone()) + .measures(Some(vec![distinct_status, total_count])) + .build(), + ); + + ctx.build_sql_from_options(options.clone()).unwrap(); + + if let Some(result) = ctx + .try_execute_pg_from_options(options, "integration_multi_fact_tables.sql") + .await + { + insta::assert_snapshot!(result); + } +} + +// The only other measure of the query is `amount_share`, which depends on a +// multi-stage measure. Such a measure plans its joins separately and so gets no +// per-measure hints entry, but its hints still count towards its view's - which +// is the only thing that gives the hint-less `COUNT(*)` a cube to query here, +// since `orders_view` has no join map to fall back to. +#[tokio::test(flavor = "multi_thread")] +async fn test_expr_measure_count_star_no_hints_beside_multi_stage_measure() { + let schema = MockSchema::from_yaml_file("common/integration_views.yaml"); + let ctx = TestContext::new(schema).unwrap(); + + let total_count = make_measure_expression("total_count", "orders_view", "COUNT(*)"); + let mut measures = members_from_strings(vec!["orders_view.amount_share"]); + measures.push(total_count); + + let options = Rc::new( + MockBaseQueryOptions::builder() + .cube_evaluator(ctx.query_tools().cube_evaluator().clone()) + .base_tools(ctx.query_tools().base_tools().clone()) + .join_graph(ctx.query_tools().join_graph().clone()) + .security_context(ctx.security_context().clone()) + .measures(Some(measures)) + .build(), + ); + + ctx.build_sql_from_options(options.clone()).unwrap(); + + if let Some(result) = ctx + .try_execute_pg_from_options(options, "integration_multi_fact_tables.sql") + .await + { + insta::assert_snapshot!(result); + } +} + +// Hint-less `COUNT(*)` on a view in a genuinely multi-fact query: `orders_count` +// and `returns_count` sit on two different facts that fan out from `customers`. +// Both are members of this view, so its hints are the union over both facts, and +// the member expression forms a third group over the fan-out tree, counting the +// joined row set of the view - +// which is what `COUNT(*)` over a view means. The legacy planner renders the +// whole query over that same fan-out tree, so it counts the same rows. +#[tokio::test(flavor = "multi_thread")] +async fn test_expr_measure_count_star_no_hints_on_multi_fact_view() { + let schema = MockSchema::from_yaml_file("common/integration_views.yaml"); + let ctx = TestContext::new(schema).unwrap(); + + let total_count = make_measure_expression("total_count", "customer_overview", "COUNT(*)"); + let mut measures = members_from_strings(vec![ + "customer_overview.orders_count", + "customer_overview.returns_count", + ]); + measures.push(total_count); + + let options = Rc::new( + MockBaseQueryOptions::builder() + .cube_evaluator(ctx.query_tools().cube_evaluator().clone()) + .base_tools(ctx.query_tools().base_tools().clone()) + .join_graph(ctx.query_tools().join_graph().clone()) + .security_context(ctx.security_context().clone()) + .measures(Some(measures)) + .build(), + ); + + ctx.build_sql_from_options(options.clone()).unwrap(); + + if let Some(result) = ctx + .try_execute_pg_from_options(options, "integration_multi_fact_tables.sql") + .await + { + insta::assert_snapshot!(result); + } +} + +// A hint-less `COUNT(*)` member expression on a view as the *only* query member, +// where the view has a join map: every path in it starts at `customers`, so that +// is the cube to query. BI tools send such member-less profiling queries against +// a view, so they must resolve. +#[tokio::test(flavor = "multi_thread")] +async fn test_expr_measure_count_star_only_member_on_view_with_join_map() { + let schema = MockSchema::from_yaml_file("common/integration_views.yaml"); + let ctx = TestContext::new(schema).unwrap(); + + let total_count = make_measure_expression("total_count", "customer_overview", "COUNT(*)"); + + let options = Rc::new( + MockBaseQueryOptions::builder() + .cube_evaluator(ctx.query_tools().cube_evaluator().clone()) + .base_tools(ctx.query_tools().base_tools().clone()) + .join_graph(ctx.query_tools().join_graph().clone()) + .security_context(ctx.security_context().clone()) + .measures(Some(vec![total_count])) + .build(), + ); + + ctx.build_sql_from_options(options.clone()).unwrap(); + + if let Some(result) = ctx + .try_execute_pg_from_options(options, "integration_multi_fact_tables.sql") + .await + { + insta::assert_snapshot!(result); + } +} + +// The join map of `nested_root_view` holds paths headed by different cubes, but +// `customers` heads one path only because it is reached from `returns` in +// another - so `returns` is the single root and the query resolves against it. +#[test] +fn test_expr_measure_count_star_only_member_on_view_with_nested_join_map() { + let schema = MockSchema::from_yaml_file("common/integration_views.yaml"); + let ctx = TestContext::new(schema).unwrap(); + + let total_count = make_measure_expression("total_count", "nested_root_view", "COUNT(*)"); + + let options = Rc::new( + MockBaseQueryOptions::builder() + .cube_evaluator(ctx.query_tools().cube_evaluator().clone()) + .base_tools(ctx.query_tools().base_tools().clone()) + .join_graph(ctx.query_tools().join_graph().clone()) + .security_context(ctx.security_context().clone()) + .measures(Some(vec![total_count])) + .build(), + ); + + let sql = normalize_sql(&ctx.build_sql_from_options(options).unwrap()); + // Anchored on the join structure rather than on a cube name being absent from + // the text: the point is that the tree is `returns` by itself. A rule that + // took the first head instead of the unreached one would root at `customers` + // and join `orders` onto it. + assert!( + sql.contains("FROM returns AS") && !sql.contains("JOIN"), + "expected the query to resolve against the root cube `returns` alone, got: {sql}" + ); +} + +// The same shape as the test below, but with a *dimension* of the other view +// instead of a measure - and it is not rejected. Dimensions land in `base_hints`, +// which is not view-scoped, so the member expression never reaches the same-view +// fallback and counts rows of `customers`, a cube `orders_view` is not built on. +// +// This pins a known hole rather than desired behaviour: the legacy planner does +// the same, and closing it is a wider change than this fix (see the note on +// `MultiFactJoinGroups::fallback_hints_for_measure`). If it is ever closed, this +// test flips to expecting the same rejection as the one below. +#[test] +fn test_expr_measure_count_star_no_hints_beside_other_view_dimension() { + let schema = MockSchema::from_yaml_file("common/integration_views.yaml"); + let ctx = TestContext::new(schema).unwrap(); + + let total_count = make_measure_expression("total_count", "orders_view", "COUNT(*)"); + + let options = Rc::new( + MockBaseQueryOptions::builder() + .cube_evaluator(ctx.query_tools().cube_evaluator().clone()) + .base_tools(ctx.query_tools().base_tools().clone()) + .join_graph(ctx.query_tools().join_graph().clone()) + .security_context(ctx.security_context().clone()) + .measures(Some(vec![total_count])) + .dimensions(Some(members_from_strings(vec!["customer_overview.city"]))) + .build(), + ); + + let sql = normalize_sql(&ctx.build_sql_from_options(options).unwrap()); + assert!( + sql.contains("FROM customers AS") && !sql.contains("JOIN"), + "expected the hole to stand: the count resolves against `customers` alone, got: {sql}" + ); +} + +// A hint-less `COUNT(*)` on `orders_view` next to a measure of a *different* +// view. `customer_overview.returns_count` pulls in `customers` and `returns`, +// neither of which `orders_view` is built on, so those hints must not be +// borrowed - counting rows of `customers` joined to `returns` would answer a +// question nobody asked. Nothing is left to resolve from, so the query is +// rejected. +#[test] +fn test_expr_measure_count_star_no_hints_beside_other_view_measure() { + let schema = MockSchema::from_yaml_file("common/integration_views.yaml"); + let ctx = TestContext::new(schema).unwrap(); + + let total_count = make_measure_expression("total_count", "orders_view", "COUNT(*)"); + let mut measures = members_from_strings(vec!["customer_overview.returns_count"]); + measures.push(total_count); + + let options = Rc::new( + MockBaseQueryOptions::builder() + .cube_evaluator(ctx.query_tools().cube_evaluator().clone()) + .base_tools(ctx.query_tools().base_tools().clone()) + .join_graph(ctx.query_tools().join_graph().clone()) + .security_context(ctx.security_context().clone()) + .measures(Some(measures)) + .build(), + ); + + let err = ctx + .build_sql_from_options(options) + .expect_err("a view member expression must not borrow the join of an unrelated view"); + assert!( + err.message.contains("Can't resolve the cube to query"), + "expected a clear unresolvable-cube error, got: {}", + err.message + ); +} + +// The join map of `two_roots_view` holds paths under two different roots, so +// there is no one cube the view is rooted at. Which one gets counted would come +// down to the order the view lists its cubes, so the query is rejected instead. +#[test] +fn test_expr_measure_count_star_only_member_on_view_with_ambiguous_join_map() { + let schema = MockSchema::from_yaml_file("common/integration_views.yaml"); + let ctx = TestContext::new(schema).unwrap(); + + let total_count = make_measure_expression("total_count", "two_roots_view", "COUNT(*)"); + + let options = Rc::new( + MockBaseQueryOptions::builder() + .cube_evaluator(ctx.query_tools().cube_evaluator().clone()) + .base_tools(ctx.query_tools().base_tools().clone()) + .join_graph(ctx.query_tools().join_graph().clone()) + .security_context(ctx.security_context().clone()) + .measures(Some(vec![total_count])) + .build(), + ); + + let err = ctx + .build_sql_from_options(options) + .expect_err("a view whose join map has several roots should be rejected"); + assert!( + err.message.contains("don't share a single root") + && err.message.contains("orders") + && err.message.contains("returns"), + "expected an ambiguous-root error naming both roots, got: {}", + err.message + ); +} + +// Every path of `cyclic_paths_view` is headed by a cube another path reaches, so +// the paths lead in a circle. That is a different fault from several roots and +// says so, rather than degrading into the generic nothing-to-join-from error. +#[test] +fn test_expr_measure_count_star_only_member_on_view_with_cyclic_join_map() { + let schema = MockSchema::from_yaml_file("common/integration_views.yaml"); + let ctx = TestContext::new(schema).unwrap(); + + let total_count = make_measure_expression("total_count", "cyclic_paths_view", "COUNT(*)"); + + let options = Rc::new( + MockBaseQueryOptions::builder() + .cube_evaluator(ctx.query_tools().cube_evaluator().clone()) + .base_tools(ctx.query_tools().base_tools().clone()) + .join_graph(ctx.query_tools().join_graph().clone()) + .security_context(ctx.security_context().clone()) + .measures(Some(vec![total_count])) + .build(), + ); + + let err = ctx + .build_sql_from_options(options) + .expect_err("a view whose join paths are cyclic should be rejected"); + assert!( + err.message.contains("join paths of that view are cyclic") + && err.message.contains("orders.customers"), + "expected a cyclic-join-map error listing the paths, got: {}", + err.message + ); +} + +// A hint-less `COUNT(*)` member expression on a view as the *only* query member: +// the view is not a joinable cube, nothing else seeds the join, and the view has +// no join map to fall back to, because a view over a single directly joinable cube +// records no path. So there is no cube to query. +// +// This is a known limitation, not the desired end state: a one-cube view is the +// most common shape, and the cube is knowable - the view does record `orders` as +// an included cube, it is just dropped from the join map as "no path needed". +// Lifting it means either exposing the view's included cubes on the cube bridge or +// keeping single-element paths in the join map, and the latter turns every root +// cube hint from `Single` into `Vector` across both planners - too wide to carry +// here. The legacy planner fails on this query too, so nothing regresses; what +// this test locks in is a clear error instead of a bridge deserialization failure +// on the null join tree. +#[test] +fn test_expr_measure_count_star_only_member_on_view() { + let schema = MockSchema::from_yaml_file("common/integration_views.yaml"); + let ctx = TestContext::new(schema).unwrap(); + + let total_count = make_measure_expression("total_count", "orders_view", "COUNT(*)"); + + let options = Rc::new( + MockBaseQueryOptions::builder() + .cube_evaluator(ctx.query_tools().cube_evaluator().clone()) + .base_tools(ctx.query_tools().base_tools().clone()) + .join_graph(ctx.query_tools().join_graph().clone()) + .security_context(ctx.security_context().clone()) + .measures(Some(vec![total_count])) + .build(), + ); + + let err = ctx + .build_sql_from_options(options) + .expect_err("a view member expression with nothing to join from should be rejected"); + assert!( + err.message.contains("Can't resolve the cube to query"), + "expected a clear unresolvable-cube error, got: {}", + err.message + ); +} + // Multiplied dim-only ME: a measure expression evaluating to a // dimension expression (MAX over `customers.city`) used together // with an `orders` dimension. `orders→customers` is many_to_one, so diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__member_expressions__expr_measure_count_star_no_hints_beside_multi_stage_measure.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__member_expressions__expr_measure_count_star_no_hints_beside_multi_stage_measure.snap new file mode 100644 index 0000000000000..d6b0511a1fa7f --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__member_expressions__expr_measure_count_star_no_hints_beside_multi_stage_measure.snap @@ -0,0 +1,8 @@ +--- +source: cubesqlplanner/src/tests/integration/member_expressions.rs +expression: result +--- +orders_view__amount_share | total_count +--------------------------+------------ +1.1956521739130435 | 8 +6.1111111111111111 | 8 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__member_expressions__expr_measure_count_star_no_hints_on_multi_fact_view.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__member_expressions__expr_measure_count_star_no_hints_on_multi_fact_view.snap new file mode 100644 index 0000000000000..255027543d188 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__member_expressions__expr_measure_count_star_no_hints_on_multi_fact_view.snap @@ -0,0 +1,7 @@ +--- +source: cubesqlplanner/src/tests/integration/member_expressions.rs +expression: result +--- +customer_overview__orders_count | customer_overview__returns_count | total_count +--------------------------------+----------------------------------+------------ +8 | 5 | 13 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__member_expressions__expr_measure_count_star_no_hints_on_view.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__member_expressions__expr_measure_count_star_no_hints_on_view.snap new file mode 100644 index 0000000000000..651ad6d427342 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__member_expressions__expr_measure_count_star_no_hints_on_view.snap @@ -0,0 +1,7 @@ +--- +source: cubesqlplanner/src/tests/integration/member_expressions.rs +expression: result +--- +distinct_status | total_count +----------------+------------ +2 | 8 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__member_expressions__expr_measure_count_star_only_member_on_view_with_join_map.snap b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__member_expressions__expr_measure_count_star_only_member_on_view_with_join_map.snap new file mode 100644 index 0000000000000..02ea023f61d07 --- /dev/null +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/integration/snapshots/cubesqlplanner__tests__integration__member_expressions__expr_measure_count_star_only_member_on_view_with_join_map.snap @@ -0,0 +1,7 @@ +--- +source: cubesqlplanner/src/tests/integration/member_expressions.rs +expression: result +--- +total_count +----------- +4 diff --git a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/join_hints_collector.rs b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/join_hints_collector.rs index f210e2634ed40..22ea79ef1eb96 100644 --- a/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/join_hints_collector.rs +++ b/rust/cube/cubesqlplanner/cubesqlplanner/src/tests/join_hints_collector.rs @@ -52,16 +52,18 @@ fn test_collect_join_hints_view_symbols() { assert_eq!(hints.len(), 1); assert_eq!(hints.items(), &[v(&["cube_a", "cube_b", "cube_c"])]); + // Members of the root cube of the view are enriched with the view join map + // into the prefix of the path they sit on, so they come back as a vector. let dim = ctx.create_dimension("a_with_b_and_c.name").unwrap(); let hints = collect_join_hints(&dim).unwrap(); assert_eq!(hints.len(), 1); - assert_eq!(hints.items(), &[s("cube_a")]); + assert_eq!(hints.items(), &[v(&["cube_a"])]); // View measure from root join_path let measure = ctx.create_measure("a_with_b_and_c.total_value").unwrap(); let hints = collect_join_hints(&measure).unwrap(); assert_eq!(hints.len(), 1); - assert_eq!(hints.items(), &[s("cube_a")]); + assert_eq!(hints.items(), &[v(&["cube_a"])]); } #[test] @@ -95,7 +97,8 @@ fn test_join_hints_many_to_one_view_root_dim() { let dim = ctx.create_dimension("many_to_one_view.root_dim").unwrap(); let hints = collect_join_hints(&dim).unwrap(); assert_eq!(hints.len(), 1); - assert_eq!(hints.items(), &[s("many_to_one_root")]); + // Enriched with the view join map into the prefix of the path it sits on. + assert_eq!(hints.items(), &[v(&["many_to_one_root"])]); } #[test] @@ -116,7 +119,8 @@ fn test_join_hints_many_to_one_view_root_measure() { let measure = ctx.create_measure("many_to_one_view.root_val_avg").unwrap(); let hints = collect_join_hints(&measure).unwrap(); assert_eq!(hints.len(), 1); - assert_eq!(hints.items(), &[s("many_to_one_root")]); + // Enriched with the view join map into the prefix of the path it sits on. + assert_eq!(hints.items(), &[v(&["many_to_one_root"])]); } #[test] From 2f1674022549f842fd86a4af1e6e36f6805e30c3 Mon Sep 17 00:00:00 2001 From: Gleb Sologub Date: Tue, 11 Aug 2026 20:31:10 +0200 Subject: [PATCH 5/6] docs: "dashboards as code" guide for programmatic dashboards (CUB-3521) (#11493) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: add "dashboards as code" guide for programmatic dashboards (CUB-3521) * docs: rewrite "dashboards as code" guide around the merged REST upsert API (CUB-3521) The first draft documented a runtime-owned YAML-in-project model that never shipped. Rewrite it to reflect what actually merged in cubejs-enterprise#13361: the idempotent public REST upserts PUT /deployments/{id}/workbooks/by-slug/{slug} PUT /deployments/{id}/reports/by-public-id/{publicId} keyed by portable slug / publicId. Covers the author→export→upsert→publish apply flow, idempotency and the three 409 conflict codes, write-once publicId adoption (incl. legacy synthesized ids), and cross-links to the generated endpoint reference pages. Fix the dashboards index cross-link accordingly. --- docs-mintlify/docs.json | 1 + .../dashboards/dashboards-as-code.mdx | 200 ++++++++++++++++++ .../docs/explore-analyze/dashboards/index.mdx | 4 + 3 files changed, 205 insertions(+) create mode 100644 docs-mintlify/docs/explore-analyze/dashboards/dashboards-as-code.mdx diff --git a/docs-mintlify/docs.json b/docs-mintlify/docs.json index 55b1751daa766..d70f3248bd9c8 100644 --- a/docs-mintlify/docs.json +++ b/docs-mintlify/docs.json @@ -118,6 +118,7 @@ ] }, "docs/explore-analyze/dashboards/styling", + "docs/explore-analyze/dashboards/dashboards-as-code", "docs/explore-analyze/dashboards/dashboard-agent" ] }, diff --git a/docs-mintlify/docs/explore-analyze/dashboards/dashboards-as-code.mdx b/docs-mintlify/docs/explore-analyze/dashboards/dashboards-as-code.mdx new file mode 100644 index 0000000000000..6684ea98ebfdb --- /dev/null +++ b/docs-mintlify/docs/explore-analyze/dashboards/dashboards-as-code.mdx @@ -0,0 +1,200 @@ +--- +title: Dashboards as code +description: Manage workbooks, dashboards, and reports as code with idempotent REST endpoints keyed by portable identifiers, so a CI/CD pipeline can apply the same definitions across deployments. +--- + +**Dashboards as code** lets you manage the reporting assets in a deployment — +[workbooks][ref-workbooks], their [dashboards][ref-dashboards], and the +[reports][ref-reports] the dashboard widgets render — from source control instead +of only through the UI. You keep each asset's definition in Git and apply it to a +deployment with the [Cube Cloud REST API][ref-api], the same way you might manage +Superset assets with `preset-cli` or infrastructure with Terraform. + +Two **idempotent upsert** endpoints make this possible. Instead of tracking the +per-deployment numeric id that a `POST` returns, you address each asset by a +**portable identifier you choose** and re-apply its definition as often as you +like: + +| Endpoint | Keyed by | Upserts | +| --- | --- | --- | +| [`PUT /deployments/{deploymentId}/workbooks/by-slug/{slug}`][ref-upsert-workbook] | a deployment-scoped **slug** | a workbook (and its dashboard draft) | +| [`PUT /deployments/{deploymentId}/reports/by-public-id/{publicId}`][ref-upsert-report] | an account-unique **`publicId`** | a report | + +Because the identifier is stable and lives in your repository, applying the same +definition twice is a no-op, and applying it to a second deployment (staging → +production) reproduces the same assets there. + + + +This page covers the REST primitives available today. They are the building +blocks for an as-code workflow you assemble in your own pipeline — Cube does not +yet ship a single bundle export/apply command that wraps them. + + + +## How the pieces fit + +Three assets are involved, each with its own identity: + +- A **report** is a saved query plus its visualization. Its portable identity is + a **`publicId`**: a 12-character alphanumeric (`[0-9A-Za-z]`) id that is unique + across your account. You mint it when you author the report and keep it fixed + for the report's lifetime. +- A **workbook** is the container that holds a dashboard. Its portable identity + is a **slug**: a human-readable, deployment-scoped id (the same slug a data + model targets with `links: [{ dashboard: }]` for drill-in). +- A **dashboard** is the layout — which widgets sit where. It is stored on its + workbook as `meta.dashboardDraft` and is made visible by **publishing** the + workbook. Each chart widget references a report. + +The identifiers you control (`publicId`, `slug`) are what make a definition +portable. The numeric ids that `POST` responses return are per-deployment and are +resolved at apply time — you never store them in Git. + +## Authenticating + +These are public REST endpoints. Authenticate with a deployment API key exactly +as for the rest of the [REST API][ref-api] — see [Authentication][ref-auth] for +how to create a key and pass it. The examples below assume: + +```bash +export CUBE_API_URL="https://" +export CUBE_API_TOKEN="" +export DEPLOYMENT_ID="" +``` + +## The apply flow + +An as-code pipeline applies a dashboard bottom-up: reports first, then the +workbook that lays them out, then publish. + +### 1. Author once, then export + +The report and dashboard-draft definitions are large and are not meant to be +hand-written. Build the reports and dashboard once in the UI, then read them back +over the API and commit the results: + +- [`GET /deployments/{deploymentId}/reports/{reportId}`][ref-get-report] returns a + report's definition. +- [`GET /deployments/{deploymentId}/workbooks/{workbookId}`][ref-get-workbook] + returns the workbook, including its `dashboardDraft`. + +Assign each report a `publicId` and the workbook a `slug` of your choosing, store +those alongside the exported definitions in your repository, and treat that as the +source of truth. + +### 2. Upsert each report + +For every report, [upsert it by `publicId`][ref-upsert-report]. If a report with +that `publicId` already exists in the deployment it is updated with the fields you +send (same semantics as [`PUT /reports/{reportId}`][ref-update-report]); +otherwise it is created with that `publicId`. + +```bash +curl -X PUT \ + "$CUBE_API_URL/api/v1/deployments/$DEPLOYMENT_ID/reports/by-public-id/revqZ1x8Kp0a" \ + -H "Authorization: $CUBE_API_TOKEN" \ + -H "Content-Type: application/json" \ + -d @report-revenue-by-month.json +``` + +The path `publicId` is the report's identity; the request body is the report +definition you exported (its query in `sqlQuery` / `jsonQuery`, pivot in +`pivotItems`, and visualization config in `meta`). Keep track of the numeric +`id` each response returns — the dashboard draft references reports by that +per-deployment id. + +### 3. Upsert the workbook and its dashboard + +[Upsert the workbook by `slug`][ref-upsert-workbook], carrying the dashboard +layout in `meta.dashboardDraft`. Only the fields you send are changed, and `meta` +is **merged** into the existing metadata rather than replacing it. The +`dashboardDraft` is validated the same way the builder validates it. + +```bash +curl -X PUT \ + "$CUBE_API_URL/api/v1/deployments/$DEPLOYMENT_ID/workbooks/by-slug/revenue-overview" \ + -H "Authorization: $CUBE_API_TOKEN" \ + -H "Content-Type: application/json" \ + -d '{ + "name": "Revenue Overview", + "meta": { "dashboardDraft": { "...": "the exported dashboard config" } } + }' +``` + +Because each chart widget inside `dashboardDraft` points at a report by its +per-deployment numeric id, rewrite those references to the ids returned in +step 2 before applying the workbook to a **new** deployment. Re-applying to +the same deployment needs no rewriting — the ids are stable there. + +### 4. Publish + +Upserting the workbook writes the dashboard **draft**. Publish it to make it +visible to viewers with [`POST /workbooks/{workbookId}/publish`][ref-publish], +using the workbook id returned in step 3. Publishing is itself idempotent per +workbook, so it is safe to run on every apply. + +## Idempotency and conflicts + +Re-applying an unchanged definition is a no-op — that is the property that makes +these endpoints safe to run on every pipeline execution. When something does go +wrong, both upserts fail with a `409` rather than guessing, and the report upsert +distinguishes three cases by a `code` field in the response body so your pipeline +can react correctly: + +| Endpoint | `code` | Meaning | What to do | +| --- | --- | --- | --- | +| workbook & report | `upsert_branch_changed` | A concurrent writer created or deleted the asset between the access check and the write, so the request would have applied under the wrong permission check. | **Retry.** Transient; happens only under concurrent applies of the same key. | +| report | _(none)_ | The `publicId` already belongs to a report in a **different** deployment. `publicId` is unique across the account. | **Permanent.** Use a different `publicId`. | +| report | `ambiguous_legacy_id` | The id matches more than one legacy report (see below), so it can't identify one. | **Permanent.** Give the intended report a `publicId` of your own (see below), then key on that. | + +The upserts serialize per key (per slug, per `publicId`), so two pipeline runs +applying the same bundle at once can't create a duplicate — the loser gets a +retryable `upsert_branch_changed` instead. + +## Choosing and adopting `publicId`s + +A report's `publicId` is **write-once**: you can assign one to a report that +doesn't have one yet, but a report's existing `publicId` can never be changed, +because clients may already have stored it. You can supply a `publicId`: + +- **On create** — pass it in the body to [`POST /reports`][ref-create-report], or + just call the [upsert endpoint][ref-upsert-report] with the id in the path. +- **On an existing report** — assign one with + [`PUT /reports/{reportId}`][ref-update-report]. This is how you bring a report + that was authored in the UI under as-code management. + +Pick any distinct 12-character `[0-9A-Za-z]` id. The auto-generated placeholder +ids shown for reports that don't have a stable id yet are a reserved, non-unique +shape and are rejected with `400` — you must choose your own. + +### Reports created before stable ids + +Reports created before `publicId` existed don't store one; the API **synthesizes** +one from the report's internal id so every report has an id on the wire. These +synthesized ids are **not unique** — several reports can share one. The upsert +endpoint resolves a synthesized id only when it is unambiguous, adopting it as the +report's real `publicId` at that point; if it matches more than one report it +returns the `ambiguous_legacy_id` conflict above. For anything you manage as code, +don't rely on a synthesized id — assign a `publicId` you chose and key on that. + +## Reference + +- [Create or update a workbook by slug][ref-upsert-workbook] +- [Create or update a report by publicId][ref-upsert-report] +- [Create a report][ref-create-report] · [Update a report][ref-update-report] +- [Publish dashboard][ref-publish] +- [Building dashboards in the UI][ref-dashboards] + +[ref-dashboards]: /docs/explore-analyze/dashboards +[ref-workbooks]: /docs/explore-analyze/workbooks +[ref-reports]: /docs/explore-analyze/workbooks/querying-data +[ref-api]: /api-reference/introduction +[ref-auth]: /api-reference/authentication +[ref-upsert-workbook]: /api-reference/workbooks/create-or-update-a-workbook-by-slug +[ref-upsert-report]: /api-reference/reports/create-or-update-a-report-by-publicid +[ref-create-report]: /api-reference/reports/create-a-report +[ref-update-report]: /api-reference/reports/update-a-report +[ref-get-report]: /api-reference/reports/get-report +[ref-get-workbook]: /api-reference/workbooks/get-workbook +[ref-publish]: /api-reference/workbooks/publish-dashboard diff --git a/docs-mintlify/docs/explore-analyze/dashboards/index.mdx b/docs-mintlify/docs/explore-analyze/dashboards/index.mdx index bbfa7a64d494b..7b27c48a450bb 100644 --- a/docs-mintlify/docs/explore-analyze/dashboards/index.mdx +++ b/docs-mintlify/docs/explore-analyze/dashboards/index.mdx @@ -18,6 +18,10 @@ Dashboards enable you to: In the dashboard builder inside your [workbook][ref-workbooks], select the reports you want to include and arrange them on the canvas alongside other [widgets][ref-widgets] to tell your data story, then publish the dashboard. This gives stakeholders direct access to the insights that matter most, without the complexity of the underlying analysis. +Prefer to manage dashboards from source control? You can also apply dashboards, +workbooks, and reports to a deployment through the REST API and keep their +definitions in Git — see [Dashboards as code](/docs/explore-analyze/dashboards/dashboards-as-code). + ## Data freshness Each widget shows a [freshness](/docs/explore-analyze/workbooks/querying-data#result-freshness-and-provenance) leaf indicating how recently its data was refreshed. The dashboard's own leaf reflects its least-recently-refreshed widget, so you can see at a glance whether everything on the dashboard is up to date. From b1c87dc40f9c7183d7eb6a5c0e08682092aac077 Mon Sep 17 00:00:00 2001 From: Alex Qyoun-ae <4062971+MazterQyou@users.noreply.github.com> Date: Tue, 11 Aug 2026 23:11:48 +0400 Subject: [PATCH 6/6] fix(cubesql): Resolve data source for synthetic-only queries (#11526) --- .../cubesql/src/compile/engine/df/wrapper.rs | 135 ++++++++++++ .../cubesql/src/compile/test/test_wrapper.rs | 44 ++++ rust/cubesql/cubesql/src/transport/ctx.rs | 196 ++++++++++++++++++ 3 files changed, 375 insertions(+) diff --git a/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs b/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs index a61b1a9d1fd2f..d09de2207999b 100644 --- a/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs +++ b/rust/cubesql/cubesql/src/compile/engine/df/wrapper.rs @@ -962,6 +962,13 @@ impl CubeScanWrapperNode { }) .map(|mem| mem.member.as_str()), ) + // A scan can have no members to resolve from, like when it selects only + // synthetic fields, so fall back to the cubes it scans. + .and_then(|data_source| { + data_source.or_else_try(|| { + meta.data_source_for_cube_names(node.used_cubes.iter().map(|c| c.as_str())) + }) + }) .map_err(|err| { CubeError::internal(format!( "Can't generate SQL for node; error: {err}; node: {node:?}" @@ -3643,6 +3650,15 @@ impl WrappedSelectNode { } meta.data_source_for_member_names(every_used_member.iter().map(|m| m.as_str())) + // A query can reference no members to resolve from, like when it + // selects only synthetic fields, so fall back to the cubes it scans. + .and_then(|data_source| { + data_source.or_else_try(|| { + meta.data_source_for_cube_names( + ungrouped_scan_node.used_cubes.iter().map(|c| c.as_str()), + ) + }) + }) .map_err(|err| { CubeError::internal(format!("Could not determine data source: {err}")) })? @@ -4490,6 +4506,125 @@ impl<'ctx, 'mem> ExpressionVisitor for CollectMembersVisitor<'ctx, 'mem> { #[cfg(test)] mod tests { use super::*; + use crate::{ + compile::engine::df::scan::CubeScanOptions, + sql::HttpAuthContext, + transport::{CubeMeta, CubeMetaDimension, CubeMetaType}, + }; + use datafusion::logical_plan::DFField; + use std::collections::HashMap; + + /// Each entry is a cube with one dimension, on a data source of its own. + fn meta_context_with_cubes(cubes: &[(&str, &str, &str)]) -> MetaContext { + MetaContext::new( + cubes + .iter() + .map(|(cube_name, member, _)| CubeMeta { + name: cube_name.to_string(), + description: None, + title: None, + r#type: CubeMetaType::Cube, + dimensions: vec![CubeMetaDimension::new( + member.to_string(), + "string".to_string(), + )], + measures: vec![], + segments: vec![], + joins: None, + folders: None, + nested_folders: None, + hierarchies: None, + meta: None, + }) + .collect(), + cubes + .iter() + .map(|(_, member, data_source)| (member.to_string(), data_source.to_string())) + .collect(), + HashMap::new(), + uuid::Uuid::new_v4(), + ) + } + + fn cube_scan_node(member_fields: Vec, used_cubes: Vec) -> CubeScanNode { + let schema = Arc::new( + DFSchema::new_with_metadata( + member_fields + .iter() + .enumerate() + .map(|(i, _)| DFField::new(None, &format!("c{i}"), DataType::Utf8, true)) + .collect::>(), + HashMap::new(), + ) + .unwrap(), + ); + + CubeScanNode::new( + schema, + member_fields, + V1LoadRequestQuery::new(), + Arc::new(HttpAuthContext { + access_token: "token".to_string(), + base_path: "path".to_string(), + }), + CubeScanOptions { + change_user: None, + max_records: None, + cache_mode: None, + throw_continue_wait: false, + }, + used_cubes, + None, + ) + } + + /// A plain wrapped `CubeScan` whose columns are all literals has no member to + /// resolve a data source from, so it falls back to the cubes it scans. The + /// push-to-cube path has `test_wrapper_only_system_fields` for this; the SQL + /// surface never reaches this one, so it is covered here directly. + #[test] + fn test_data_source_for_cube_scan_without_members() { + let meta = meta_context_with_cubes(&[("Orders", "Orders.status", "warehouse")]); + let node = cube_scan_node( + vec![MemberField::Literal(ScalarValue::Utf8(Some( + "anything".to_string(), + )))], + vec!["Orders".to_string()], + ); + + let data_source = CubeScanWrapperNode::data_source_for_cube_scan(&meta, &node).unwrap(); + assert!(matches!(data_source, DataSource::Specific("warehouse"))); + } + + /// With a member to resolve from, that member decides and the fallback must not + /// take over. `used_cubes` holds a second cube on another data source, so + /// falling back would merge the two into a conflict instead - that is what + /// makes the precedence observable rather than assumed. + #[test] + fn test_data_source_for_cube_scan_with_members() { + let meta = meta_context_with_cubes(&[ + ("Orders", "Orders.status", "warehouse"), + ("Visits", "Visits.url", "analytics"), + ]); + let node = cube_scan_node( + vec![MemberField::regular("Orders.status".to_string())], + vec!["Orders".to_string(), "Visits".to_string()], + ); + + let data_source = CubeScanWrapperNode::data_source_for_cube_scan(&meta, &node).unwrap(); + assert!(matches!(data_source, DataSource::Specific("warehouse"))); + } + + /// Literal-only columns and no cubes to fall back to: nothing restricts the + /// data source, and the caller raises its own error. + #[test] + fn test_data_source_for_cube_scan_without_members_or_cubes() { + let meta = meta_context_with_cubes(&[("Orders", "Orders.status", "warehouse")]); + let node = cube_scan_node(vec![MemberField::Literal(ScalarValue::Utf8(None))], vec![]); + + let data_source = CubeScanWrapperNode::data_source_for_cube_scan(&meta, &node).unwrap(); + assert!(matches!(data_source, DataSource::Unrestricted)); + } #[test] fn test_member_expression_sql() { diff --git a/rust/cubesql/cubesql/src/compile/test/test_wrapper.rs b/rust/cubesql/cubesql/src/compile/test/test_wrapper.rs index c0971d3c3ea24..e202347037baa 100644 --- a/rust/cubesql/cubesql/src/compile/test/test_wrapper.rs +++ b/rust/cubesql/cubesql/src/compile/test/test_wrapper.rs @@ -2809,3 +2809,47 @@ async fn test_case_wrapper_sum_case_date_only_string() { displayable(physical_plan.as_ref()).indent() ); } + +/// Query can reference no members at all, only synthetic fields. +/// Data source is resolved from cubes of the scan node in that case. +#[tokio::test] +async fn test_wrapper_only_system_fields() { + if !Rewriter::sql_push_down_enabled() { + return; + } + init_testing_logger(); + + let query_plan = convert_select_to_query_plan( + r#" + SELECT COUNT(DISTINCT "F0"."__user") AS "user_count", + COUNT(1) AS "row_count", + MIN("F0"."__user") AS "user_min", + MAX("F0"."__user") AS "user_max", + COUNT(DISTINCT "F0"."__cubeJoinField") AS "join_field_count", + MIN("F0"."__cubeJoinField") AS "join_field_min", + MAX("F0"."__cubeJoinField") AS "join_field_max" + FROM ( + SELECT "T1"."__user" AS "__user", "T1"."__cubeJoinField" AS "__cubeJoinField" + FROM KibanaSampleDataEcommerce AS "T1" + ) AS "F0" + LIMIT 1 + "# + .to_string(), + DatabaseProtocol::PostgreSQL, + ) + .await; + + let logical_plan = query_plan.as_logical_plan(); + let sql = logical_plan.find_cube_scan_wrapped_sql().wrapped_sql.sql; + assert!( + sql.contains(r#"\"cubeName\":\"KibanaSampleDataEcommerce\""#), + "SQL contains member expressions for the scanned cube: {}", + sql + ); + + let physical_plan = query_plan.as_physical_plan().await.unwrap(); + println!( + "Physical plan: {}", + displayable(physical_plan.as_ref()).indent() + ); +} diff --git a/rust/cubesql/cubesql/src/transport/ctx.rs b/rust/cubesql/cubesql/src/transport/ctx.rs index 63b001d70501d..655baeb29baa0 100644 --- a/rust/cubesql/cubesql/src/transport/ctx.rs +++ b/rust/cubesql/cubesql/src/transport/ctx.rs @@ -49,6 +49,10 @@ pub enum DataSourceError { Conflict(String, String), #[error("Data source not found for member '{0}'")] Missing(String), + #[error("Data source not found for cube '{0}'")] + MissingForCube(String), + #[error("Data source not found for any of the cubes: {0}")] + MissingForEveryCube(String), } impl<'meta> DataSource<'meta> { @@ -59,6 +63,14 @@ impl<'meta> DataSource<'meta> { } } + /// Resolve again with `f` when nothing has restricted the data source yet. + pub fn or_else_try(self, f: impl FnOnce() -> Result) -> Result { + match self { + Self::Unrestricted => f(), + specific => Ok(specific), + } + } + pub fn merge(&self, other: &Self) -> Result { match (self, other) { (Self::Unrestricted, ds) | (ds, Self::Unrestricted) => Ok(ds.clone()), @@ -138,6 +150,57 @@ impl MetaContext { .try_fold(DataSource::Unrestricted, |l, r| l.merge(&r?)) } + /// Data source for a cube or a view as a whole. + /// A query can reference a cube without referencing any of its members, + /// like when it selects only synthetic fields (`__user`, `__cubeJoinField`). + /// Members of a single view can come from different data sources, + /// so the first member with a known data source wins. + pub fn data_source_for_cube_name( + &self, + cube_name: &str, + ) -> Result, DataSourceError> { + let cube = self + .find_cube_with_name(cube_name) + .ok_or_else(|| DataSourceError::MissingForCube(cube_name.to_string()))?; + + cube.dimensions + .iter() + .map(|dimension| &dimension.name) + .chain(cube.measures.iter().map(|measure| &measure.name)) + .chain(cube.segments.iter().map(|segment| &segment.name)) + .find_map(|member| self.member_to_data_source.get(member)) + .map(|data_source| DataSource::Specific(data_source.as_ref())) + .ok_or_else(|| DataSourceError::MissingForCube(cube_name.to_string())) + } + + /// Data source shared by `cube_names`, as a fallback for a query that + /// references no members at all. Cubes without a data source of their own are + /// skipped, so that one unresolvable cube does not mask a resolvable one; the + /// rest are merged, which reports two different data sources as a conflict + /// like every other resolution path here. An empty `cube_names` is + /// `Unrestricted`; `cube_names` where nothing at all resolves names them all, + /// since the caller has no other way to tell what was tried. + pub fn data_source_for_cube_names<'names>( + &self, + cube_names: impl IntoIterator, + ) -> Result, DataSourceError> { + let mut tried = Vec::new(); + let data_source = cube_names + .into_iter() + .filter_map(|cube_name| { + tried.push(cube_name); + self.data_source_for_cube_name(cube_name).ok() + }) + .try_fold(DataSource::Unrestricted, |l, r| l.merge(&r))?; + + match data_source { + DataSource::Unrestricted if !tried.is_empty() => { + Err(DataSourceError::MissingForEveryCube(tried.join(", "))) + } + data_source => Ok(data_source), + } + } + pub fn find_cube_with_name(&self, name: &str) -> Option<&CubeMeta> { self.cubes.iter().find(|&cube| cube.name == name) } @@ -310,4 +373,137 @@ mod tests { _ => panic!("wrong name!"), } } + + fn cube_with_members(name: &str, members: &[&str]) -> CubeMeta { + CubeMeta { + name: name.to_string(), + description: None, + title: None, + r#type: CubeMetaType::Cube, + dimensions: members + .iter() + .map(|member| CubeMetaDimension::new(member.to_string(), "string".to_string())) + .collect(), + measures: vec![], + segments: vec![], + joins: None, + folders: None, + nested_folders: None, + hierarchies: None, + meta: None, + } + } + + /// `orders` resolves through its member, `logs` has a member with no data + /// source of its own, and `events` is not in the schema at all. + fn data_source_test_context() -> MetaContext { + MetaContext::new( + vec![ + cube_with_members("orders", &["orders.status"]), + cube_with_members("logs", &["logs.line"]), + ], + HashMap::from([("orders.status".to_string(), "warehouse".to_string())]), + HashMap::new(), + Uuid::new_v4(), + ) + } + + #[test] + fn test_data_source_for_cube_name() { + let ctx = data_source_test_context(); + + assert!(matches!( + ctx.data_source_for_cube_name("orders"), + Ok(DataSource::Specific("warehouse")) + )); + assert!(matches!( + ctx.data_source_for_cube_name("logs"), + Err(DataSourceError::MissingForCube(cube)) if cube == "logs" + )); + assert!(matches!( + ctx.data_source_for_cube_name("events"), + Err(DataSourceError::MissingForCube(cube)) if cube == "events" + )); + } + + #[test] + fn test_data_source_for_cube_names() { + let ctx = data_source_test_context(); + + // Nothing to resolve from leaves the data source open, so that the caller + // can raise its own error. + assert!(matches!( + ctx.data_source_for_cube_names(Vec::<&str>::new()), + Ok(DataSource::Unrestricted) + )); + // A cube without a data source of its own does not mask a cube that has + // one, whichever order they come in. + assert!(matches!( + ctx.data_source_for_cube_names(vec!["logs", "orders"]), + Ok(DataSource::Specific("warehouse")) + )); + assert!(matches!( + ctx.data_source_for_cube_names(vec!["orders", "logs"]), + Ok(DataSource::Specific("warehouse")) + )); + // When nothing resolves, the error names everything that was tried. + let err = ctx + .data_source_for_cube_names(vec!["logs", "events"]) + .expect_err("neither cube has a data source"); + assert!( + matches!(&err, DataSourceError::MissingForEveryCube(cubes) if cubes == "logs, events"), + "expected every tried cube to be named, got: {}", + err + ); + } + + /// A view can include members from cubes on different data sources. There is + /// no single right answer then, and resolving a whole cube is only ever a + /// fallback for a query that names no members, so the first member that has a + /// data source wins. This pins that choice rather than endorsing it. + #[test] + fn test_data_source_for_cube_name_of_multi_data_source_view() { + let mut view = cube_with_members("everything", &["everything.url", "everything.status"]); + view.r#type = CubeMetaType::View; + + let ctx = MetaContext::new( + vec![view], + HashMap::from([ + ("everything.url".to_string(), "analytics".to_string()), + ("everything.status".to_string(), "warehouse".to_string()), + ]), + HashMap::new(), + Uuid::new_v4(), + ); + + assert!(matches!( + ctx.data_source_for_cube_name("everything"), + Ok(DataSource::Specific("analytics")) + )); + } + + #[test] + fn test_data_source_for_cube_names_reports_conflict() { + let ctx = MetaContext::new( + vec![ + cube_with_members("orders", &["orders.status"]), + cube_with_members("visits", &["visits.url"]), + ], + HashMap::from([ + ("orders.status".to_string(), "warehouse".to_string()), + ("visits.url".to_string(), "analytics".to_string()), + ]), + HashMap::new(), + Uuid::new_v4(), + ); + + let err = ctx + .data_source_for_cube_names(vec!["orders", "visits"]) + .expect_err("two data sources should conflict"); + assert!( + matches!(&err, DataSourceError::Conflict(..)), + "expected a conflict, got: {}", + err + ); + } }