From 78abd1b9b3075943f1019b53fb0c0082e43b7c1a Mon Sep 17 00:00:00 2001 From: Andre Baaij Date: Wed, 12 Aug 2026 21:12:45 -0400 Subject: [PATCH 1/3] feat(dbt): emit Hex's semantic-model table binding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hex's Semantic Model Sync parses dbt MetricFlow YAML straight from a git repo (not via an API), and binds each semantic model to a physical table through `config.meta.hex.table`. Without that key the sync imports models that resolve to nothing: the agent then answers about no data, which a benchmark grades as a wrong answer rather than the setup failure it is. That exact failure mode cost a full Lightdash run earlier. Adds two options, both off by default: table-prefix logical -> physical name (ClickHouse materialises fct_orders as marts__fct_orders) dbt-hex-meta emit the config.meta.hex.table binding `dbt` becomes Configurable to receive them. A plain dbt build is unchanged and carries no Hex key — asserted by a test, because the dbt target also writes the shared ground-truth reference every arm reads, and one vendor's meta must not leak into it. Also fixes a defaulting bug this surfaced. `schema` defaulted to MAIN unconditionally, so the binding came out as `ecomm.MAIN.marts__fct_orders`, which does not resolve on ClickHouse's two-part namespace. `schema` is now a pointer so an EXPLICIT empty value is distinguishable from an omitted one; omitted still defaults to MAIN. Changing the default globally was the wrong fix: ossie, nao-yaml, supersimple and the warehouse dialects all read Schema, and Snowflake targets genuinely need the qualified form. Verified against the pinned eval reference: 38 semantic models, 22 measures, 14 metrics, all 38 bound to ecomm.marts__. Co-Authored-By: Claude Opus 5 --- cmd/semglot/config.go | 23 +++++++++++++++--- cmd/semglot/main.go | 2 ++ dialect/dbt.go | 7 +++++- dialect/dbt_emit.go | 43 ++++++++++++++++++++++++++++++--- dialect/dbt_test.go | 55 +++++++++++++++++++++++++++++++++++++++++++ dialect/dialect.go | 11 +++++++++ 6 files changed, 134 insertions(+), 7 deletions(-) diff --git a/cmd/semglot/config.go b/cmd/semglot/config.go index 0ab1bee..3e2a52b 100644 --- a/cmd/semglot/config.go +++ b/cmd/semglot/config.go @@ -33,11 +33,21 @@ type profile struct { TargetDialect string `yaml:"target-dialect"` Output string `yaml:"output"` Database string `yaml:"database"` - Schema string `yaml:"schema"` + // Schema is a pointer so an EXPLICIT empty value is distinguishable from an + // omitted one. Omitted defaults to MAIN (Snowflake-shaped targets need a + // qualified name); explicitly empty means a two-part namespace such as + // ClickHouse, where "db.MAIN.table" would not resolve. + Schema *string `yaml:"schema"` ViewSchema string `yaml:"view-schema"` ModelName string `yaml:"model-name"` Description string `yaml:"description"` DbtMetaKeyPath string `yaml:"dbt-meta-key-path"` + // TablePrefix maps a logical table name to its physical one (ClickHouse + // materialises fct_orders as marts__fct_orders). + TablePrefix string `yaml:"table-prefix"` + // DbtHexMeta emits Hex's config.meta.hex.table binding on each semantic + // model; Hex's Semantic Model Sync cannot resolve the physical table without it. + DbtHexMeta bool `yaml:"dbt-hex-meta"` } // configFile is the top-level shape of semglot.yaml. @@ -57,6 +67,8 @@ type buildSpec struct { ModelName string Description string DbtMetaKeyPath string + TablePrefix string + DbtHexMeta bool } // warehouseTargets emit into a physical warehouse (Snowflake, or a Databricks @@ -94,16 +106,21 @@ func loadProfile(configPath, name string) (buildSpec, error) { TargetDialect: p.TargetDialect, Output: p.Output, Database: p.Database, - Schema: p.Schema, + Schema: "", ViewSchema: p.ViewSchema, ModelName: p.ModelName, Description: p.Description, DbtMetaKeyPath: p.DbtMetaKeyPath, + TablePrefix: p.TablePrefix, + DbtHexMeta: p.DbtHexMeta, } if spec.SourceDialect == "" { spec.SourceDialect = "dbt" } - if spec.Schema == "" { + switch { + case p.Schema != nil: + spec.Schema = *p.Schema + default: spec.Schema = "MAIN" } if spec.ModelName == "" { diff --git a/cmd/semglot/main.go b/cmd/semglot/main.go index 4fa6193..2cf35b5 100644 --- a/cmd/semglot/main.go +++ b/cmd/semglot/main.go @@ -72,6 +72,8 @@ func buildCmd(args []string) int { Name: spec.ModelName, Description: spec.Description, DbtMetaKeyPath: spec.DbtMetaKeyPath, + TablePrefix: spec.TablePrefix, + DbtHexMeta: spec.DbtHexMeta, }) } diff --git a/dialect/dbt.go b/dialect/dbt.go index 1b3e747..5dffcb5 100644 --- a/dialect/dbt.go +++ b/dialect/dbt.go @@ -17,7 +17,12 @@ func init() { Register(dbt{}) } // model properties (`models:` — table/column descriptions, data types, key and // relationship constraints/tests) and the semantic layer (`semantic_models:` + // `metrics:` — measures, aggregations, metrics). Either may be present alone. -type dbt struct{} +type dbt struct{ opts Options } + +// WithOptions makes dbt Configurable so a consumer-specific binding (Hex's +// config.meta.hex.table) and the physical table prefix can be supplied per +// build rather than baked in. +func (d dbt) WithOptions(o Options) Emitter { d.opts = o; return d } func (dbt) Name() string { return "dbt" } diff --git a/dialect/dbt_emit.go b/dialect/dbt_emit.go index 0cb5e4f..4b53dff 100644 --- a/dialect/dbt_emit.go +++ b/dialect/dbt_emit.go @@ -72,12 +72,30 @@ type dbtEmitRelTest struct { type dbtEmitSemantic struct { Name string `yaml:"name"` Model string `yaml:"model"` + Config *dbtEmitSemConfig `yaml:"config,omitempty"` Defaults *dbtEmitDefaults `yaml:"defaults,omitempty"` Entities []dbtEmitEntity `yaml:"entities,omitempty"` Dimensions []dbtEmitDimension `yaml:"dimensions,omitempty"` Measures []dbtEmitMeasure `yaml:"measures,omitempty"` } +// dbtEmitSemConfig carries the `config: meta:` block on a semantic model. Only +// Hex's binding lives here today; the shape is nested rather than flat so other +// consumers' meta keys can sit alongside without moving Hex's. +type dbtEmitSemConfig struct { + Meta dbtEmitSemMeta `yaml:"meta"` +} + +type dbtEmitSemMeta struct { + Hex *dbtEmitHexMeta `yaml:"hex,omitempty"` +} + +// dbtEmitHexMeta is Hex's semantic-model binding: the fully qualified physical +// table the model reads. +type dbtEmitHexMeta struct { + Table string `yaml:"table"` +} + type dbtEmitDefaults struct { AggTimeDimension string `yaml:"agg_time_dimension,omitempty"` } @@ -145,7 +163,7 @@ type dbtEmitConversionParams struct { // inlined aggregates are emitted rather than reported; what remains is a // genuine dbt limit. It is applied before emitModel too, so a physical column // that only a synthesised measure references still gets a columns[] entry. -func (dbt) Emit(m *ir.Model, dir string) ([]string, error) { +func (d dbt) Emit(m *ir.Model, dir string) ([]string, error) { var f dbtEmitFile var warnings []string // A relationship reaches dbt as a `relationships` data test on the FK @@ -200,7 +218,7 @@ func (dbt) Emit(m *ir.Model, dir string) ([]string, error) { } f.Models = append(f.Models, emitModel(m, t, pk, fk)) - f.SemanticModels = append(f.SemanticModels, emitSemantic(t, pk, fk)) + f.SemanticModels = append(f.SemanticModels, emitSemantic(t, pk, fk, d.opts)) metrics, warn := emitMetrics(t) f.Metrics = append(f.Metrics, metrics...) warnings = append(warnings, warn...) @@ -338,12 +356,31 @@ func emitModel(m *ir.Model, t ir.Table, pk, fk map[string]bool) dbtEmitModel { return em } +// physicalTable renders the warehouse-qualified name a semantic model reads: +// [database.][schema.]prefix+name. The IR holds logical names only, so the +// prefix and container come from Options. +func physicalTable(name string, opts Options) string { + out := opts.TablePrefix + name + if opts.Schema != "" { + out = opts.Schema + "." + out + } + if opts.Database != "" { + out = opts.Database + "." + out + } + return out +} + // emitSemantic builds the semantic_models block: a primary entity per PK column, // every non-PK/non-FK dimension as a semantic dimension (FK columns round-trip // as plain model columns + the relationship test, so they are NOT re-emitted as // entities), and every measure. -func emitSemantic(t ir.Table, pk, fk map[string]bool) dbtEmitSemantic { +func emitSemantic(t ir.Table, pk, fk map[string]bool, opts Options) dbtEmitSemantic { sm := dbtEmitSemantic{Name: t.Name, Model: "ref('" + t.Name + "')"} + if opts.DbtHexMeta { + sm.Config = &dbtEmitSemConfig{Meta: dbtEmitSemMeta{ + Hex: &dbtEmitHexMeta{Table: physicalTable(t.Name, opts)}, + }} + } if t.Grain != "" { sm.Defaults = &dbtEmitDefaults{AggTimeDimension: t.Grain} } diff --git a/dialect/dbt_test.go b/dialect/dbt_test.go index 83e64b3..764e069 100644 --- a/dialect/dbt_test.go +++ b/dialect/dbt_test.go @@ -510,3 +510,58 @@ func TestDBTParseLabelAndGrain(t *testing.T) { t.Fatalf("Def = %#v, want %#v", m.Def, wantDef) } } + +// TestDbtEmitsHexBinding covers Hex's Semantic Model Sync requirement: it parses +// MetricFlow YAML straight from a repo and cannot resolve which physical table a +// semantic model reads without config.meta.hex.table. Without the binding the +// sync imports models that resolve to nothing — the agent then answers about no +// data, which grades as a wrong answer rather than a setup failure. +func TestDbtEmitsHexBinding(t *testing.T) { + m := &ir.Model{Tables: []ir.Table{{ + Name: "fct_orders", + PrimaryKey: []string{"order_id"}, + Dimensions: []ir.Field{{Name: "status", Expr: "status"}}, + Measures: []ir.Measure{{Field: ir.Field{Name: "gross", Expr: "gross"}, Agg: "sum"}}, + }}} + dir := t.TempDir() + d := dbt{}.WithOptions(Options{Database: "ecomm", TablePrefix: "marts__", DbtHexMeta: true}) + if _, err := d.Emit(m, dir); err != nil { + t.Fatalf("Emit: %v", err) + } + got := readFile(t, filepath.Join(dir, "ecommerce.yml")) + for _, want := range []string{"config:", "meta:", "hex:", "table: ecomm.marts__fct_orders"} { + if !strings.Contains(got, want) { + t.Errorf("emitted YAML missing %q:\n%s", want, got) + } + } +} + +// TestDbtOmitsHexBindingByDefault: the binding is one consumer's key, so a plain +// dbt build must stay clean. Emitting it unasked would put a vendor's meta into +// the shared ground-truth reference every arm reads. +func TestDbtOmitsHexBindingByDefault(t *testing.T) { + m := &ir.Model{Tables: []ir.Table{{Name: "fct_orders", PrimaryKey: []string{"order_id"}}}} + dir := t.TempDir() + if _, err := (dbt{}).Emit(m, dir); err != nil { + t.Fatalf("Emit: %v", err) + } + if got := readFile(t, filepath.Join(dir, "ecommerce.yml")); strings.Contains(got, "hex:") { + t.Errorf("default dbt emit must not carry Hex meta:\n%s", got) + } +} + +func TestPhysicalTable(t *testing.T) { + cases := []struct { + opts Options + want string + }{ + {Options{Database: "ecomm", TablePrefix: "marts__"}, "ecomm.marts__fct_orders"}, + {Options{Database: "EVAL_MARTS", Schema: "MAIN"}, "EVAL_MARTS.MAIN.fct_orders"}, + {Options{}, "fct_orders"}, + } + for _, c := range cases { + if got := physicalTable("fct_orders", c.opts); got != c.want { + t.Errorf("physicalTable(%+v) = %q, want %q", c.opts, got, c.want) + } + } +} diff --git a/dialect/dialect.go b/dialect/dialect.go index 3b639e1..7c97586 100644 --- a/dialect/dialect.go +++ b/dialect/dialect.go @@ -45,6 +45,17 @@ type Options struct { // DbtMetaKeyPath selects where Lightdash meta lives: "" / "meta" nests under // meta: (dbt <=1.9); "config.meta" nests under config.meta: (dbt 1.10+). DbtMetaKeyPath string + // TablePrefix is prepended to a table's IR name to form its PHYSICAL name. + // The IR carries logical names (fct_orders); a warehouse may materialise + // them under a prefix (ClickHouse marts__fct_orders). Consumers that bind a + // semantic model to a physical table need the latter. + TablePrefix string + // DbtHexMeta emits Hex's `config.meta.hex.table` binding on every semantic + // model. Hex's Semantic Model Sync parses dbt MetricFlow YAML straight from + // a repo and CANNOT resolve which physical table a semantic model reads + // without it, so without this the sync imports models that resolve to + // nothing. + DbtHexMeta bool } // Configurable is an Emitter that accepts model/view identity options. From aa1136a1abb637de8bf548a357197b87b75d96e7 Mon Sep 17 00:00:00 2001 From: Andre Baaij Date: Wed, 12 Aug 2026 21:23:39 -0400 Subject: [PATCH 2/3] style: gofmt config.go The Schema *string change altered the struct's field-alignment column, which gofmt reflows. CI checks gofmt -l and I did not run it before pushing. Co-Authored-By: Claude Opus 5 --- cmd/semglot/config.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/semglot/config.go b/cmd/semglot/config.go index 3e2a52b..ba8f21d 100644 --- a/cmd/semglot/config.go +++ b/cmd/semglot/config.go @@ -28,20 +28,20 @@ func (s *sourcePaths) UnmarshalYAML(node *yaml.Node) error { // profile is one named build in semglot.yaml. type profile struct { - Source sourcePaths `yaml:"source"` - SourceDialect string `yaml:"source-dialect"` - TargetDialect string `yaml:"target-dialect"` - Output string `yaml:"output"` - Database string `yaml:"database"` + Source sourcePaths `yaml:"source"` + SourceDialect string `yaml:"source-dialect"` + TargetDialect string `yaml:"target-dialect"` + Output string `yaml:"output"` + Database string `yaml:"database"` // Schema is a pointer so an EXPLICIT empty value is distinguishable from an // omitted one. Omitted defaults to MAIN (Snowflake-shaped targets need a // qualified name); explicitly empty means a two-part namespace such as // ClickHouse, where "db.MAIN.table" would not resolve. - Schema *string `yaml:"schema"` - ViewSchema string `yaml:"view-schema"` - ModelName string `yaml:"model-name"` - Description string `yaml:"description"` - DbtMetaKeyPath string `yaml:"dbt-meta-key-path"` + Schema *string `yaml:"schema"` + ViewSchema string `yaml:"view-schema"` + ModelName string `yaml:"model-name"` + Description string `yaml:"description"` + DbtMetaKeyPath string `yaml:"dbt-meta-key-path"` // TablePrefix maps a logical table name to its physical one (ClickHouse // materialises fct_orders as marts__fct_orders). TablePrefix string `yaml:"table-prefix"` From 2d21fbc9f9d50bd0ad072a388a11e4ad5ae449b6 Mon Sep 17 00:00:00 2001 From: Andre Baaij Date: Thu, 13 Aug 2026 19:47:37 -0400 Subject: [PATCH 3/3] fix(supersimple): two-sided join keys, table prefix, and case control MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three changes needed to target a case-sensitive warehouse with prefixed tables. The first is a correctness bug that would have produced wrong numbers silently. TWO-SIDED JOIN KEYS. The emitter wrote `join_key: ` for every relation. That is Supersimple's shorthand for "both sides share this column name", which held for 20 of 23 relations and was WRONG for the rest: fct_orders.order_date -> dim_date.date_day was emitted as `join_key: date_day`, addressing a column the child does not have. `supersimple validate` caught three of these ("references unknown property 'date_day' on related model"), but the dangerous case is the one it could not catch — the golden encoded a role-playing relation joining BILLING_CUSTOMER_SK as `join_key: CUSTOMER_SK`, a column that DOES exist on both sides, so it would have joined on the wrong one and returned plausible wrong answers rather than erroring. The vendor's own discovery snapshot uses join_key_on_base/join_key_on_related; we now emit the shorthand only when both sides genuinely agree. Golden updated: it captured the bug. TablePrefix maps the IR's logical name to the physical one (ClickHouse materialises fct_orders as marts__fct_orders). LowerCaseIdentifiers keeps identifiers as spelled instead of upper-casing them: the upper-case default suits Snowflake, where unquoted identifiers fold up, but ClickHouse is case-SENSITIVE and MARTS__FCT_ORDERS does not exist. Both default off, so Snowflake output is byte-identical. Schema now honours an EXPLICIT empty value rather than always defaulting to MAIN — ClickHouse has a two-part namespace, so MAIN.marts__fct_orders resolves to nothing. Verified end to end: 38 models, 338 properties, 23 relations, 14 metrics (on_time_delivery_rate intact as a real operations expression, not degraded to prose), no NOTES sidecar, `supersimple validate` reports Configuration is valid. Co-Authored-By: Claude Opus 5 --- cmd/semglot/config.go | 53 ++++++----- cmd/semglot/main.go | 17 ++-- dialect/dialect.go | 5 ++ dialect/supersimple.go | 89 +++++++++++++++---- dialect/supersimple_test.go | 18 ++++ .../dbt/supersimple/DIM_CUSTOMER.yaml | 3 +- 6 files changed, 136 insertions(+), 49 deletions(-) diff --git a/cmd/semglot/config.go b/cmd/semglot/config.go index ba8f21d..f20ae1e 100644 --- a/cmd/semglot/config.go +++ b/cmd/semglot/config.go @@ -48,6 +48,9 @@ type profile struct { // DbtHexMeta emits Hex's config.meta.hex.table binding on each semantic // model; Hex's Semantic Model Sync cannot resolve the physical table without it. DbtHexMeta bool `yaml:"dbt-hex-meta"` + // LowerCaseIdentifiers keeps identifiers as spelled rather than upper-casing + // them; required for case-sensitive warehouses such as ClickHouse. + LowerCaseIdentifiers bool `yaml:"lowercase-identifiers"` } // configFile is the top-level shape of semglot.yaml. @@ -57,18 +60,19 @@ type configFile struct { // buildSpec is a fully-resolved build: a validated profile with defaults applied. type buildSpec struct { - Sources []string - SourceDialect string - TargetDialect string - Output string - Database string - Schema string - ViewSchema string - ModelName string - Description string - DbtMetaKeyPath string - TablePrefix string - DbtHexMeta bool + Sources []string + SourceDialect string + TargetDialect string + Output string + Database string + Schema string + ViewSchema string + ModelName string + Description string + DbtMetaKeyPath string + TablePrefix string + DbtHexMeta bool + LowerCaseIdentifiers bool } // warehouseTargets emit into a physical warehouse (Snowflake, or a Databricks @@ -101,18 +105,19 @@ func loadProfile(configPath, name string) (buildSpec, error) { return buildSpec{}, fmt.Errorf("profile %q: output is required", name) } spec := buildSpec{ - Sources: []string(p.Source), - SourceDialect: p.SourceDialect, - TargetDialect: p.TargetDialect, - Output: p.Output, - Database: p.Database, - Schema: "", - ViewSchema: p.ViewSchema, - ModelName: p.ModelName, - Description: p.Description, - DbtMetaKeyPath: p.DbtMetaKeyPath, - TablePrefix: p.TablePrefix, - DbtHexMeta: p.DbtHexMeta, + Sources: []string(p.Source), + SourceDialect: p.SourceDialect, + TargetDialect: p.TargetDialect, + Output: p.Output, + Database: p.Database, + Schema: "", + ViewSchema: p.ViewSchema, + ModelName: p.ModelName, + Description: p.Description, + DbtMetaKeyPath: p.DbtMetaKeyPath, + TablePrefix: p.TablePrefix, + DbtHexMeta: p.DbtHexMeta, + LowerCaseIdentifiers: p.LowerCaseIdentifiers, } if spec.SourceDialect == "" { spec.SourceDialect = "dbt" diff --git a/cmd/semglot/main.go b/cmd/semglot/main.go index 2cf35b5..f85f796 100644 --- a/cmd/semglot/main.go +++ b/cmd/semglot/main.go @@ -66,14 +66,15 @@ func buildCmd(args []string) int { } if c, ok := emitter.(dialect.Configurable); ok { emitter = c.WithOptions(dialect.Options{ - Database: spec.Database, - Schema: spec.Schema, - ViewSchema: spec.ViewSchema, - Name: spec.ModelName, - Description: spec.Description, - DbtMetaKeyPath: spec.DbtMetaKeyPath, - TablePrefix: spec.TablePrefix, - DbtHexMeta: spec.DbtHexMeta, + Database: spec.Database, + Schema: spec.Schema, + ViewSchema: spec.ViewSchema, + Name: spec.ModelName, + Description: spec.Description, + DbtMetaKeyPath: spec.DbtMetaKeyPath, + TablePrefix: spec.TablePrefix, + DbtHexMeta: spec.DbtHexMeta, + LowerCaseIdentifiers: spec.LowerCaseIdentifiers, }) } diff --git a/dialect/dialect.go b/dialect/dialect.go index 7c97586..7f73868 100644 --- a/dialect/dialect.go +++ b/dialect/dialect.go @@ -56,6 +56,11 @@ type Options struct { // without it, so without this the sync imports models that resolve to // nothing. DbtHexMeta bool + // LowerCaseIdentifiers keeps identifiers as the IR spells them instead of + // upper-casing. Snowflake folds unquoted identifiers up, so upper-case is + // the safe default there; ClickHouse is case-sensitive and would not + // resolve them. + LowerCaseIdentifiers bool } // Configurable is an Emitter that accepts model/view identity options. diff --git a/dialect/supersimple.go b/dialect/supersimple.go index 5e8468b..d9ca1d5 100644 --- a/dialect/supersimple.go +++ b/dialect/supersimple.go @@ -15,9 +15,18 @@ import ( func init() { Register(supersimple{}) } // supersimple emits one supersimple config YAML per model. Zero value usable; -// the build command sets Schema from the profile's schema field. +// the build command sets Schema and TablePrefix from the profile. type supersimple struct { Schema string + // TablePrefix maps a logical table name to its PHYSICAL one. The IR carries + // logical names (fct_orders); a warehouse may materialise them under a + // prefix (ClickHouse marts__fct_orders), and `table:` must address the + // physical object or every query fails to resolve. + TablePrefix string + // LowerCase emits identifiers as-is rather than upper-casing them. The + // upper-case default suits Snowflake, where unquoted identifiers fold up; + // ClickHouse is case-SENSITIVE, so MARTS__FCT_ORDERS does not exist. + LowerCase bool } func (supersimple) Name() string { return "supersimple" } @@ -40,7 +49,7 @@ func isRatioDef(def ir.Expr) bool { // WithOptions lets the CLI pass the profile's schema (other identity fields are unused). func (supersimple) WithOptions(o Options) Emitter { - return supersimple{Schema: o.Schema} + return supersimple{Schema: o.Schema, TablePrefix: o.TablePrefix, LowerCase: o.LowerCaseIdentifiers} } const ssHeader = "# yaml-language-server: $schema=https://assets.supersimple.io/configuration_schema/1.0.0/supersimple_configuration_schema.json\n" @@ -69,9 +78,31 @@ type ssRelation struct { ModelID string `yaml:"model_id"` JoinStrategy ssJoinStrategy `yaml:"join_strategy"` } + +// ssJoinStrategy addresses a join. `join_key` is Supersimple's shorthand for +// "both sides share this column name"; when the foreign key and the primary key +// are named differently the two-sided form is REQUIRED, and the shorthand +// silently addresses the wrong column. +// +// The vendor's own discovery emits the two-sided form (dim_date joins +// fct_orders on join_key_on_base: DATE_DAY / join_key_on_related: ORDER_DATE). +// Emitting only the shorthand made `supersimple validate` reject three +// relations with "references unknown property 'date_day' on related model". type ssJoinStrategy struct { - JoinKey string `yaml:"join_key"` + JoinKey string `yaml:"join_key,omitempty"` + JoinKeyOnBase string `yaml:"join_key_on_base,omitempty"` + JoinKeyOnRelated string `yaml:"join_key_on_related,omitempty"` } + +// joinStrategy renders the shorthand when both sides agree and the explicit +// two-sided form when they differ. +func joinStrategy(base, related string) ssJoinStrategy { + if base == related || related == "" { + return ssJoinStrategy{JoinKey: base} + } + return ssJoinStrategy{JoinKeyOnBase: base, JoinKeyOnRelated: related} +} + type ssMetric struct { Name string `yaml:"name"` ModelID string `yaml:"model_id"` @@ -122,11 +153,30 @@ type ssRelationRef struct { // Emit does not mutate m; it reads m.Notes and accumulates its own degrade // notes locally before writing the combined text to NOTES.md. +// fold renders an identifier in the casing the target warehouse resolves. +// Snowflake folds unquoted identifiers to upper case, so upper is the safe +// default; ClickHouse is case-sensitive and MARTS__FCT_ORDERS does not exist. +func (s supersimple) fold(v string) string { + if s.LowerCase { + return v + } + return strings.ToUpper(v) +} + +// foldAll is fold over a slice. +func (s supersimple) foldAll(vs []string) []string { + out := make([]string, len(vs)) + for i, v := range vs { + out[i] = s.fold(v) + } + return out +} + func (s supersimple) Emit(m *ir.Model, dir string) ([]string, error) { + // An EXPLICIT empty schema is honoured (ClickHouse has a two-part namespace, + // so "MAIN.marts__fct_orders" resolves to nothing); only an unset one + // defaults. schema := s.Schema - if schema == "" { - schema = "MAIN" - } // relationships grouped by parent (Right) table relsByParent := map[string][]ir.Relationship{} for _, r := range m.Relationships { @@ -182,7 +232,7 @@ func (s supersimple) Emit(m *ir.Model, dir string) ([]string, error) { // and relations) and register its simple metrics. for _, t := range m.Tables { t.Metrics = hoist.metricsFor(t) // t is the range's own copy; m is untouched - id := strings.ToUpper(t.Name) + id := s.fold(t.Name) // Prefer the source dialect's own declared physical address. `table:` // holds ONE opaque string, unlike cortexBaseTable's separate // Database/Schema/Table fields, so a genuine table reference is used @@ -192,7 +242,10 @@ func (s supersimple) Emit(m *ir.Model, dir string) ([]string, error) { // permits) can't go here as-is — that falls back to the profile // reconstruction with a warning rather than pasting a query into // `table:` as if it were an address. - table := schema + "." + id + table := s.TablePrefix + id + if schema != "" { + table = schema + "." + table + } if t.Source != "" { if looksLikeQuery(t.Source) { degradeNotes = append(degradeNotes, querySourceWarning("supersimple", t.Name, t.Source)) @@ -203,12 +256,12 @@ func (s supersimple) Emit(m *ir.Model, dir string) ([]string, error) { model := ssModel{ Name: prettify(t.Name), Table: table, - PrimaryKey: upperAll(t.PrimaryKey), + PrimaryKey: s.foldAll(t.PrimaryKey), Description: appendClause(t.Description, synonymClause(t.Synonyms)), Properties: map[string]ssProperty{}, } addProp := func(f ir.Field, typ string) { - col := strings.ToUpper(f.Expr) + col := s.fold(f.Expr) if _, ok := model.Properties[col]; ok { return } @@ -229,9 +282,13 @@ func (s supersimple) Emit(m *ir.Model, dir string) ([]string, error) { } for _, r := range relsByParent[t.Name] { child := r.Left - join := "" + // r.Right is the PARENT (this model, the base); r.Left is the child. + // Column pairs are (Left=child column, Right=parent column), so the + // base key is the parent's and the related key is the child's. + var baseKey, relatedKey string if len(r.Columns) > 0 { - join = strings.ToUpper(r.Columns[0].Right) + baseKey = s.fold(r.Columns[0].Right) + relatedKey = s.fold(r.Columns[0].Left) } if model.Relations == nil { model.Relations = map[string]ssRelation{} @@ -258,8 +315,8 @@ func (s supersimple) Emit(m *ir.Model, dir string) ([]string, error) { } } model.Relations[key] = ssRelation{ - Name: label, Type: "hasMany", ModelID: strings.ToUpper(child), - JoinStrategy: ssJoinStrategy{JoinKey: join}, + Name: label, Type: "hasMany", ModelID: s.fold(child), + JoinStrategy: joinStrategy(baseKey, relatedKey), } } @@ -271,12 +328,12 @@ func (s supersimple) Emit(m *ir.Model, dir string) ([]string, error) { var key string switch a := arg.(type) { case ir.Col: - key = strings.ToUpper(a.Name) + key = s.fold(a.Name) case ir.Raw: // raw.SQL is unqualified; wrap its columns and synthesize a // property keyed by the metric name, guarding against clobbering // a physical column that already owns that key. - key = strings.ToUpper(mt.Name) + key = s.fold(mt.Name) for { if _, taken := model.Properties[key]; !taken { break diff --git a/dialect/supersimple_test.go b/dialect/supersimple_test.go index ea77d88..9e7d31b 100644 --- a/dialect/supersimple_test.go +++ b/dialect/supersimple_test.go @@ -398,3 +398,21 @@ func TestSupersimpleCrossTableRatioEmit(t *testing.T) { t.Fatal("NOTES.md should not exist when nothing is deferred") } } + +// TestSupersimpleTwoSidedJoinKey pins the two-sided join form. `join_key` is +// Supersimple's shorthand for "both sides share this column name"; when the FK +// and PK differ it addresses the wrong column, and `supersimple validate` +// rejected three dim_date relations with "references unknown property +// 'date_day' on related model" because of exactly this. +func TestSupersimpleTwoSidedJoinKey(t *testing.T) { + if got := joinStrategy("date_day", "order_date"); got.JoinKeyOnBase != "date_day" || + got.JoinKeyOnRelated != "order_date" || got.JoinKey != "" { + t.Errorf("differing keys must use the two-sided form; got %+v", got) + } + // Matching keys keep the shorthand, which is what the vendor emits and what + // the other 20 relations already validated with. + if got := joinStrategy("order_id", "order_id"); got.JoinKey != "order_id" || + got.JoinKeyOnBase != "" { + t.Errorf("matching keys should use the shorthand; got %+v", got) + } +} diff --git a/test/models/ecommerce/dbt/supersimple/DIM_CUSTOMER.yaml b/test/models/ecommerce/dbt/supersimple/DIM_CUSTOMER.yaml index b7eb42e..501d6aa 100644 --- a/test/models/ecommerce/dbt/supersimple/DIM_CUSTOMER.yaml +++ b/test/models/ecommerce/dbt/supersimple/DIM_CUSTOMER.yaml @@ -25,7 +25,8 @@ models: type: hasMany model_id: FCT_ORDERS join_strategy: - join_key: CUSTOMER_SK + join_key_on_base: CUSTOMER_SK + join_key_on_related: BILLING_CUSTOMER_SK orders_customer_sk: name: Orders (Customer sk) type: hasMany