From ec0436e5681974525e8e8ccdcf4e54960aba6f5a Mon Sep 17 00:00:00 2001 From: Haoran Li Date: Fri, 17 Jul 2026 16:07:28 +0000 Subject: [PATCH 1/3] Add Databricks Unity Catalog Metric View converter (converters/databricks) Bidirectional, offline converter between Apache Ossie semantic models and Databricks Unity Catalog Metric Views (YAML v1.1), filling the DATABRICKS spoke already listed in converters/README.md. Packaged like the sibling spokes: pyproject.toml (apache-ossie-databricks), an ossie_databricks package under src/, ASF license headers, and a tests/ suite (example-based + Hypothesis property-based round-trip; 74 tests). PyYAML is the only runtime dependency. Co-authored-by: jackstein21 <82542300+jackstein21@users.noreply.github.com> --- converters/databricks/README.md | 124 ++++ converters/databricks/pyproject.toml | 52 ++ .../src/ossie_databricks/__init__.py | 32 + .../src/ossie_databricks/_common.py | 288 ++++++++ .../databricks/src/ossie_databricks/cli.py | 78 ++ .../ossie_databricks/metric_view_to_ossie.py | 366 ++++++++++ .../ossie_databricks/ossie_to_metric_view.py | 660 +++++++++++++++++ .../databricks/tests/_roundtrip_helpers.py | 355 ++++++++++ converters/databricks/tests/_util.py | 76 ++ converters/databricks/tests/conftest.py | 23 + .../tests/fixtures/fixtureA_metric_view.yaml | 53 ++ .../tests/fixtures/fixtureA_ossie.yaml | 79 +++ .../tests/fixtures/fixtureB_metric_view.yaml | 51 ++ .../tests/fixtures/fixtureB_ossie.yaml | 72 ++ .../tests/fixtures/tpcds_metric_view.yaml | 67 ++ .../tests/fixtures/tpcds_ossie.yaml | 89 +++ .../tests/test_metric_view_to_ossie.py | 347 +++++++++ .../tests/test_ossie_to_metric_view.py | 670 ++++++++++++++++++ converters/databricks/tests/test_roundtrip.py | 95 +++ .../tests/test_roundtrip_properties.py | 107 +++ 20 files changed, 3684 insertions(+) create mode 100644 converters/databricks/README.md create mode 100644 converters/databricks/pyproject.toml create mode 100644 converters/databricks/src/ossie_databricks/__init__.py create mode 100644 converters/databricks/src/ossie_databricks/_common.py create mode 100644 converters/databricks/src/ossie_databricks/cli.py create mode 100644 converters/databricks/src/ossie_databricks/metric_view_to_ossie.py create mode 100644 converters/databricks/src/ossie_databricks/ossie_to_metric_view.py create mode 100644 converters/databricks/tests/_roundtrip_helpers.py create mode 100644 converters/databricks/tests/_util.py create mode 100644 converters/databricks/tests/conftest.py create mode 100644 converters/databricks/tests/fixtures/fixtureA_metric_view.yaml create mode 100644 converters/databricks/tests/fixtures/fixtureA_ossie.yaml create mode 100644 converters/databricks/tests/fixtures/fixtureB_metric_view.yaml create mode 100644 converters/databricks/tests/fixtures/fixtureB_ossie.yaml create mode 100644 converters/databricks/tests/fixtures/tpcds_metric_view.yaml create mode 100644 converters/databricks/tests/fixtures/tpcds_ossie.yaml create mode 100644 converters/databricks/tests/test_metric_view_to_ossie.py create mode 100644 converters/databricks/tests/test_ossie_to_metric_view.py create mode 100644 converters/databricks/tests/test_roundtrip.py create mode 100644 converters/databricks/tests/test_roundtrip_properties.py diff --git a/converters/databricks/README.md b/converters/databricks/README.md new file mode 100644 index 00000000..e2fe6ccb --- /dev/null +++ b/converters/databricks/README.md @@ -0,0 +1,124 @@ + + +# Apache Ossie Databricks Converter + +Bidirectional, offline conversion between an [Apache Ossie](https://github.com/apache/ossie) +semantic model and a Databricks +[Unity Catalog Metric View](https://docs.databricks.com/aws/en/metric-views/) (YAML +`1.1`). No Databricks connection required. + +- **Export** (`ossie-databricks export`): Apache Ossie -> Metric View (one fact + `source` with a nested `joins` tree and a flat `dimensions` list). +- **Import** (`ossie-databricks import`): Metric View -> Apache Ossie. Metric View features Apache Ossie has + no native field for are preserved in `custom_extensions[DATABRICKS]`, so + `MV -> Apache Ossie -> MV` is lossless. + +On **export** (Apache Ossie -> Metric View), Apache Ossie features with no Metric View slot -- relationship +`ai_context`, `dimension.is_time`, non-`DATABRICKS`/`ANSI_SQL` dialects, foreign-vendor +`custom_extensions` -- are **dropped with a warning**. On **import** (Metric View -> Apache Ossie), +Metric View only features (filter, window, format, rely, ...) are instead **preserved** in +`custom_extensions[DATABRICKS]`, so `MV -> Apache Ossie -> MV` is lossless. Any input that breaks a +[requirement](#requirements) **raises a `ConversionError`** -- the converter never +silently drops a field or produces an invalid result. + +## Installation + +```bash +pip install apache-ossie-databricks # once published to PyPI +# or, from a checkout of this directory: +pip install -e . +``` + +The only runtime dependency is `PyYAML`. Python 3.11+. + +## Usage + +### Command line + +```bash +ossie-databricks export -i model.yaml -o view.yaml [--source orders] # Apache Ossie -> Metric View +ossie-databricks import -i view.yaml -o model.yaml [--name my_model] # Metric View -> Apache Ossie +``` + +With no `-o`, output goes to stdout. `--source` (export) picks the fact/grain (default: +the FK-sink dataset; naming a coarser-grain dataset produces `one_to_many` joins); +`--name` (import) sets the Apache Ossie model name (default: the source's last identifier). + +### Python API + +```python +from ossie_databricks import convert_ossie_to_metric_view, convert_metric_view_to_ossie + +metric_view_yaml = convert_ossie_to_metric_view(ossie_yaml_str) # optionally choose the fact/grain, e.g. (ossie_yaml_str, source="orders") +ossie_yaml = convert_metric_view_to_ossie(metric_view_yaml_str, model_name="sales") +``` + +## Mapping + +Each row maps in both directions; the **Notes** flag where a behavior is specific to +**export** (Apache Ossie -> Metric View) or **import** (Metric View -> Apache Ossie). + +| Apache Ossie | Metric View (v1.1) | Notes | +|---|---|---| +| `semantic_model.description` | `comment` | Model-level description only. | +| root dataset | `source` | The fact/grain. | +| other `datasets` | nested `joins[]` | Export: the relationship graph is reassembled into the join tree; a dataset reached by two paths (a diamond) fans out into one aliased join per path. | +| `relationship` `from_columns`/`to_columns` | join `on` (differing names) / `using` (shared names) | Decomposed into columns on import; rebuilt into `on`/`using` on export. | +| `relationship.from`/`to` direction | join `cardinality` | Export: source on the many (`from`) side -> `many_to_one`; on the one (`to`) side -> `one_to_many`. | +| `dataset.primary_key` / `unique_keys` | join `rely.at_most_one_match` | Both directions: export sets `at_most_one_match` when a key covers the join columns; import recovers a `unique_keys` from it. | +| `dataset.fields[]` | `dimensions[]` | Export: fields flatten into one list and a joined column is qualified by its full join path (`customer.c_name`; `customer.region.r_name` when nested). | +| `field.expression.dialects[]` | `expr` | Export: prefer the `DATABRICKS` dialect, else `ANSI_SQL`. | +| `metrics[]` | `measures[]` | Export: fact columns are referenced bare (`SUM(amount)`). | +| `field.label` | `display_name` | | +| `field` / `metric` `description` | `comment` | | +| `ai_context.synonyms` | `synonyms` | | +| `custom_extensions[DATABRICKS]` | `filter`, `window`, `format`, `rely`, `materialization` | Import stashes Metric View only features here; export restores them -- keeping `MV -> Apache Ossie -> MV` lossless. | + +## Requirements + +Conversion raises a `ConversionError` (rather than guessing or emitting something +invalid) when an input breaks one of these: + +- the Metric View `version` is not `1.1`; +- a `source` is not a 3-part `catalog.schema.table` name or a `SELECT`/`WITH` subquery; +- the relationship graph is not acyclic and resolvable to a single fact -- a cycle, or + multiple candidate facts without `--source`, is rejected (a diamond is allowed and + fanned out); +- a join has no condition (a cross join has no Apache Ossie relationship form); +- a join condition is non-equi or otherwise can't be decomposed into equi-join columns + (Apache Ossie relationships are equi-joins, so the join has no Apache Ossie representation); +- the input YAML is malformed. + +## Development + +```bash +pip install -e ".[dev]" +python3 -m pytest tests/ +``` + +Example-based unit tests plus Hypothesis property-based round-trip tests +(`test_roundtrip_properties.py`, which skip if `hypothesis` is not installed). + +## Future effort + +Both the Apache Ossie specification and the Databricks Unity Catalog Metric View YAML are still +evolving. As either side adds or changes fields, this converter will be updated to track +them -- extending the mapping and coverage in both directions to keep the conversion +current and to support as much as each format allows over time. diff --git a/converters/databricks/pyproject.toml b/converters/databricks/pyproject.toml new file mode 100644 index 00000000..d4ed3511 --- /dev/null +++ b/converters/databricks/pyproject.toml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "apache-ossie-databricks" +version = "0.2.0.dev0" +description = "Databricks Unity Catalog Metric View <> Apache Ossie converter" +requires-python = ">=3.11" +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", +] +dependencies = [ + "PyYAML>=6.0", +] + +[project.license] +text = "Apache-2.0" + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "hypothesis>=6.0", +] + +[project.scripts] +ossie-databricks = "ossie_databricks.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_databricks"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] diff --git a/converters/databricks/src/ossie_databricks/__init__.py b/converters/databricks/src/ossie_databricks/__init__.py new file mode 100644 index 00000000..379a2aec --- /dev/null +++ b/converters/databricks/src/ossie_databricks/__init__.py @@ -0,0 +1,32 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Bidirectional converter between Apache Ossie semantic models and Databricks Unity Catalog +Metric Views (YAML v1.1). Pure offline string-in / string-out transforms. + + from ossie_databricks import convert_ossie_to_metric_view, convert_metric_view_to_ossie +""" + +from ._common import ConversionError +from .metric_view_to_ossie import convert_metric_view_to_ossie +from .ossie_to_metric_view import convert_ossie_to_metric_view + +__all__ = [ + "ConversionError", + "convert_metric_view_to_ossie", + "convert_ossie_to_metric_view", +] diff --git a/converters/databricks/src/ossie_databricks/_common.py b/converters/databricks/src/ossie_databricks/_common.py new file mode 100644 index 00000000..4f17a5b3 --- /dev/null +++ b/converters/databricks/src/ossie_databricks/_common.py @@ -0,0 +1,288 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared helpers for the Apache Ossie <-> Databricks Metric View converters. + +Both directions are pure offline YAML transforms. The only cross-cutting concerns +live here: version constants, the dialect preference order, the `custom_extensions` +stash protocol, and small SQL-string helpers. +""" + +import json +import re + +import yaml + +# Apache Ossie semantic model spec version this converter targets (see core-spec). +# +# NOTE: this is an exact-match check (see convert_ossie_to_metric_view). Unlike the +# dbt converter, this spoke intentionally has no `apache-ossie` package dependency, so +# nothing updates this automatically -- it MUST be bumped in lockstep with the +# `version` in `core-spec/` whenever the spec version moves, or the converter will +# reject otherwise-valid Apache Ossie files. +OSSIE_VERSION = "0.2.0.dev0" + +# Databricks Unity Catalog Metric View YAML version. Only 1.1 supports joins, +# per-column comments, synonyms, and the format/window/parameters surface. +MV_VERSION = "1.1" + +# Vendor id used for the `custom_extensions` stash and for dialect selection. +VENDOR = "DATABRICKS" + +# Expression dialects this converter understands, in preference order. +DIALECT_DATABRICKS = "DATABRICKS" +DIALECT_ANSI = "ANSI_SQL" + +# Metric Views cap the number of synonyms per column. +SYNONYM_LIMIT = 10 + +# Bump when the shape of a stashed `data` blob changes. +STASH_VERSION = 1 + +# Metric View join cardinalities (the only two values v1.1 defines). Apache Ossie has no +# cardinality field; the value is implied by relationship direction -- `from` is the +# many side, `to` is the one side -- so the converter derives it from / writes it +# into the from/to orientation rather than relying on a dedicated field. +CARD_MANY_TO_ONE = "many_to_one" +CARD_ONE_TO_MANY = "one_to_many" + +# Model-level stash key recording which dataset was the Metric View `source` (its +# grain). Needed only when a one_to_many join puts the source on a relationship's +# `to` side, where the natural FK-sink heuristic would otherwise pick the wrong +# fact on re-export. Absent for plain many-to-one stars, so they stay clean. +STASH_SOURCE_KEY = "source_dataset" + +# A bare SQL identifier (single column reference), e.g. `c_name`. Used to decide +# whether an expression can be safely alias-prefixed on export / de-prefixed on +# import. +_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +class ConversionError(Exception): + """Raised when an input cannot be converted.""" + + +def require(obj, key, what): + """Return `obj[key]`, or raise a clean ConversionError if it's missing/empty -- so + malformed input surfaces as an error message rather than a raw KeyError traceback. + + Presence is tested by key (not truthiness), so a legitimately falsy value such as + `0` or `False` is returned; a missing key, a null, or an empty/whitespace string is + rejected. + """ + if not isinstance(obj, dict) or key not in obj or obj[key] is None: + raise ConversionError(f"{what} is missing required '{key}'") + value = obj[key] + if isinstance(value, str) and not value.strip(): + raise ConversionError(f"{what} has an empty '{key}'") + return value + + +def require_str(obj, key, what): + """Like require(), but also enforce the value is a string -- so a non-string scalar + (e.g. a YAML number for a name or expression) raises a clean ConversionError instead + of crashing later in a string operation.""" + value = require(obj, key, what) + if not isinstance(value, str): + raise ConversionError( + f"{what}: '{key}' must be a string, got {type(value).__name__}") + return value + + +# YAML 1.1 (PyYAML's default) treats bare on/off/yes/no/y/n as booleans, so a metric +# view join's `on:` key would parse as the boolean True and silently lose the join +# condition. Databricks (Jackson) uses YAML 1.2 booleans (only true/false). The Loader +# below uses 1.2 semantics, so it reads DBR's bare `on:` (and any "on"/"off" value) as a +# string. The Dumper additionally force-quotes those tokens on output (see below), so the +# YAML it emits round-trips the same way through a YAML 1.1 reader too (e.g. stock +# yaml.safe_load) rather than turning an "on"/"off" synonym/label into a boolean. +class _Yaml12Loader(yaml.SafeLoader): + """SafeLoader with YAML 1.2 boolean semantics.""" + + +class _Yaml12Dumper(yaml.SafeDumper): + """SafeDumper with YAML 1.2 boolean semantics.""" + + +_YAML12_BOOL = re.compile(r"^(?:true|True|TRUE|false|False|FALSE)$") +for _cls in (_Yaml12Loader, _Yaml12Dumper): + # Drop the YAML 1.1 bool resolver (yes/no/on/off/y/n) and re-add a 1.2 one. + _cls.yaml_implicit_resolvers = { + ch: [(tag, rx) for (tag, rx) in resolvers if tag != "tag:yaml.org,2002:bool"] + for ch, resolvers in _cls.yaml_implicit_resolvers.items() + } + _cls.add_implicit_resolver("tag:yaml.org,2002:bool", _YAML12_BOOL, list("tTfF")) + + +# Force-quote string scalars that a YAML 1.1 reader would otherwise interpret as booleans +# (yes/no/on/off/y/n/true/false, any case). Number- and null-like strings are already +# quoted by PyYAML's surviving resolvers; only these bool tokens need it. Without this, a +# synonym/label/comment like "on" emits bare and a 1.1 consumer reads it back as `True`. +_YAML11_BOOL_STRS = frozenset( + variant + for word in ("y", "n", "yes", "no", "on", "off", "true", "false") + for variant in (word, word.capitalize(), word.upper()) +) + + +def _represent_str(dumper, data): + style = "'" if data in _YAML11_BOOL_STRS else None + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style=style) + + +_Yaml12Dumper.add_representer(str, _represent_str) + + +def load_yaml(text): + """Parse YAML with 1.2 boolean semantics, so a join `on:` key stays the string + `on` rather than becoming the boolean True. A syntax error is surfaced as a + ConversionError so callers (and the CLI) get a clean message, not a raw traceback.""" + try: + return yaml.load(text, Loader=_Yaml12Loader) + except yaml.YAMLError as e: + raise ConversionError(f"Invalid YAML: {e}") from e + + +def dump_yaml(obj): + """Serialize to YAML with 1.2 boolean semantics. The bool-like token `on` -- whether + a join condition key or an "on"/"off"/"yes"/... string value -- is force-quoted as + `'on'` by the str representer (see `_represent_str`), so a YAML 1.1 reader of this + output reads it as the string, not the boolean True. Databricks' Jackson (1.2) parser + reads the quoted key/value correctly too.""" + return yaml.dump(obj, Dumper=_Yaml12Dumper, sort_keys=False, default_flow_style=False) + + +def is_simple_identifier(expr): + """True if `expr` is a single bare column reference (no operators/functions). + + A non-string input is simply not an identifier (returns False) rather than raising.""" + return isinstance(expr, str) and bool(_IDENTIFIER_RE.match(expr.strip())) + + +def read_stash(obj): + """Return the DATABRICKS stash dict on an Apache Ossie object, or {} if absent. + + The `_v` version marker is stripped from the returned dict. + """ + for ext in (obj or {}).get("custom_extensions") or []: + if ext.get("vendor_name") == VENDOR: + try: + data = json.loads(ext.get("data") or "{}") + except json.JSONDecodeError as e: + raise ConversionError( + f"DATABRICKS custom_extensions data is not valid JSON: {e}") from e + data.pop("_v", None) + return data + return {} + + +def write_stash(obj, data): + """Attach a DATABRICKS `custom_extensions` entry holding `data` (a dict). + + No-op when `data` is empty, so hand-authored Apache Ossie stays clean. Merges into an + existing DATABRICKS entry if one is already present. + """ + if not data: + return + payload = {"_v": STASH_VERSION} + payload.update(data) + blob = json.dumps(payload) + exts = obj.setdefault("custom_extensions", []) + for ext in exts: + if ext.get("vendor_name") == VENDOR: + ext["data"] = blob + return + exts.append({"vendor_name": VENDOR, "data": blob}) + + +def foreign_vendor_extensions(obj): + """Return non-DATABRICKS custom_extensions (dropped on export, with a warning).""" + return [ + ext + for ext in (obj or {}).get("custom_extensions") or [] + if ext.get("vendor_name") != VENDOR + ] + + +def pick_expression(ossie_expression): + """Choose the SQL string for an Apache Ossie expression: DATABRICKS, else ANSI_SQL. + + Returns None if neither dialect is present (the caller warns and skips). Does + not warn about other dialects here -- only the absence of a usable one matters. + """ + dialects = { + d.get("dialect"): d.get("expression") + for d in (ossie_expression or {}).get("dialects") or [] + } + expr = dialects.get(DIALECT_DATABRICKS) or dialects.get(DIALECT_ANSI) + if expr is not None and not isinstance(expr, str): + raise ConversionError( + f"expression must be a string, got {type(expr).__name__}") + return expr + + +def synonyms_of(ai_context): + """Extract the synonyms list from an Apache Ossie ai_context (object form only).""" + if isinstance(ai_context, dict): + return list(ai_context.get("synonyms") or []) + return [] + + +def merge_description(description, ai_context): + """Fold a string-form ai_context into a description. + + The Apache Ossie schema allows ai_context to be either a string or an object. A string + has no Metric View home of its own, so it is appended to the description + (which maps to `comment`). Object-form ai_context is handled separately + (synonyms map natively; instructions/examples are dropped). + """ + if isinstance(ai_context, str) and ai_context.strip(): + return f"{description}\n{ai_context}" if description else ai_context + return description + + +def validate_source(source, dataset_name): + """Validate and normalize a dataset source for a Metric View. + + Accepts a 3-part `catalog.schema.table` identifier or a `SELECT`/`WITH` + subquery. Raises ConversionError otherwise. + """ + if not source or not str(source).strip(): + raise ConversionError(f"Dataset '{dataset_name}': missing/empty 'source'") + s = str(source).strip() + # A SELECT/WITH subquery source. `\b` after the keyword matches `WITH(...)` (no + # space) too, but not an identifier like `WITHHELD`. + if re.match(r"(?i)(select|with)\b", s): + return s + # Exactly 3 parts, each a non-empty token with no whitespace -- so `.sch.tbl`, + # `cat..tbl`, `cat.sch.`, and `cat . sch . tbl` are all rejected (an empty or + # space-laden part is not a valid catalog/schema/table identifier). + parts = s.split(".") + if len(parts) == 3 and all(p and not any(ch.isspace() for ch in p) for p in parts): + return s + raise ConversionError( + f"Dataset '{dataset_name}': source '{source}' must be a 3-part " + f"catalog.schema.table identifier or a SELECT/WITH subquery" + ) + + +def last_identifier(source): + """Last dotted part of a table reference, e.g. `samples.tpch.lineitem` -> `lineitem`. + + Coerces to str so a malformed (non-string) source doesn't crash here -- it gets a + clean error from validate_source instead.""" + return str(source).strip().split(".")[-1].strip("`") if source else source diff --git a/converters/databricks/src/ossie_databricks/cli.py b/converters/databricks/src/ossie_databricks/cli.py new file mode 100644 index 00000000..a3751ba4 --- /dev/null +++ b/converters/databricks/src/ossie_databricks/cli.py @@ -0,0 +1,78 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Command-line interface for the Apache Ossie <-> Databricks Metric View converter. + + ossie-databricks export -i model.yaml [-o view.yaml] [--source orders] + ossie-databricks import -i view.yaml [-o model.yaml] [--name my_model] + +`export` converts an Apache Ossie semantic model to a Databricks Metric View; `import` does the +reverse. With no `-o`, the result is written to stdout. Conversions that drop +information emit warnings to stderr. +""" + +import argparse +import sys + +from ._common import ConversionError +from .metric_view_to_ossie import convert_metric_view_to_ossie +from .ossie_to_metric_view import convert_ossie_to_metric_view + + +def _build_parser(): + parser = argparse.ArgumentParser(prog="ossie-databricks", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command") + sub.required = True # set as attribute (the add_subparsers kwarg is 3.7+) + + exp = sub.add_parser("export", help="Apache Ossie semantic model -> Databricks Metric View YAML") + exp.add_argument("-i", "--input", required=True, help="Apache Ossie YAML file") + exp.add_argument("-o", "--output", help="output Metric View YAML (default: stdout)") + exp.add_argument("-s", "--source", + help="dataset to use as the fact/grain (default: the FK-sink dataset); " + "naming a coarser-grain dataset unlocks one_to_many joins") + + imp = sub.add_parser("import", help="Databricks Metric View YAML -> Apache Ossie semantic model") + imp.add_argument("-i", "--input", required=True, help="Metric View YAML file") + imp.add_argument("-o", "--output", help="output Apache Ossie YAML (default: stdout)") + imp.add_argument("--name", help="Apache Ossie model name (default: derived from the source)") + return parser + + +def main(argv=None): + args = _build_parser().parse_args(argv) + try: + with open(args.input) as fh: + text = fh.read() + if args.command == "export": + out = convert_ossie_to_metric_view(text, source=args.source) + else: + out = convert_metric_view_to_ossie(text, model_name=args.name) + except (ConversionError, OSError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + + if args.output: + with open(args.output, "w") as fh: + fh.write(out) + else: + sys.stdout.write(out) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/converters/databricks/src/ossie_databricks/metric_view_to_ossie.py b/converters/databricks/src/ossie_databricks/metric_view_to_ossie.py new file mode 100644 index 00000000..75cdb752 --- /dev/null +++ b/converters/databricks/src/ossie_databricks/metric_view_to_ossie.py @@ -0,0 +1,366 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert a Databricks Unity Catalog Metric View (v1.1) to an Apache Ossie semantic model. + +Pure offline conversion. Accepts a Metric View (one `source` with a nested `joins` +tree). Metric View features Apache Ossie has no native field for -- filter, window, format, +rely, cardinality, parameters, materialization -- are preserved in +`custom_extensions[DATABRICKS]` so that converting back reproduces the original view. +A join condition an Apache Ossie relationship cannot represent (a non-equi or cross join) is +rejected, not stashed. See README.md. + +Usage (CLI): + ossie-databricks import -i view.yaml [-o model.yaml] [--name NAME] +""" + +import re +import warnings + +from ._common import ( + CARD_MANY_TO_ONE, + CARD_ONE_TO_MANY, + ConversionError, + DIALECT_DATABRICKS, + MV_VERSION, + OSSIE_VERSION, + STASH_SOURCE_KEY, + dump_yaml, + is_simple_identifier, + last_identifier, + load_yaml, + require, + require_str, + validate_source, + write_stash, +) + +# Metric View fields with no native Apache Ossie home -> stashed verbatim. +_MODEL_STASH_KEYS = ("filter", "parameters", "materialization") +_JOIN_STASH_KEYS = ("rely", "cardinality") +_COLUMN_STASH_KEYS = ("format", "window") + + +def _warn(scope, msg): + warnings.warn(f"[{scope}] {msg}") + + +# Operators that mean a join condition is NOT a simple equi-join (so it cannot be +# expressed as from_columns/to_columns, and the join is rejected on import). +_NON_EQUI_RE = re.compile(r"[<>!]=|<>|[<>]") + + +def _is_wildcard(col): + """A wildcard column (`expr: source.*`) is projected without a `name`; Apache Ossie has + no representation for it. Detected by the absence of a `name` key (a named column, + even one whose name is falsy like `0` or whose expression contains `*` as + multiplication, is not a wildcard).""" + return "name" not in col + + +def convert_metric_view_to_ossie(mv_yaml_str, model_name=None): + """Parse Metric View v1.1 YAML and return Apache Ossie semantic model YAML (string).""" + # load_yaml uses YAML 1.2 booleans, so a join `on:` key stays the string "on" + # (PyYAML's default 1.1 would parse it as the boolean True and drop the condition). + view = load_yaml(mv_yaml_str) + if not isinstance(view, dict): + raise ConversionError("Invalid Metric View YAML: expected a mapping at the root") + + version = str(view.get("version", "")) + if version != MV_VERSION: + raise ConversionError( + f"Unsupported Metric View version '{version}'. This converter targets " + f"v{MV_VERSION} only." + ) + + model = _convert_view(view, model_name) + return dump_yaml({"version": OSSIE_VERSION, "semantic_model": [model]}) + + +def _convert_view(view, model_name): + source = view.get("source") + if not source: + raise ConversionError("Metric View is missing required 'source'") + + # Derive the model/fact name from a table source's last identifier. A SELECT/WITH + # subquery source has no meaningful table name, so use a stable default instead of + # slicing a token out of the SQL text (override with --name). + is_sql = str(source).strip().split(None, 1)[0].upper() in ("SELECT", "WITH") + last_id = last_identifier(source) + fact_name = model_name or ( + last_id if (not is_sql and last_id and is_simple_identifier(last_id)) + else "metric_view" + ) + # Validate the source shape up front (3-part table or SELECT/WITH), mirroring the + # exporter -- so a malformed source fails here with a clean error instead of passing + # silently through to Apache Ossie and only erroring on a later re-export. + validate_source(source, fact_name) + + datasets = [{"name": fact_name, "source": source}] + relationships = [] + alias_to_dataset = {"source": fact_name, fact_name: fact_name} + # Names are compared case-insensitively (DBR identifiers are case-insensitive), so a + # `Fact`/`fact` or `dim`/`Dim` collision is caught here instead of producing two + # datasets DBR would reject on re-export. + seen_names = {fact_name.strip().lower()} + + # Walk the join tree, emitting one dataset + one relationship per join. + def walk(parent_name, parent_alias, joins): + for join in joins or []: + child = require_str(join, "name", "join") + # `source` is the reserved fact qualifier; reject any casing. + if child.strip().lower() == "source": + raise ConversionError( + "Join name 'source' is reserved for the fact source; rename the join." + ) + if child.strip().lower() in seen_names: + raise ConversionError( + f"Duplicate dataset/join name '{child}'; Metric View join names " + f"and the source must be distinct (case-insensitively)." + ) + seen_names.add(child.strip().lower()) + child_ds = {"name": child, "source": require_str(join, "source", f"join '{child}'")} + datasets.append(child_ds) + alias_to_dataset[child] = child + rel = _convert_join(join, parent_name, parent_alias, child) + relationships.append(rel) + # rely.at_most_one_match asserts the join key is unique on the joined + # (one) side, so record those columns as a unique key on the child dataset + # -- recovering key info Apache Ossie would otherwise lack. Only a many_to_one join + # has the child on the `to` side (one_to_many flips it), so this naturally + # skips one_to_many joins. + if (rel["to"] == child and rel.get("to_columns") + and (join.get("rely") or {}).get("at_most_one_match")): + child_ds["unique_keys"] = [list(rel["to_columns"])] + walk(child, child, join.get("joins")) + + walk(fact_name, "source", view.get("joins")) + + # Dimensions -> fields, grouped onto the dataset their alias points at. + # `fields` is a v1.1 alias for `dimensions` (and the form the DBR docs use), + # so accept either key. + if view.get("dimensions") and view.get("fields"): + _warn("view", "both 'dimensions' and 'fields' are set; 'fields' is a v1.1 alias " + "for 'dimensions', so the 'fields' list is ignored") + fields_by_dataset = {d["name"]: [] for d in datasets} + for dim in (view.get("dimensions") or view.get("fields") or []): + if _is_wildcard(dim): + _warn("dimension", f"wildcard column '{dim.get('expr')}' has no Apache Ossie field " + f"representation; skipped") + continue + ds_name, field = _convert_dimension(dim, alias_to_dataset, fact_name) + fields_by_dataset[ds_name].append(field) + for d in datasets: + flds = fields_by_dataset[d["name"]] + if flds: + d["fields"] = flds + + metrics = [] + for m in view.get("measures", []) or []: + if _is_wildcard(m): + _warn("measure", f"wildcard measure '{m.get('expr')}' has no Apache Ossie metric " + f"representation; skipped") + continue + metrics.append(_convert_measure(m, fact_name)) + + model = {"name": fact_name} + if view.get("comment"): + model["description"] = view["comment"] + model["datasets"] = datasets + if relationships: + model["relationships"] = relationships + if metrics: + model["metrics"] = metrics + + # Model-level stash: filter / parameters / materialization, plus the source + # dataset's identity when a one_to_many join is present -- without it the + # exporter's FK-sink heuristic would re-root at the wrong (many-side) dataset. + model_stash = {k: view[k] for k in _MODEL_STASH_KEYS if k in view} + if _has_otm(view.get("joins")): + model_stash[STASH_SOURCE_KEY] = fact_name + write_stash(model, model_stash) + return model + + +def _has_otm(joins): + """True if any join in the (nested) tree is one_to_many.""" + for j in joins or []: + if str(j.get("cardinality") or "").lower() == CARD_ONE_TO_MANY: + return True + if _has_otm(j.get("joins")): + return True + return False + + +def _convert_join(join, parent_name, parent_alias, child): + if not join.get("using") and not join.get("on"): + raise ConversionError( + f"Join '{child}' has no join condition (empty or absent 'on'/'using'); " + f"condition-less (cross) joins have no Apache Ossie relationship representation." + ) + # _decompose_on returns (parent-side columns, child-side columns). + parent_cols, child_cols, raw_on = _decompose_on(join, parent_alias, parent_name, child) + if raw_on is not None: + raise ConversionError( + f"Join '{child}' uses a non-equi or unsupported join condition ('on: {raw_on}') " + f"that an Apache Ossie relationship cannot represent. Apache Ossie joins are equi-joins of simple " + f"`alias.column` pairs (the fact side may be qualified with `source`, the source " + f"table name, or left bare). Cannot import." + ) + if "using" in join and not parent_cols: + # `using: [cols]` -> equal lists on both sides. Two distinct list objects, so the + # emitted YAML doesn't serialize one as an anchor/alias of the other. + parent_cols, child_cols = list(join["using"]), list(join["using"]) + + # Cardinality (default many_to_one) decides the Apache Ossie direction, since `from` is + # always the many side. many_to_one -> parent is many (from=parent); one_to_many + # -> the joined child is many (from=child, to=parent). Compared case-insensitively. + cardinality = join.get("cardinality") or CARD_MANY_TO_ONE + if str(cardinality).lower() == CARD_ONE_TO_MANY: + rel = {"name": f"{child}_to_{parent_name}", "from": child, "to": parent_name, + "from_columns": child_cols, "to_columns": parent_cols} + else: + rel = {"name": f"{parent_name}_to_{child}", "from": parent_name, "to": child, + "from_columns": parent_cols, "to_columns": child_cols} + + stash = {k: join[k] for k in _JOIN_STASH_KEYS if k in join} + write_stash(rel, stash) + return rel + + +def _decompose_on(join, parent_alias, parent_name, child_alias): + """Return (from_columns, to_columns, raw_on). + + raw_on is None when `on` decomposes cleanly into equi-join column pairs; it + holds the original string otherwise (a non-equi/complex condition the caller + rejects). `using` short-circuits to empty columns here and is handled by the caller. + + The child side of a clause is always referenced by its join name. The parent side + may be referenced by its alias (`source` at the top level, else the parent join + name) or by the parent dataset's own name. A bare (unqualified) operand is read as + the fact only at the top level; inside a nested join it is ambiguous (parent vs. + fact) and is rejected rather than guessed. + """ + if "using" in join: + return [], [], None + on = join.get("on") + if not on: + return [], [], None + + parent_aliases = {parent_alias, parent_name} + from_cols, to_cols = [], [] + for clause in re.split(r"\s+AND\s+", on, flags=re.IGNORECASE): + if _NON_EQUI_RE.search(clause): # >=, <=, !=, <>, <, > -> not an equi-join + return [], [], on + m = re.match(r"^\s*(.+?)\s*=\s*(.+?)\s*$", clause) + if not m: + return [], [], on + la, lc = _split_alias(m.group(1)) + ra, rc = _split_alias(m.group(2)) + # Both sides must be `.` (or a bare fact column). If an + # operand is a SQL fragment (e.g. `dim.b + 1`, or the trailing half of an + # OR/`=`-laden clause), `_split_alias` yields a non-identifier "column" -- + # that can't be an FK column pair, so stash the whole condition verbatim. + if not (is_simple_identifier(lc) and is_simple_identifier(rc)): + return [], [], on + # The parent side: its alias or the source table name. A *bare* (unqualified) + # operand is read as the fact only at the top level (`source`); inside a nested + # join an unqualified column is ambiguous (parent vs. fact), so don't guess -- + # leave it for rejection rather than silently attributing it to the parent. + allow_bare = parent_alias == "source" + l_parent = la in parent_aliases or (la is None and allow_bare) + r_parent = ra in parent_aliases or (ra is None and allow_bare) + if la == child_alias and r_parent: + from_cols.append(rc) + to_cols.append(lc) + elif ra == child_alias and l_parent: + from_cols.append(lc) + to_cols.append(rc) + else: + return [], [], on + return from_cols, to_cols, None + + +def _split_alias(operand): + """`customer.c_custkey` -> ('customer', 'c_custkey'); `x` -> (None, 'x').""" + operand = operand.strip() + if "." in operand: + alias, col = operand.split(".", 1) + return alias.strip(), col.strip() + return None, operand + + +def _convert_dimension(dim, alias_to_dataset, fact_name): + name = require_str(dim, "name", "dimension") + expr = require_str(dim, "expr", f"dimension '{name}'") + ds_name, ossie_expr = _resolve_column(expr, alias_to_dataset, fact_name) + + field = { + "name": name, + "expression": {"dialects": [{"dialect": DIALECT_DATABRICKS, "expression": ossie_expr}]}, + } + if dim.get("comment"): + field["description"] = dim["comment"] + if dim.get("display_name"): + field["label"] = dim["display_name"] + if dim.get("synonyms"): + field["ai_context"] = {"synonyms": list(dim["synonyms"])} + write_stash(field, {k: dim[k] for k in _COLUMN_STASH_KEYS if k in dim}) + return ds_name, field + + +def _resolve_column(expr, alias_to_dataset, fact_name): + """Map a dimension expression to (dataset_name, de-aliased_expression). + + A leading join path of known aliases files the field under the **deepest** one and + de-qualifies a bare column -- mirroring the exporter's nested-join qualification: + `partsupp.supplier.nation.n_name` -> `n_name` on `nation`; `customer.c_name` -> + `c_name` on `customer`; `source.x` -> `x` on the fact. A complex expression is filed + under that dataset but kept verbatim. A bare column (no leading alias) is a fact column. + """ + segments = [s.strip() for s in expr.split(".")] + ds = None + i = 0 + # Consume leading segments that are known join/source aliases (but never the last + # segment -- that is the column). The deepest alias is the owning dataset. + while i < len(segments) - 1 and segments[i] in alias_to_dataset: + ds = alias_to_dataset[segments[i]] + i += 1 + if ds is None: + return fact_name, expr + rest = ".".join(segments[i:]) + return (ds, rest) if is_simple_identifier(rest) else (ds, expr) + + +def _convert_measure(measure, fact_name): + name = require_str(measure, "name", "measure") + # Mirror the exporter's word-boundary handling: a `source.` fact qualifier (if + # an author used one) maps back to the fact dataset name. The replacement is a + # lambda so `fact_name` is inserted literally (a `--name` containing backslashes + # is not interpreted as a regex backreference). + raw_expr = require_str(measure, "expr", f"measure '{name}'") + expr = re.sub(r"\bsource\.", lambda _m: f"{fact_name}.", raw_expr) + metric = { + "name": name, + "expression": {"dialects": [{"dialect": DIALECT_DATABRICKS, "expression": expr}]}, + } + if measure.get("comment"): + metric["description"] = measure["comment"] + if measure.get("synonyms"): + metric["ai_context"] = {"synonyms": list(measure["synonyms"])} + write_stash(metric, {k: measure[k] for k in _COLUMN_STASH_KEYS if k in measure}) + return metric diff --git a/converters/databricks/src/ossie_databricks/ossie_to_metric_view.py b/converters/databricks/src/ossie_databricks/ossie_to_metric_view.py new file mode 100644 index 00000000..3c535edc --- /dev/null +++ b/converters/databricks/src/ossie_databricks/ossie_to_metric_view.py @@ -0,0 +1,660 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert an Apache Ossie semantic model to a Databricks Unity Catalog Metric View (v1.1). + +Pure offline conversion -- no Databricks connection required. Produces the +Metric View: one fact `source` with a nested `joins` tree and +all fields flattened into one `dimensions` list. See README.md for the +capability summary and limitations. + +Usage (CLI): + ossie-databricks export -i model.yaml [-o view.yaml] [--source orders] +""" + +import re +import warnings + +from ._common import ( + CARD_ONE_TO_MANY, + ConversionError, + MV_VERSION, + OSSIE_VERSION, + STASH_SOURCE_KEY, + SYNONYM_LIMIT, + dump_yaml, + foreign_vendor_extensions, + is_simple_identifier, + load_yaml, + merge_description, + pick_expression, + read_stash, + require, + require_str, + synonyms_of, + validate_source, +) + + +def _warn(scope, msg): + warnings.warn(f"[{scope}] {msg}") + + +# Fanning a diamond out into per-path joins can expand exponentially on a pathological +# lattice; real snowflakes are tiny, so cap total joins to catch runaway inputs. +_MAX_JOIN_NODES = 200 + + +def convert_ossie_to_metric_view(ossie_yaml_str, source=None): + """Parse Apache Ossie YAML and return Databricks Metric View v1.1 YAML (string). + + `source` names the dataset to use as the view's fact/grain. When omitted, the + fact is the dataset that is never a relationship `to` (the FK sink of a plain + many-to-one star). Naming a coarser-grain dataset as the source is what unlocks + `one_to_many` joins (the joined detail tables sit on the `from`/many side). + """ + root = load_yaml(ossie_yaml_str) + if not isinstance(root, dict): + raise ConversionError("Invalid Apache Ossie YAML: expected a mapping at the root") + + version = str(root.get("version", "")) + if version != OSSIE_VERSION: + raise ConversionError( + f"Unsupported Apache Ossie version '{version}'. Supported: {OSSIE_VERSION}" + ) + + models = root.get("semantic_model") + if not isinstance(models, list) or not models: + raise ConversionError("'semantic_model' must be a non-empty list") + if len(models) > 1: + _warn("model", "multiple semantic models found; converting only the first") + + view = _convert_model(models[0], explicit_source=source) + return dump_yaml(view) + + +def _convert_model(model, explicit_source=None): + name = model.get("name", "") + dataset_list = model.get("datasets", []) or [] + if not dataset_list: + raise ConversionError(f"Model '{name}' has no datasets") + + seen = set() + for d in dataset_list: + ds_name = require_str(d, "name", f"Model '{name}': dataset") + if ds_name.strip().lower() in seen: # case-insensitive: DBR identifiers are + raise ConversionError(f"Model '{name}': duplicate dataset name '{ds_name}'") + seen.add(ds_name.strip().lower()) + datasets = {d["name"]: d for d in dataset_list} + relationships = model.get("relationships", []) or [] + + # Model-level stash: filter / parameters / materialization, plus an optional + # `source_dataset` recording the original grain (written on import only when a + # one_to_many join made the fact ambiguous). An explicit `source` arg wins. + model_stash = read_stash(model) + fact_hint = explicit_source or model_stash.get(STASH_SOURCE_KEY) + root, fact = _build_join_tree(name, datasets, relationships, fact_hint) + counts = _assign_aliases(root, fact) + + # Mark one_to_many nodes (parent on the `to`/one side); their columns can't be + # dimensions. Also validates one_to_many subtree uniformity. + _mark_otm(name, root) + + fact_ds = datasets[fact] + view = {"version": MV_VERSION, "source": validate_source(fact_ds.get("source"), fact)} + + # The view comment is simply the model's top-level description -- the closest + # match. Model ai_context and dataset descriptions are not merged in (dropped; + # see _warn_dropped_model), which keeps model.description round-trippable. + comment = model.get("description") + if comment: + view["comment"] = comment + + if "filter" in model_stash: + view["filter"] = model_stash["filter"] + + joins = [_build_join(child, "source", datasets) for child in root["children"]] + if joins: + view["joins"] = joins + + # Dimensions: every field across every join instance, fact first then join order. + # A dataset joined under more than one alias (fanned out) is emitted once per + # instance with alias-prefixed names. Track dropped names so we can cascade-drop + # anything that references them. + dropped_dims, dropped_measures = set(), set() + dimensions = [] + seen_dims = set() + for node, join_path in _node_order(root): + is_fact = node is root + prefix = node["alias"] if counts[node["dataset"]] > 1 else None + for field in datasets[node["dataset"]].get("fields", []) or []: + fname = require_str(field, "name", f"dataset '{node['dataset']}': field") + if node["is_otm"]: + _warn( + f"field '{fname}'", + "column on a one-to-many-joined table cannot be a dimension " + "(must resolve to one value per source row); dropped", + ) + dropped_dims.add(fname) + continue + dim = _convert_field(field, fname, ".".join(join_path), is_fact, prefix) + if dim is None: + dropped_dims.add(fname) + continue + if dim["name"].lower() in seen_dims: # case-insensitive + raise ConversionError( + f"dataset '{node['dataset']}': dimension name '{dim['name']}' " + f"collides with another dimension/measure; Metric Views require " + f"unique dimension/measure names -- rename before use" + ) + seen_dims.add(dim["name"].lower()) + dimensions.append(dim) + + measures = [] + for metric in model.get("metrics", []) or []: + measure = _convert_metric(metric, fact, seen_dims) + if measure is None: + dropped_measures.add(metric.get("name")) + continue + measures.append(measure) + + # Cascade: drop any dimension/measure whose expression references a dropped + # name (transitively), so we never emit a dangling reference. + _cascade_drop(dimensions, measures, dropped_dims, dropped_measures) + + if dimensions: + view["dimensions"] = dimensions + if measures: + view["measures"] = measures + + if "parameters" in model_stash: + view["parameters"] = model_stash["parameters"] + if "materialization" in model_stash: + view["materialization"] = model_stash["materialization"] + + _warn_dropped_model(model) + return view + + +def _build_join_tree(model_name, datasets, relationships, fact_hint=None): + """Build the Metric View join tree from the Apache Ossie relationship graph; return + (root_node, fact_name). + + Each node is a dict: {alias, dataset, rel, parent_is_from, children, is_otm}. + Edges are oriented away from the fact (the nearer endpoint is the parent), so a + dataset reachable by more than one path -- a diamond, e.g. two facts sharing a + dimension, or a dimension reached via two parents -- is fanned out into one node + per path. Each instance is later given a unique alias, mirroring how a Metric View + joins the same table more than once. Non-tree (cyclic) shapes are rejected. + """ + for rel in relationships: + scope = f"Model '{model_name}': relationship '{rel.get('name', '')}'" + if require(rel, "from", scope) not in datasets or require(rel, "to", scope) not in datasets: + raise ConversionError( + f"Model '{model_name}': relationship '{rel.get('name')}' references " + f"an unknown dataset" + ) + + # Re-orient any relationship whose declared keys show `from`/`to` is mislabeled + # (the `from` columns are a unique key, the `to` columns are not). Done before + # fact selection so cardinality, columns, and fact choice all use the key-derived + # orientation. The join condition is unchanged (it is symmetric). + relationships = [_orient_by_key(model_name, rel, datasets) for rel in relationships] + + fact = _pick_fact(model_name, datasets, relationships, fact_hint) + + # BFS (undirected) measures each dataset's distance from the fact; that distance + # orients every edge away from the fact (parent = the nearer endpoint). + adj = {name: [] for name in datasets} + for rel in relationships: + adj[rel["from"]].append(rel["to"]) + adj[rel["to"]].append(rel["from"]) + dist = {fact: 0} + queue = [fact] + while queue: + cur = queue.pop(0) + for neighbor in adj[cur]: + if neighbor not in dist: + dist[neighbor] = dist[cur] + 1 + queue.append(neighbor) + + unreachable = set(datasets) - set(dist) + if unreachable: + raise ConversionError( + f"Model '{model_name}': datasets {sorted(unreachable)} are not reachable " + f"from fact '{fact}' via relationships." + ) + + # Orient each edge nearer->farther. An edge between two equidistant datasets has + # no fact-ward direction -- that only happens in a cyclic / non-tree graph. + children_of = {name: [] for name in datasets} + for rel in relationships: + a, b = rel["from"], rel["to"] + if dist[a] == dist[b]: + raise ConversionError( + f"Model '{model_name}': relationship '{rel.get('name')}' joins two " + f"datasets equidistant from the fact; the graph is not tree-shaped " + f"(it contains a cycle)." + ) + parent, child = (a, b) if dist[a] < dist[b] else (b, a) + children_of[parent].append((child, rel, parent == rel["from"])) + + counter = [0] + + def build(dataset, rel, parent_is_from): + counter[0] += 1 + if counter[0] > _MAX_JOIN_NODES: + raise ConversionError( + f"Model '{model_name}': join graph fans out to more than " + f"{_MAX_JOIN_NODES} joins; check for an unintended diamond explosion." + ) + node = {"alias": None, "dataset": dataset, "rel": rel, + "parent_is_from": parent_is_from, "children": [], "is_otm": False} + for child, crel, cfrom in children_of[dataset]: + node["children"].append(build(child, crel, cfrom)) + return node + + return build(fact, None, None), fact + + +def _assign_aliases(root, fact): + """Give every node a unique join alias and return per-dataset instance counts. + + A dataset with a single instance keeps its bare name (so non-diamond graphs are + unchanged); a fanned-out dataset's instances are disambiguated by parent alias + (e.g. `customers_regions` / `suppliers_regions`). The fact's alias is `source`. + """ + counts = {} + + def count(node): + counts[node["dataset"]] = counts.get(node["dataset"], 0) + 1 + for c in node["children"]: + count(c) + + count(root) + + used = {"source"} # reserved for the fact, so a dataset named `source` gets renamed + + def assign(node, parent_alias): + if node["dataset"] == fact: + alias = "source" + else: + # Single-instance datasets keep their bare name; fanned-out ones are + # qualified by the parent alias. Either way the result is deduped against + # `used` (which reserves `source`), so no two joins ever share an alias. + if counts[node["dataset"]] == 1: + base = node["dataset"] + else: + base = (f"{parent_alias}_{node['dataset']}" + if parent_alias and parent_alias != "source" else node["dataset"]) + alias, n = base, 2 + while alias in used: + alias, n = f"{base}_{n}", n + 1 + node["alias"] = alias + used.add(alias) + for c in node["children"]: + assign(c, alias) + + assign(root, None) + return counts + + +def _pick_fact(model_name, datasets, relationships, fact_hint): + """Choose the fact/root: an explicit hint if given, else the dataset that is + never a relationship `to` (the FK sink of a plain many-to-one star).""" + if fact_hint is not None: + if fact_hint not in datasets: + raise ConversionError( + f"Model '{model_name}': requested source '{fact_hint}' is not a dataset" + ) + return fact_hint + if len(datasets) > 1 and not relationships: + raise ConversionError( + f"Model '{model_name}': {len(datasets)} datasets but no relationships; " + f"cannot determine the fact table." + ) + incoming = {name: 0 for name in datasets} + for rel in relationships: + incoming[rel["to"]] += 1 + roots = [n for n, c in incoming.items() if c == 0] + if not roots: + raise ConversionError( + f"Model '{model_name}': join graph contains a cycle (no root dataset). " + f"A Metric View requires an acyclic, tree-shaped graph." + ) + if len(roots) > 1: + raise ConversionError( + f"Model '{model_name}': multiple candidate fact datasets {sorted(roots)}. " + f"Name the grain with --source -- e.g. for multiple facts sharing a " + f"dimension, name that dimension so each fact becomes a one_to_many join." + ) + return roots[0] + + +def _mark_otm(model_name, root): + """Mark each node `is_otm` (reached through a one_to_many join -- a parent on the + `to`/one side). Their columns can't be dimensions. Enforces the DBR rule that + every descendant of a one_to_many join is itself one_to_many.""" + + def visit(node, under_otm): + for child in node["children"]: + is_otm = not child["parent_is_from"] # parent on the `to` (one) side + if under_otm and not is_otm: + raise ConversionError( + f"Model '{model_name}': join '{child['alias']}' is many-to-one but " + f"descends from a one-to-many join; all descendants of a one-to-many " + f"join must also be one-to-many (Databricks Metric View rule)." + ) + child["is_otm"] = under_otm or is_otm + visit(child, child["is_otm"]) + + visit(root, False) + + +def _node_order(root): + """Fact first, then a stable depth-first walk of the join tree (one node per join + instance, so a fanned-out dataset appears once per path). Yields (node, join_path): + `join_path` is the tuple of join aliases from the source down to and including the + node (empty for the fact). A joined column is qualified in a dimension/measure by this + full path (`parent.child.col`) -- the Databricks nested-join rule -- which for a + depth-1 join is just the join's own name.""" + order = [] + + def visit(node, path): + order.append((node, path)) + for child in node["children"]: + visit(child, path + (child["alias"],)) + + visit(root, ()) + return order + + +def _build_join(node, parent_alias, datasets): + """Build one Metric View join entry from a tree node (recursively for nested joins). + + `node['parent_is_from']` is True when the parent is the relationship's `from` + (many) side -> a many_to_one join (the default, left implicit). When the parent is + the `to` (one) side the join is one_to_many and the column roles flip. + """ + rel, alias = node["rel"], node["alias"] + join = {"name": alias, + "source": validate_source(datasets[node["dataset"]].get("source"), node["dataset"])} + + stash = read_stash(rel) + from_cols = rel.get("from_columns") or [] + to_cols = rel.get("to_columns") or [] + # Apache Ossie relationships are equi-joins; a relationship without usable equi columns + # (e.g. a non-equi join the importer would have rejected) is rejected here too. + _validate_join_columns(rel, from_cols, to_cols) + # Write parent-side = child-side: the parent uses whichever list belongs to + # it -- from_columns when it is the `from`, to_columns when it is the `to`. + parent_cols, child_cols = ( + (from_cols, to_cols) if node["parent_is_from"] else (to_cols, from_cols)) + if parent_cols == child_cols: + # Equal column lists are an equi-join on shared names -> `using`, which + # round-trips faithfully (the importer maps `using` to equal lists). + join["using"] = list(parent_cols) + else: + join["on"] = " AND ".join( + f"{parent_alias}.{pc} = {alias}.{cc}" for pc, cc in zip(parent_cols, child_cols) + ) + # rely.at_most_one_match: a stashed value round-trips verbatim; otherwise derive it + # for a many_to_one join whose `to_columns` cover a declared primary/unique key of + # the joined dataset (joining on a key matches at most one row -- no fan-out). + if "rely" in stash: + join["rely"] = stash["rely"] + elif node["parent_is_from"] and _covers_unique_key(datasets[node["dataset"]], to_cols): + join["rely"] = {"at_most_one_match": True} + # Cardinality: an explicit stashed value round-trips verbatim; otherwise derive + # from orientation -- parent on the `to` (one) side means one_to_many. The + # many_to_one default is left implicit. + if "cardinality" in stash: + join["cardinality"] = stash["cardinality"] + elif not node["parent_is_from"]: + join["cardinality"] = CARD_ONE_TO_MANY + + nested = [_build_join(c, alias, datasets) for c in node["children"]] + if nested: + join["joins"] = nested + return join + + +def _covers_unique_key(dataset, join_cols): + """True if `join_cols` include a declared `primary_key` or one of `unique_keys` of + `dataset` -- i.e. joining on them matches at most one target row, so a many_to_one + join can assert `rely.at_most_one_match`.""" + cols = set(join_cols) + keys = [dataset.get("primary_key")] if dataset.get("primary_key") else [] + keys += dataset.get("unique_keys") or [] + return any(key and set(key) <= cols for key in keys) + + +def _orient_by_key(model_name, rel, datasets): + """`to` should be the unique 'one' side (per spec `to_columns` are key columns). If + the declared keys say otherwise -- the `from` columns are a unique key while the + `to` side declares keys its `to_columns` don't cover -- `from`/`to` is mislabeled. + Return a copy with `from`/`to` (and their columns) swapped, and warn. The swap is + symmetric, so the join condition is unchanged; only the orientation is corrected. + + When the `from` columns cover a unique key but the `to` side declares no key at all, + the orientation can't be verified either way (the `to` side may or may not be + unique); leave it as-is but warn, since the resulting cardinality may be inverted.""" + from_cols = rel.get("from_columns") or [] + to_cols = rel.get("to_columns") or [] + if not from_cols or not to_cols: + return rel # non-equi / column-less: nothing to deduce from + to_ds = datasets[rel["to"]] + to_has_keys = bool(to_ds.get("primary_key") or to_ds.get("unique_keys")) + from_covers = _covers_unique_key(datasets[rel["from"]], from_cols) + if from_covers and to_has_keys and not _covers_unique_key(to_ds, to_cols): + _warn( + f"relationship '{rel.get('name')}'", + "from/to looks mislabeled (the `from` columns are a declared key, the `to` " + "columns are not); re-orienting so the key side is the `to`/one side", + ) + return {**rel, "from": rel["to"], "to": rel["from"], + "from_columns": to_cols, "to_columns": from_cols} + if from_covers and not to_has_keys: + _warn( + f"relationship '{rel.get('name')}'", + "the `from` columns are a declared key but the `to` side declares none, so " + "from/to orientation can't be verified; using it as-is -- check the join " + "direction if the resulting cardinality looks inverted", + ) + return rel + + +def _validate_join_columns(rel, from_cols, to_cols): + if not from_cols or not to_cols: + raise ConversionError( + f"Relationship '{rel.get('name')}': from_columns and to_columns are required" + ) + if not isinstance(from_cols, list) or not isinstance(to_cols, list): + raise ConversionError( + f"Relationship '{rel.get('name')}': from_columns and to_columns must be lists" + ) + if len(from_cols) != len(to_cols): + raise ConversionError( + f"Relationship '{rel.get('name')}': from_columns ({len(from_cols)}) and " + f"to_columns ({len(to_cols)}) must have the same length" + ) + + +def _convert_field(field, name, qualifier, is_fact, prefix=None): + scope = f"field '{name}'" + expr = pick_expression(field.get("expression")) + if expr is None: + _warn(scope, "no DATABRICKS/ANSI_SQL dialect; dropping field") + return None + + # Requalify a joined-table column with its full join-name path from the source + # (`parent.child.col`); a depth-1 join is just its own name. Only safe for bare + # columns. A complex expression on a single join is emitted as-is (likely resolves; + # warned). On a fanned-out (diamond) dataset it cannot be attributed to one of the + # instances, so it is dropped rather than emitted as an ambiguous dimension. + if not is_fact: + if is_simple_identifier(expr): + expr = f"{qualifier}.{expr}" + elif prefix: + _warn(scope, "complex expression on a fanned-out (diamond) join cannot be " + "unambiguously qualified; dropped") + return None + else: + _warn(scope, "complex expression on a joined table; emitted as-is, verify qualification") + + # A fanned-out dataset (joined under more than one alias) needs unique dimension + # names, so prefix with the instance alias (e.g. customer_region's r_name -> + # customer_region_r_name). Single-instance datasets keep the bare field name. + if prefix: + name = f"{prefix}_{name}" + + dim = {"name": name, "expr": expr} + comment = merge_description(field.get("description"), field.get("ai_context")) + if comment: + dim["comment"] = comment + if field.get("label"): + dim["display_name"] = field["label"] + syns = synonyms_of(field.get("ai_context")) + if syns: + dim["synonyms"] = _truncate_synonyms(syns, scope) + + stash = read_stash(field) + if "format" in stash: + dim["format"] = stash["format"] + _warn_dropped_field(field, scope) + return dim + + +def _convert_metric(metric, fact, seen_names): + name = require_str(metric, "name", "metric") + scope = f"metric '{name}'" + if name.lower() in seen_names: # case-insensitive (shares seen_dims with dimensions) + raise ConversionError( + f"metric '{name}' collides with another dimension/measure; Metric Views " + f"require unique dimension/measure names -- rename before use") + seen_names.add(name.lower()) + expr = pick_expression(metric.get("expression")) + if expr is None: + _warn(scope, "no DATABRICKS/ANSI_SQL dialect; dropping metric") + return None + + # Fact-table columns are referenced by bare name in measure expressions (DBR + # idiom: `SUM(amount)`, not `SUM(source.amount)`); strip a `.` qualifier. + # Joined-table columns keep their alias. Word boundary avoids touching a table + # whose name merely ends with the fact name (e.g. fact 'sales' vs 'store_sales'). + expr = re.sub(r"\b" + re.escape(fact) + r"\.", "", expr) + + measure = {"name": name, "expr": expr} + comment = merge_description(metric.get("description"), metric.get("ai_context")) + if comment: + measure["comment"] = comment + syns = synonyms_of(metric.get("ai_context")) + if syns: + measure["synonyms"] = _truncate_synonyms(syns, scope) + + stash = read_stash(metric) + if "format" in stash: + measure["format"] = stash["format"] + if "window" in stash: + measure["window"] = stash["window"] + return measure + + +def _references_dropped(expr, self_name, dropped_dims, dropped_measures): + """Return a dropped name referenced by `expr`, or None. + + Measures are only referenceable via `measure()` (exact). Dimensions are + referenced by their bare, *unqualified* name: a name that is part of a qualified + path (`alias.name` or `name.col`) is ignored, so a join alias or joined column + that merely shares a dropped dimension's name is not over-dropped. The one + ambiguity the regex can't resolve without a SQL parser is a bare, unqualified + *source column* sharing a dropped dimension's name -- there it errs on dropping. + """ + for m in dropped_measures: + if re.search(r"measure\(\s*" + re.escape(m) + r"\s*\)", expr): + return m + for d in dropped_dims: + # Match only a bare, unqualified token: the negative look-behind/ahead for a + # word char or `.` excludes both substrings of a larger identifier and + # qualified paths (`alias.name` / `name.col`), so a join alias or joined + # column sharing a dropped name is not falsely cascade-dropped. + if d != self_name and re.search( + r"(? SYNONYM_LIMIT: + _warn(scope, f"{len(syns)} synonyms exceeds Metric View limit; keeping first {SYNONYM_LIMIT}") + return syns[:SYNONYM_LIMIT] + return syns + + +def _warn_dropped_model(model): + if foreign_vendor_extensions(model): + _warn("model", "foreign-vendor custom_extensions dropped") + if model.get("ai_context"): # string or object -- only the description maps to comment + _warn("model", "model-level ai_context dropped (only the description maps to the view comment)") + for ds in model.get("datasets", []) or []: + scope = f"dataset '{ds['name']}'" + if ds.get("primary_key") or ds.get("unique_keys"): + _warn(scope, "primary_key/unique_keys not stored as columns; used to set " + "rely.at_most_one_match on a matching many_to_one join where applicable") + if isinstance(ds.get("ai_context"), dict) and ds["ai_context"]: + _warn(scope, "dataset-level ai_context (object) dropped") + # Dataset descriptions are not merged into the view comment (a Metric View + # has no per-source comment); only the model description is used. + if ds.get("description"): + _warn(scope, "dataset-level description dropped (no per-source comment field)") + if foreign_vendor_extensions(ds): + _warn(scope, "foreign-vendor custom_extensions dropped") + for rel in model.get("relationships", []) or []: + if rel.get("ai_context"): + _warn(f"relationship '{rel.get('name', '')}'", "relationship ai_context dropped") + + +def _warn_dropped_field(field, scope): + dim = field.get("dimension") + if isinstance(dim, dict) and "is_time" in dim: + _warn(scope, "dimension.is_time has no Metric View counterpart; dropped") + if foreign_vendor_extensions(field): + _warn(scope, "foreign-vendor custom_extensions dropped") diff --git a/converters/databricks/tests/_roundtrip_helpers.py b/converters/databricks/tests/_roundtrip_helpers.py new file mode 100644 index 00000000..6c4ddfb8 --- /dev/null +++ b/converters/databricks/tests/_roundtrip_helpers.py @@ -0,0 +1,355 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared model builders and round-trip assertions for property-based tests. + +This module is deliberately free of any third-party test dependency (no hypothesis, +no pytest) so the generation + assertion logic can run two ways: + + - driven by Hypothesis strategies (see test_roundtrip_properties.py), and + - driven by a plain seeded `random.Random` (RandomRnd below), which is how the + logic is exercised in environments where hypothesis is not installed. + +Both drivers implement the small `Rnd` interface (chance/count/pick/text/colname); +the builders below depend only on that interface, so the generated model space is +identical regardless of driver. + +The builders intentionally generate within the *round-trippable subset* of models - +the shapes the converter reproduces exactly. Known normalizations are avoided by +construction (documented inline) rather than asserted around, e.g.: + - join `on` conditions use distinct parent/child column names, so an equi-join on + shared names is never silently rewritten to `using`; + - measure expressions are emitted without a `source.` qualifier, which the exporter + would otherwise strip; + - in the Apache Ossie direction, joined-dataset fields are bare identifiers, so they survive + the alias requalify/de-qualify trip on the same dataset. +Name fuzzing (reserved words, collisions) is left to the targeted unit tests, which +assert the converter *rejects* those inputs. +""" + +import random +import re +import string +import warnings + +from ossie_databricks import metric_view_to_ossie as importer +from ossie_databricks import ossie_to_metric_view as exporter +from ossie_databricks._common import MV_VERSION, OSSIE_VERSION, dump_yaml, load_yaml + +_AGGS = ["SUM", "COUNT", "AVG", "MIN", "MAX"] + + +# --- Rnd backend for offline (no hypothesis) runs -------------------------------- + +class RandomRnd: + """The `Rnd` interface backed by a seeded `random.Random`.""" + + def __init__(self, seed): + self.r = random.Random(seed) + + def chance(self, p=0.5): + return self.r.random() < p + + def count(self, lo, hi): + return self.r.randint(lo, hi) + + def pick(self, seq): + return self.r.choice(list(seq)) + + def text(self): + # Alphanumeric with optional interior spaces; no leading/trailing space and + # no YAML-special characters, so the value is preserved verbatim through a + # dump/load cycle (any failure then reflects the converter, not PyYAML). + alnum = string.ascii_letters + string.digits + n = self.r.randint(0, 10) + body = "".join(self.r.choice(alnum + " ") for _ in range(n)) + return (self.r.choice(alnum) + body).strip() or "x" + + def colname(self): + first = self.r.choice(string.ascii_lowercase + "_") + rest = "".join( + self.r.choice(string.ascii_lowercase + string.digits + "_") + for _ in range(self.r.randint(0, 7)) + ) + return first + rest + + +# --- Small generation helpers ---------------------------------------------------- + +class _Names: + """Hands out globally-unique names with a given prefix.""" + + def __init__(self): + self._n = {} + + def next(self, prefix): + i = self._n.get(prefix, 0) + self._n[prefix] = i + 1 + return f"{prefix}{i}" + + +def _maybe_meta(rnd, target): + """Attach optional comment/display_name/synonyms/format to a dim/measure dict.""" + if rnd.chance(0.4): + target["comment"] = rnd.text() + if rnd.chance(0.3): + target["display_name"] = rnd.text() + if rnd.chance(0.3): + target["synonyms"] = [rnd.text() for _ in range(rnd.count(1, 3))] + if rnd.chance(0.25): + fmt = {"type": rnd.pick(["number", "currency", "date"])} + if fmt["type"] == "currency": + fmt["currency_code"] = "USD" + target["format"] = fmt + + +# --- Metric View builder (for MV -> Apache Ossie -> MV) ----------------------------------- + +def _build_join(rnd, names, parent_alias, depth, ancestor_path): + name = names.next("j") + # Full join-name path from the source -> how dimensions/measures qualify this join's + # columns (DBR nested-join rule). `on:` conditions instead use the immediate names. + qual = ".".join(ancestor_path + [name]) + join = {"name": name, "source": _three_part(rnd)} + if rnd.chance(0.5): + ncols = rnd.count(1, 2) + join["using"] = [f"u{i}_{rnd.colname()}" for i in range(ncols)] + else: + ncols = rnd.count(1, 2) + # distinct parent/child names so the equi-join stays `on`, not `using` + pairs = [(f"fk{i}_{rnd.colname()}", f"pk{i}_{rnd.colname()}") for i in range(ncols)] + join["on"] = " AND ".join(f"{parent_alias}.{pc} = {name}.{cc}" for pc, cc in pairs) + if rnd.chance(0.4): + join["cardinality"] = "many_to_one" # only the lossless cardinality (see module doc) + if rnd.chance(0.3): + join["rely"] = {"at_most_one_match": True} + # dimensions on this join (qualified by the full join path) + dims = [] + for _ in range(rnd.count(0, 2)): + col = rnd.colname() + expr = (f"{qual}.{col}" if rnd.chance(0.7) + else f"{qual}.{col} + {qual}.{rnd.colname()}") + dim = {"name": names.next("c"), "expr": expr} + _maybe_meta(rnd, dim) + dims.append(dim) + if depth < 2 and rnd.chance(0.35): + child, child_dims = _build_join(rnd, names, name, depth + 1, ancestor_path + [name]) + join["joins"] = [child] + dims.extend(child_dims) + return join, dims + + +def build_metric_view(rnd): + """Generate a Metric View YAML dict in the round-trippable subset.""" + names = _Names() + mv = {"version": MV_VERSION, "source": _three_part(rnd)} + if rnd.chance(0.4): + mv["comment"] = rnd.text() + if rnd.chance(0.3): + mv["filter"] = f"{rnd.colname()} > 0" + + fields, joins = [], [] + for _ in range(rnd.count(0, 3)): # source dimensions (bare or function exprs) + col = rnd.colname() + expr = col if rnd.chance(0.7) else f"UPPER({col})" + dim = {"name": names.next("c"), "expr": expr} + _maybe_meta(rnd, dim) + fields.append(dim) + for _ in range(rnd.count(0, 2)): # join subtrees + join, jdims = _build_join(rnd, names, "source", 0, []) + joins.append(join) + fields.extend(jdims) + + measures = [] + for _ in range(rnd.count(0, 2)): + m = {"name": names.next("c"), "expr": f"{rnd.pick(_AGGS)}({rnd.colname()})"} + if rnd.chance(0.4): + m["comment"] = rnd.text() + if rnd.chance(0.3): + m["synonyms"] = [rnd.text() for _ in range(rnd.count(1, 3))] + if rnd.chance(0.3): + m["window"] = [{"order": rnd.colname(), "range": "trailing 7 day"}] + measures.append(m) + + if joins: + mv["joins"] = joins + if fields: + mv["fields"] = fields + if measures: + mv["measures"] = measures + if rnd.chance(0.2): + mv["materialization"] = {"schedule": "every 6 hours", + "mode": rnd.pick(["relaxed", "strict"])} + return mv + + +# --- Apache Ossie builder (for Apache Ossie -> MV -> Apache Ossie) ------------------------------------------ + +def _ossie_field(name, expr): + return {"name": name, + "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": expr}]}} + + +def build_ossie(rnd): + """Generate an Apache Ossie semantic model dict in the round-trippable subset.""" + names = _Names() + fact = "fact" # fact name must equal its source's last identifier to round-trip + datasets = [{"name": fact, "source": f"c.s.{fact}"}] + relationships = [] + + n_dims = rnd.count(0, 3) + dim_names = [names.next("dim") for _ in range(n_dims)] + reachable = [fact] + for i, dname in enumerate(dim_names): + parent = rnd.pick(reachable) # star, or snowflake off an earlier node + ds = {"name": dname, "source": f"c.s.{rnd.colname()}{i}"} + datasets.append(ds) + reachable.append(dname) + if rnd.chance(0.5): # equal column names -> `using`; else distinct -> `on` + cols = [rnd.colname() for _ in range(rnd.count(1, 2))] + relationships.append({"name": names.next("r"), "from": parent, "to": dname, + "from_columns": list(cols), "to_columns": list(cols)}) + else: + n = rnd.count(1, 2) + fcols = [f"fk{j}_{rnd.colname()}" for j in range(n)] + tcols = [f"pk{j}_{rnd.colname()}" for j in range(n)] + relationships.append({"name": names.next("r"), "from": parent, "to": dname, + "from_columns": fcols, "to_columns": tcols}) + + # fields, bare identifiers, filed onto a random dataset (globally unique names) + for ds in datasets: + flds = [_ossie_field(names.next("c"), rnd.colname()) for _ in range(rnd.count(0, 3))] + if flds: + ds["fields"] = flds + + metrics = [{"name": names.next("c"), + "expression": {"dialects": [{"dialect": "DATABRICKS", + "expression": f"{rnd.pick(_AGGS)}({rnd.colname()})"}]}} + for _ in range(rnd.count(0, 2))] + + model = {"name": names.next("m")} + if rnd.chance(0.4): + model["description"] = rnd.text() + model["datasets"] = datasets + if relationships: + model["relationships"] = relationships + if metrics: + model["metrics"] = metrics + return {"version": OSSIE_VERSION, "semantic_model": [model]} + + +def _three_part(rnd): + return f"{rnd.colname()}.{rnd.colname()}.{rnd.colname()}" + + +# --- Round-trip assertions ------------------------------------------------------- + +def _convert(fn, text): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return fn(text) + + +def _cond_canon(join): + if join.get("using"): + return ("using", tuple(sorted(join["using"]))) + on = join.get("on") + if not on: + return (None, None) + pairs = set() + for clause in re.split(r"\s+AND\s+", on, flags=re.IGNORECASE): + left, right = clause.split("=", 1) + pairs.add((left.strip(), right.strip())) + return ("on", frozenset(pairs)) + + +def _flatten_joins(joins, parent="source", acc=None, edges=None): + acc = {} if acc is None else acc + edges = set() if edges is None else edges + for j in joins or []: + acc[j["name"]] = {"source": j["source"], "cond": _cond_canon(j), + "cardinality": j.get("cardinality"), "rely": j.get("rely")} + edges.add((parent, j["name"])) + _flatten_joins(j.get("joins"), j["name"], acc, edges) + return acc, edges + + +def _dims(mv): + # The exporter emits the canonical `dimensions:` key; the importer also accepts the + # `fields:` alias. Read either so the comparison is key-name agnostic. + return mv.get("dimensions") or mv.get("fields") or [] + + +def _dim_norm(d): + return (d["expr"], d.get("comment"), d.get("display_name"), + d.get("synonyms"), d.get("format")) + + +def _meas_norm(m): + return (m["expr"], m.get("comment"), m.get("synonyms"), m.get("format"), m.get("window")) + + +def assert_mv_roundtrip(mv): + """A Metric View dict survives MV -> Apache Ossie -> MV with content preserved.""" + ossie_yaml = _convert(importer.convert_metric_view_to_ossie, dump_yaml(mv)) + mv2 = load_yaml(_convert(exporter.convert_ossie_to_metric_view, ossie_yaml)) + + assert mv2["source"] == mv["source"], "source" + assert mv2.get("comment") == mv.get("comment"), "comment" + assert mv2.get("filter") == mv.get("filter"), "filter" + assert mv2.get("materialization") == mv.get("materialization"), "materialization" + + assert ({d["name"]: _dim_norm(d) for d in _dims(mv)} + == {d["name"]: _dim_norm(d) for d in _dims(mv2)}), "fields" + assert ({m["name"]: _meas_norm(m) for m in mv.get("measures", [])} + == {m["name"]: _meas_norm(m) for m in mv2.get("measures", [])}), "measures" + + a1, e1 = _flatten_joins(mv.get("joins")) + a2, e2 = _flatten_joins(mv2.get("joins")) + assert a1 == a2, "joins" + assert e1 == e2, "join nesting" + + +def _expr_of(obj): + for d in obj["expression"]["dialects"]: + if d["dialect"] == "DATABRICKS": + return d["expression"] + return None + + +def _fields_map(ds): + return {f["name"]: _expr_of(f) for f in ds.get("fields", [])} + + +def _rel_set(model): + return {(r["from"], r["to"], tuple(r.get("from_columns") or []), + tuple(r.get("to_columns") or [])) + for r in model.get("relationships", [])} + + +def assert_ossie_roundtrip(ossie): + """An Apache Ossie model dict survives Apache Ossie -> MV -> Apache Ossie with content preserved.""" + mv_yaml = _convert(exporter.convert_ossie_to_metric_view, dump_yaml(ossie)) + ossie2 = load_yaml(_convert(importer.convert_metric_view_to_ossie, mv_yaml)) + + m1, m2 = ossie["semantic_model"][0], ossie2["semantic_model"][0] + assert ({d["name"]: (d["source"], _fields_map(d)) for d in m1["datasets"]} + == {d["name"]: (d["source"], _fields_map(d)) for d in m2["datasets"]}), "datasets" + assert _rel_set(m1) == _rel_set(m2), "relationships" + assert ({x["name"]: _expr_of(x) for x in m1.get("metrics", [])} + == {x["name"]: _expr_of(x) for x in m2.get("metrics", [])}), "metrics" + assert m1.get("description") == m2.get("description"), "description" diff --git a/converters/databricks/tests/_util.py b/converters/databricks/tests/_util.py new file mode 100644 index 00000000..e82a1d0b --- /dev/null +++ b/converters/databricks/tests/_util.py @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared test helpers: fixture loading and structural normalization.""" + +import copy +import json +import pathlib + +from ossie_databricks._common import load_yaml # src is on sys.path via conftest.py + +FIXTURES = pathlib.Path(__file__).resolve().parent / "fixtures" + + +def load_fixture(name): + with open(FIXTURES / name) as fh: + return fh.read() + + +def parse(yaml_str): + # YAML 1.2 booleans so a join `on:` key parses as the string "on" (matching the + # converter's own load/dump), not the YAML-1.1 boolean True. + return load_yaml(yaml_str) + + +def canon(obj): + """Deep-copy with every `custom_extensions[].data` JSON string parsed into a + dict, so comparisons are insensitive to JSON key order / whitespace.""" + obj = copy.deepcopy(obj) + + def walk(node): + if isinstance(node, dict): + for ext in node.get("custom_extensions") or []: + if isinstance(ext.get("data"), str): + ext["data"] = json.loads(ext["data"]) + for v in node.values(): + walk(v) + elif isinstance(node, list): + for v in node: + walk(v) + + walk(obj) + return obj + + +def strip_dropped(ossie): + """Normalize away what the Apache Ossie -> MV -> Apache Ossie trip changes, so a round-trip + comparison reflects the documented limitations. Besides outright losses (model + name, descriptions), a declared key transforms across the trip: `primary_key` -> + `rely.at_most_one_match` (MV) -> `unique_keys` + a relationship rely-stash. We drop + both key forms and the relationship stash so the key info is compared as 'gone'.""" + ossie = copy.deepcopy(ossie) + for model in ossie.get("semantic_model", []): + model.pop("name", None) # MV carries no model name + model.pop("description", None) # model + fact descriptions merge into one comment + for ds in model.get("datasets", []): + ds.pop("primary_key", None) + ds.pop("unique_keys", None) + ds.pop("description", None) # no per-source comment in single-source MV + for rel in model.get("relationships", []): + rel.pop("custom_extensions", None) # derived rely-stash from a declared key + return ossie diff --git a/converters/databricks/tests/conftest.py b/converters/databricks/tests/conftest.py new file mode 100644 index 00000000..0bcde537 --- /dev/null +++ b/converters/databricks/tests/conftest.py @@ -0,0 +1,23 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pathlib +import sys + +# Make the converter modules in ../src importable from the tests. +_SRC = pathlib.Path(__file__).resolve().parent.parent / "src" +sys.path.insert(0, str(_SRC)) diff --git a/converters/databricks/tests/fixtures/fixtureA_metric_view.yaml b/converters/databricks/tests/fixtures/fixtureA_metric_view.yaml new file mode 100644 index 00000000..4ffe6819 --- /dev/null +++ b/converters/databricks/tests/fixtures/fixtureA_metric_view.yaml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Fixture A -- expected UC Metric View (v1.1, single-source) produced from +# fixtureA_ossie.yaml. Must parse under the v1.1 strict schema. + +version: '1.1' +source: samples.tpch.orders +comment: Sales orders with customer attributes +joins: +- name: customer + source: samples.tpch.customer + on: source.o_custkey = customer.c_custkey + rely: + at_most_one_match: true +dimensions: +- name: o_orderkey + expr: o_orderkey + comment: Order identifier +- name: o_orderdate + expr: o_orderdate + display_name: Order Date + synonyms: + - order date + - date +- name: c_name + expr: customer.c_name + comment: Customer name +measures: +- name: total_revenue + expr: SUM(o_totalprice) + comment: Total order revenue + synonyms: + - revenue + - total revenue + - sales +- name: order_count + expr: COUNT(*) + comment: Number of orders diff --git a/converters/databricks/tests/fixtures/fixtureA_ossie.yaml b/converters/databricks/tests/fixtures/fixtureA_ossie.yaml new file mode 100644 index 00000000..a53942f3 --- /dev/null +++ b/converters/databricks/tests/fixtures/fixtureA_ossie.yaml @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yaml-language-server: $schema=../../../../core-spec/osi-schema.json +# +# Fixture A -- all-native round-trip (Apache Ossie -> MV -> Apache Ossie). +# Star schema; every field maps to a native MV field. Documented losses on the +# Apache Ossie -> MV -> Apache Ossie trip: the model name (MV carries none) and primary_key. + +version: "0.2.0.dev0" + +semantic_model: + - name: sales + description: Sales orders with customer attributes + datasets: + - name: orders # fact: no incoming relationship -> becomes `source` + source: samples.tpch.orders + primary_key: [o_orderkey] # dropped on export (Apache Ossie-only) + description: One row per order + fields: + - name: o_orderkey + expression: + dialects: + - dialect: DATABRICKS + expression: o_orderkey + description: Order identifier + - name: o_orderdate + expression: + dialects: + - dialect: DATABRICKS + expression: o_orderdate + label: Order Date + ai_context: + synonyms: [order date, date] + - name: customer + source: samples.tpch.customer + primary_key: [c_custkey] + fields: + - name: c_name + expression: + dialects: + - dialect: DATABRICKS + expression: c_name + description: Customer name + relationships: + - name: orders_to_customer + from: orders + to: customer + from_columns: [o_custkey] + to_columns: [c_custkey] + metrics: + - name: total_revenue + expression: + dialects: + - dialect: DATABRICKS + expression: SUM(o_totalprice) # fact columns are bare in measures + description: Total order revenue + ai_context: + synonyms: [revenue, total revenue, sales] + - name: order_count + expression: + dialects: + - dialect: DATABRICKS + expression: COUNT(*) + description: Number of orders diff --git a/converters/databricks/tests/fixtures/fixtureB_metric_view.yaml b/converters/databricks/tests/fixtures/fixtureB_metric_view.yaml new file mode 100644 index 00000000..bbbbabc8 --- /dev/null +++ b/converters/databricks/tests/fixtures/fixtureB_metric_view.yaml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Fixture B -- stash round-trip (MV -> Apache Ossie -> MV, lossless). +# Exercises the stash at every placement level: model (filter), relationship +# (rely), dimension (format), measure (format). Must parse under v1.1. + +version: '1.1' +source: samples.tpch.lineitem +filter: l_returnflag = 'N' +comment: Line item shipping metrics +joins: +- name: orders + source: samples.tpch.orders + on: source.l_orderkey = orders.o_orderkey + rely: + at_most_one_match: true +dimensions: +- name: line_number + expr: l_linenumber + format: + type: number + decimal_places: + type: exact + places: 0 +measures: +- name: revenue + expr: SUM(l_extendedprice * (1 - l_discount)) + comment: Net revenue + format: + type: currency + currency_code: USD + decimal_places: + type: exact + places: 2 +- name: order_count + expr: COUNT(DISTINCT l_orderkey) diff --git a/converters/databricks/tests/fixtures/fixtureB_ossie.yaml b/converters/databricks/tests/fixtures/fixtureB_ossie.yaml new file mode 100644 index 00000000..f2c11d9c --- /dev/null +++ b/converters/databricks/tests/fixtures/fixtureB_ossie.yaml @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yaml-language-server: $schema=../../../../core-spec/osi-schema.json +# +# Fixture B -- expected Apache Ossie produced from fixtureB_metric_view.yaml. MV-only +# features are stashed in custom_extensions[DATABRICKS], keyed by their exact v1.1 +# field name. Exporting this back must reproduce fixtureB_metric_view.yaml. +# Model name is derived from the fact table (`lineitem`). + +version: "0.2.0.dev0" + +semantic_model: + - name: lineitem + description: Line item shipping metrics + datasets: + - name: lineitem + source: samples.tpch.lineitem + fields: + - name: line_number + expression: + dialects: + - dialect: DATABRICKS + expression: l_linenumber + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "format": {"type": "number", "decimal_places": {"type": "exact", "places": 0}}}' + - name: orders + source: samples.tpch.orders + unique_keys: + - [o_orderkey] + relationships: + - name: lineitem_to_orders + from: lineitem + to: orders + from_columns: [l_orderkey] + to_columns: [o_orderkey] + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "rely": {"at_most_one_match": true}}' + metrics: + - name: revenue + expression: + dialects: + - dialect: DATABRICKS + expression: SUM(l_extendedprice * (1 - l_discount)) + description: Net revenue + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "format": {"type": "currency", "currency_code": "USD", "decimal_places": {"type": "exact", "places": 2}}}' + - name: order_count + expression: + dialects: + - dialect: DATABRICKS + expression: COUNT(DISTINCT l_orderkey) + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "filter": "l_returnflag = ''N''"}' diff --git a/converters/databricks/tests/fixtures/tpcds_metric_view.yaml b/converters/databricks/tests/fixtures/tpcds_metric_view.yaml new file mode 100644 index 00000000..b22fc698 --- /dev/null +++ b/converters/databricks/tests/fixtures/tpcds_metric_view.yaml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: '1.1' +source: tpcds.public.store_sales +comment: Store sales enriched with date, item, and customer dimensions +filter: ss_net_profit > 0 +joins: +- name: date_dim + source: tpcds.public.date_dim + on: source.ss_sold_date_sk = date_dim.d_date_sk + rely: + at_most_one_match: true +- name: item + source: tpcds.public.item + on: source.ss_item_sk = item.i_item_sk + rely: + at_most_one_match: true +- name: customer + source: tpcds.public.customer + on: source.ss_customer_sk = customer.c_customer_sk + rely: + at_most_one_match: true +dimensions: +- name: ticket_number + expr: ss_ticket_number +- name: sold_year + expr: date_dim.d_year + display_name: Year + synonyms: + - year + - yr +- name: sold_date + expr: date_dim.d_date +- name: item_category + expr: item.i_category + synonyms: + - category + - product type +- name: item_brand + expr: item.i_brand +- name: birth_country + expr: customer.c_birth_country +measures: +- name: total_sales + expr: SUM(ss_ext_sales_price) + comment: Total sales revenue + format: + type: currency + currency_code: USD +- name: total_quantity + expr: SUM(ss_quantity) + comment: Total units sold diff --git a/converters/databricks/tests/fixtures/tpcds_ossie.yaml b/converters/databricks/tests/fixtures/tpcds_ossie.yaml new file mode 100644 index 00000000..e055eab1 --- /dev/null +++ b/converters/databricks/tests/fixtures/tpcds_ossie.yaml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: "0.2.0.dev0" +semantic_model: + - name: tpcds_store_sales + description: Store sales enriched with date, item, and customer dimensions + datasets: + - name: store_sales + source: tpcds.public.store_sales + fields: + - name: ticket_number + expression: + dialects: [{dialect: DATABRICKS, expression: ss_ticket_number}] + - name: date_dim + source: tpcds.public.date_dim + primary_key: [d_date_sk] + fields: + - name: sold_year + expression: + dialects: [{dialect: DATABRICKS, expression: d_year}] + label: Year + ai_context: {synonyms: [year, yr]} + - name: sold_date + expression: + dialects: [{dialect: DATABRICKS, expression: d_date}] + - name: item + source: tpcds.public.item + primary_key: [i_item_sk] + fields: + - name: item_category + expression: + dialects: [{dialect: DATABRICKS, expression: i_category}] + ai_context: {synonyms: [category, product type]} + - name: item_brand + expression: + dialects: [{dialect: DATABRICKS, expression: i_brand}] + - name: customer + source: tpcds.public.customer + primary_key: [c_customer_sk] + fields: + - name: birth_country + expression: + dialects: [{dialect: DATABRICKS, expression: c_birth_country}] + relationships: + - name: store_sales_to_date_dim + from: store_sales + to: date_dim + from_columns: [ss_sold_date_sk] + to_columns: [d_date_sk] + - name: store_sales_to_item + from: store_sales + to: item + from_columns: [ss_item_sk] + to_columns: [i_item_sk] + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: [ss_customer_sk] + to_columns: [c_customer_sk] + metrics: + - name: total_sales + expression: + dialects: [{dialect: DATABRICKS, expression: SUM(ss_ext_sales_price)}] + description: Total sales revenue + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "format": {"type": "currency", "currency_code": "USD"}}' + - name: total_quantity + expression: + dialects: [{dialect: DATABRICKS, expression: SUM(ss_quantity)}] + description: Total units sold + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "filter": "ss_net_profit > 0"}' diff --git a/converters/databricks/tests/test_metric_view_to_ossie.py b/converters/databricks/tests/test_metric_view_to_ossie.py new file mode 100644 index 00000000..02890037 --- /dev/null +++ b/converters/databricks/tests/test_metric_view_to_ossie.py @@ -0,0 +1,347 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the Databricks Metric View -> Apache Ossie importer.""" + +import pytest + +from ossie_databricks import ConversionError +from ossie_databricks import metric_view_to_ossie as importer +from _util import canon, load_fixture, parse + + +def test_fixtureB_import_matches_expected(): + out = importer.convert_metric_view_to_ossie(load_fixture("fixtureB_metric_view.yaml")) + assert canon(parse(out)) == canon(parse(load_fixture("fixtureB_ossie.yaml"))) + + +def test_fields_is_accepted_as_alias_for_dimensions(): + """`fields:` is a v1.1 alias for `dimensions:` (the form the DBR docs use); the + importer must read it, not silently drop the columns.""" + mv = ( + "version: '1.1'\nsource: c.s.orders\n" + "fields:\n- {name: region, expr: region}\n" + ) + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + fields = ossie["semantic_model"][0]["datasets"][0].get("fields", []) + assert [f["name"] for f in fields] == ["region"] + + +def test_unsupported_version_rejected(): + with pytest.raises(ConversionError): + importer.convert_metric_view_to_ossie("version: '0.1'\nsource: c.s.t\n") + + +def test_both_dimensions_and_fields_present_warns_and_uses_dimensions(): + """`fields` is a v1.1 alias for `dimensions`; if a (malformed) view sets both, the + importer uses `dimensions` and warns that the `fields` list is ignored.""" + import warnings + mv = ( + "version: '1.1'\nsource: c.s.orders\n" + "dimensions:\n- {name: kept, expr: kept}\n" + "fields:\n- {name: ignored, expr: ignored}\n" + ) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + names = [f["name"] for f in ossie["semantic_model"][0]["datasets"][0].get("fields", [])] + assert names == ["kept"] + assert any("fields" in str(w.message) and "ignored" in str(w.message) for w in caught) + + +def test_stash_written_at_each_level(): + ossie = parse(importer.convert_metric_view_to_ossie(load_fixture("fixtureB_metric_view.yaml"))) + model = ossie["semantic_model"][0] + + # model-level filter + assert any(e["vendor_name"] == "DATABRICKS" and "filter" in e["data"] + for e in model["custom_extensions"]) + # relationship-level rely + rel = model["relationships"][0] + assert any("rely" in e["data"] for e in rel["custom_extensions"]) + # metric-level format + revenue = next(m for m in model["metrics"] if m["name"] == "revenue") + assert any("format" in e["data"] for e in revenue["custom_extensions"]) + + +def test_name_override(): + ossie = parse(importer.convert_metric_view_to_ossie( + load_fixture("fixtureB_metric_view.yaml"), model_name="custom")) + assert ossie["semantic_model"][0]["name"] == "custom" + + +def test_cross_join_rejected(): + mv = "version: '1.1'\nsource: c.s.fact\njoins:\n- name: dim\n source: c.s.dim\n" + with pytest.raises(ConversionError, match="cross"): + importer.convert_metric_view_to_ossie(mv) + + +def test_duplicate_join_name_rejected(): + # a join named like the fact (derived from the source's last identifier) collides + mv = "version: '1.1'\nsource: c.s.fact\njoins:\n- name: fact\n source: c.s.other\n using: [id]\n" + with pytest.raises(ConversionError, match="Duplicate"): + importer.convert_metric_view_to_ossie(mv) + + +def test_complex_joined_dimension_filed_under_join_dataset(): + mv = ( + "version: '1.1'\nsource: c.s.fact\n" + "joins:\n- name: cust\n source: c.s.cust\n on: source.cid = cust.id\n" + "dimensions:\n- name: full\n expr: cust.a || cust.b\n" + ) + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + cust = next(d for d in ossie["semantic_model"][0]["datasets"] if d["name"] == "cust") + assert any(f["name"] == "full" for f in cust.get("fields", [])) + + +def test_non_equi_on_rejected(): + """Apache Ossie relationships are equi-joins (from_columns/to_columns required, minItems 1), so a + non-equi `on` has no Apache Ossie representation and is rejected on import (rather than emitting + a relationship with empty column lists, which is invalid per the Apache Ossie schema).""" + mv = ( + "version: '1.1'\n" + "source: c.s.fact\n" + "joins:\n" + "- name: dim\n" + " source: c.s.dim\n" + " on: source.a >= dim.b\n" + ) + with pytest.raises(ConversionError, match="non-equi"): + importer.convert_metric_view_to_ossie(mv) + + +def test_complex_equi_on_rejected(): + """An equi `on` whose operand is a SQL fragment (OR, computed) can't be decomposed + into from/to columns, so it's rejected rather than producing schema-invalid Apache Ossie with + empty column lists.""" + for cond in ("source.a = dim.b OR source.c = dim.d", "source.a = dim.b + 1"): + mv = ( + "version: '1.1'\nsource: c.s.fact\n" + f"joins:\n- name: dim\n source: c.s.dim\n on: {cond}\n" + ) + with pytest.raises(ConversionError, match="non-equi"): + importer.convert_metric_view_to_ossie(mv) + + +def test_one_to_many_join_flips_from_to_and_stashes_source(): + """A one_to_many MV join becomes an Apache Ossie relationship with the MANY side as `from` + (the joined table), the source/grain on the `to` side, and the grain recorded in + the model-level stash so re-export re-roots correctly.""" + mv = ( + "version: '1.1'\nsource: c.s.orders\n" + "joins:\n- name: line_items\n source: c.s.line_items\n" + " on: source.order_id = line_items.l_order_id\n" + " cardinality: one_to_many\n" + "measures:\n- {name: order_count, expr: COUNT(*)}\n" + ) + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + model = ossie["semantic_model"][0] + rel = model["relationships"][0] + assert rel["from"] == "line_items" # many side (holds the FK) + assert rel["to"] == "orders" # one side (holds the PK) + assert rel["from_columns"] == ["l_order_id"] + assert rel["to_columns"] == ["order_id"] + assert any(e["vendor_name"] == "DATABRICKS" and "source_dataset" in e["data"] + for e in model["custom_extensions"]) + + +def test_at_most_one_match_recovers_unique_key(): + """A many_to_one join with rely.at_most_one_match records the join key as a + unique_keys entry on the joined dataset (recovering key info Apache Ossie would lack).""" + mv = ( + "version: '1.1'\nsource: c.s.orders\n" + "joins:\n- name: customer\n source: c.s.customer\n" + " on: source.cid = customer.id\n" + " rely: {at_most_one_match: true}\n" + ) + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + cust = next(d for d in ossie["semantic_model"][0]["datasets"] if d["name"] == "customer") + assert cust.get("unique_keys") == [["id"]] + + +def test_join_named_source_rejected(): + """`source` is reserved for the fact; a join named `source` is rejected (DBR + forbids it too) rather than silently overwriting the fact alias.""" + mv = ("version: '1.1'\nsource: c.s.fact\n" + "joins:\n- name: source\n source: c.s.dim\n using: [id]\n") + with pytest.raises(ConversionError, match="reserved"): + importer.convert_metric_view_to_ossie(mv) + + +def test_sql_source_name_defaults_to_metric_view(): + """A SELECT/WITH source has no table name, so the model name defaults to + `metric_view` (not a token sliced out of the SQL).""" + mv = "version: '1.1'\nsource: SELECT a, b FROM main.sales.orders\n" + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + assert ossie["semantic_model"][0]["name"] == "metric_view" + + +def test_join_missing_source_raises(): + """Missing required keys surface as ConversionError, not a raw KeyError.""" + mv = "version: '1.1'\nsource: c.s.fact\njoins:\n- name: dim\n using: [id]\n" + with pytest.raises(ConversionError, match="missing required 'source'"): + importer.convert_metric_view_to_ossie(mv) + + +def test_measure_rewrite_with_regex_special_name(): + r"""The `source.` -> fact-name rewrite in measures inserts the name literally, so a + --name containing regex backreference syntax (e.g. \1) does not raise a re.error.""" + mv = "version: '1.1'\nsource: c.s.fact\nmeasures:\n- {name: rev, expr: SUM(source.amount)}\n" + ossie = parse(importer.convert_metric_view_to_ossie(mv, model_name=r"a\1b")) + expr = ossie["semantic_model"][0]["metrics"][0]["expression"]["dialects"][0]["expression"] + assert expr == r"SUM(a\1b.amount)" + + +def test_invalid_source_rejected(): + """A malformed source (not 3-part / not SELECT) is rejected on import, matching the + exporter -- rather than passing through and only failing on a later re-export.""" + mv = "version: '1.1'\nsource: a.b\n" # 2-part, invalid + with pytest.raises(ConversionError, match="3-part"): + importer.convert_metric_view_to_ossie(mv) + + +def test_malformed_yaml_raises_conversion_error(): + """Invalid YAML surfaces as ConversionError, not a raw yaml.YAMLError traceback.""" + with pytest.raises(ConversionError, match="Invalid YAML"): + importer.convert_metric_view_to_ossie("source: c.s.t\njoins: [oops\n") + + +def test_empty_using_rejected(): + """An empty `using: []` is a condition-less join -- rejected at import rather than + silently producing a relationship with empty key columns (which would then fail on + re-export).""" + mv = "version: '1.1'\nsource: c.s.fact\njoins:\n- name: dim\n source: c.s.dim\n using: []\n" + with pytest.raises(ConversionError, match="cross"): + importer.convert_metric_view_to_ossie(mv) + + +def test_boollike_string_values_stay_strings_for_a_yaml_1_1_reader(): + """Bool-like string scalars (e.g. the synonyms `on`/`off`) must be emitted quoted so a + stock YAML 1.1 reader reads them back as strings, not booleans.""" + import yaml + mv = ("version: '1.1'\nsource: c.s.t\n" + "dimensions:\n- {name: status, expr: status, synonyms: [on, off]}\n") + ossie_out = importer.convert_metric_view_to_ossie(mv) + field = yaml.safe_load(ossie_out)["semantic_model"][0]["datasets"][0]["fields"][0] + assert field["ai_context"]["synonyms"] == ["on", "off"] + + +def test_fact_qualifier_variants_in_on_decompose(): + """The fact side of an `on` is valid MV YAML whether qualified with `source`, the + source table name, or left bare; all decompose to the same equi-join columns rather + than being wrongly rejected as a non-equi condition (bug-bash finding).""" + base = ("version: '1.1'\nsource: c.s.orders\n" + "joins:\n- name: customer\n source: c.s.customer\n on: {cond}\n") + for cond in ( + "source.o_custkey = customer.c_custkey", # `source` qualifier + "orders.o_custkey = customer.c_custkey", # source table name + "o_custkey = customer.c_custkey", # bare fact column + "customer.c_custkey = o_custkey", # reversed operand order, bare fact + ): + rel = parse(importer.convert_metric_view_to_ossie( + base.format(cond=cond)))["semantic_model"][0]["relationships"][0] + assert rel["from"] == "orders" and rel["to"] == "customer" + assert rel["from_columns"] == ["o_custkey"] + assert rel["to_columns"] == ["c_custkey"] + + +def test_multi_column_on_with_bare_and_tablename_fact(): + """Composite keys decompose with bare / source-table-name fact qualifiers too.""" + mv = ("version: '1.1'\nsource: c.s.orders\n" + "joins:\n- name: customer\n source: c.s.customer\n" + " on: o_a = customer.c_a AND orders.o_b = customer.c_b\n") + rel = parse(importer.convert_metric_view_to_ossie(mv))["semantic_model"][0]["relationships"][0] + assert rel["from_columns"] == ["o_a", "o_b"] + assert rel["to_columns"] == ["c_a", "c_b"] + + +def test_join_named_source_rejected_any_case(): + """`source` is reserved case-insensitively (DBR identifiers are case-insensitive), + so `Source`/`SOURCE` are rejected too (bug-bash finding).""" + for name in ("Source", "SOURCE", "SoUrCe"): + mv = (f"version: '1.1'\nsource: c.s.fact\n" + f"joins:\n- name: {name}\n source: c.s.dim\n using: [id]\n") + with pytest.raises(ConversionError, match="reserved"): + importer.convert_metric_view_to_ossie(mv) + + +def test_empty_source_part_rejected(): + """A 3-dot source with an empty part (`.s.t`, `c..t`, `c.s.`) is not a valid 3-part + identifier and is rejected -- the dot count alone is not enough (bug-bash finding).""" + for src in (".s.t", "c..t", "c.s."): + mv = f"version: '1.1'\nsource: {src}\n" + with pytest.raises(ConversionError, match="3-part"): + importer.convert_metric_view_to_ossie(mv) + + +def test_whitespace_source_part_rejected(): + """A 3-dot source with a whitespace-laden part (`cat . sch . tbl`) is rejected -- + a space is not part of a valid identifier (review finding).""" + with pytest.raises(ConversionError, match="3-part"): + importer.convert_metric_view_to_ossie("version: '1.1'\nsource: cat . sch . tbl\n") + + +def test_with_paren_subquery_source_accepted(): + """A `WITH(...)` subquery with no space after the keyword is recognized as SQL, + not mistaken for a (non-3-part) identifier (review finding).""" + mv = "version: '1.1'\nsource: WITH(t AS (SELECT 1 AS a)) SELECT a FROM t\n" + ossie = parse(importer.convert_metric_view_to_ossie(mv)) + assert ossie["semantic_model"][0]["datasets"][0]["source"].startswith("WITH(") + + +def test_nested_join_bare_column_rejected(): + """A bare (unqualified) operand in a NESTED join's `on` is ambiguous (parent vs. + fact), so it is rejected rather than silently attributed to the immediate parent. + A bare fact column is still accepted at the top level (review finding).""" + mv = ("version: '1.1'\nsource: c.s.orders\n" + "joins:\n- name: customer\n source: c.s.customer\n on: source.ckey = customer.c_key\n" + " joins:\n - name: nation\n source: c.s.nation\n on: o_nkey = nation.n_key\n") + with pytest.raises(ConversionError, match="non-equi or unsupported"): + importer.convert_metric_view_to_ossie(mv) + + +def test_case_variant_duplicate_name_rejected(): + """DBR identifiers are case-insensitive, so `dim`/`Dim` collide and must be rejected + (consistent with the case-insensitive reserved-`source` check) (review finding).""" + mv = ("version: '1.1'\nsource: c.s.x\n" + "joins:\n- {name: dim, source: c.s.a, using: [id]}\n- {name: Dim, source: c.s.b, using: [id]}\n") + with pytest.raises(ConversionError, match="[Dd]uplicate"): + importer.convert_metric_view_to_ossie(mv) + + +def test_falsy_dimension_name_is_not_a_wildcard(): + """A present-but-falsy name (`0`) is a malformed column, not a wildcard projection; + it raises a clean error rather than being silently dropped (review finding).""" + with pytest.raises(ConversionError): + importer.convert_metric_view_to_ossie("version: '1.1'\nsource: c.s.o\ndimensions:\n- {name: 0, expr: x}\n") + + +def test_non_string_scalars_raise_clean_error(): + """Non-string scalars where a string is required (join name, measure expr) raise a + ConversionError, not a raw AttributeError/TypeError (review finding).""" + for mv in ("version: '1.1'\nsource: c.s.f\njoins:\n- {name: 5, source: c.s.d, using: [id]}\n", + "version: '1.1'\nsource: c.s.o\nmeasures:\n- {name: rev, expr: 5}\n"): + with pytest.raises(ConversionError): + importer.convert_metric_view_to_ossie(mv) + + +def test_using_join_emits_no_yaml_anchor(): + """`from_columns`/`to_columns` of a `using` join are distinct list objects, so the + output has no YAML anchor/alias (`&id`/`*id`) -- safer for other Apache Ossie consumers.""" + out = importer.convert_metric_view_to_ossie( + "version: '1.1'\nsource: c.s.a\njoins:\n- {name: b, source: c.s.b, using: [id]}\n") + assert "&id" not in out and "*id" not in out diff --git a/converters/databricks/tests/test_ossie_to_metric_view.py b/converters/databricks/tests/test_ossie_to_metric_view.py new file mode 100644 index 00000000..8b8808b2 --- /dev/null +++ b/converters/databricks/tests/test_ossie_to_metric_view.py @@ -0,0 +1,670 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the Apache Ossie -> Databricks Metric View exporter.""" + +import json + +import pytest + +from ossie_databricks import ConversionError +from ossie_databricks import ossie_to_metric_view as exporter +from _util import canon, load_fixture, parse + + +def test_fixtureA_export_matches_expected(): + out = exporter.convert_ossie_to_metric_view(load_fixture("fixtureA_ossie.yaml")) + assert parse(out) == parse(load_fixture("fixtureA_metric_view.yaml")) + + +def test_tpcds_export_matches_expected(): + """A normalized TPC-DS star (fact + date/item/customer dims) exports to the expected + Metric View: a join tree, primary keys bridged to rely.at_most_one_match, joined + columns alias-qualified, and the filter/format stash carried through.""" + out = exporter.convert_ossie_to_metric_view(load_fixture("tpcds_ossie.yaml")) + assert parse(out) == parse(load_fixture("tpcds_metric_view.yaml")) + + +def test_unsupported_version_rejected(): + ossie = "version: '9.9.9'\nsemantic_model:\n - name: m\n datasets:\n - {name: d, source: c.s.t}\n" + with pytest.raises(ConversionError): + exporter.convert_ossie_to_metric_view(ossie) + + +def _model(rels): + return { + "version": exporter.OSSIE_VERSION, + "semantic_model": [ + { + "name": "m", + "datasets": [ + {"name": "a", "source": "c.s.a"}, + {"name": "b", "source": "c.s.b"}, + {"name": "x", "source": "c.s.x"}, + ], + "relationships": rels, + } + ], + } + + +def _rel(name, frm, to): + return {"name": name, "from": frm, "to": to, + "from_columns": ["k"], "to_columns": ["k"]} + + +def test_multiple_roots_raises(): + import yaml + # a->b leaves x as a second root. + ossie = yaml.safe_dump(_model([_rel("r1", "a", "b")])) + with pytest.raises(ConversionError, match="multiple candidate fact"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_triangle_is_rejected_as_cycle(): + import yaml + # a->b, a->x, b->x : b and x are equidistant from a (a triangle) -> not a tree. + ossie = yaml.safe_dump(_model([_rel("r1", "a", "b"), _rel("r2", "a", "x"), + _rel("r3", "b", "x")])) + with pytest.raises(ConversionError, match="cycle"): + exporter.convert_ossie_to_metric_view(ossie) + + +def _field(name, col): + return {"name": name, "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": col}]}} + + +def test_mto_diamond_fans_out(): + """A shared dimension reached by two parents (orders->customers->regions and + orders->suppliers->regions) is fanned out into two aliased joins, not rejected.""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "fields": [_field("amt", "amount")]}, + {"name": "customers", "source": "c.s.customers"}, + {"name": "suppliers", "source": "c.s.suppliers"}, + {"name": "regions", "source": "c.s.regions", "fields": [_field("rname", "r_name")]}, + ], + "relationships": [ + {"name": "r1", "from": "orders", "to": "customers", "from_columns": ["cid"], "to_columns": ["id"]}, + {"name": "r2", "from": "orders", "to": "suppliers", "from_columns": ["sid"], "to_columns": ["id"]}, + {"name": "r3", "from": "customers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, + {"name": "r4", "from": "suppliers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, + ], + }]}) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + region_joins = [j for top in out["joins"] for j in top.get("joins", []) if j["source"] == "c.s.regions"] + assert {j["name"] for j in region_joins} == {"customers_regions", "suppliers_regions"} + dims = {d["name"]: d["expr"] for d in out["dimensions"]} + # the fanned `regions` is a depth-2 join, so its column is qualified by the full path + assert dims["customers_regions_rname"] == "customers.customers_regions.r_name" + assert dims["suppliers_regions_rname"] == "suppliers.suppliers_regions.r_name" + + +def test_otm_diamond_fans_out(): + """customers (fact) -> past_orders/future_orders -> line_items: the shared + line_items is fanned out, and every join is one_to_many.""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "customers", "source": "c.s.customers"}, + {"name": "past_orders", "source": "c.s.past_orders"}, + {"name": "future_orders", "source": "c.s.future_orders"}, + {"name": "line_items", "source": "c.s.line_items"}, + ], + "relationships": [ + {"name": "r1", "from": "past_orders", "to": "customers", "from_columns": ["cid"], "to_columns": ["id"]}, + {"name": "r2", "from": "future_orders", "to": "customers", "from_columns": ["cid"], "to_columns": ["id"]}, + {"name": "r3", "from": "line_items", "to": "past_orders", "from_columns": ["oid"], "to_columns": ["id"]}, + {"name": "r4", "from": "line_items", "to": "future_orders", "from_columns": ["oid"], "to_columns": ["id"]}, + ], + "metrics": [{"name": "cnt", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "COUNT(*)"}]}}], + }]}) + out = parse(exporter.convert_ossie_to_metric_view(ossie, source="customers")) + leaf_names = {j["name"] for top in out["joins"] for j in top.get("joins", [])} + assert leaf_names == {"past_orders_line_items", "future_orders_line_items"} + + def all_otm(joins): + return all(j.get("cardinality") == "one_to_many" and all_otm(j.get("joins", [])) + for j in joins) + assert all_otm(out["joins"]) + + +def test_cycle_raises(): + import yaml + # a->b->x->a : cycle, and no root. + ossie = yaml.safe_dump(_model([_rel("r1", "a", "b"), _rel("r2", "b", "x"), + _rel("r3", "x", "a")])) + with pytest.raises(ConversionError, match="cycle"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_no_unknown_keys_leak(): + """Exporter output must contain no key outside the v1.1 schema (the strict-parse + guard: no `custom_extensions`, no `sql_on`).""" + out = exporter.convert_ossie_to_metric_view(load_fixture("fixtureA_ossie.yaml")) + assert "custom_extensions" not in out + assert "sql_on" not in out + + +def test_primary_key_is_dropped(): + out = parse(exporter.convert_ossie_to_metric_view(load_fixture("fixtureA_ossie.yaml"))) + assert "primary_key" not in json.dumps(out) + + +def _single_fact_model(metric_expr): + import yaml + return yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [{"name": "orders", "source": "c.s.orders", + "fields": [{"name": "k", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": "k"}]}}]}], + "metrics": [{"name": "rev", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": metric_expr}]}}], + }], + }) + + +def test_measure_strips_fact_prefix(): + """Fact columns are bare in measure expressions (DBR idiom), not `source.`-qualified.""" + out = parse(exporter.convert_ossie_to_metric_view(_single_fact_model("SUM(orders.amount)"))) + assert out["measures"][0]["expr"] == "SUM(amount)" + + +def test_measure_keeps_lookalike_table_prefix(): + """A table whose name merely ends with the fact name is not stripped.""" + out = parse(exporter.convert_ossie_to_metric_view(_single_fact_model("SUM(store_orders.amount)"))) + assert out["measures"][0]["expr"] == "SUM(store_orders.amount)" + + +def test_invalid_source_rejected(): + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{"name": "m", "datasets": [{"name": "d", "source": "justatable"}]}], + }) + with pytest.raises(ConversionError, match="source"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_duplicate_dimension_name_rejected(): + # Two datasets each contributing a field named `id` collide in the single flat + # `dimensions` list. Metric Views require unique dimension/measure names, so this + # is rejected (rather than emitting a duplicate the user would hand to Databricks). + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", + "fields": [{"name": "id", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "id"}]}}]}, + {"name": "customer", "source": "c.s.customer", + "fields": [{"name": "id", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "id"}]}}]}, + ], + "relationships": [{"name": "r", "from": "orders", "to": "customer", + "from_columns": ["cid"], "to_columns": ["id"]}], + }], + }) + with pytest.raises(ConversionError, match="collides"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_measure_name_collides_with_dimension_rejected(): + # A measure whose name matches an existing dimension (case-insensitively) collides + # in the shared name space; Metric Views require uniqueness, so this is rejected. + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", + "fields": [{"name": "total", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "total"}]}}]}, + ], + "metrics": [ + {"name": "Total", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "SUM(total)"}]}}, + ], + }], + }) + with pytest.raises(ConversionError, match="collides"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_cascade_drop_downstream_measure_reference(): + """A measure that references a dropped measure via measure() is itself dropped + (transitively), so no dangling reference is emitted.""" + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [{"name": "f", "source": "c.s.f"}], + "metrics": [ + {"name": "base", "expression": {"dialects": [{"dialect": "SNOWFLAKE", "expression": "SUM(x)"}]}}, # dropped (no DBX/ANSI) + {"name": "derived", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "measure(base) * 2"}]}}, + {"name": "derived2", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "measure(derived) + 1"}]}}, # transitive + {"name": "ok", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "COUNT(*)"}]}}, + ], + }], + }) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + names = [m["name"] for m in out.get("measures", [])] + assert names == ["ok"] # base dropped; derived + derived2 cascade-dropped; ok survives + + +def test_cascade_drop_downstream_dimension_reference(): + """A field/measure referencing a dropped dimension by name is also dropped.""" + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [{"name": "f", "source": "c.s.f", "fields": [ + {"name": "region", "expression": {"dialects": [{"dialect": "SNOWFLAKE", "expression": "r"}]}}, # dropped dim + {"name": "label", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "upper(region)"}]}}, # references region + {"name": "keep", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "id"}]}}, + ]}], + }], + }) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + dims = [d["name"] for d in out.get("dimensions", [])] + assert dims == ["keep"] # region dropped; label cascade-dropped; keep survives + + +def test_orientation_unverifiable_when_to_side_has_no_key_warns(): + """If the `from` columns are a declared key but the `to` side declares no key, the + from/to orientation can't be verified; the converter leaves it as-is (no reorient) + and warns, rather than silently producing a possibly-inverted cardinality.""" + import warnings + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "a", "source": "c.s.a", "primary_key": ["a_id"], "fields": [ + {"name": "a_name", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "a_name"}]}}]}, + {"name": "b", "source": "c.s.b", "fields": [ + {"name": "b_name", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "b_name"}]}}]}, + ], + # from columns cover a's PK, but b (the `to` side) declares no key + "relationships": [{"name": "a_to_b", "from": "a", "to": "b", + "from_columns": ["a_id"], "to_columns": ["b_x"]}], + }], + }) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + exporter.convert_ossie_to_metric_view(ossie) + assert any("orientation can't be verified" in str(w.message) for w in caught) + + +def test_cascade_drop_skips_qualified_join_alias_collision(): + """A dropped field whose name collides with a join alias must NOT cascade-drop a + *qualified* `alias.col` reference. A genuine dimension reference is unqualified; + `region.r_name` points at the join `region`, a different thing than a dropped bare + `region`, so the cascade must leave it (and the joined column) alone.""" + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "fields": [ + # dropped (no DBX/ANSI dialect); its name collides with the `region` join + {"name": "region", "expression": {"dialects": [{"dialect": "SNOWFLAKE", "expression": "r"}]}}, + # references the join alias `region`, not the dropped field -> must survive + {"name": "summary", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "region.r_name"}]}}, + ]}, + {"name": "region", "source": "c.s.region", "primary_key": ["r_key"], "fields": [ + {"name": "r_name", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "r_name"}]}}, + ]}, + ], + "relationships": [{"name": "orr", "from": "orders", "to": "region", + "from_columns": ["o_rkey"], "to_columns": ["r_key"]}], + }], + }) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + dims = [d["name"] for d in out.get("dimensions", [])] + # `region` field dropped; `summary` (refs alias region.r_name) and the joined + # `r_name` (qualified to region.r_name on export) both survive -- no false cascade. + assert "region" not in dims + assert "summary" in dims and "r_name" in dims + + +def _orders_lineitems_ossie(): + """Apache Ossie: line_items (many, FK l_order_id) -> orders (one, PK order_id).""" + import yaml + return yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "sales", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "primary_key": ["order_id"], + "fields": [{"name": "order_date", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": "o_order_date"}]}}]}, + {"name": "line_items", "source": "c.s.line_items", + "fields": [{"name": "product_sk", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": "l_product_sk"}]}}]}, + ], + "relationships": [{"name": "li_to_order", "from": "line_items", "to": "orders", + "from_columns": ["l_order_id"], "to_columns": ["order_id"]}], + "metrics": [{"name": "order_count", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": "COUNT(*)"}]}}], + }], + }) + + +def test_source_on_to_side_derives_one_to_many(): + """Naming the PK/one-side dataset as the source makes its join one_to_many, and + the many-side table's columns drop (a field must resolve to one value/source row).""" + out = parse(exporter.convert_ossie_to_metric_view(_orders_lineitems_ossie(), source="orders")) + assert out["source"] == "c.s.orders" + join = out["joins"][0] + assert join["name"] == "line_items" + assert join["cardinality"] == "one_to_many" + assert join["on"] == "source.order_id = line_items.l_order_id" + assert [d["name"] for d in out.get("dimensions", [])] == ["order_date"] # product_sk dropped + + +def test_default_fact_is_fk_sink_and_many_to_one(): + """Without an explicit source the fact is the FK-sink (line_items) and the join to + orders is the default many_to_one (no explicit cardinality).""" + out = parse(exporter.convert_ossie_to_metric_view(_orders_lineitems_ossie())) + assert out["source"] == "c.s.line_items" + join = out["joins"][0] + assert join["name"] == "orders" + assert "cardinality" not in join + assert join["on"] == "source.l_order_id = orders.order_id" + + +def test_unknown_source_rejected(): + with pytest.raises(ConversionError, match="not a dataset"): + exporter.convert_ossie_to_metric_view(_orders_lineitems_ossie(), source="nope") + + +def test_one_to_many_subtree_must_stay_one_to_many(): + """A many_to_one join descending from a one_to_many join is rejected (DBR rule).""" + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders"}, + {"name": "line_items", "source": "c.s.line_items"}, + {"name": "product", "source": "c.s.product"}, + ], + "relationships": [ + {"name": "li_to_order", "from": "line_items", "to": "orders", # orders->li : OTM + "from_columns": ["l_order_id"], "to_columns": ["order_id"]}, + {"name": "li_to_product", "from": "line_items", "to": "product", # li->product : MTO + "from_columns": ["l_product_sk"], "to_columns": ["p_sk"]}, + ], + }], + }) + with pytest.raises(ConversionError, match="one-to-many"): + exporter.convert_ossie_to_metric_view(ossie, source="orders") + + +def test_primary_key_deduces_at_most_one_match(): + """A many_to_one join whose to_columns cover the target's declared primary_key + gets rely.at_most_one_match; a join to a key-less dataset does not.""" + import yaml + + def model(dim_extra): + dim = {"name": "customer", "source": "c.s.customer"} + dim.update(dim_extra) + return yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [{"name": "orders", "source": "c.s.orders"}, dim], + "relationships": [{"name": "r", "from": "orders", "to": "customer", + "from_columns": ["cid"], "to_columns": ["id"]}], + }]}) + + join = parse(exporter.convert_ossie_to_metric_view(model({"primary_key": ["id"]})))["joins"][0] + assert join.get("rely") == {"at_most_one_match": True} + join2 = parse(exporter.convert_ossie_to_metric_view(model({})))["joins"][0] + assert "rely" not in join2 + + +def test_mislabeled_from_to_reoriented_by_key(): + """When from/to is swapped but the declared keys show the real one-side, the + converter re-orients to the key side (warns) -- so fact selection and cardinality + come out identical to the well-formed model.""" + import warnings + import yaml + + def model(frm, to, from_cols, to_cols): + return yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "primary_key": ["order_id"], + "fields": [_field("amt", "amount")]}, + {"name": "customer", "source": "c.s.customer", "primary_key": ["c_id"], + "fields": [_field("cname", "c_name")]}, + ], + "relationships": [{"name": "r", "from": frm, "to": to, + "from_columns": from_cols, "to_columns": to_cols}], + }]}) + + well = parse(exporter.convert_ossie_to_metric_view( + model("orders", "customer", ["cust_id"], ["c_id"]))) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + swapped = parse(exporter.convert_ossie_to_metric_view( + model("customer", "orders", ["c_id"], ["cust_id"]))) + assert any("mislabeled" in str(w.message) for w in caught) + assert swapped["source"] == "c.s.orders" # fact selection corrected to the FK holder + assert swapped == well # identical to the well-formed model + + +def test_dataset_named_source_is_renamed(): + """A dataset literally named `source` must not collide with the fact's reserved + `source` alias (would otherwise emit an ambiguous join).""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders"}, + {"name": "source", "source": "c.s.dim", "fields": [_field("x", "xcol")]}, + ], + "relationships": [{"name": "r", "from": "orders", "to": "source", + "from_columns": ["sid"], "to_columns": ["id"]}], + }]}) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + join = out["joins"][0] + assert join["name"] != "source" + assert join["on"] == f"source.sid = {join['name']}.id" + assert out["dimensions"][0]["expr"] == f"{join['name']}.xcol" + + +def test_fanout_alias_collision_deduped(): + """A real dataset whose name equals a synthesized fan-out alias still gets a + distinct alias -- no two joins share a name.""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders"}, + {"name": "customers", "source": "c.s.customers"}, + {"name": "suppliers", "source": "c.s.suppliers"}, + {"name": "regions", "source": "c.s.regions"}, + {"name": "customers_regions", "source": "c.s.cr"}, # collides with fan-out alias + ], + "relationships": [ + {"name": "r1", "from": "orders", "to": "customers", "from_columns": ["cid"], "to_columns": ["id"]}, + {"name": "r2", "from": "orders", "to": "suppliers", "from_columns": ["sid"], "to_columns": ["id"]}, + {"name": "r3", "from": "customers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, + {"name": "r4", "from": "suppliers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, + {"name": "r5", "from": "orders", "to": "customers_regions", "from_columns": ["xid"], "to_columns": ["id"]}, + ], + }]}) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + names = [] + + def collect(joins): + for j in joins: + names.append(j["name"]) + collect(j.get("joins", [])) + + collect(out["joins"]) + assert len(names) == len(set(names)), names # all join names unique + + +def test_malformed_input_raises_conversion_error(): + """Missing required keys surface as ConversionError, not a raw KeyError traceback.""" + import yaml + bad = yaml.safe_dump({"version": exporter.OSSIE_VERSION, + "semantic_model": [{"name": "m", "datasets": [{"source": "c.s.t"}]}]}) + with pytest.raises(ConversionError, match="missing required 'name'"): + exporter.convert_ossie_to_metric_view(bad) + + +def test_nameless_relationship_with_ai_context_does_not_crash(): + """A relationship may omit `name`; the dropped-ai_context warning must not raise a + raw KeyError when it has ai_context but no name.""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "fields": [_field("amt", "amt")]}, + {"name": "customers", "source": "c.s.customers"}, + ], + "relationships": [ + {"from": "orders", "to": "customers", "from_columns": ["cid"], + "to_columns": ["id"], "ai_context": "joins orders to customers"}, + ], + }]}) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) # must not raise + assert out["joins"][0]["name"] == "customers" + + +def _cfield(name, expr): + return {"name": name, "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": expr}]}} + + +def test_fanout_complex_expr_dropped_not_emitted_ambiguously(): + """On a fanned-out (diamond) dataset, a simple column fans out into one aliased + dimension per instance, but a complex expression -- which cannot be attributed to a + single instance -- is dropped rather than emitted ambiguously.""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders"}, + {"name": "customers", "source": "c.s.customers"}, + {"name": "suppliers", "source": "c.s.suppliers"}, + {"name": "regions", "source": "c.s.regions", + "fields": [_cfield("r_name", "r_name"), _cfield("rfull", "r_a || r_b")]}, + ], + "relationships": [ + {"name": "r1", "from": "orders", "to": "customers", "from_columns": ["cid"], "to_columns": ["id"]}, + {"name": "r2", "from": "orders", "to": "suppliers", "from_columns": ["sid"], "to_columns": ["id"]}, + {"name": "r3", "from": "customers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, + {"name": "r4", "from": "suppliers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, + ], + }]}) + dims = parse(exporter.convert_ossie_to_metric_view(ossie)).get("dimensions", []) + # the simple column fans out into two unambiguous, alias-qualified dimensions ... + assert sum(1 for d in dims if d["name"].endswith("_r_name")) == 2 + # ... while the ambiguous complex expression is dropped (never emitted unqualified) + assert not any("||" in d["expr"] for d in dims) + + +def test_malformed_yaml_raises_conversion_error(): + """Invalid YAML surfaces as ConversionError, not a raw yaml.YAMLError traceback.""" + with pytest.raises(ConversionError, match="Invalid YAML"): + exporter.convert_ossie_to_metric_view("semantic_model: [oops\n") + + +def test_nested_join_uses_full_path_qualification(): + """A snowflake (orders -> customer -> nation) qualifies a column from the nested + `nation` join by its full join path from the source (`customer.nation.n_name`) -- the + Databricks nested-join rule -- not the single-level `nation.n_name`. A depth-1 join + stays single-name.""" + import yaml + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "fields": [_field("amt", "amount")]}, + {"name": "customer", "source": "c.s.customer", "fields": [_field("cname", "c_name")]}, + {"name": "nation", "source": "c.s.nation", "fields": [_field("nname", "n_name")]}, + ], + "relationships": [ + {"name": "r1", "from": "orders", "to": "customer", "from_columns": ["ckey"], "to_columns": ["c_key"]}, + {"name": "r2", "from": "customer", "to": "nation", "from_columns": ["nkey"], "to_columns": ["n_key"]}, + ], + }]}) + out = parse(exporter.convert_ossie_to_metric_view(ossie)) + exprs = {d["name"]: d["expr"] for d in out["dimensions"]} + assert exprs["cname"] == "customer.c_name" # depth-1: the join's own name + assert exprs["nname"] == "customer.nation.n_name" # depth-2: full path from source + # the nested join's `on:` still uses immediate names (single-level) + nation_join = out["joins"][0]["joins"][0] + assert nation_join["on"] == "customer.nkey = nation.n_key" + + +def test_case_variant_dataset_name_rejected(): + """DBR identifiers are case-insensitive, so two datasets differing only in case + (`customer`/`Customer`) collide and are rejected (review finding).""" + ossie = ("version: 0.2.0.dev0\nsemantic_model:\n- name: m\n datasets:\n" + " - {name: customer, source: c.s.c}\n - {name: Customer, source: c.s.c2}\n") + with pytest.raises(ConversionError, match="duplicate"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_non_string_field_expression_raises_clean_error(): + """A non-string dialect expression raises a ConversionError, not a raw crash + (review finding).""" + ossie = ("version: 0.2.0.dev0\nsemantic_model:\n- name: m\n datasets:\n" + " - name: o\n source: c.s.o\n fields:\n - name: d\n expression:\n" + " dialects:\n - {dialect: DATABRICKS, expression: 123}\n") + with pytest.raises(ConversionError, match="must be a string"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_scalar_join_columns_rejected(): + """`from_columns`/`to_columns` given as a scalar string (not a list) raise a clear + 'must be lists' error rather than a misleading character-count length error.""" + ossie = ("version: 0.2.0.dev0\nsemantic_model:\n- name: m\n datasets:\n" + " - {name: a, source: c.s.a}\n - {name: b, source: c.s.b}\n relationships:\n" + " - {name: ab, from: a, to: b, from_columns: cid, to_columns: id}\n") + with pytest.raises(ConversionError, match="must be lists"): + exporter.convert_ossie_to_metric_view(ossie) + + +def test_malformed_stash_json_raises_conversion_error(): + # A hand-edited DATABRICKS custom_extensions with invalid JSON in `data` must + # surface as a clean ConversionError, not a raw json.JSONDecodeError traceback. + import yaml + ossie = yaml.safe_dump({ + "version": exporter.OSSIE_VERSION, + "semantic_model": [{ + "name": "m", + "custom_extensions": [{"vendor_name": "DATABRICKS", "data": "{not valid json"}], + "datasets": [{"name": "f", "source": "c.s.f", + "fields": [{"name": "x", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "x"}]}}]}], + "metrics": [{"name": "n", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "COUNT(*)"}]}}], + }], + }) + with pytest.raises(ConversionError, match="not valid JSON"): + exporter.convert_ossie_to_metric_view(ossie) diff --git a/converters/databricks/tests/test_roundtrip.py b/converters/databricks/tests/test_roundtrip.py new file mode 100644 index 00000000..c9d5629a --- /dev/null +++ b/converters/databricks/tests/test_roundtrip.py @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Round-trip tests in both directions.""" + +from ossie_databricks import metric_view_to_ossie as importer +from ossie_databricks import ossie_to_metric_view as exporter +from _util import canon, load_fixture, parse, strip_dropped + + +def test_ossie_to_mv_to_ossie(): + """Apache Ossie -> MV -> Apache Ossie preserves everything except the documented drops + (model name, primary_key/unique_keys).""" + ossie_in = load_fixture("fixtureA_ossie.yaml") + mv = exporter.convert_ossie_to_metric_view(ossie_in) + ossie_out = importer.convert_metric_view_to_ossie(mv) + assert strip_dropped(parse(ossie_out)) == strip_dropped(parse(ossie_in)) + + +def test_using_clause_round_trips(): + """A `using` join survives MV -> Apache Ossie -> MV (equal column lists re-emit as `using`).""" + mv_in = ( + "version: '1.1'\nsource: c.s.fact\n" + "joins:\n- name: dim\n source: c.s.dim\n using: [id]\n" + "measures:\n- name: n\n expr: count(*)\n" + ) + ossie = importer.convert_metric_view_to_ossie(mv_in) + join = parse(exporter.convert_ossie_to_metric_view(ossie))["joins"][0] + assert join.get("using") == ["id"] + assert "on" not in join + + +def test_mv_to_ossie_to_mv_is_lossless(): + """MV -> Apache Ossie -> MV is byte-faithful (structurally): the stash carries every + MV-only feature through Apache Ossie and back.""" + mv_in = load_fixture("fixtureB_metric_view.yaml") + ossie = importer.convert_metric_view_to_ossie(mv_in) + mv_out = exporter.convert_ossie_to_metric_view(ossie) + assert parse(mv_out) == parse(mv_in) + + +def test_tpcds_mv_round_trips(): + """The TPC-DS Metric View (multi-join star with rely/filter/format) survives + MV -> Apache Ossie -> MV unchanged.""" + mv_in = load_fixture("tpcds_metric_view.yaml") + ossie = importer.convert_metric_view_to_ossie(mv_in) + mv_out = exporter.convert_ossie_to_metric_view(ossie) + assert parse(mv_out) == parse(mv_in) + + +def test_one_to_many_round_trips_mv_ossie_mv(): + """A one_to_many Metric View survives MV -> Apache Ossie -> MV: cardinality rides the + relationship direction (+ stash), and the source/grain rides the model stash, so + the exporter re-roots at `orders` rather than the FK-sink `line_items`.""" + mv_in = ( + "version: '1.1'\nsource: c.s.orders\ncomment: Orders\n" + "joins:\n- name: line_items\n source: c.s.line_items\n" + " on: source.order_id = line_items.l_order_id\n" + " cardinality: one_to_many\n" + "dimensions:\n- {name: order_date, expr: o_order_date}\n" + "measures:\n- {name: order_count, expr: COUNT(*)}\n" + ) + ossie = importer.convert_metric_view_to_ossie(mv_in) + mv_out = exporter.convert_ossie_to_metric_view(ossie) + assert parse(mv_out) == parse(mv_in) + + +# Property-based round-trip coverage. The Hypothesis driver lives in +# test_roundtrip_properties.py; these run the same generators/assertions under a plain +# seeded RNG so the property coverage also holds where Hypothesis is not installed. + +def test_property_mv_to_ossie_to_mv_seeded(): + from _roundtrip_helpers import RandomRnd, assert_mv_roundtrip, build_metric_view + for seed in range(250): + assert_mv_roundtrip(build_metric_view(RandomRnd(seed))) + + +def test_property_ossie_to_mv_to_ossie_seeded(): + from _roundtrip_helpers import RandomRnd, assert_ossie_roundtrip, build_ossie + for seed in range(250): + assert_ossie_roundtrip(build_ossie(RandomRnd(1_000_000 + seed))) diff --git a/converters/databricks/tests/test_roundtrip_properties.py b/converters/databricks/tests/test_roundtrip_properties.py new file mode 100644 index 00000000..d022ebb1 --- /dev/null +++ b/converters/databricks/tests/test_roundtrip_properties.py @@ -0,0 +1,107 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Property-based round-trip tests (Hypothesis). + +For any generated model in the round-trippable subset, converting one direction and +back preserves content: + + - MV -> Apache Ossie -> MV : source, every dimension/measure name+expression+metadata, every + join (name, source, condition, cardinality, rely) and the join nesting, and the + model-level filter/comment/materialization. + - Apache Ossie -> MV -> Apache Ossie : dataset names+sources+fields, relationship from/to/columns, + metric name+expression, and the model description. + +The model generation and the assertions live in `_roundtrip_helpers` (no test-framework +dependency) so the exact same logic also runs under a plain seeded RNG where Hypothesis +is unavailable (see the `*_seeded` tests in test_roundtrip.py). This file is the thin +Hypothesis driver: it maps draws into the shared `Rnd` interface and runs the properties. + +Run: `pytest test_roundtrip_properties.py` (needs `hypothesis`). +""" + +import pytest + +pytest.importorskip("hypothesis") # skip cleanly if hypothesis is not installed + +from hypothesis import HealthCheck, given, settings +from hypothesis import strategies as st + +from _roundtrip_helpers import ( + assert_mv_roundtrip, + assert_ossie_roundtrip, + build_metric_view, + build_ossie, +) + +# Alphanumeric metadata text with optional interior spaces (no leading/trailing space, +# no YAML-special characters), so values survive a dump/load cycle verbatim. +_safe_text = st.from_regex(r"[A-Za-z0-9]([A-Za-z0-9 ]{0,18}[A-Za-z0-9])?", fullmatch=True) +# A SQL column-style identifier. +_colident = st.from_regex(r"[a-z_][a-z0-9_]{0,7}", fullmatch=True) + +_SETTINGS = settings( + max_examples=300, + suppress_health_check=[HealthCheck.too_slow, HealthCheck.data_too_large], +) + + +class _HypothesisRnd: + """The `Rnd` interface backed by a Hypothesis `draw`. `chance(p)` ignores `p` + (Hypothesis explores both branches regardless).""" + + def __init__(self, draw): + self._draw = draw + + def chance(self, p=0.5): + return self._draw(st.booleans()) + + def count(self, lo, hi): + return self._draw(st.integers(min_value=lo, max_value=hi)) + + def pick(self, seq): + return self._draw(st.sampled_from(list(seq))) + + def text(self): + return self._draw(_safe_text) + + def colname(self): + return self._draw(_colident) + + +@st.composite +def metric_views(draw): + return build_metric_view(_HypothesisRnd(draw)) + + +@st.composite +def ossie_models(draw): + return build_ossie(_HypothesisRnd(draw)) + + +class TestMetricViewRoundTrip: + @given(mv=metric_views()) + @_SETTINGS + def test_mv_to_ossie_to_mv(self, mv): + assert_mv_roundtrip(mv) + + +class TestOSIRoundTrip: + @given(ossie=ossie_models()) + @_SETTINGS + def test_ossie_to_mv_to_ossie(self, ossie): + assert_ossie_roundtrip(ossie) From efeb18e0f2fd6485e770ff9234867e13061bc2f1 Mon Sep 17 00:00:00 2001 From: Haoran Li Date: Wed, 19 Aug 2026 16:50:36 +0000 Subject: [PATCH 2/3] [OSSIE] Add Java converter and restructure converters/databricks Move the Python converter under converters/databricks/python/ (content unchanged) and add a Maven Java module -- library, CLI (OssieDatabricksConverter), JUnit tests, and fixtures under java/, package org.apache.ossie.converter.databricks -- as the maintained implementation. Add a root README describing the two-language layout, and a Java build job (mvn -B verify, JDK 21) in converter-databricks-ci.yml, mirroring the polaris converter. Signed-off-by: Haoran Li --- .github/workflows/converter-databricks-ci.yml | 85 ++ converters/databricks/README.md | 133 +-- converters/databricks/java/README.md | 137 +++ converters/databricks/java/pom.xml | 131 +++ .../databricks/MetricViewToOssie.java | 474 +++++++++ .../converter/databricks/OssieConverter.java | 103 ++ .../databricks/OssieConverterCommon.java | 498 ++++++++++ .../databricks/OssieDatabricksConverter.java | 179 ++++ .../databricks/OssieToMetricView.java | 903 ++++++++++++++++++ .../OssieConverterRoundTripSuite.java | 560 +++++++++++ .../databricks/OssieConverterSuite.java | 774 +++++++++++++++ .../ossie_fixtureA_metric_view.yaml} | 0 .../test/resources/ossie_fixtureA_ossie.yaml} | 0 .../resources/ossie_fixtureB_metric_view.yaml | 60 ++ .../test/resources/ossie_fixtureB_ossie.yaml | 75 ++ .../resources/ossie_tpcds_metric_view.yaml} | 0 .../test/resources/ossie_tpcds_ossie.yaml} | 0 converters/databricks/python/README.md | 124 +++ .../databricks/{ => python}/pyproject.toml | 36 +- .../src/ossie_databricks/__init__.py | 0 .../src/ossie_databricks/_common.py | 0 .../{ => python}/src/ossie_databricks/cli.py | 0 .../ossie_databricks/metric_view_to_ossie.py | 0 .../ossie_databricks/ossie_to_metric_view.py | 0 .../{ => python}/tests/_roundtrip_helpers.py | 0 .../databricks/{ => python}/tests/_util.py | 0 .../databricks/{ => python}/tests/conftest.py | 0 .../tests/fixtures/fixtureA_metric_view.yaml | 53 + .../python/tests/fixtures/fixtureA_ossie.yaml | 79 ++ .../tests/fixtures/fixtureB_metric_view.yaml | 0 .../tests/fixtures/fixtureB_ossie.yaml | 0 .../tests/fixtures/tpcds_metric_view.yaml | 67 ++ .../python/tests/fixtures/tpcds_ossie.yaml | 89 ++ .../tests/test_metric_view_to_ossie.py | 0 .../tests/test_ossie_to_metric_view.py | 0 .../{ => python}/tests/test_roundtrip.py | 0 .../tests/test_roundtrip_properties.py | 0 converters/databricks/python/uv.lock | 214 +++++ 38 files changed, 4641 insertions(+), 133 deletions(-) create mode 100644 .github/workflows/converter-databricks-ci.yml create mode 100644 converters/databricks/java/README.md create mode 100644 converters/databricks/java/pom.xml create mode 100644 converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/MetricViewToOssie.java create mode 100644 converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/OssieConverter.java create mode 100644 converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/OssieConverterCommon.java create mode 100644 converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/OssieDatabricksConverter.java create mode 100644 converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/OssieToMetricView.java create mode 100644 converters/databricks/java/src/test/java/org/apache/ossie/converter/databricks/OssieConverterRoundTripSuite.java create mode 100644 converters/databricks/java/src/test/java/org/apache/ossie/converter/databricks/OssieConverterSuite.java rename converters/databricks/{tests/fixtures/fixtureA_metric_view.yaml => java/src/test/resources/ossie_fixtureA_metric_view.yaml} (100%) rename converters/databricks/{tests/fixtures/fixtureA_ossie.yaml => java/src/test/resources/ossie_fixtureA_ossie.yaml} (100%) create mode 100644 converters/databricks/java/src/test/resources/ossie_fixtureB_metric_view.yaml create mode 100644 converters/databricks/java/src/test/resources/ossie_fixtureB_ossie.yaml rename converters/databricks/{tests/fixtures/tpcds_metric_view.yaml => java/src/test/resources/ossie_tpcds_metric_view.yaml} (100%) rename converters/databricks/{tests/fixtures/tpcds_ossie.yaml => java/src/test/resources/ossie_tpcds_ossie.yaml} (100%) create mode 100644 converters/databricks/python/README.md rename converters/databricks/{ => python}/pyproject.toml (77%) rename converters/databricks/{ => python}/src/ossie_databricks/__init__.py (100%) rename converters/databricks/{ => python}/src/ossie_databricks/_common.py (100%) rename converters/databricks/{ => python}/src/ossie_databricks/cli.py (100%) rename converters/databricks/{ => python}/src/ossie_databricks/metric_view_to_ossie.py (100%) rename converters/databricks/{ => python}/src/ossie_databricks/ossie_to_metric_view.py (100%) rename converters/databricks/{ => python}/tests/_roundtrip_helpers.py (100%) rename converters/databricks/{ => python}/tests/_util.py (100%) rename converters/databricks/{ => python}/tests/conftest.py (100%) create mode 100644 converters/databricks/python/tests/fixtures/fixtureA_metric_view.yaml create mode 100644 converters/databricks/python/tests/fixtures/fixtureA_ossie.yaml rename converters/databricks/{ => python}/tests/fixtures/fixtureB_metric_view.yaml (100%) rename converters/databricks/{ => python}/tests/fixtures/fixtureB_ossie.yaml (100%) create mode 100644 converters/databricks/python/tests/fixtures/tpcds_metric_view.yaml create mode 100644 converters/databricks/python/tests/fixtures/tpcds_ossie.yaml rename converters/databricks/{ => python}/tests/test_metric_view_to_ossie.py (100%) rename converters/databricks/{ => python}/tests/test_ossie_to_metric_view.py (100%) rename converters/databricks/{ => python}/tests/test_roundtrip.py (100%) rename converters/databricks/{ => python}/tests/test_roundtrip_properties.py (100%) create mode 100644 converters/databricks/python/uv.lock diff --git a/.github/workflows/converter-databricks-ci.yml b/.github/workflows/converter-databricks-ci.yml new file mode 100644 index 00000000..d4061961 --- /dev/null +++ b/.github/workflows/converter-databricks-ci.yml @@ -0,0 +1,85 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Converters Databricks CI + +on: + push: + branches: [ "main" ] + paths: + - 'converters/databricks/**' + - '.github/workflows/converter-databricks-ci.yml' + pull_request: + branches: [ "main" ] + paths: + - 'converters/databricks/**' + - '.github/workflows/converter-databricks-ci.yml' + +jobs: + python: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + defaults: + run: + working-directory: converters/databricks/python + + steps: + - name: Checkout project + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" + + - name: Sync dependencies + run: | + uv sync + + - name: Unit Tests + run: | + uv run pytest + + java: + runs-on: ubuntu-latest + defaults: + run: + working-directory: converters/databricks/java + + steps: + - name: Checkout project + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up JDK 21 + uses: actions/setup-java@03ad4de0992f5dab5e18fcb136590ce7c4a0ac95 # v5.6.0 + with: + java-version: '21' + distribution: 'temurin' + cache: 'maven' + + - name: Build and test + run: | + mvn -B verify diff --git a/converters/databricks/README.md b/converters/databricks/README.md index e2fe6ccb..ff7997ca 100644 --- a/converters/databricks/README.md +++ b/converters/databricks/README.md @@ -1,124 +1,17 @@ - +Layout +------ -# Apache Ossie Databricks Converter +| Path | Language | Role | +|------|----------|------| +| [`java/`](java/) | Java | The maintained implementation; also ships a command-line tool (`OssieDatabricksConverter`). | +| [`python/`](python/) | Python | The original reference implementation. To be deprecated. | -Bidirectional, offline conversion between an [Apache Ossie](https://github.com/apache/ossie) -semantic model and a Databricks -[Unity Catalog Metric View](https://docs.databricks.com/aws/en/metric-views/) (YAML -`1.1`). No Databricks connection required. - -- **Export** (`ossie-databricks export`): Apache Ossie -> Metric View (one fact - `source` with a nested `joins` tree and a flat `dimensions` list). -- **Import** (`ossie-databricks import`): Metric View -> Apache Ossie. Metric View features Apache Ossie has - no native field for are preserved in `custom_extensions[DATABRICKS]`, so - `MV -> Apache Ossie -> MV` is lossless. - -On **export** (Apache Ossie -> Metric View), Apache Ossie features with no Metric View slot -- relationship -`ai_context`, `dimension.is_time`, non-`DATABRICKS`/`ANSI_SQL` dialects, foreign-vendor -`custom_extensions` -- are **dropped with a warning**. On **import** (Metric View -> Apache Ossie), -Metric View only features (filter, window, format, rely, ...) are instead **preserved** in -`custom_extensions[DATABRICKS]`, so `MV -> Apache Ossie -> MV` is lossless. Any input that breaks a -[requirement](#requirements) **raises a `ConversionError`** -- the converter never -silently drops a field or produces an invalid result. - -## Installation - -```bash -pip install apache-ossie-databricks # once published to PyPI -# or, from a checkout of this directory: -pip install -e . -``` - -The only runtime dependency is `PyYAML`. Python 3.11+. - -## Usage - -### Command line - -```bash -ossie-databricks export -i model.yaml -o view.yaml [--source orders] # Apache Ossie -> Metric View -ossie-databricks import -i view.yaml -o model.yaml [--name my_model] # Metric View -> Apache Ossie -``` - -With no `-o`, output goes to stdout. `--source` (export) picks the fact/grain (default: -the FK-sink dataset; naming a coarser-grain dataset produces `one_to_many` joins); -`--name` (import) sets the Apache Ossie model name (default: the source's last identifier). - -### Python API - -```python -from ossie_databricks import convert_ossie_to_metric_view, convert_metric_view_to_ossie - -metric_view_yaml = convert_ossie_to_metric_view(ossie_yaml_str) # optionally choose the fact/grain, e.g. (ossie_yaml_str, source="orders") -ossie_yaml = convert_metric_view_to_ossie(metric_view_yaml_str, model_name="sales") -``` - -## Mapping - -Each row maps in both directions; the **Notes** flag where a behavior is specific to -**export** (Apache Ossie -> Metric View) or **import** (Metric View -> Apache Ossie). - -| Apache Ossie | Metric View (v1.1) | Notes | -|---|---|---| -| `semantic_model.description` | `comment` | Model-level description only. | -| root dataset | `source` | The fact/grain. | -| other `datasets` | nested `joins[]` | Export: the relationship graph is reassembled into the join tree; a dataset reached by two paths (a diamond) fans out into one aliased join per path. | -| `relationship` `from_columns`/`to_columns` | join `on` (differing names) / `using` (shared names) | Decomposed into columns on import; rebuilt into `on`/`using` on export. | -| `relationship.from`/`to` direction | join `cardinality` | Export: source on the many (`from`) side -> `many_to_one`; on the one (`to`) side -> `one_to_many`. | -| `dataset.primary_key` / `unique_keys` | join `rely.at_most_one_match` | Both directions: export sets `at_most_one_match` when a key covers the join columns; import recovers a `unique_keys` from it. | -| `dataset.fields[]` | `dimensions[]` | Export: fields flatten into one list and a joined column is qualified by its full join path (`customer.c_name`; `customer.region.r_name` when nested). | -| `field.expression.dialects[]` | `expr` | Export: prefer the `DATABRICKS` dialect, else `ANSI_SQL`. | -| `metrics[]` | `measures[]` | Export: fact columns are referenced bare (`SUM(amount)`). | -| `field.label` | `display_name` | | -| `field` / `metric` `description` | `comment` | | -| `ai_context.synonyms` | `synonyms` | | -| `custom_extensions[DATABRICKS]` | `filter`, `window`, `format`, `rely`, `materialization` | Import stashes Metric View only features here; export restores them -- keeping `MV -> Apache Ossie -> MV` lossless. | - -## Requirements - -Conversion raises a `ConversionError` (rather than guessing or emitting something -invalid) when an input breaks one of these: - -- the Metric View `version` is not `1.1`; -- a `source` is not a 3-part `catalog.schema.table` name or a `SELECT`/`WITH` subquery; -- the relationship graph is not acyclic and resolvable to a single fact -- a cycle, or - multiple candidate facts without `--source`, is rejected (a diamond is allowed and - fanned out); -- a join has no condition (a cross join has no Apache Ossie relationship form); -- a join condition is non-equi or otherwise can't be decomposed into equi-join columns - (Apache Ossie relationships are equi-joins, so the join has no Apache Ossie representation); -- the input YAML is malformed. - -## Development - -```bash -pip install -e ".[dev]" -python3 -m pytest tests/ -``` - -Example-based unit tests plus Hypothesis property-based round-trip tests -(`test_roundtrip_properties.py`, which skip if `hypothesis` is not installed). - -## Future effort - -Both the Apache Ossie specification and the Databricks Unity Catalog Metric View YAML are still -evolving. As either side adds or changes fields, this converter will be updated to track -them -- extending the mapping and coverage in both directions to keep the conversion -current and to support as much as each format allows over time. +See [`java/README.md`](java/README.md) and [`python/README.md`](python/README.md) for building and +using each implementation. diff --git a/converters/databricks/java/README.md b/converters/databricks/java/README.md new file mode 100644 index 00000000..b24e9895 --- /dev/null +++ b/converters/databricks/java/README.md @@ -0,0 +1,137 @@ +# Apache Ossie Databricks Converter + +Bidirectional, offline conversion between an [Apache Ossie](https://github.com/apache/ossie) +semantic model and a Databricks +[Unity Catalog Metric View](https://docs.databricks.com/aws/en/metric-views/) (YAML `1.1`). Pure +YAML text in, YAML text out: it reads and writes the two formats as parsed maps and lists. + +- **Export** (`MetricViewToOssie`): Metric View -> Apache Ossie. The direction is named from the + Metric View's point of view -- it takes a Metric View *out* to Ossie. Metric-View-only features + Apache Ossie has no native field for are preserved in `custom_extensions[DATABRICKS]`, so + `MV -> Apache Ossie -> MV` is lossless. +- **Import** (`OssieToMetricView`): Apache Ossie -> Metric View (one fact `source` with a nested + `joins` tree and a flat `dimensions` list). + +On **import** (Apache Ossie -> Metric View), Apache Ossie features with no Metric View slot -- +relationship `ai_context`, `dimension.is_time`, non-`DATABRICKS`/`ANSI_SQL` dialects, foreign-vendor +`custom_extensions` -- are **dropped with a notice**. On **export** (Metric View -> Apache Ossie), +Metric-View-only features (`filter`, `parameters`, `materialization`, per-column `format`, measure +`window` / `partition`) are instead **preserved** in `custom_extensions[DATABRICKS]`, so +`MV -> Apache Ossie -> MV` is lossless. Any input that breaks a [requirement](#requirements) +**raises a `ConversionException`** -- the converter never silently drops a field or produces an +invalid result. + +## Requirements + +- **Java 21+** +- **Maven 3.6+** -- required to build the jar + +## Building + +Build the self-contained executable jar from source: + +```bash +mvn clean package +``` + +This produces `target/ossie-databricks-converter-0.1.0-SNAPSHOT.jar` with all dependencies +(Jackson and SnakeYAML) bundled. + +## Usage + +### Command line + +```bash +# import: Apache Ossie -> Metric View +java -jar target/ossie-databricks-converter-0.1.0-SNAPSHOT.jar import model.yaml -o view.yaml + +# export: Metric View -> Apache Ossie +java -jar target/ossie-databricks-converter-0.1.0-SNAPSHOT.jar export view.yaml -o model.yaml +``` + +With no `-o`, output goes to stdout. `--source` (import) picks the fact/grain (default: the FK-sink +dataset; naming a coarser-grain dataset produces `one_to_many` joins); `--name` (export) sets the +Apache Ossie model name (default: the source's last identifier). Conversion notices (features +dropped on import) are written to stderr; a non-convertible input exits non-zero. + +### Java API + +```java +import org.apache.ossie.converter.databricks.OssieConverter; + +// export: Metric View -> Apache Ossie (optionally name the model; default: the source's last part) +OssieConverter.Result ossie = OssieConverter.convertMetricViewToOssie(metricViewYaml, "sales"); + +// import: Apache Ossie -> Metric View (optionally choose the fact/grain; default: the FK-sink +// dataset -- naming a coarser-grain dataset produces one_to_many joins) +OssieConverter.Result view = OssieConverter.convertOssieToMetricView(ossieYaml, "orders"); +``` + +Each `Result` carries the output YAML (`result.yaml`) and any notices raised (`result.notices`, +the features dropped on import). A broken [requirement](#requirements) throws a +`ConversionException` instead. + +## Mapping + +Each row maps in both directions; the **Notes** flag where a behavior is specific to +**export** (Metric View -> Apache Ossie) or **import** (Apache Ossie -> Metric View). + +| Apache Ossie | Metric View (v1.1) | Notes | +|---|---|---| +| `semantic_model.description` | `comment` | Model-level description only. | +| root dataset | `source` | The fact/grain. | +| other `datasets` | nested `joins[]` | Import: the relationship graph is reassembled into the join tree; a dataset reached by two paths (a diamond) fans out into one aliased join per path. | +| `relationship` `from_columns`/`to_columns` | join `on` (differing names) / `using` (shared names) | Decomposed into columns on export; rebuilt into `on`/`using` on import. | +| `relationship.from`/`to` direction | join `cardinality` | Import: source on the many (`from`) side -> `many_to_one`; on the one (`to`) side -> `one_to_many`. | +| `dataset.primary_key` / `unique_keys` | join `rely.at_most_one_match` | Both directions: import sets `at_most_one_match` when a key covers the join columns; export recovers a `unique_keys` from it. | +| `dataset.fields[]` | `dimensions[]` | Import: fields flatten into one list and a joined column is qualified by its full join path (`customer.c_name`; `customer.region.r_name` when nested). | +| `field.expression.dialects[]` | `expr` | Import: prefer the `DATABRICKS` dialect, else `ANSI_SQL`. | +| `metrics[]` | `measures[]` | Import: fact columns are referenced bare (`SUM(amount)`). | +| `field.label` | `display_name` | | +| `field` / `metric` `description` | `comment` | | +| `ai_context.synonyms` | `synonyms` | | +| `custom_extensions[DATABRICKS]` | `filter`, `parameters`, `materialization`, per-column `format`, measure `window` / `partition` | Export stashes Metric-View-only features here; import restores them -- keeping `MV -> Apache Ossie -> MV` lossless. | + +## Requirements + +Conversion throws a `ConversionException` (rather than guessing or emitting something invalid) when +an input breaks one of these: + +- the Metric View `version` is not `1.1`; +- a `source` is not a 3-part `catalog.schema.table` name or a `SELECT`/`WITH` subquery; +- the relationship graph is not acyclic and resolvable to a single fact -- a cycle, or multiple + candidate facts without a chosen source, is rejected (a diamond is allowed and fanned out); +- a join has no condition (a cross join has no Apache Ossie relationship form); +- a join condition is non-equi or otherwise can't be decomposed into equi-join columns (Apache + Ossie relationships are equi-joins, so the join has no Apache Ossie representation); +- the input YAML is malformed. + +## Development + +Run the test suite: + +```bash +mvn test +``` + +JUnit 5 suites (unit + round-trip) live under `src/test/java/`, with the YAML fixtures in +`src/test/resources/`. The source layout: + +``` +src/main/java/org/apache/ossie/converter/databricks/ + OssieConverter.java public facade: entry points + ConversionException/Notices/Result + OssieConverterCommon.java shared constants, YAML I/O, map accessors, the stash codec + MetricViewToOssie.java export: Metric View v1.1 -> Apache Ossie + OssieToMetricView.java import: Apache Ossie -> Metric View v1.1 + OssieDatabricksConverter.java command-line entry point (import / export) +``` + +The authoritative contract is Metric View YAML v1.1 as Databricks defines it; the checked-in +fixtures under `src/test/resources/` pin the expected output of both directions. + +## Future effort + +Both the Apache Ossie specification and the Databricks Unity Catalog Metric View YAML are still +evolving. As either side adds or changes fields, this converter will be updated to track them -- +extending the mapping and coverage in both directions to keep the conversion current and to support +as much as each format allows over time. diff --git a/converters/databricks/java/pom.xml b/converters/databricks/java/pom.xml new file mode 100644 index 00000000..84120eb1 --- /dev/null +++ b/converters/databricks/java/pom.xml @@ -0,0 +1,131 @@ + + + + + 4.0.0 + + + org.apache + apache + 39 + + + + org.apache.ossie + ossie-databricks-converter + 0.1.0-SNAPSHOT + jar + + Apache Ossie Databricks Converter + Converts between Apache Ossie semantic models and Databricks Unity Catalog Metric Views + + + 21 + UTF-8 + 2.2 + 2.18.9 + 5.10.2 + + + + + + org.yaml + snakeyaml + ${snakeyaml.version} + + + + + com.fasterxml.jackson.dataformat + jackson-dataformat-yaml + ${jackson.version} + + + com.fasterxml.jackson.core + jackson-databind + ${jackson.version} + + + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + + + + + org.apache.maven.plugins + maven-jar-plugin + 3.3.0 + + + + org.apache.ossie.converter.databricks.OssieDatabricksConverter + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.5.1 + + + package + + shade + + + + + + + + org.apache.rat + apache-rat-plugin + + + verify + + check + + + + + true + + **/target/** + + **/*.md + + + + + + diff --git a/converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/MetricViewToOssie.java b/converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/MetricViewToOssie.java new file mode 100644 index 00000000..39a269af --- /dev/null +++ b/converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/MetricViewToOssie.java @@ -0,0 +1,474 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ossie.converter.databricks; + +import static org.apache.ossie.converter.databricks.OssieConverterCommon.CARD_MANY_TO_ONE; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.CARD_ONE_TO_MANY; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.DIALECT_DATABRICKS; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.MAPPER; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.MV_VERSION; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.OSSIE_VERSION; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.STASH_SOURCE_KEY; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.asList; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.asMap; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.get; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.isSimpleIdentifier; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.loadYaml; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.lastIdentifier; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.replaceOutsideLiterals; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.requireStr; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.str; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.strList; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.truthy; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.validateSource; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.writeStash; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.ossie.converter.databricks.OssieConverter.ConversionException; +import org.apache.ossie.converter.databricks.OssieConverter.Notices; +import org.apache.ossie.converter.databricks.OssieConverter.Result; + +/** + * IMPORT direction: Metric View v1.1 YAML -> Apache Ossie semantic model. Shared helpers come + * from {@link OssieConverterCommon}; the public entry point is re-exported through + * {@link OssieConverter}. + */ +// Map-based YAML manipulation: casts of the parsed Object graph to Map/List are inherently +// unchecked; the asMap/asList helpers guard them, so unchecked warnings here are expected. +@SuppressWarnings("unchecked") +final class MetricViewToOssie { + + private static final String[] MODEL_STASH_KEYS = {"filter", "parameters", "materialization"}; + private static final String[] JOIN_STASH_KEYS = {"rely", "cardinality"}; + // Metric-View-only column fields with no Apache Ossie representation, preserved verbatim in the + // DATABRICKS stash so an import can restore them. `window` and `partition` are measure-only + // (they live on MeasureExpression); a dimension never carries them, so the shared key list is + // simply never hit for those on the dimension path. + private static final String[] COLUMN_STASH_KEYS = {"format", "window", "partition"}; + private static final Pattern NON_EQUI_RE = Pattern.compile("[<>!]=|<>|[<>]"); + private static final Pattern AND_SPLIT_RE = Pattern.compile("\\s+AND\\s+", Pattern.CASE_INSENSITIVE); + private static final Pattern EQ_CLAUSE_RE = Pattern.compile("^\\s*(.+?)\\s*=\\s*(.+?)\\s*$"); + private static final Pattern SOURCE_QUALIFIER_RE = Pattern.compile("\\bsource\\."); + + private MetricViewToOssie() {} + + static Result convertMetricViewToOssie(String mvYamlStr, String modelName) { + Notices notices = new Notices(); + Map view; + try { + view = asMap(loadYaml(mvYamlStr)); + } catch (Exception e) { + throw new ConversionException("Invalid Metric View YAML: " + e.getMessage(), e); + } + if (view.isEmpty()) { + throw new ConversionException("Invalid Metric View YAML: expected a mapping at the root"); + } + String version = str(get(view, "version")); + if (!MV_VERSION.equals(version)) { + throw new ConversionException("Unsupported Metric View version '" + version + + "'. This converter targets v" + MV_VERSION + " only."); + } + Map model = convertView(view, modelName, notices); + Map out = new LinkedHashMap<>(); + out.put("version", OSSIE_VERSION); + List models = new ArrayList<>(); + models.add(model); + out.put("semantic_model", models); + try { + return new Result(MAPPER.writeValueAsString(out), notices.toList()); + } catch (Exception e) { + throw new ConversionException("failed to serialize Apache Ossie YAML: " + e.getMessage(), e); + } + } + + private static Map convertView( + Map view, String modelName, Notices notices) { + Object source = get(view, "source"); + if (source == null || source.toString().isEmpty()) { + throw new ConversionException("Metric View is missing required 'source'"); + } + String s = source.toString().trim(); + String firstToken = s.split("\\s+", 2)[0].toUpperCase(Locale.ROOT); + boolean isSql = firstToken.equals("SELECT") || firstToken.equals("WITH"); + String lastId = lastIdentifier(source); + String factName; + if (modelName != null) { + factName = modelName; + } else if (!isSql && lastId != null && isSimpleIdentifier(lastId)) { + factName = lastId; + } else { + factName = "metric_view"; + } + validateSource(source, factName); + + List> datasets = new ArrayList<>(); + Map factDs = new LinkedHashMap<>(); + factDs.put("name", factName); + factDs.put("source", source); + datasets.add(factDs); + List> relationships = new ArrayList<>(); + Map aliasToDataset = new HashMap<>(); + aliasToDataset.put("source", factName); + aliasToDataset.put(factName, factName); + Set seenNames = new HashSet<>(); + seenNames.add(factName.trim().toLowerCase(Locale.ROOT)); + + walk(factName, "source", asList(get(view, "joins")), datasets, relationships, + aliasToDataset, seenNames); + + // `fields` is a v1.1 alias for `dimensions`: an empty `dimensions: []` falls through to + // `fields`, and the "both set" warning fires only when BOTH are non-empty (not merely + // present). + List dimList = asList(get(view, "dimensions")); + List fieldList = asList(get(view, "fields")); + if (!dimList.isEmpty() && !fieldList.isEmpty()) { + notices.warn("view", "both 'dimensions' and 'fields' are set; 'fields' is a v1.1 alias " + + "for 'dimensions', so the 'fields' list is ignored"); + } + Map> fieldsByDataset = new LinkedHashMap<>(); + for (Map d : datasets) { + fieldsByDataset.put((String) d.get("name"), new ArrayList<>()); + } + List dims = !dimList.isEmpty() ? dimList : fieldList; + for (Object dimObj : dims) { + Map dim = asMap(dimObj); + if (isWildcard(dim)) { + notices.warn("dimension", "wildcard column '" + str(get(dim, "expr")) + + "' has no Apache Ossie field representation; skipped"); + continue; + } + Object[] converted = convertDimension(dim, aliasToDataset, factName); + fieldsByDataset.get((String) converted[0]).add(converted[1]); + } + for (Map d : datasets) { + List flds = fieldsByDataset.get((String) d.get("name")); + if (!flds.isEmpty()) { + d.put("fields", flds); + } + } + + List metrics = new ArrayList<>(); + for (Object mObj : asList(get(view, "measures"))) { + Map m = asMap(mObj); + if (isWildcard(m)) { + notices.warn("measure", "wildcard measure '" + str(get(m, "expr")) + + "' has no Apache Ossie metric representation; skipped"); + continue; + } + metrics.add(convertMeasure(m, factName)); + } + + Map model = new LinkedHashMap<>(); + model.put("name", factName); + if (truthy(get(view, "comment"))) { + model.put("description", get(view, "comment")); + } + model.put("datasets", datasets); + if (!relationships.isEmpty()) { + model.put("relationships", relationships); + } + if (!metrics.isEmpty()) { + model.put("metrics", metrics); + } + + Map modelStash = new LinkedHashMap<>(); + for (String k : MODEL_STASH_KEYS) { + if (view.containsKey(k)) { + modelStash.put(k, view.get(k)); + } + } + if (hasOtm(asList(get(view, "joins")))) { + modelStash.put(STASH_SOURCE_KEY, factName); + } + writeStash(model, modelStash); + return model; + } + + private static void walk(String parentName, String parentAlias, List joins, + List> datasets, List> relationships, + Map aliasToDataset, Set seenNames) { + for (Object joinObj : joins) { + Map join = asMap(joinObj); + String child = requireStr(join, "name", "join"); + if (child.trim().equalsIgnoreCase("source")) { + throw new ConversionException( + "Join name 'source' is reserved for the fact source; rename the join."); + } + if (!seenNames.add(child.trim().toLowerCase(Locale.ROOT))) { + throw new ConversionException("Duplicate dataset/join name '" + child + + "'; Metric View join names and the source must be distinct (case-insensitively)."); + } + Map childDs = new LinkedHashMap<>(); + childDs.put("name", child); + childDs.put("source", requireStr(join, "source", "join '" + child + "'")); + datasets.add(childDs); + aliasToDataset.put(child, child); + Map rel = convertJoin(join, parentName, parentAlias, child); + relationships.add(rel); + // rely.at_most_one_match on a many_to_one join -> recover a unique_key on the child. + Map rely = asMap(get(join, "rely")); + List toCols = strList(get(rel, "to_columns")); + if (child.equals(str(get(rel, "to"))) && !toCols.isEmpty() + && Boolean.TRUE.equals(rely.get("at_most_one_match"))) { + List uk = new ArrayList<>(); + uk.add(new ArrayList<>(toCols)); + childDs.put("unique_keys", uk); + } + walk(child, child, asList(get(join, "joins")), datasets, relationships, + aliasToDataset, seenNames); + } + } + + private static boolean hasOtm(List joins) { + for (Object jObj : joins) { + Map j = asMap(jObj); + if (CARD_ONE_TO_MANY.equalsIgnoreCase(str(get(j, "cardinality")))) { + return true; + } + if (hasOtm(asList(get(j, "joins")))) { + return true; + } + } + return false; + } + + private static Map convertJoin( + Map join, String parentName, String parentAlias, String child) { + boolean hasUsing = get(join, "using") != null && !asList(get(join, "using")).isEmpty(); + boolean hasOn = str(get(join, "on")) != null && !str(get(join, "on")).isEmpty(); + if (!hasUsing && !hasOn) { + throw new ConversionException("Join '" + child + "' has no join condition (empty or " + + "absent 'on'/'using'); condition-less (cross) joins have no Apache Ossie " + + "relationship representation."); + } + Object[] decomposed = decomposeOn(join, parentAlias, parentName, child); + List parentCols = (List) decomposed[0]; + List childCols = (List) decomposed[1]; + String rawOn = (String) decomposed[2]; + if (rawOn != null) { + throw new ConversionException("Join '" + child + "' uses a non-equi or unsupported join " + + "condition ('on: " + rawOn + "') that an Apache Ossie relationship cannot represent. " + + "Apache Ossie joins are equi-joins of simple `alias.column` pairs (the fact side may " + + "be qualified with `source`, the source table name, or left bare). Cannot import."); + } + // Only fall back to `using` when there is no `on` to decompose (see decomposeOn: `on` wins). + if (!hasOn && hasUsing && parentCols.isEmpty()) { + List using = strList(get(join, "using")); + parentCols = new ArrayList<>(using); + childCols = new ArrayList<>(using); + } + + String cardinality = str(get(join, "cardinality")); + if (cardinality == null) { + cardinality = CARD_MANY_TO_ONE; + } + Map rel = new LinkedHashMap<>(); + if (cardinality.toLowerCase(Locale.ROOT).equals(CARD_ONE_TO_MANY)) { + rel.put("name", child + "_to_" + parentName); + rel.put("from", child); + rel.put("to", parentName); + rel.put("from_columns", childCols); + rel.put("to_columns", parentCols); + } else { + rel.put("name", parentName + "_to_" + child); + rel.put("from", parentName); + rel.put("to", child); + rel.put("from_columns", parentCols); + rel.put("to_columns", childCols); + } + Map stash = new LinkedHashMap<>(); + for (String k : JOIN_STASH_KEYS) { + if (join.containsKey(k)) { + stash.put(k, join.get(k)); + } + } + writeStash(rel, stash); + return rel; + } + + /** Returns {parentCols, childCols, rawOn}; rawOn non-null means reject. */ + private static Object[] decomposeOn( + Map join, String parentAlias, String parentName, String childAlias) { + // A join may carry both `on` and `using` (Metric View validation only requires that at least + // one is present). `on` takes precedence, matching how Databricks resolves the join criteria + // in DataModelUtils.getJoinCriteriaExpression: `case (Some(on), _) => ...`. Falling back to + // `using` when `on` is present would silently join on different columns than the view does. + String on = str(get(join, "on")); + if (on == null || on.isEmpty()) { + // No usable `on`; the caller derives the columns from `using`. + return new Object[] {new ArrayList(), new ArrayList(), null}; + } + Set parentAliases = new HashSet<>(); + parentAliases.add(parentAlias); + parentAliases.add(parentName); + boolean allowBare = "source".equals(parentAlias); + List fromCols = new ArrayList<>(); + List toCols = new ArrayList<>(); + for (String clause : AND_SPLIT_RE.split(on)) { + if (NON_EQUI_RE.matcher(clause).find()) { + return new Object[] {null, null, on}; + } + Matcher m = EQ_CLAUSE_RE.matcher(clause); + if (!m.matches()) { + return new Object[] {null, null, on}; + } + String[] left = splitAlias(m.group(1)); + String[] right = splitAlias(m.group(2)); + String la = left[0]; + String lc = left[1]; + String ra = right[0]; + String rc = right[1]; + if (!(isSimpleIdentifier(lc) && isSimpleIdentifier(rc))) { + return new Object[] {null, null, on}; + } + boolean lParent = parentAliases.contains(la) || (la == null && allowBare); + boolean rParent = parentAliases.contains(ra) || (ra == null && allowBare); + if (childAlias.equals(la) && rParent) { + fromCols.add(rc); + toCols.add(lc); + } else if (childAlias.equals(ra) && lParent) { + fromCols.add(lc); + toCols.add(rc); + } else { + return new Object[] {null, null, on}; + } + } + return new Object[] {fromCols, toCols, null}; + } + + /** `customer.c_custkey` -> {"customer","c_custkey"}; `x` -> {null,"x"}. */ + private static String[] splitAlias(String operand) { + operand = operand.trim(); + int dot = operand.indexOf('.'); + if (dot >= 0) { + return new String[] {operand.substring(0, dot).trim(), operand.substring(dot + 1).trim()}; + } + return new String[] {null, operand}; + } + + private static Object[] convertDimension( + Map dim, Map aliasToDataset, String factName) { + String name = requireStr(dim, "name", "dimension"); + String expr = requireStr(dim, "expr", "dimension '" + name + "'"); + String[] resolved = resolveColumn(expr, aliasToDataset, factName); + String dsName = resolved[0]; + String ossieExpr = resolved[1]; + + Map field = new LinkedHashMap<>(); + field.put("name", name); + field.put("expression", dialectExpr(ossieExpr)); + if (truthy(get(dim, "comment"))) { + field.put("description", get(dim, "comment")); + } + if (truthy(get(dim, "display_name"))) { + field.put("label", get(dim, "display_name")); + } + if (truthy(get(dim, "synonyms"))) { + Map ai = new LinkedHashMap<>(); + ai.put("synonyms", new ArrayList<>(asList(get(dim, "synonyms")))); + field.put("ai_context", ai); + } + Map stash = new LinkedHashMap<>(); + for (String k : COLUMN_STASH_KEYS) { + if (dim.containsKey(k)) { + stash.put(k, dim.get(k)); + } + } + writeStash(field, stash); + return new Object[] {dsName, field}; + } + + /** Map a dimension expression to {dataset_name, de-aliased_expression}. */ + private static String[] resolveColumn( + String expr, Map aliasToDataset, String factName) { + // The -1 limit keeps trailing empty segments, so a malformed `customer.` files under + // `customer` rather than under the fact. + String[] segments = expr.split("\\.", -1); + for (int i = 0; i < segments.length; i++) { + segments[i] = segments[i].trim(); + } + String ds = null; + int i = 0; + while (i < segments.length - 1 && aliasToDataset.containsKey(segments[i])) { + ds = aliasToDataset.get(segments[i]); + i++; + } + if (ds == null) { + return new String[] {factName, expr}; + } + StringBuilder rest = new StringBuilder(); + for (int j = i; j < segments.length; j++) { + if (j > i) { + rest.append("."); + } + rest.append(segments[j]); + } + String restStr = rest.toString(); + return isSimpleIdentifier(restStr) ? new String[] {ds, restStr} : new String[] {ds, expr}; + } + + private static Map convertMeasure(Map measure, String factName) { + String name = requireStr(measure, "name", "measure"); + String rawExpr = requireStr(measure, "expr", "measure '" + name + "'"); + String expr = replaceOutsideLiterals(rawExpr, SOURCE_QUALIFIER_RE, factName + "."); + Map metric = new LinkedHashMap<>(); + metric.put("name", name); + metric.put("expression", dialectExpr(expr)); + if (truthy(get(measure, "comment"))) { + metric.put("description", get(measure, "comment")); + } + if (truthy(get(measure, "synonyms"))) { + Map ai = new LinkedHashMap<>(); + ai.put("synonyms", new ArrayList<>(asList(get(measure, "synonyms")))); + metric.put("ai_context", ai); + } + Map stash = new LinkedHashMap<>(); + for (String k : COLUMN_STASH_KEYS) { + if (measure.containsKey(k)) { + stash.put(k, measure.get(k)); + } + } + writeStash(metric, stash); + return metric; + } + + private static Map dialectExpr(String expr) { + Map dialect = new LinkedHashMap<>(); + dialect.put("dialect", DIALECT_DATABRICKS); + dialect.put("expression", expr); + List dialects = new ArrayList<>(); + dialects.add(dialect); + Map expression = new LinkedHashMap<>(); + expression.put("dialects", dialects); + return expression; + } + + private static boolean isWildcard(Map col) { + return !col.containsKey("name"); + } +} diff --git a/converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/OssieConverter.java b/converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/OssieConverter.java new file mode 100644 index 00000000..278cdadc --- /dev/null +++ b/converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/OssieConverter.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ossie.converter.databricks; + +import java.util.ArrayList; +import java.util.List; + +/** + * Bidirectional converter between Apache Ossie semantic models and Databricks Metric Views. + * This class is the public facade, re-exporting the two conversion entry points and the shared + * {@link ConversionException}/{@link Notices}/{@link Result} types. The implementation is split + * by direction so each half can be read on its own: + * + *
    + *
  • {@link OssieConverterCommon} -- shared constants, YAML I/O, and map accessors + *
  • {@link OssieToMetricView} -- EXPORT: Apache Ossie -> Metric View + *
  • {@link MetricViewToOssie} -- IMPORT: Metric View -> Apache Ossie + *
+ * + *

The authoritative contract is Metric View YAML v1.1 as defined by the Databricks serde + * ({@code com.databricks.sql.serde.v11}) and its validation rules; the checked-in YAML fixtures + * pin the expected output for both directions. + * + *

Conversion operates on parsed YAML as plain maps and lists rather than typed models, so the + * converter stays independent of the Databricks serde classes and runs standalone. Warnings + * ("drops") are collected into a Notices buffer and returned rather than written to stderr, so a + * SQL surface can present them to the caller. + */ +public final class OssieConverter { + + // Re-exported so callers can reference OssieConverter.OSSIE_VERSION / .MV_VERSION as before. + public static final String OSSIE_VERSION = OssieConverterCommon.OSSIE_VERSION; + public static final String MV_VERSION = OssieConverterCommon.MV_VERSION; + + private OssieConverter() {} + + /** Raised for any input the converter refuses to convert (Java twin of ConversionError). */ + public static final class ConversionException extends RuntimeException { + public ConversionException(String message) { + super(message); + } + public ConversionException(String message, Throwable cause) { + super(message, cause); + } + } + + /** Collects drop/rewrite notices during a conversion. */ + public static final class Notices { + private final List messages = new ArrayList<>(); + void warn(String scope, String msg) { + messages.add("[" + scope + "] " + msg); + } + public List toList() { + return new ArrayList<>(messages); + } + } + + /** Result of a conversion: the emitted YAML plus any drop notices. */ + public static final class Result { + public final String yaml; + public final List notices; + Result(String yaml, List notices) { + this.yaml = yaml; + this.notices = notices; + } + } + + /** EXPORT: Apache Ossie semantic model YAML -> Metric View v1.1 YAML. */ + public static Result convertOssieToMetricView(String osiYamlStr, String source) { + return OssieToMetricView.convertOssieToMetricView(osiYamlStr, source); + } + + /** IMPORT: Metric View v1.1 YAML -> Apache Ossie semantic model YAML. */ + public static Result convertMetricViewToOssie(String mvYamlStr, String modelName) { + return MetricViewToOssie.convertMetricViewToOssie(mvYamlStr, modelName); + } + + /** Parse YAML text into a plain value (YAML 1.2 booleans, matching the converter). */ + public static Object parseYaml(String s) { + return OssieConverterCommon.parseYaml(s); + } + + /** Serialize a value to YAML using the converter's write mapper (for tests without their + * own jackson-yaml import). */ + public static String dumpYaml(Object obj) { + return OssieConverterCommon.dumpYaml(obj); + } +} diff --git a/converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/OssieConverterCommon.java b/converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/OssieConverterCommon.java new file mode 100644 index 00000000..deefa677 --- /dev/null +++ b/converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/OssieConverterCommon.java @@ -0,0 +1,498 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ossie.converter.databricks; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; + +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.SafeConstructor; +import org.yaml.snakeyaml.nodes.Tag; +import org.yaml.snakeyaml.resolver.Resolver; + +import org.apache.ossie.converter.databricks.OssieConverter.ConversionException; + +/** + * Shared constants, YAML I/O, and typed accessors for the Apache Ossie <-> Databricks + * Metric View converter. The direction-specific logic lives in {@link OssieToMetricView} + * (export) and {@link MetricViewToOssie} (import), which static-import these members. The public + * entry points and shared types are re-exported through {@link OssieConverter}. + */ +// Map-based YAML manipulation: casts of the parsed Object graph to Map/List are inherently +// unchecked; the asMap/asList helpers guard them, so unchecked warnings here are expected. +@SuppressWarnings("unchecked") +final class OssieConverterCommon { + + // -- constants ------------------------------------------------------------- + static final String OSSIE_VERSION = "0.2.0.dev0"; + static final String MV_VERSION = "1.1"; + static final String VENDOR = "DATABRICKS"; + static final String DIALECT_DATABRICKS = "DATABRICKS"; + static final String DIALECT_ANSI = "ANSI_SQL"; + static final int SYNONYM_LIMIT = 10; + static final int STASH_VERSION = 1; + static final String STASH_SOURCE_KEY = "source_dataset"; + static final String CARD_ONE_TO_MANY = "one_to_many"; + static final String CARD_MANY_TO_ONE = "many_to_one"; + static final int MAX_JOIN_NODES = 200; + + static final Pattern IDENTIFIER_RE = Pattern.compile("^[A-Za-z_][A-Za-z0-9_]*$"); + static final Pattern SELECT_WITH_RE = + Pattern.compile("(?i)^(select|with)\\b"); + + static final ObjectMapper MAPPER = buildMapper(); + // Writer for the custom_extensions stash blob. The exact byte format is pinned by the + // checked-in fixtures: a space after ':' and ', ' between entries, as in + // {"_v": 1, "filter": "x"}. + static final com.fasterxml.jackson.databind.ObjectWriter JSON_WRITER = buildJsonWriter(); + + /** A MinimalPrettyPrinter (no newlines) using the stash blob's separators: ": " / ", ". */ + private static final class JsonDumpsPrinter + extends com.fasterxml.jackson.core.util.MinimalPrettyPrinter { + @Override + public void writeObjectFieldValueSeparator(com.fasterxml.jackson.core.JsonGenerator g) + throws java.io.IOException { + g.writeRaw(": "); + } + @Override + public void writeObjectEntrySeparator(com.fasterxml.jackson.core.JsonGenerator g) + throws java.io.IOException { + g.writeRaw(", "); + } + @Override + public void writeArrayValueSeparator(com.fasterxml.jackson.core.JsonGenerator g) + throws java.io.IOException { + g.writeRaw(", "); + } + } + + private static com.fasterxml.jackson.databind.ObjectWriter buildJsonWriter() { + // ESCAPE_NON_ASCII keeps the blob pure ASCII (every non-ASCII + // char is emitted as a \\uXXXX escape). Set on the JsonFactory so it takes effect on the + // generator. Jackson emits the hex in UPPERCASE; writeStash lowercases it so the blob is + // byte-identical to the checked-in fixtures. + com.fasterxml.jackson.core.JsonFactory jf = new com.fasterxml.jackson.core.JsonFactory(); + jf.enable(com.fasterxml.jackson.core.JsonGenerator.Feature.ESCAPE_NON_ASCII); + return new ObjectMapper(jf).writer(new JsonDumpsPrinter()); + } + + // Matches a real unicode escape in serialized JSON so writeStash can lowercase its hex digits. + // + // The escape must be preceded by an EVEN number of backslashes, otherwise the `u` belongs to an + // escaped backslash rather than to an escape sequence: a stashed value holding a literal + // backslash followed by "uABCD" serializes with a doubled backslash, where the `u` is ordinary + // text and must be left alone (lowercasing it there would corrupt the value). Group 1 captures + // the (possibly empty) run of escaped backslashes so it can be re-emitted verbatim; group 2 is + // the hex to lowercase. + private static final Pattern UNICODE_ESCAPE_RE = + Pattern.compile("(? YAML_READER = ThreadLocal.withInitial(() -> + new Yaml(new SafeConstructor(new LoaderOptions()), new org.yaml.snakeyaml.representer.Representer( + new org.yaml.snakeyaml.DumperOptions()), new org.yaml.snakeyaml.DumperOptions(), + new LoaderOptions(), new Yaml12Resolver())); + + static Object loadYaml(String s) { + return YAML_READER.get().load(s); + } + + private OssieConverterCommon() {} + + // -- typed accessors over parsed YAML (Object) ---------------------------- + @SuppressWarnings("unchecked") + static Map asMap(Object x) { + if (x instanceof Map) { + return (Map) x; + } + return new LinkedHashMap<>(); + } + + @SuppressWarnings("unchecked") + static List asList(Object x) { + if (x instanceof List) { + return (List) x; + } + return new ArrayList<>(); + } + + static Object get(Map m, String k) { + return m.get(k); + } + + static String str(Object x) { + if (x == null) { + return null; + } + return x.toString(); + } + + static List strList(Object x) { + List out = new ArrayList<>(); + for (Object o : asList(x)) { + if (o != null) { + out.add(o.toString()); + } + } + return out; + } + + // -- helpers --------------------------------------------------------------- + static boolean isSimpleIdentifier(Object expr) { + return expr instanceof String && IDENTIFIER_RE.matcher(((String) expr).trim()).matches(); + } + + /** + * Applies {@code pattern -> replacement} to {@code sql}, but only to the parts of the expression + * that are actual SQL code -- spans inside string literals ({@code '...'}, {@code "..."}), + * backquoted identifiers, {@code -- line} comments, and {@code /* block *}{@code /} comments are + * copied through untouched. + * + *

Measure expressions are rewritten to add or strip a fact qualifier, and a blind + * {@code replaceAll} over the raw text also rewrites any occurrence inside a literal: a measure + * such as {@code SUM(IF(source.region = 'source.us', amt, 0))} would silently become + * {@code ... = 'us'}, changing the predicate and therefore the measure's value. Only code spans + * may be rewritten. + * + *

{@code replacement} is treated as a literal string, not as a regex replacement template. + */ + static String replaceOutsideLiterals(String sql, Pattern pattern, String replacement) { + StringBuilder out = new StringBuilder(sql.length()); + int i = 0; + int codeStart = 0; + while (i < sql.length()) { + char c = sql.charAt(i); + int skipTo = -1; + if (c == '\'' || c == '"' || c == '`') { + skipTo = endOfQuoted(sql, i, c); + } else if (c == '-' && i + 1 < sql.length() && sql.charAt(i + 1) == '-') { + int nl = sql.indexOf('\n', i); + skipTo = nl < 0 ? sql.length() : nl; + } else if (c == '/' && i + 1 < sql.length() && sql.charAt(i + 1) == '*') { + int end = sql.indexOf("*/", i + 2); + skipTo = end < 0 ? sql.length() : end + 2; + } + if (skipTo < 0) { + i++; + continue; + } + // Rewrite the code span that precedes this literal/comment, then copy the span verbatim. + out.append(rewriteLiterally(sql.substring(codeStart, i), pattern, replacement)); + out.append(sql, i, skipTo); + i = skipTo; + codeStart = skipTo; + } + out.append(rewriteLiterally(sql.substring(codeStart), pattern, replacement)); + return out.toString(); + } + + /** Index just past the quoted span starting at {@code start}; handles doubled-quote escapes. */ + private static int endOfQuoted(String sql, int start, char quote) { + int i = start + 1; + while (i < sql.length()) { + char c = sql.charAt(i); + if (c == '\\' && quote != '`' && i + 1 < sql.length()) { + i += 2; + continue; + } + if (c == quote) { + // A doubled quote is an escaped quote, not the end of the span. + if (i + 1 < sql.length() && sql.charAt(i + 1) == quote) { + i += 2; + continue; + } + return i + 1; + } + i++; + } + // Unterminated literal: treat the remainder as part of the span rather than rewriting it. + return sql.length(); + } + + private static String rewriteLiterally(String code, Pattern pattern, String replacement) { + return pattern.matcher(code).replaceAll(Matcher.quoteReplacement(replacement)); + } + + // Presence is tested by key, so a legitimately falsy non-string value (0, false) is returned; + // a missing key, a null, or an empty/whitespace-only string is rejected. + static Object require(Map obj, String key, String what) { + if (!obj.containsKey(key) || obj.get(key) == null) { + throw new ConversionException(what + " is missing required '" + key + "'"); + } + Object value = obj.get(key); + if (value instanceof String && ((String) value).trim().isEmpty()) { + throw new ConversionException(what + " has an empty '" + key + "'"); + } + return value; + } + + // Like require(), but the value must be a string. + static String requireStr(Map obj, String key, String what) { + Object v = require(obj, key, what); + if (v instanceof String) { + return (String) v; + } + throw new ConversionException( + what + ": '" + key + "' must be a string, got " + v.getClass().getSimpleName()); + } + + static String validateSource(Object source, String datasetName) { + String s = source == null ? "" : source.toString().trim(); + if (s.isEmpty()) { + throw new ConversionException("Dataset '" + datasetName + "': missing/empty 'source'"); + } + if (SELECT_WITH_RE.matcher(s).find()) { + return s; + } + String[] parts = s.split("\\.", -1); + boolean ok = parts.length == 3; + if (ok) { + for (String p : parts) { + if (p.isEmpty() || containsWhitespace(p)) { + ok = false; + break; + } + } + } + if (ok) { + return s; + } + throw new ConversionException("Dataset '" + datasetName + "': source '" + source + + "' must be a 3-part catalog.schema.table identifier or a SELECT/WITH subquery"); + } + + private static boolean containsWhitespace(String p) { + for (int i = 0; i < p.length(); i++) { + if (Character.isWhitespace(p.charAt(i))) { + return true; + } + } + return false; + } + + static String pickExpression(Object osiExpression) { + // Keep the raw (possibly non-string) values so + // the type check below can fire; select DATABRICKS-or-ANSI by truthiness (`or`), so a + // null/empty DATABRICKS expr falls through to ANSI; and raise on a non-string chosen + // value rather than silently coercing it. + Map dialects = new LinkedHashMap<>(); + for (Object d : asList(get(asMap(osiExpression), "dialects"))) { + Map dm = asMap(d); + dialects.put(str(get(dm, "dialect")), get(dm, "expression")); + } + Object chosen = truthy(dialects.get(DIALECT_DATABRICKS)) + ? dialects.get(DIALECT_DATABRICKS) : dialects.get(DIALECT_ANSI); + if (chosen != null && !(chosen instanceof String)) { + throw new ConversionException( + "expression must be a string, got " + chosen.getClass().getSimpleName()); + } + return (String) chosen; // null when neither dialect present -> caller warns and skips + } + + /** + * Emptiness test used throughout the converter for optional YAML values: null, the empty string, + * an empty list/map, numeric zero, and boolean false are all treated as absent; everything else + * is present. Used for the DATABRICKS-or-ANSI expression fallthrough (so an empty or absent + * DATABRICKS expr falls through to ANSI) and for optional-field mapping (so an empty + * `comment`/`synonyms` is dropped rather than emitted as an empty value). + */ + static boolean truthy(Object v) { + if (v == null) { + return false; + } + if (v instanceof String) { + return !((String) v).isEmpty(); + } + if (v instanceof java.util.Collection) { + return !((java.util.Collection) v).isEmpty(); + } + if (v instanceof Map) { + return !((Map) v).isEmpty(); + } + if (v instanceof Number) { + return ((Number) v).doubleValue() != 0.0; + } + if (v instanceof Boolean) { + return (Boolean) v; + } + return true; + } + + static List synonymsOf(Object aiContext) { + if (aiContext instanceof Map) { + return strList(get(asMap(aiContext), "synonyms")); + } + return new ArrayList<>(); + } + + static String mergeDescription(Object description, Object aiContext) { + String desc = str(description); + if (aiContext instanceof String && !((String) aiContext).trim().isEmpty()) { + String s = (String) aiContext; + // When both are present they are joined with a newline; otherwise the + // `if description` is a truthiness test, so an empty (or null) description returns the + // ai_context alone rather than prepending a stray newline. + return (desc != null && !desc.isEmpty()) ? desc + "\n" + s : s; + } + return desc; + } + + static Map readStash(Map obj) { + for (Object extObj : asList(get(obj, "custom_extensions"))) { + Map ext = asMap(extObj); + if (VENDOR.equals(str(get(ext, "vendor_name")))) { + // A null or empty-string `data` is treated as an empty object rather than a parse error. + String data = str(get(ext, "data")); + if (data == null || data.isEmpty()) { + data = "{}"; + } + Map parsed; + try { + parsed = asMap(MAPPER.readValue(data, Object.class)); + } catch (Exception e) { + throw new ConversionException( + "DATABRICKS custom_extensions data is not valid JSON: " + e.getMessage(), e); + } + parsed.remove("_v"); + return parsed; + } + } + return new LinkedHashMap<>(); + } + + static List foreignVendorExtensions(Map obj) { + List out = new ArrayList<>(); + for (Object extObj : asList(get(obj, "custom_extensions"))) { + if (!VENDOR.equals(str(get(asMap(extObj), "vendor_name")))) { + out.add(extObj); + } + } + return out; + } + + /** Attach a DATABRICKS custom_extensions entry holding `data`; no-op when empty. */ + @SuppressWarnings("unchecked") + static void writeStash(Map obj, Map data) { + if (data.isEmpty()) { + return; + } + Map payload = new LinkedHashMap<>(); + payload.put("_v", STASH_VERSION); + payload.putAll(data); + String blob; + try { + blob = JSON_WRITER.writeValueAsString(payload); + } catch (Exception e) { + throw new ConversionException("failed to serialize stash: " + e.getMessage(), e); + } + // Jackson emits unicode escapes with uppercase hex; the stash format uses lowercase. Lowercase + // just the 4 hex digits of each real escape, preserving any escaped-backslash run in front of + // it (see UNICODE_ESCAPE_RE). + blob = UNICODE_ESCAPE_RE.matcher(blob) + .replaceAll(m -> m.group(1) + "\\\\u" + m.group(2).toLowerCase(Locale.ROOT)); + List exts = (List) obj.computeIfAbsent("custom_extensions", k -> new ArrayList<>()); + for (Object extObj : exts) { + Map ext = asMap(extObj); + if (VENDOR.equals(str(get(ext, "vendor_name")))) { + ext.put("data", blob); + return; + } + } + Map ext = new LinkedHashMap<>(); + ext.put("vendor_name", VENDOR); + ext.put("data", blob); + exts.add(ext); + } + + /** Last dotted part of a table reference: `samples.tpch.lineitem` -> `lineitem`. + * Trim the whole reference, take the final dotted + * segment, then strip any surrounding backticks (so `cat.sch.`t`` -> `t`). */ + static String lastIdentifier(Object source) { + if (source == null) { + return null; + } + String s = source.toString().trim(); + int dot = s.lastIndexOf('.'); + String last = dot >= 0 ? s.substring(dot + 1) : s; + int start = 0; + int end = last.length(); + while (start < end && last.charAt(start) == '`') { + start++; + } + while (end > start && last.charAt(end - 1) == '`') { + end--; + } + return last.substring(start, end); + } + + /** Parse YAML text into a plain value (YAML 1.2 booleans, matching the converter). */ + static Object parseYaml(String s) { + try { + return loadYaml(s); + } catch (Exception e) { + throw new ConversionException("failed to parse YAML: " + e.getMessage(), e); + } + } + + /** Serialize a value to YAML using the converter's write mapper (for tests without their + * own jackson-yaml import). */ + static String dumpYaml(Object obj) { + try { + return MAPPER.writeValueAsString(obj); + } catch (Exception e) { + throw new ConversionException("failed to serialize YAML: " + e.getMessage(), e); + } + } +} diff --git a/converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/OssieDatabricksConverter.java b/converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/OssieDatabricksConverter.java new file mode 100644 index 00000000..51e1acbf --- /dev/null +++ b/converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/OssieDatabricksConverter.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ossie.converter.databricks; + +import java.io.IOException; +import java.io.PrintStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +/** + * Command-line entry point for the converter between Apache Ossie and Databricks Metric Views. + * + * 
{@code
+ *   ossie-databricks import  [-o ] [--source ]
+ *   ossie-databricks export   [-o ] [--name ]
+ * }
+ * + *

{@code import} converts an Apache Ossie semantic model to a Metric View; {@code export} + * converts a Metric View to an Apache Ossie model. Output goes to the {@code -o} file, or to stdout + * when omitted. Conversion notices (features dropped on import) are written to stderr. A broken + * input raises {@link OssieConverter.ConversionException}, reported as a non-zero exit. + * + *

This is a thin command-line wrapper around {@link OssieConverter}: it parses arguments, reads + * the input YAML, invokes the library, and writes the result. Programmatic callers should use + * {@link OssieConverter} directly. + */ +public final class OssieDatabricksConverter { + + private OssieDatabricksConverter() {} + + public static void main(String[] args) { + try { + run(args, System.out, System.err); + } catch (ExitException e) { + System.err.println(e.getMessage()); + System.exit(e.code); + } catch (OssieConverter.ConversionException e) { + System.err.println("Conversion failed: " + e.getMessage()); + System.exit(1); + } + } + + /** Testable core: parses args, runs the conversion, and writes output. */ + static void run(String[] args, PrintStream out, PrintStream err) { + if (args.length == 0) { + throw new ExitException(2, usage()); + } + String command = args[0]; + Args parsed = Args.parse(args); + + String input = read(parsed.inputPath); + OssieConverter.Result result; + switch (command) { + case "import": + // Apache Ossie -> Metric View. `--source` picks the fact/grain (optional). + result = OssieConverter.convertOssieToMetricView(input, parsed.option); + break; + case "export": + // Metric View -> Apache Ossie. `--name` sets the model name (optional). + result = OssieConverter.convertMetricViewToOssie(input, parsed.option); + break; + default: + throw new ExitException(2, "Unknown command '" + command + "'.\n" + usage()); + } + + write(parsed.outputPath, result.yaml, out); + List notices = result.notices; + if (!notices.isEmpty()) { + err.println("Conversion notices (" + notices.size() + "):"); + for (String notice : notices) { + err.println(" " + notice); + } + } + } + + private static String read(String path) { + try { + return Files.readString(Path.of(path), StandardCharsets.UTF_8); + } catch (IOException e) { + throw new ExitException(1, "Cannot read input file '" + path + "': " + e.getMessage()); + } + } + + private static void write(String path, String content, PrintStream out) { + if (path == null) { + out.println(content); + return; + } + try { + Files.writeString(Path.of(path), content, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new ExitException(1, "Cannot write output file '" + path + "': " + e.getMessage()); + } + } + + private static String usage() { + return "Usage:\n" + + " ossie-databricks import [-o ] [--source ]\n" + + " ossie-databricks export [-o ] [--name ]"; + } + + /** Parsed command-line arguments: the input file, an optional output file, and the option. */ + private static final class Args { + final String inputPath; + final String outputPath; + final String option; + + private Args(String inputPath, String outputPath, String option) { + this.inputPath = inputPath; + this.outputPath = outputPath; + this.option = option; + } + + static Args parse(String[] args) { + String inputPath = null; + String outputPath = null; + String option = null; + for (int i = 1; i < args.length; i++) { + String arg = args[i]; + switch (arg) { + case "-o": + case "--output": + outputPath = requireValue(args, ++i, arg); + break; + case "--source": + case "--name": + option = requireValue(args, ++i, arg); + break; + default: + if (arg.startsWith("-")) { + throw new ExitException(2, "Unknown option '" + arg + "'.\n" + usage()); + } + if (inputPath != null) { + throw new ExitException(2, "Unexpected extra argument '" + arg + "'.\n" + usage()); + } + inputPath = arg; + } + } + if (inputPath == null) { + throw new ExitException(2, "Missing input file.\n" + usage()); + } + return new Args(inputPath, outputPath, option); + } + + private static String requireValue(String[] args, int index, String flag) { + if (index >= args.length) { + throw new ExitException(2, "Option '" + flag + "' requires a value.\n" + usage()); + } + return args[index]; + } + } + + /** Signals a clean CLI exit with a message and status code (kept out of the library core). */ + static final class ExitException extends RuntimeException { + final int code; + + ExitException(int code, String message) { + super(message); + this.code = code; + } + } +} diff --git a/converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/OssieToMetricView.java b/converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/OssieToMetricView.java new file mode 100644 index 00000000..c1940c50 --- /dev/null +++ b/converters/databricks/java/src/main/java/org/apache/ossie/converter/databricks/OssieToMetricView.java @@ -0,0 +1,903 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ossie.converter.databricks; + +import static org.apache.ossie.converter.databricks.OssieConverterCommon.CARD_MANY_TO_ONE; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.CARD_ONE_TO_MANY; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.MAPPER; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.MAX_JOIN_NODES; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.MV_VERSION; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.OSSIE_VERSION; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.STASH_SOURCE_KEY; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.SYNONYM_LIMIT; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.asList; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.asMap; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.foreignVendorExtensions; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.get; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.isSimpleIdentifier; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.loadYaml; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.mergeDescription; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.pickExpression; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.readStash; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.replaceOutsideLiterals; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.require; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.requireStr; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.str; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.strList; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.synonymsOf; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.truthy; +import static org.apache.ossie.converter.databricks.OssieConverterCommon.validateSource; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.regex.Pattern; + +import org.apache.ossie.converter.databricks.OssieConverter.ConversionException; +import org.apache.ossie.converter.databricks.OssieConverter.Notices; +import org.apache.ossie.converter.databricks.OssieConverter.Result; + +/** + * EXPORT direction: Apache Ossie semantic model -> Databricks Metric View v1.1 YAML. This is the + * harder direction: it reassembles a relationship graph into a join tree. Shared helpers come from + * {@link OssieConverterCommon}; the public entry point is re-exported through + * {@link OssieConverter}. + */ +// Map-based YAML manipulation: casts of the parsed Object graph to Map/List are inherently +// unchecked; the asMap/asList helpers guard them, so unchecked warnings here are expected. +@SuppressWarnings("unchecked") +final class OssieToMetricView { + + private OssieToMetricView() {} + + // -- tree node ------------------------------------------------------------ + private static final class Node { + final String dataset; + final Map rel; // empty for the fact/root + final boolean parentIsFrom; + String alias; + boolean isOtm; + final List children = new ArrayList<>(); + + Node(String dataset, Map rel, boolean parentIsFrom) { + this.dataset = dataset; + this.rel = rel; + this.parentIsFrom = parentIsFrom; + } + } + + // -- public entry --------------------------------------------------------- + static Result convertOssieToMetricView(String osiYamlStr, String source) { + Notices notices = new Notices(); + Map root; + try { + root = asMap(loadYaml(osiYamlStr)); + } catch (Exception e) { + throw new ConversionException("Invalid Apache Ossie YAML: " + e.getMessage(), e); + } + if (root.isEmpty()) { + throw new ConversionException("Invalid Apache Ossie YAML: expected a mapping at the root"); + } + String version = str(get(root, "version")); + if (!OSSIE_VERSION.equals(version)) { + throw new ConversionException( + "Unsupported Apache Ossie version '" + version + "'. Supported: " + OSSIE_VERSION); + } + List models = asList(get(root, "semantic_model")); + if (models.isEmpty()) { + throw new ConversionException("'semantic_model' must be a non-empty list"); + } + if (models.size() > 1) { + notices.warn("model", "multiple semantic models found; converting only the first"); + } + Map view = convertModel(asMap(models.get(0)), source, notices); + try { + return new Result(MAPPER.writeValueAsString(view), notices.toList()); + } catch (Exception e) { + throw new ConversionException("failed to serialize Metric View YAML: " + e.getMessage(), e); + } + } + + private static Map convertModel( + Map model, String explicitSource, Notices notices) { + String name = model.containsKey("name") ? str(get(model, "name")) : ""; + List datasetList = asList(get(model, "datasets")); + if (datasetList.isEmpty()) { + throw new ConversionException("Model '" + name + "' has no datasets"); + } + Set seen = new HashSet<>(); + Map> datasets = new LinkedHashMap<>(); + for (Object dObj : datasetList) { + Map d = asMap(dObj); + String dsName = requireStr(d, "name", "Model '" + name + "': dataset"); + if (!seen.add(dsName.trim().toLowerCase(Locale.ROOT))) { + throw new ConversionException("Model '" + name + "': duplicate dataset name '" + dsName + "'"); + } + datasets.put(dsName, d); + } + List> relationships = new ArrayList<>(); + for (Object r : asList(get(model, "relationships"))) { + relationships.add(asMap(r)); + } + + Map modelStash = readStash(model); + String factHint = explicitSource != null ? explicitSource : str(get(modelStash, STASH_SOURCE_KEY)); + Object[] built = buildJoinTree(name, datasets, relationships, factHint, notices); + Node root = (Node) built[0]; + String fact = (String) built[1]; + Map counts = assignAliases(root, fact); + markOtm(name, root); + + Map factDs = datasets.get(fact); + Map view = new LinkedHashMap<>(); + view.put("version", MV_VERSION); + view.put("source", validateSource(get(factDs, "source"), fact)); + + String comment = str(get(model, "description")); + if (truthy(comment)) { + view.put("comment", comment); + } + if (modelStash.containsKey("filter")) { + view.put("filter", modelStash.get("filter")); + } + + List joins = new ArrayList<>(); + for (Node child : root.children) { + joins.add(buildJoin(child, "source", datasets, notices)); + } + if (!joins.isEmpty()) { + view.put("joins", joins); + } + + Set droppedDims = new HashSet<>(); + Set droppedMeasures = new HashSet<>(); + List> dimensions = new ArrayList<>(); + Set seenDims = new HashSet<>(); + // Dataset name -> the alias path that addresses its columns from the primary source, collected + // from the same walk the dimensions use so measures qualify identically (see qualifyMeasure). + Map datasetAliasPath = new LinkedHashMap<>(); + for (Object[] entry : nodeOrder(root)) { + Node node = (Node) entry[0]; + @SuppressWarnings("unchecked") + List joinPath = (List) entry[1]; + boolean isFact = node == root; + String prefix = counts.get(node.dataset) > 1 ? node.alias : null; + // Fixed per node, like `prefix`: hoisted so it is not rebuilt for every field. + String qualifier = String.join(".", joinPath); + if (!isFact) { + // A dataset reachable by more than one path (diamond) appears once per path; the first + // wins, matching the order dimensions are emitted in. + datasetAliasPath.putIfAbsent(node.dataset, qualifier); + } + for (Object fObj : asList(get(datasets.get(node.dataset), "fields"))) { + Map field = asMap(fObj); + String fname = requireStr(field, "name", "dataset '" + node.dataset + "': field"); + if (node.isOtm) { + notices.warn("field '" + fname + "'", + "column on a one-to-many-joined table cannot be a dimension " + + "(must resolve to one value per source row); dropped"); + droppedDims.add(fname); + continue; + } + Map dim = + convertField(field, fname, qualifier, isFact, prefix, notices); + if (dim == null) { + droppedDims.add(fname); + continue; + } + String dn = (String) dim.get("name"); + if (!seenDims.add(dn.toLowerCase(Locale.ROOT))) { + throw new ConversionException("dataset '" + node.dataset + "': dimension name '" + dn + + "' collides with another dimension/measure; Metric Views require unique " + + "dimension/measure names -- rename before use"); + } + dimensions.add(dim); + } + } + + List> measures = new ArrayList<>(); + // Depends only on the fact name, so compile it once rather than per metric. + Pattern factQualifier = Pattern.compile("\\b" + Pattern.quote(fact) + "\\."); + for (Object mObj : asList(get(model, "metrics"))) { + Map measure = + convertMetric(asMap(mObj), factQualifier, datasetAliasPath, seenDims, notices); + if (measure == null) { + droppedMeasures.add(str(get(asMap(mObj), "name"))); + continue; + } + measures.add(measure); + } + + cascadeDrop(dimensions, measures, droppedDims, droppedMeasures, notices); + + // A Metric View must define at least one dimension or measure + // (SingleSourceMetricView.validate rejects an empty `select`), so a view with neither is one + // Databricks refuses at CREATE. Fail here instead, naming the dropped columns: after + // cascadeDrop the emptiness is usually a consequence of earlier drops rather than an empty + // input, and those names are the actionable part. + if (dimensions.isEmpty() && measures.isEmpty()) { + StringBuilder msg = new StringBuilder("Model '" + name + + "' produced no dimensions or measures; a Metric View requires at least one."); + if (!droppedDims.isEmpty() || !droppedMeasures.isEmpty()) { + msg.append(" Dropped during conversion:"); + if (!droppedDims.isEmpty()) { + msg.append(" dimensions ").append(sortedNames(droppedDims)); + } + if (!droppedMeasures.isEmpty()) { + msg.append(" measures ").append(sortedNames(droppedMeasures)); + } + msg.append(" -- see the warnings for why each was dropped."); + } + throw new ConversionException(msg.toString()); + } + + if (!dimensions.isEmpty()) { + view.put("dimensions", dimensions); + } + if (!measures.isEmpty()) { + view.put("measures", measures); + } + if (modelStash.containsKey("parameters")) { + view.put("parameters", modelStash.get("parameters")); + } + if (modelStash.containsKey("materialization")) { + view.put("materialization", modelStash.get("materialization")); + } + + warnDroppedModel(model, notices); + return view; + } + + private static Object[] buildJoinTree( + String modelName, Map> datasets, + List> relationships0, String factHint, Notices notices) { + for (Map rel : relationships0) { + String scope = "Model '" + modelName + "': relationship '" + + (rel.containsKey("name") ? str(get(rel, "name")) : "") + "'"; + Object f = require(rel, "from", scope); + Object t = require(rel, "to", scope); + if (!datasets.containsKey(str(f)) || !datasets.containsKey(str(t))) { + throw new ConversionException("Model '" + modelName + "': relationship '" + + str(get(rel, "name")) + "' references an unknown dataset"); + } + } + List> relationships = new ArrayList<>(); + for (Map rel : relationships0) { + relationships.add(orientByKey(rel, datasets, notices)); + } + String fact = pickFact(modelName, datasets, relationships, factHint); + rejectDirectedCycle(modelName, datasets, relationships); + + Map> adj = new HashMap<>(); + for (String n : datasets.keySet()) { + adj.put(n, new ArrayList<>()); + } + for (Map rel : relationships) { + adj.get(str(get(rel, "from"))).add(str(get(rel, "to"))); + adj.get(str(get(rel, "to"))).add(str(get(rel, "from"))); + } + Map dist = new HashMap<>(); + dist.put(fact, 0); + Deque queue = new ArrayDeque<>(); + queue.add(fact); + while (!queue.isEmpty()) { + String cur = queue.poll(); + for (String nb : adj.get(cur)) { + if (!dist.containsKey(nb)) { + dist.put(nb, dist.get(cur) + 1); + queue.add(nb); + } + } + } + List unreachable = new ArrayList<>(); + for (String n : datasets.keySet()) { + if (!dist.containsKey(n)) { + unreachable.add(n); + } + } + if (!unreachable.isEmpty()) { + java.util.Collections.sort(unreachable); + throw new ConversionException("Model '" + modelName + "': datasets " + unreachable + + " are not reachable from fact '" + fact + "' via relationships."); + } + Map> childrenOf = new HashMap<>(); + for (String n : datasets.keySet()) { + childrenOf.put(n, new ArrayList<>()); + } + for (Map rel : relationships) { + String a = str(get(rel, "from")); + String b = str(get(rel, "to")); + if (dist.get(a).equals(dist.get(b))) { + throw new ConversionException("Model '" + modelName + "': relationship '" + + str(get(rel, "name")) + "' joins two datasets equidistant from the fact; " + + "the graph is not tree-shaped (it contains a cycle)."); + } + String parent = dist.get(a) < dist.get(b) ? a : b; + String child = dist.get(a) < dist.get(b) ? b : a; + childrenOf.get(parent).add(new Object[] {child, rel, parent.equals(str(get(rel, "from")))}); + } + int[] counter = {0}; + Node root = build(modelName, fact, new LinkedHashMap<>(), false, childrenOf, counter); + return new Object[] {root, fact}; + } + + private static Node build(String modelName, String dataset, Map rel, + boolean parentIsFrom, Map> childrenOf, int[] counter) { + counter[0]++; + if (counter[0] > MAX_JOIN_NODES) { + throw new ConversionException("Model '" + modelName + "': join graph fans out to more than " + + MAX_JOIN_NODES + " joins; check for an unintended diamond explosion."); + } + Node node = new Node(dataset, rel, parentIsFrom); + for (Object[] c : childrenOf.get(dataset)) { + @SuppressWarnings("unchecked") + Map crel = (Map) c[1]; + node.children.add(build(modelName, (String) c[0], crel, (Boolean) c[2], childrenOf, counter)); + } + return node; + } + + private static Map assignAliases(Node root, String fact) { + Map counts = new HashMap<>(); + countNode(root, counts); + Set used = new HashSet<>(); + used.add("source"); + assign(root, null, fact, counts, used); + return counts; + } + + private static void countNode(Node node, Map counts) { + counts.merge(node.dataset, 1, Integer::sum); + for (Node c : node.children) { + countNode(c, counts); + } + } + + private static void assign(Node node, String parentAlias, String fact, + Map counts, Set used) { + String alias; + if (node.dataset.equals(fact)) { + alias = "source"; + } else { + String base; + if (counts.get(node.dataset) == 1) { + base = node.dataset; + } else if (parentAlias != null && !parentAlias.equals("source")) { + base = parentAlias + "_" + node.dataset; + } else { + base = node.dataset; + } + alias = base; + int n = 2; + while (used.contains(alias)) { + alias = base + "_" + n; + n++; + } + } + node.alias = alias; + used.add(alias); + for (Node c : node.children) { + assign(c, alias, fact, counts, used); + } + } + + private static String pickFact(String modelName, Map> datasets, + List> relationships, String factHint) { + if (factHint != null) { + if (!datasets.containsKey(factHint)) { + throw new ConversionException( + "Model '" + modelName + "': requested source '" + factHint + "' is not a dataset"); + } + return factHint; + } + if (datasets.size() > 1 && relationships.isEmpty()) { + throw new ConversionException("Model '" + modelName + "': " + datasets.size() + + " datasets but no relationships; cannot determine the fact table."); + } + Map incoming = new LinkedHashMap<>(); + for (String n : datasets.keySet()) { + incoming.put(n, 0); + } + for (Map rel : relationships) { + incoming.merge(str(get(rel, "to")), 1, Integer::sum); + } + List roots = new ArrayList<>(); + for (Map.Entry e : incoming.entrySet()) { + if (e.getValue() == 0) { + roots.add(e.getKey()); + } + } + if (roots.isEmpty()) { + throw new ConversionException("Model '" + modelName + "': join graph contains a cycle " + + "(no root dataset). A Metric View requires an acyclic, tree-shaped graph."); + } + if (roots.size() > 1) { + java.util.Collections.sort(roots); + throw new ConversionException("Model '" + modelName + "': multiple candidate fact datasets " + + roots + ". Name the grain with --source."); + } + return roots.get(0); + } + + /** + * Marks each joined node with its branch's cardinality and rejects a branch that mixes the two. + * + *

A Metric View requires every join within one top-level branch to share a single cardinality: + * `Join.validateSubJoinCardinalities` seeds the expected value from the top-level join and fails + * any descendant that differs (an absent `cardinality` reads as `many_to_one`). So a mixed branch + * is rejected in *either* direction -- a many-to-one nested under one-to-many, and equally a + * one-to-many nested under many-to-one. Emitting one would produce a view Databricks refuses at + * CREATE, so reject it here with a converter-level error instead. + */ + private static void markOtm(String modelName, Node root) { + // Each top-level join starts a branch and sets that branch's expected cardinality. + for (Node top : root.children) { + boolean branchIsOtm = !top.parentIsFrom; + top.isOtm = branchIsOtm; + markOtmVisit(modelName, top, branchIsOtm); + } + } + + private static void markOtmVisit(String modelName, Node node, boolean branchIsOtm) { + for (Node child : node.children) { + boolean isOtm = !child.parentIsFrom; + if (isOtm != branchIsOtm) { + throw new ConversionException("Model '" + modelName + "': join '" + child.alias + "' is " + + cardinalityName(isOtm) + " but descends from a " + cardinalityName(branchIsOtm) + + " join; a Metric View requires every join within one top-level branch to share the " + + "same cardinality."); + } + child.isOtm = branchIsOtm; + markOtmVisit(modelName, child, branchIsOtm); + } + } + + private static String cardinalityName(boolean isOtm) { + return isOtm ? CARD_ONE_TO_MANY : CARD_MANY_TO_ONE; + } + + /** + * Rejects a directed cycle in the relationship graph. + * + *

Checked on the DIRECTED graph, and specifically for an edge back to a dataset on the current + * DFS stack: a dataset reachable by two distinct paths (a diamond, e.g. `a -> b -> d` plus + * `a -> c -> d`) is directed-acyclic and legitimately supported via the fan-out aliases, so a + * test for "reached twice" would wrongly reject it. Only a genuine directed cycle is an error. + * + *

This replaces relying on the equidistance heuristic below, which only detects a cycle whose + * closing edge happens to join two datasets at the same BFS distance from the fact: a cycle such + * as `a -> b -> c -> d -> e -> b` has no equidistant edge, so it used to expand into duplicate + * join paths (`d` emitted under both `c` and `e`) and fabricate a tree from a cyclic model. + * `pickFact` does not catch it either -- it only fails when no dataset has zero incoming edges, + * and here `a` has none. With this check, MAX_JOIN_NODES is purely a fan-out bound rather than + * the last defense against a cycle. + */ + private static void rejectDirectedCycle(String modelName, + Map> datasets, List> relationships) { + Map> out = new HashMap<>(); + for (String n : datasets.keySet()) { + out.put(n, new ArrayList<>()); + } + for (Map rel : relationships) { + out.get(str(get(rel, "from"))).add(str(get(rel, "to"))); + } + Set done = new HashSet<>(); + Set onStack = new LinkedHashSet<>(); + for (String n : datasets.keySet()) { + List cycle = findCycle(n, out, done, onStack); + if (cycle != null) { + throw new ConversionException("Model '" + modelName + "': relationships form a directed" + + " cycle " + String.join(" -> ", cycle) + + "; a Metric View join graph must be acyclic."); + } + } + } + + /** The cycle path (closing dataset repeated at the end), or null if this subtree is clean. */ + private static List findCycle( + String node, Map> out, Set done, Set onStack) { + if (done.contains(node)) { + return null; + } + if (!onStack.add(node)) { + // Back-edge: report from the first occurrence of `node` so the message shows just the cycle. + List cycle = new ArrayList<>(); + boolean seen = false; + for (String s : onStack) { + if (s.equals(node)) { + seen = true; + } + if (seen) { + cycle.add(s); + } + } + cycle.add(node); + return cycle; + } + for (String next : out.get(node)) { + List cycle = findCycle(next, out, done, onStack); + if (cycle != null) { + return cycle; + } + } + onStack.remove(node); + done.add(node); + return null; + } + + /** + * Rewrites `.` heads in a measure expression to the alias path that addresses that + * dataset's columns from the primary source (`parentJoin.nestedJoin.`). + * + *

A dataset joined directly to the primary already maps to its own alias, so those rewrites + * are no-ops; only a nested dataset actually changes. Datasets are processed longest-name-first + * so a shorter name is never rewritten inside a longer one (`nation` must not match + * `nation_x.`), and the rewrite skips string literals and comments, as the fact strip does. + */ + private static String qualifyMeasure(String expr, Map datasetAliasPath) { + List datasets = new ArrayList<>(datasetAliasPath.keySet()); + datasets.sort((a, b) -> b.length() - a.length()); + String out = expr; + for (String dataset : datasets) { + String aliasPath = datasetAliasPath.get(dataset); + if (aliasPath == null || aliasPath.isEmpty() || aliasPath.equals(dataset)) { + continue; + } + out = replaceOutsideLiterals( + out, Pattern.compile("\\b" + Pattern.quote(dataset) + "\\."), aliasPath + "."); + } + return out; + } + + /** Sorted so the error message is deterministic (the dropped-name sets are unordered). */ + private static String sortedNames(Set names) { + List sorted = new ArrayList<>(); + for (String n : names) { + if (n != null) { + sorted.add(n); + } + } + java.util.Collections.sort(sorted); + return sorted.toString(); + } + + private static List nodeOrder(Node root) { + List order = new ArrayList<>(); + nodeOrderVisit(root, new ArrayList<>(), order); + return order; + } + + private static void nodeOrderVisit(Node node, List path, List order) { + order.add(new Object[] {node, new ArrayList<>(path)}); + for (Node child : node.children) { + List childPath = new ArrayList<>(path); + childPath.add(child.alias); + nodeOrderVisit(child, childPath, order); + } + } + + private static Map buildJoin(Node node, String parentAlias, + Map> datasets, Notices notices) { + Map rel = node.rel; + String alias = node.alias; + Map join = new LinkedHashMap<>(); + join.put("name", alias); + join.put("source", validateSource(get(datasets.get(node.dataset), "source"), node.dataset)); + + Map stash = readStash(rel); + List fromCols = strList(get(rel, "from_columns")); + List toCols = strList(get(rel, "to_columns")); + validateJoinColumns(rel, fromCols, toCols); + List parentCols = node.parentIsFrom ? fromCols : toCols; + List childCols = node.parentIsFrom ? toCols : fromCols; + if (parentCols.equals(childCols)) { + join.put("using", new ArrayList<>(parentCols)); + } else { + List clauses = new ArrayList<>(); + for (int i = 0; i < parentCols.size(); i++) { + clauses.add(parentAlias + "." + parentCols.get(i) + " = " + alias + "." + childCols.get(i)); + } + join.put("on", String.join(" AND ", clauses)); + } + if (stash.containsKey("rely")) { + join.put("rely", stash.get("rely")); + } else if (node.parentIsFrom && coversUniqueKey(datasets.get(node.dataset), toCols)) { + Map rely = new LinkedHashMap<>(); + rely.put("at_most_one_match", true); + join.put("rely", rely); + } + if (stash.containsKey("cardinality")) { + join.put("cardinality", stash.get("cardinality")); + } else if (!node.parentIsFrom) { + join.put("cardinality", CARD_ONE_TO_MANY); + } + List nested = new ArrayList<>(); + for (Node c : node.children) { + nested.add(buildJoin(c, alias, datasets, notices)); + } + if (!nested.isEmpty()) { + join.put("joins", nested); + } + return join; + } + + private static boolean coversUniqueKey(Map dataset, List joinCols) { + Set cols = new HashSet<>(joinCols); + List> keys = new ArrayList<>(); + List pk = strList(get(dataset, "primary_key")); + if (!pk.isEmpty()) { + keys.add(pk); + } + for (Object k : asList(get(dataset, "unique_keys"))) { + keys.add(strList(k)); + } + for (List key : keys) { + if (!key.isEmpty() && cols.containsAll(key)) { + return true; + } + } + return false; + } + + private static Map orientByKey( + Map rel, Map> datasets, Notices notices) { + List fromCols = strList(get(rel, "from_columns")); + List toCols = strList(get(rel, "to_columns")); + if (fromCols.isEmpty() || toCols.isEmpty()) { + return rel; + } + Map toDs = datasets.get(str(get(rel, "to"))); + boolean toHasKeys = !strList(get(toDs, "primary_key")).isEmpty() + || !asList(get(toDs, "unique_keys")).isEmpty(); + boolean fromCovers = coversUniqueKey(datasets.get(str(get(rel, "from"))), fromCols); + if (fromCovers && toHasKeys && !coversUniqueKey(toDs, toCols)) { + notices.warn("relationship '" + str(get(rel, "name")) + "'", + "from/to looks mislabeled (the `from` columns are a declared key, the `to` columns " + + "are not); re-orienting so the key side is the `to`/one side"); + Map swapped = new LinkedHashMap<>(rel); + swapped.put("from", get(rel, "to")); + swapped.put("to", get(rel, "from")); + swapped.put("from_columns", toCols); + swapped.put("to_columns", fromCols); + return swapped; + } + if (fromCovers && !toHasKeys) { + notices.warn("relationship '" + str(get(rel, "name")) + "'", + "the `from` columns are a declared key but the `to` side declares none, so from/to " + + "orientation can't be verified; using it as-is -- check the join direction if " + + "the resulting cardinality looks inverted"); + } + return rel; + } + + private static void validateJoinColumns( + Map rel, List fromCols, List toCols) { + String name = str(get(rel, "name")); + if (fromCols.isEmpty() || toCols.isEmpty()) { + throw new ConversionException( + "Relationship '" + name + "': from_columns and to_columns are required"); + } + if (fromCols.size() != toCols.size()) { + throw new ConversionException("Relationship '" + name + "': from_columns (" + fromCols.size() + + ") and to_columns (" + toCols.size() + ") must have the same length"); + } + } + + private static Map convertField(Map field, String name0, + String qualifier, boolean isFact, String prefix, Notices notices) { + String scope = "field '" + name0 + "'"; + String expr = pickExpression(get(field, "expression")); + if (expr == null) { + notices.warn(scope, "no DATABRICKS/ANSI_SQL dialect; dropping field"); + return null; + } + if (!isFact) { + if (isSimpleIdentifier(expr)) { + expr = qualifier + "." + expr; + } else if (prefix != null) { + notices.warn(scope, "complex expression on a fanned-out (diamond) join cannot be " + + "unambiguously qualified; dropped"); + return null; + } else { + notices.warn(scope, "complex expression on a joined table; emitted as-is, verify qualification"); + } + } + String name = prefix != null ? prefix + "_" + name0 : name0; + Map dim = new LinkedHashMap<>(); + dim.put("name", name); + dim.put("expr", expr); + String comment = mergeDescription(get(field, "description"), get(field, "ai_context")); + if (truthy(comment)) { + dim.put("comment", comment); + } + String label = str(get(field, "label")); + if (truthy(label)) { + dim.put("display_name", label); + } + List syns = synonymsOf(get(field, "ai_context")); + if (!syns.isEmpty()) { + dim.put("synonyms", truncateSynonyms(syns, scope, notices)); + } + Map stash = readStash(field); + if (stash.containsKey("format")) { + dim.put("format", stash.get("format")); + } + warnDroppedField(field, scope, notices); + return dim; + } + + private static Map convertMetric( + Map metric, + Pattern factQualifier, + Map datasetAliasPath, + Set seenNames, + Notices notices) { + String name = requireStr(metric, "name", "metric"); + String scope = "metric '" + name + "'"; + if (!seenNames.add(name.toLowerCase(Locale.ROOT))) { + throw new ConversionException("metric '" + name + "' collides with another dimension/measure; " + + "Metric Views require unique dimension/measure names -- rename before use"); + } + String expr = pickExpression(get(metric, "expression")); + if (expr == null) { + notices.warn(scope, "no DATABRICKS/ANSI_SQL dialect; dropping metric"); + return null; + } + // Re-qualify a joined dataset's columns with the alias path that addresses them from the + // primary source. A Metric View addresses a nested join column by its full path + // (`parentJoin.nestedJoin.col`), so a bare nested alias at the head is read as struct access on + // a parameter rather than as a join column -- it fails silently. Dimensions already qualify + // this way via the joinPath handed to convertField, so this keeps the two directions + // consistent on the same input. + expr = qualifyMeasure(expr, datasetAliasPath); + // Strip a `.` qualifier so fact columns are bare in measures (the Metric View idiom). + // Only outside string literals / comments: a literal such as 'customer.us' must not be + // rewritten, or the measure's predicate changes. + expr = replaceOutsideLiterals(expr, factQualifier, ""); + Map measure = new LinkedHashMap<>(); + measure.put("name", name); + measure.put("expr", expr); + String comment = mergeDescription(get(metric, "description"), get(metric, "ai_context")); + if (truthy(comment)) { + measure.put("comment", comment); + } + List syns = synonymsOf(get(metric, "ai_context")); + if (!syns.isEmpty()) { + measure.put("synonyms", truncateSynonyms(syns, scope, notices)); + } + Map stash = readStash(metric); + if (stash.containsKey("format")) { + measure.put("format", stash.get("format")); + } + if (stash.containsKey("window")) { + measure.put("window", stash.get("window")); + } + if (stash.containsKey("partition")) { + measure.put("partition", stash.get("partition")); + } + return measure; + } + + private static String referencesDropped( + String expr, String selfName, Set droppedDims, Set droppedMeasures) { + for (String m : droppedMeasures) { + if (m != null && Pattern.compile("measure\\(\\s*" + Pattern.quote(m) + "\\s*\\)") + .matcher(expr).find()) { + return m; + } + } + for (String d : droppedDims) { + if (d != null && !d.equals(selfName) + && Pattern.compile("(?> dimensions, + List> measures, Set droppedDims, + Set droppedMeasures, Notices notices) { + boolean changed = true; + while (changed) { + changed = false; + changed |= cascadePass(dimensions, "dimension", droppedDims, droppedDims, droppedMeasures, notices); + changed |= cascadePass(measures, "measure", droppedMeasures, droppedDims, droppedMeasures, notices); + } + } + + private static boolean cascadePass(List> coll, String kind, + Set droppedSet, Set droppedDims, Set droppedMeasures, Notices notices) { + boolean changed = false; + List> survivors = new ArrayList<>(); + for (Map col : coll) { + String nm = (String) col.get("name"); + String ref = referencesDropped((String) col.get("expr"), nm, droppedDims, droppedMeasures); + if (ref != null) { + notices.warn(kind + " '" + nm + "'", + "references dropped '" + ref + "'; dropping (downstream of a dropped field/metric)"); + droppedSet.add(nm); + changed = true; + } else { + survivors.add(col); + } + } + coll.clear(); + coll.addAll(survivors); + return changed; + } + + private static List truncateSynonyms(List syns, String scope, Notices notices) { + if (syns.size() > SYNONYM_LIMIT) { + notices.warn(scope, syns.size() + " synonyms exceeds Metric View limit; keeping first " + SYNONYM_LIMIT); + return new ArrayList<>(syns.subList(0, SYNONYM_LIMIT)); + } + return syns; + } + + private static void warnDroppedModel(Map model, Notices notices) { + if (!foreignVendorExtensions(model).isEmpty()) { + notices.warn("model", "foreign-vendor custom_extensions dropped"); + } + if (truthy(get(model, "ai_context"))) { + notices.warn("model", "model-level ai_context dropped (only the description maps to the view comment)"); + } + for (Object dsObj : asList(get(model, "datasets"))) { + Map ds = asMap(dsObj); + String scope = "dataset '" + str(get(ds, "name")) + "'"; + if (!strList(get(ds, "primary_key")).isEmpty() || !asList(get(ds, "unique_keys")).isEmpty()) { + notices.warn(scope, "primary_key/unique_keys not stored as columns; used to set " + + "rely.at_most_one_match on a matching many_to_one join where applicable"); + } + if (get(ds, "ai_context") instanceof Map && !asMap(get(ds, "ai_context")).isEmpty()) { + notices.warn(scope, "dataset-level ai_context (object) dropped"); + } + if (truthy(get(ds, "description"))) { + notices.warn(scope, "dataset-level description dropped (no per-source comment field)"); + } + if (!foreignVendorExtensions(ds).isEmpty()) { + notices.warn(scope, "foreign-vendor custom_extensions dropped"); + } + } + for (Object relObj : asList(get(model, "relationships"))) { + Map rel = asMap(relObj); + if (truthy(get(rel, "ai_context"))) { + String rn = rel.containsKey("name") ? str(get(rel, "name")) : ""; + notices.warn("relationship '" + rn + "'", "relationship ai_context dropped"); + } + } + } + + private static void warnDroppedField(Map field, String scope, Notices notices) { + Object dim = get(field, "dimension"); + if (dim instanceof Map && asMap(dim).containsKey("is_time")) { + notices.warn(scope, "dimension.is_time has no Metric View counterpart; dropped"); + } + if (!foreignVendorExtensions(field).isEmpty()) { + notices.warn(scope, "foreign-vendor custom_extensions dropped"); + } + } +} diff --git a/converters/databricks/java/src/test/java/org/apache/ossie/converter/databricks/OssieConverterRoundTripSuite.java b/converters/databricks/java/src/test/java/org/apache/ossie/converter/databricks/OssieConverterRoundTripSuite.java new file mode 100644 index 00000000..c6dfdfae --- /dev/null +++ b/converters/databricks/java/src/test/java/org/apache/ossie/converter/databricks/OssieConverterRoundTripSuite.java @@ -0,0 +1,560 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ossie.converter.databricks; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.TreeSet; + +import org.junit.jupiter.api.Test; + +/** + * Property-based round-trip tests. For any generated model in the round-trippable subset, + * converting + * one direction and back preserves content: + * + * MV -> Ossie -> MV : source, every dimension/measure name+expr+metadata, every join + * (name/source/condition/cardinality/rely) and nesting, and model filter/comment/ + * materialization. + * Ossie -> MV -> Ossie : dataset names+sources+fields, relationship from/to/columns, + * metric name+expr, and model description. + * + * Uses a seeded java.util.Random so no + * property-testing library is needed; each of NUM_SEEDS seeds is one generated model. + */ +public class OssieConverterRoundTripSuite { + + private static final int NUM_SEEDS = 300; + private static final String[] AGGS = {"SUM", "COUNT", "AVG", "MIN", "MAX"}; + + // --- Rnd: the small interface the builders depend on --------------------- + private static final class Rnd { + private final Random r; + Rnd(long seed) { + this.r = new Random(seed); + } + boolean chance(double p) { + return r.nextDouble() < p; + } + int count(int lo, int hi) { + return lo + r.nextInt(hi - lo + 1); + } + T pick(List seq) { + return seq.get(r.nextInt(seq.size())); + } + String text() { + String alnum = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; + int n = r.nextInt(11); + StringBuilder b = new StringBuilder(); + b.append(alnum.charAt(r.nextInt(alnum.length()))); + String alnumSpace = alnum + " "; + for (int i = 0; i < n; i++) { + b.append(alnumSpace.charAt(r.nextInt(alnumSpace.length()))); + } + String s = b.toString().trim(); + return s.isEmpty() ? "x" : s; + } + String colname() { + String lower = "abcdefghijklmnopqrstuvwxyz_"; + String rest = lower + "0123456789"; + StringBuilder b = new StringBuilder(); + b.append(lower.charAt(r.nextInt(lower.length()))); + int n = r.nextInt(8); + for (int i = 0; i < n; i++) { + b.append(rest.charAt(r.nextInt(rest.length()))); + } + return b.toString(); + } + } + + private static final class Names { + private final Map n = new HashMap<>(); + String next(String prefix) { + int i = n.getOrDefault(prefix, 0); + n.put(prefix, i + 1); + return prefix + i; + } + } + + private static String threePart(Rnd rnd) { + return rnd.colname() + "." + rnd.colname() + "." + rnd.colname(); + } + + private static void maybeMeta(Rnd rnd, Map target) { + if (rnd.chance(0.4)) { + target.put("comment", rnd.text()); + } + if (rnd.chance(0.3)) { + target.put("display_name", rnd.text()); + } + if (rnd.chance(0.3)) { + List syns = new ArrayList<>(); + int k = rnd.count(1, 3); + for (int i = 0; i < k; i++) { + syns.add(rnd.text()); + } + target.put("synonyms", syns); + } + if (rnd.chance(0.25)) { + Map fmt = new LinkedHashMap<>(); + String type = rnd.pick(List.of("number", "currency", "date")); + fmt.put("type", type); + if (type.equals("currency")) { + fmt.put("currency_code", "USD"); + } + target.put("format", fmt); + } + } + + // --- Metric View builder (for MV -> Ossie -> MV) ------------------------- + private static Object[] buildJoin( + Rnd rnd, Names names, String parentAlias, int depth, List ancestorPath) { + String name = names.next("j"); + List path = new ArrayList<>(ancestorPath); + path.add(name); + String qual = String.join(".", path); + Map join = new LinkedHashMap<>(); + join.put("name", name); + join.put("source", threePart(rnd)); + if (rnd.chance(0.5)) { + int ncols = rnd.count(1, 2); + List using = new ArrayList<>(); + for (int i = 0; i < ncols; i++) { + using.add("u" + i + "_" + rnd.colname()); + } + join.put("using", using); + } else { + int ncols = rnd.count(1, 2); + List clauses = new ArrayList<>(); + for (int i = 0; i < ncols; i++) { + String pc = "fk" + i + "_" + rnd.colname(); + String cc = "pk" + i + "_" + rnd.colname(); + clauses.add(parentAlias + "." + pc + " = " + name + "." + cc); + } + join.put("on", String.join(" AND ", clauses)); + } + if (rnd.chance(0.4)) { + join.put("cardinality", "many_to_one"); + } + if (rnd.chance(0.3)) { + Map rely = new LinkedHashMap<>(); + rely.put("at_most_one_match", true); + join.put("rely", rely); + } + List> dims = new ArrayList<>(); + int nd = rnd.count(0, 2); + for (int i = 0; i < nd; i++) { + String col = rnd.colname(); + String expr = rnd.chance(0.7) ? qual + "." + col + : qual + "." + col + " + " + qual + "." + rnd.colname(); + Map dim = new LinkedHashMap<>(); + dim.put("name", names.next("c")); + dim.put("expr", expr); + maybeMeta(rnd, dim); + dims.add(dim); + } + if (depth < 2 && rnd.chance(0.35)) { + Object[] childResult = buildJoin(rnd, names, name, depth + 1, path); + List childJoins = new ArrayList<>(); + childJoins.add(childResult[0]); + join.put("joins", childJoins); + @SuppressWarnings("unchecked") + List> childDims = (List>) childResult[1]; + dims.addAll(childDims); + } + return new Object[] {join, dims}; + } + + private static Map buildMetricView(Rnd rnd) { + Names names = new Names(); + Map mv = new LinkedHashMap<>(); + mv.put("version", OssieConverter.MV_VERSION); + mv.put("source", threePart(rnd)); + if (rnd.chance(0.4)) { + mv.put("comment", rnd.text()); + } + if (rnd.chance(0.3)) { + mv.put("filter", rnd.colname() + " > 0"); + } + List> fields = new ArrayList<>(); + List joins = new ArrayList<>(); + int nsrc = rnd.count(0, 3); + for (int i = 0; i < nsrc; i++) { + String col = rnd.colname(); + String expr = rnd.chance(0.7) ? col : "UPPER(" + col + ")"; + Map dim = new LinkedHashMap<>(); + dim.put("name", names.next("c")); + dim.put("expr", expr); + maybeMeta(rnd, dim); + fields.add(dim); + } + int njoins = rnd.count(0, 2); + for (int i = 0; i < njoins; i++) { + Object[] jr = buildJoin(rnd, names, "source", 0, new ArrayList<>()); + joins.add(jr[0]); + @SuppressWarnings("unchecked") + List> jdims = (List>) jr[1]; + fields.addAll(jdims); + } + List> measures = new ArrayList<>(); + int nmeas = rnd.count(0, 2); + for (int i = 0; i < nmeas; i++) { + Map m = new LinkedHashMap<>(); + m.put("name", names.next("c")); + m.put("expr", rnd.pick(List.of(AGGS)) + "(" + rnd.colname() + ")"); + if (rnd.chance(0.4)) { + m.put("comment", rnd.text()); + } + if (rnd.chance(0.3)) { + List syns = new ArrayList<>(); + int k = rnd.count(1, 3); + for (int j = 0; j < k; j++) { + syns.add(rnd.text()); + } + m.put("synonyms", syns); + } + if (rnd.chance(0.3)) { + Map w = new LinkedHashMap<>(); + w.put("order", rnd.colname()); + w.put("range", "trailing 7 day"); + List window = new ArrayList<>(); + window.add(w); + m.put("window", window); + } + measures.add(m); + } + if (!joins.isEmpty()) { + mv.put("joins", joins); + } + // A Metric View requires at least one dimension or measure, so a model with neither is outside + // the round-trippable subset (the converter rejects it, as Databricks would). Both counts can + // independently come out zero, so add one dimension when that happens. + if (fields.isEmpty() && measures.isEmpty()) { + Map dim = new LinkedHashMap<>(); + dim.put("name", names.next("c")); + dim.put("expr", rnd.colname()); + fields.add(dim); + } + if (!fields.isEmpty()) { + mv.put("fields", fields); + } + if (!measures.isEmpty()) { + mv.put("measures", measures); + } + if (rnd.chance(0.2)) { + Map mat = new LinkedHashMap<>(); + mat.put("schedule", "every 6 hours"); + mat.put("mode", rnd.pick(List.of("relaxed", "strict"))); + mv.put("materialization", mat); + } + return mv; + } + + // --- Ossie builder (for Ossie -> MV -> Ossie) ---------------------------- + private static Map ossieField(String name, String expr) { + Map dialect = new LinkedHashMap<>(); + dialect.put("dialect", "DATABRICKS"); + dialect.put("expression", expr); + List dialects = new ArrayList<>(); + dialects.add(dialect); + Map expression = new LinkedHashMap<>(); + expression.put("dialects", dialects); + Map field = new LinkedHashMap<>(); + field.put("name", name); + field.put("expression", expression); + return field; + } + + private static Map buildOssie(Rnd rnd) { + Names names = new Names(); + String fact = "fact"; + List> datasets = new ArrayList<>(); + Map factDs = new LinkedHashMap<>(); + factDs.put("name", fact); + factDs.put("source", "c.s." + fact); + datasets.add(factDs); + List> relationships = new ArrayList<>(); + + int nDims = rnd.count(0, 3); + List reachable = new ArrayList<>(); + reachable.add(fact); + for (int i = 0; i < nDims; i++) { + String dname = names.next("dim"); + String parent = rnd.pick(reachable); + Map ds = new LinkedHashMap<>(); + ds.put("name", dname); + ds.put("source", "c.s." + rnd.colname() + i); + datasets.add(ds); + reachable.add(dname); + Map rel = new LinkedHashMap<>(); + rel.put("name", names.next("r")); + rel.put("from", parent); + rel.put("to", dname); + if (rnd.chance(0.5)) { + List cols = new ArrayList<>(); + int k = rnd.count(1, 2); + for (int j = 0; j < k; j++) { + cols.add(rnd.colname()); + } + rel.put("from_columns", new ArrayList<>(cols)); + rel.put("to_columns", new ArrayList<>(cols)); + } else { + int n = rnd.count(1, 2); + List fcols = new ArrayList<>(); + List tcols = new ArrayList<>(); + for (int j = 0; j < n; j++) { + fcols.add("fk" + j + "_" + rnd.colname()); + tcols.add("pk" + j + "_" + rnd.colname()); + } + rel.put("from_columns", fcols); + rel.put("to_columns", tcols); + } + relationships.add(rel); + } + for (Map ds : datasets) { + List flds = new ArrayList<>(); + int nf = rnd.count(0, 3); + for (int j = 0; j < nf; j++) { + flds.add(ossieField(names.next("c"), rnd.colname())); + } + if (!flds.isEmpty()) { + ds.put("fields", flds); + } + } + List metrics = new ArrayList<>(); + int nm = rnd.count(0, 2); + for (int i = 0; i < nm; i++) { + metrics.add(ossieField(names.next("c"), rnd.pick(List.of(AGGS)) + "(" + rnd.colname() + ")")); + } + // The converted Metric View needs at least one dimension or measure, so a model whose datasets + // have no fields and which declares no metrics is outside the round-trippable subset. Give the + // first dataset a field when nothing else would produce a column. + boolean anyField = false; + for (Map ds : datasets) { + if (!asList(ds.get("fields")).isEmpty()) { + anyField = true; + break; + } + } + if (!anyField && metrics.isEmpty()) { + List flds = new ArrayList<>(); + flds.add(ossieField(names.next("c"), rnd.colname())); + datasets.get(0).put("fields", flds); + } + Map model = new LinkedHashMap<>(); + model.put("name", names.next("m")); + if (rnd.chance(0.4)) { + model.put("description", rnd.text()); + } + model.put("datasets", datasets); + if (!relationships.isEmpty()) { + model.put("relationships", relationships); + } + if (!metrics.isEmpty()) { + model.put("metrics", metrics); + } + Map out = new LinkedHashMap<>(); + out.put("version", OssieConverter.OSSIE_VERSION); + List models = new ArrayList<>(); + models.add(model); + out.put("semantic_model", models); + return out; + } + + // --- Round-trip assertions ----------------------------------------------- + + @SuppressWarnings("unchecked") + private static Map asMap(Object x) { + return x instanceof Map ? (Map) x : new LinkedHashMap<>(); + } + + @SuppressWarnings("unchecked") + private static List asList(Object x) { + return x instanceof List ? (List) x : new ArrayList<>(); + } + + private static String dumpYaml(Map obj) { + return OssieConverter.dumpYaml(obj); + } + + private static String condCanon(Map join) { + List using = asList(join.get("using")); + if (!using.isEmpty()) { + Set sorted = new TreeSet<>(); + for (Object u : using) { + sorted.add(u.toString()); + } + return "using:" + sorted; + } + Object on = join.get("on"); + if (on == null) { + return "none"; + } + Set pairs = new TreeSet<>(); + for (String clause : on.toString().split("(?i)\\s+AND\\s+")) { + String[] lr = clause.split("=", 2); + pairs.add(lr[0].trim() + "=" + lr[1].trim()); + } + return "on:" + pairs; + } + + private static void flattenJoins( + List joins, String parent, Map acc, Set edges) { + for (Object jObj : joins) { + Map j = asMap(jObj); + String name = (String) j.get("name"); + acc.put(name, j.get("source") + "|" + condCanon(j) + "|" + j.get("cardinality") + + "|" + j.get("rely")); + edges.add(parent + "->" + name); + flattenJoins(asList(j.get("joins")), name, acc, edges); + } + } + + private static List dims(Map mv) { + List d = asList(mv.get("dimensions")); + return !d.isEmpty() ? d : asList(mv.get("fields")); + } + + private static String dimNorm(Map d) { + return d.get("expr") + "|" + d.get("comment") + "|" + d.get("display_name") + + "|" + d.get("synonyms") + "|" + d.get("format"); + } + + private static String measNorm(Map m) { + return m.get("expr") + "|" + m.get("comment") + "|" + m.get("synonyms") + + "|" + m.get("format") + "|" + m.get("window"); + } + + private static Map byName(List items, boolean measure) { + Map out = new LinkedHashMap<>(); + for (Object o : items) { + Map m = asMap(o); + out.put((String) m.get("name"), measure ? measNorm(m) : dimNorm(m)); + } + return out; + } + + private void assertMvRoundTrip(Map mv, long seed) { + String ossieYaml = OssieConverter.convertMetricViewToOssie(dumpYaml(mv), null).yaml; + Map mv2 = + asMap(OssieConverter.parseYaml(OssieConverter.convertOssieToMetricView(ossieYaml, null).yaml)); + + String ctx = " (seed " + seed + ")"; + assertEquals(mv.get("source"), mv2.get("source"), "source" + ctx); + assertEquals(mv.get("comment"), mv2.get("comment"), "comment" + ctx); + assertEquals(mv.get("filter"), mv2.get("filter"), "filter" + ctx); + assertEquals(mv.get("materialization"), mv2.get("materialization"), "materialization" + ctx); + assertEquals(byName(dims(mv), false), byName(dims(mv2), false), "fields" + ctx); + assertEquals(byName(asList(mv.get("measures")), true), + byName(asList(mv2.get("measures")), true), "measures" + ctx); + + Map a1 = new LinkedHashMap<>(); + Set e1 = new LinkedHashSet<>(); + flattenJoins(asList(mv.get("joins")), "source", a1, e1); + Map a2 = new LinkedHashMap<>(); + Set e2 = new LinkedHashSet<>(); + flattenJoins(asList(mv2.get("joins")), "source", a2, e2); + assertEquals(a1, a2, "joins" + ctx); + assertEquals(e1, e2, "join nesting" + ctx); + } + + private static String exprOf(Map obj) { + for (Object dObj : asList(asMap(obj.get("expression")).get("dialects"))) { + Map d = asMap(dObj); + if ("DATABRICKS".equals(d.get("dialect"))) { + return (String) d.get("expression"); + } + } + return null; + } + + private static Map fieldsMap(Map ds) { + Map out = new LinkedHashMap<>(); + for (Object fObj : asList(ds.get("fields"))) { + Map f = asMap(fObj); + out.put((String) f.get("name"), exprOf(f)); + } + return out; + } + + private static Set relSet(Map model) { + Set out = new LinkedHashSet<>(); + for (Object rObj : asList(model.get("relationships"))) { + Map r = asMap(rObj); + out.add(r.get("from") + "->" + r.get("to") + "|" + asList(r.get("from_columns")) + + "|" + asList(r.get("to_columns"))); + } + return out; + } + + private void assertOssieRoundTrip(Map ossie, long seed) { + String mvYaml = OssieConverter.convertOssieToMetricView(dumpYaml(ossie), null).yaml; + Map ossie2 = + asMap(OssieConverter.parseYaml(OssieConverter.convertMetricViewToOssie(mvYaml, null).yaml)); + + String ctx = " (seed " + seed + ")"; + Map m1 = asMap(asList(ossie.get("semantic_model")).get(0)); + Map m2 = asMap(asList(ossie2.get("semantic_model")).get(0)); + + Map d1 = new LinkedHashMap<>(); + for (Object dsObj : asList(m1.get("datasets"))) { + Map ds = asMap(dsObj); + d1.put((String) ds.get("name"), ds.get("source") + "|" + fieldsMap(ds)); + } + Map d2 = new LinkedHashMap<>(); + for (Object dsObj : asList(m2.get("datasets"))) { + Map ds = asMap(dsObj); + d2.put((String) ds.get("name"), ds.get("source") + "|" + fieldsMap(ds)); + } + assertEquals(d1, d2, "datasets" + ctx); + assertEquals(relSet(m1), relSet(m2), "relationships" + ctx); + + Map met1 = new LinkedHashMap<>(); + for (Object x : asList(m1.get("metrics"))) { + met1.put((String) asMap(x).get("name"), exprOf(asMap(x))); + } + Map met2 = new LinkedHashMap<>(); + for (Object x : asList(m2.get("metrics"))) { + met2.put((String) asMap(x).get("name"), exprOf(asMap(x))); + } + assertEquals(met1, met2, "metrics" + ctx); + assertEquals(m1.get("description"), m2.get("description"), "description" + ctx); + } + + @Test + public void metricViewRoundTripAcrossSeeds() { + for (long seed = 0; seed < NUM_SEEDS; seed++) { + assertMvRoundTrip(buildMetricView(new Rnd(seed)), seed); + } + } + + @Test + public void ossieRoundTripAcrossSeeds() { + for (long seed = 0; seed < NUM_SEEDS; seed++) { + assertOssieRoundTrip(buildOssie(new Rnd(seed)), seed); + } + } +} diff --git a/converters/databricks/java/src/test/java/org/apache/ossie/converter/databricks/OssieConverterSuite.java b/converters/databricks/java/src/test/java/org/apache/ossie/converter/databricks/OssieConverterSuite.java new file mode 100644 index 00000000..349ce92d --- /dev/null +++ b/converters/databricks/java/src/test/java/org/apache/ossie/converter/databricks/OssieConverterSuite.java @@ -0,0 +1,774 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ossie.converter.databricks; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; + +import org.junit.jupiter.api.Test; + +/** + * Example-based tests for the Apache Ossie <-> Metric View converter, plus the fixture + * comparisons that pin the expected output of both directions. + */ +public class OssieConverterSuite { + + private static Object export(String osi, String source) { + return OssieConverter.parseYaml( + OssieConverter.convertOssieToMetricView(osi, source).yaml); + } + + @Test + public void fixtureAStarSchemaExportsToExpectedMetricView() { + String osi = + "version: \"0.2.0.dev0\"\n" + + "semantic_model:\n" + + " - name: sales\n" + + " description: Sales orders with customer attributes\n" + + " datasets:\n" + + " - name: orders\n" + + " source: samples.tpch.orders\n" + + " primary_key: [o_orderkey]\n" + + " description: One row per order\n" + + " fields:\n" + + " - name: o_orderkey\n" + + " expression:\n" + + " dialects:\n" + + " - dialect: DATABRICKS\n" + + " expression: o_orderkey\n" + + " description: Order identifier\n" + + " - name: o_orderdate\n" + + " expression:\n" + + " dialects:\n" + + " - dialect: DATABRICKS\n" + + " expression: o_orderdate\n" + + " label: Order Date\n" + + " ai_context:\n" + + " synonyms: [order date, date]\n" + + " - name: customer\n" + + " source: samples.tpch.customer\n" + + " primary_key: [c_custkey]\n" + + " fields:\n" + + " - name: c_name\n" + + " expression:\n" + + " dialects:\n" + + " - dialect: DATABRICKS\n" + + " expression: c_name\n" + + " description: Customer name\n" + + " relationships:\n" + + " - name: orders_to_customer\n" + + " from: orders\n" + + " to: customer\n" + + " from_columns: [o_custkey]\n" + + " to_columns: [c_custkey]\n" + + " metrics:\n" + + " - name: total_revenue\n" + + " expression:\n" + + " dialects:\n" + + " - dialect: DATABRICKS\n" + + " expression: SUM(o_totalprice)\n" + + " description: Total order revenue\n" + + " ai_context:\n" + + " synonyms: [revenue, total revenue, sales]\n" + + " - name: order_count\n" + + " expression:\n" + + " dialects:\n" + + " - dialect: DATABRICKS\n" + + " expression: COUNT(*)\n" + + " description: Number of orders\n"; + + String expected = + "version: '1.1'\n" + + "source: samples.tpch.orders\n" + + "comment: Sales orders with customer attributes\n" + + "joins:\n" + + "- name: customer\n" + + " source: samples.tpch.customer\n" + + " on: source.o_custkey = customer.c_custkey\n" + + " rely:\n" + + " at_most_one_match: true\n" + + "dimensions:\n" + + "- name: o_orderkey\n" + + " expr: o_orderkey\n" + + " comment: Order identifier\n" + + "- name: o_orderdate\n" + + " expr: o_orderdate\n" + + " display_name: Order Date\n" + + " synonyms:\n" + + " - order date\n" + + " - date\n" + + "- name: c_name\n" + + " expr: customer.c_name\n" + + " comment: Customer name\n" + + "measures:\n" + + "- name: total_revenue\n" + + " expr: SUM(o_totalprice)\n" + + " comment: Total order revenue\n" + + " synonyms:\n" + + " - revenue\n" + + " - total revenue\n" + + " - sales\n" + + "- name: order_count\n" + + " expr: COUNT(*)\n" + + " comment: Number of orders\n"; + + assertEquals(OssieConverter.parseYaml(expected), export(osi, null)); + } + + @Test + public void unsupportedVersionIsRejected() { + OssieConverter.ConversionException e = assertThrows(OssieConverter.ConversionException.class, + () -> OssieConverter.convertOssieToMetricView("version: '9.9'\nsemantic_model: []\n", null)); + assertTrue(e.getMessage().contains("Unsupported Apache Ossie version")); + } + + @Test + public void multipleCandidateFactsWithoutSourceIsRejected() { + String osi = + "version: 0.2.0.dev0\n" + + "semantic_model:\n" + + "- name: m\n" + + " datasets:\n" + + " - {name: orders, source: c.s.orders}\n" + + " - {name: returns, source: c.s.returns}\n" + + " - {name: customer, source: c.s.customer, primary_key: [c_custkey]}\n" + + " relationships:\n" + + " - {name: oc, from: orders, to: customer, from_columns: [o_custkey], to_columns: [c_custkey]}\n" + + " - {name: rc, from: returns, to: customer, from_columns: [re_custkey], to_columns: [c_custkey]}\n"; + OssieConverter.ConversionException e = assertThrows(OssieConverter.ConversionException.class, + () -> OssieConverter.convertOssieToMetricView(osi, null)); + assertTrue(e.getMessage().contains("multiple candidate fact datasets")); + } + + @Test + @SuppressWarnings("unchecked") + public void oneToManyEmitsCardinality() { + String osi = + "version: 0.2.0.dev0\n" + + "semantic_model:\n" + + "- name: m\n" + + " datasets:\n" + + " - name: orders\n" + + " source: c.s.orders\n" + + " primary_key: [o_orderkey]\n" + + " fields:\n" + + " - {name: o_orderstatus, expression: {dialects: [{dialect: DATABRICKS, expression: o_orderstatus}]}}\n" + + " - name: lineitem\n" + + " source: c.s.lineitem\n" + + " relationships:\n" + + " - {name: lio, from: lineitem, to: orders, from_columns: [l_orderkey], to_columns: [o_orderkey]}\n" + + " metrics:\n" + + " - {name: qty, expression: {dialects: [{dialect: DATABRICKS, expression: SUM(lineitem.l_quantity)}]}}\n"; + Map view = (Map) export(osi, "orders"); + List joins = (List) view.get("joins"); + Map join = (Map) joins.get(0); + assertEquals("one_to_many", join.get("cardinality")); + } + + @Test + public void oneToManyNestedUnderManyToOneIsRejected() { + // A Metric View requires one cardinality per top-level branch, so a one-to-many join nested + // under a many-to-one parent is rejected by Databricks just as the reverse nesting is. Fact + // `d`; `f -> d` points at d (many-to-one from d's perspective), and `g -> f` points away from + // f (one-to-many), which would nest one_to_many inside the many-to-one branch. + String osi = + "version: 0.2.0.dev0\n" + + "semantic_model:\n" + + "- name: m\n" + + " datasets:\n" + + " - name: d\n" + + " source: c.s.d\n" + + " fields:\n" + + " - {name: dcol, expression: {dialects: [{dialect: DATABRICKS, expression: dcol}]}}\n" + + " - name: f\n" + + " source: c.s.f\n" + + " primary_key: [fk]\n" + + " - name: g\n" + + " source: c.s.g\n" + + " relationships:\n" + + " - {name: df, from: d, to: f, from_columns: [fk], to_columns: [fk]}\n" + + " - {name: gf, from: g, to: f, from_columns: [fk], to_columns: [fk]}\n" + + " metrics:\n" + + " - {name: c, expression: {dialects: [{dialect: DATABRICKS, expression: COUNT(1)}]}}\n"; + OssieConverter.ConversionException e = assertThrows(OssieConverter.ConversionException.class, + () -> OssieConverter.convertOssieToMetricView(osi, "d")); + assertTrue(e.getMessage().contains("share the same cardinality"), + "expected a mixed-cardinality rejection, got: " + e.getMessage()); + } + + @Test + public void directedCycleWithNoEquidistantEdgeIsRejected() { + // a -> b -> c -> d -> e -> b with fact `a`: every edge spans adjacent BFS levels, so the + // equidistance heuristic sees nothing, and `a` has zero incoming edges so pickFact finds a + // root. Without a real acyclicity check this expanded into duplicate join paths (`d` under + // both `c` and `e`), fabricating a tree from a cyclic model. + String osi = + "version: 0.2.0.dev0\n" + + "semantic_model:\n" + + "- name: m\n" + + " datasets:\n" + + " - name: a\n" + + " source: c.s.a\n" + + " fields:\n" + + " - {name: acol, expression: {dialects: [{dialect: DATABRICKS, expression: acol}]}}\n" + + " - name: b\n" + + " source: c.s.b\n" + + " - name: c\n" + + " source: c.s.c\n" + + " - name: d\n" + + " source: c.s.d\n" + + " - name: e\n" + + " source: c.s.e\n" + + " relationships:\n" + + " - {name: ab, from: a, to: b, from_columns: [k], to_columns: [k]}\n" + + " - {name: bc, from: b, to: c, from_columns: [k], to_columns: [k]}\n" + + " - {name: cd, from: c, to: d, from_columns: [k], to_columns: [k]}\n" + + " - {name: de, from: d, to: e, from_columns: [k], to_columns: [k]}\n" + + " - {name: eb, from: e, to: b, from_columns: [k], to_columns: [k]}\n"; + OssieConverter.ConversionException ex = assertThrows(OssieConverter.ConversionException.class, + () -> OssieConverter.convertOssieToMetricView(osi, "a")); + assertTrue(ex.getMessage().contains("directed cycle"), + "expected a directed-cycle rejection, got: " + ex.getMessage()); + } + + @Test + @SuppressWarnings("unchecked") + public void diamondIsStillAcceptedByTheCycleCheck() { + // A diamond (`a -> b -> d` plus `a -> c -> d`) is an UNDIRECTED cycle but directed-acyclic, and + // is supported via the fan-out aliases. The cycle check must not reject it. + String osi = + "version: 0.2.0.dev0\n" + + "semantic_model:\n" + + "- name: m\n" + + " datasets:\n" + + " - name: a\n" + + " source: c.s.a\n" + + " - name: b\n" + + " source: c.s.b\n" + + " - name: c\n" + + " source: c.s.c\n" + + " - name: d\n" + + " source: c.s.d\n" + + " fields:\n" + + " - {name: dcol, expression: {dialects: [{dialect: DATABRICKS, expression: dcol}]}}\n" + + " relationships:\n" + + " - {name: ab, from: a, to: b, from_columns: [k], to_columns: [k]}\n" + + " - {name: ac, from: a, to: c, from_columns: [k], to_columns: [k]}\n" + + " - {name: bd, from: b, to: d, from_columns: [k], to_columns: [k]}\n" + + " - {name: cd, from: c, to: d, from_columns: [k], to_columns: [k]}\n"; + Map view = (Map) export(osi, "a"); + assertEquals("c.s.a", view.get("source")); + // `d` is reached by two paths, so its column is emitted once per fan-out alias. + List dims = (List) view.get("dimensions"); + assertEquals(2, dims.size(), "diamond should fan out to one dimension per path, got: " + dims); + } + + @Test + @SuppressWarnings("unchecked") + public void nestedJoinColumnInMeasureGetsFullAliasPath() { + // orders -> customer -> nation: `nation` nests under `customer`, so its columns are addressed + // as `customer.nation.col`. A bare `nation.` head would be read as struct access on a + // parameter, so the measure must be re-qualified -- and identically to the dimension path, + // which already emits `customer.nation.population` for the same column. + String osi = + "version: 0.2.0.dev0\n" + + "semantic_model:\n" + + "- name: m\n" + + " datasets:\n" + + " - name: orders\n" + + " source: c.s.orders\n" + + " - name: customer\n" + + " source: c.s.customer\n" + + " primary_key: [c_custkey]\n" + + " - name: nation\n" + + " source: c.s.nation\n" + + " primary_key: [n_nationkey]\n" + + " fields:\n" + + " - {name: population, expression: {dialects: [{dialect: DATABRICKS, expression: population}]}}\n" + + " relationships:\n" + + " - {name: oc, from: orders, to: customer, from_columns: [c_custkey], to_columns: [c_custkey]}\n" + + " - {name: cn, from: customer, to: nation, from_columns: [n_nationkey], to_columns: [n_nationkey]}\n" + + " metrics:\n" + + " - {name: pop, expression: {dialects: [{dialect: DATABRICKS, expression: SUM(nation.population)}]}}\n"; + Map view = (Map) export(osi, "orders"); + + List dims = (List) view.get("dimensions"); + Map dim = (Map) dims.get(0); + assertEquals("customer.nation.population", dim.get("expr"), + "dimension path should qualify with the full alias path"); + + List measures = (List) view.get("measures"); + Map measure = (Map) measures.get(0); + assertEquals("SUM(customer.nation.population)", measure.get("expr"), + "measure must use the same full alias path as the dimension, not a bare nested alias"); + } + + @Test + public void modelWithNoFieldsOrMetricsIsRejected() { + // A Metric View requires at least one dimension or measure, so emitting a version+source-only + // view would just fail at CREATE. Fail at conversion time instead. + String osi = + "version: 0.2.0.dev0\n" + + "semantic_model:\n" + + "- name: m\n" + + " datasets:\n" + + " - name: d\n" + + " source: c.s.d\n"; + OssieConverter.ConversionException e = assertThrows(OssieConverter.ConversionException.class, + () -> OssieConverter.convertOssieToMetricView(osi, null)); + assertTrue(e.getMessage().contains("no dimensions or measures"), + "expected an empty-view rejection, got: " + e.getMessage()); + } + + @Test + public void emptyAfterDropsNamesTheDroppedColumns() { + // Here the input is non-empty but everything drops: the only metric has no DATABRICKS/ANSI_SQL + // dialect. The error must name the dropped column so the cause is actionable, since after the + // cascade the emptiness is a consequence of the drop rather than an empty input. + String osi = + "version: 0.2.0.dev0\n" + + "semantic_model:\n" + + "- name: m\n" + + " datasets:\n" + + " - name: d\n" + + " source: c.s.d\n" + + " metrics:\n" + + " - {name: only_metric, expression: {dialects: [{dialect: SNOWFLAKE, expression: SUM(x)}]}}\n"; + OssieConverter.ConversionException e = assertThrows(OssieConverter.ConversionException.class, + () -> OssieConverter.convertOssieToMetricView(osi, null)); + assertTrue(e.getMessage().contains("no dimensions or measures"), + "expected an empty-view rejection, got: " + e.getMessage()); + assertTrue(e.getMessage().contains("only_metric"), + "the message must name the dropped column, got: " + e.getMessage()); + } + + @Test + public void duplicateDimensionNameIsRejected() { + String osi = + "version: 0.2.0.dev0\n" + + "semantic_model:\n" + + "- name: m\n" + + " datasets:\n" + + " - name: orders\n" + + " source: c.s.orders\n" + + " fields:\n" + + " - {name: id, expression: {dialects: [{dialect: DATABRICKS, expression: id}]}}\n" + + " - name: customer\n" + + " source: c.s.customer\n" + + " fields:\n" + + " - {name: id, expression: {dialects: [{dialect: DATABRICKS, expression: id}]}}\n" + + " relationships:\n" + + " - {name: r, from: orders, to: customer, from_columns: [cid], to_columns: [id]}\n"; + OssieConverter.ConversionException e = assertThrows(OssieConverter.ConversionException.class, + () -> OssieConverter.convertOssieToMetricView(osi, null)); + assertTrue(e.getMessage().contains("collides")); + } + + @Test + public void foreignVendorExtensionDroppedWithNotice() { + String osi = + "version: 0.2.0.dev0\n" + + "semantic_model:\n" + + "- name: m\n" + + " custom_extensions:\n" + + " - {vendor_name: SNOWFLAKE, data: '{}'}\n" + + " datasets:\n" + + " - name: orders\n" + + " source: c.s.orders\n" + + " fields:\n" + + " - {name: s, expression: {dialects: [{dialect: DATABRICKS, expression: s}]}}\n" + + " metrics:\n" + + " - {name: n, expression: {dialects: [{dialect: DATABRICKS, expression: COUNT(*)}]}}\n"; + OssieConverter.Result r = OssieConverter.convertOssieToMetricView(osi, null); + assertTrue(r.notices.stream().anyMatch(m -> m.contains("foreign-vendor custom_extensions dropped"))); + } + + // -- import direction (Metric View -> Apache Ossie) ----------------------- + + private static Object importMv(String mv) { + return OssieConverter.parseYaml( + OssieConverter.convertMetricViewToOssie(mv, null).yaml); + } + + @Test + @SuppressWarnings("unchecked") + public void importDecomposesJoinIntoRelationship() { + String mv = + "version: '1.1'\n" + + "source: c.s.orders\n" + + "joins:\n" + + "- name: customer\n" + + " source: c.s.customer\n" + + " on: source.o_custkey = customer.c_custkey\n" + + " rely: {at_most_one_match: true}\n" + + "dimensions:\n" + + "- {name: o_status, expr: o_orderstatus}\n" + + "- {name: c_name, expr: customer.c_name}\n" + + "measures:\n" + + "- {name: revenue, expr: SUM(o_totalprice)}\n"; + Map out = (Map) importMv(mv); + List models = (List) out.get("semantic_model"); + Map model = (Map) models.get(0); + List rels = (List) model.get("relationships"); + Map rel = (Map) rels.get(0); + assertEquals("orders", rel.get("from")); + assertEquals("customer", rel.get("to")); + assertEquals(List.of("o_custkey"), rel.get("from_columns")); + assertEquals(List.of("c_custkey"), rel.get("to_columns")); + } + + @Test + public void importRejectsNonEquiJoin() { + String mv = + "version: '1.1'\n" + + "source: c.s.orders\n" + + "joins:\n" + + "- name: customer\n" + + " source: c.s.customer\n" + + " on: source.o_custkey >= customer.c_custkey\n"; + OssieConverter.ConversionException e = assertThrows(OssieConverter.ConversionException.class, + () -> OssieConverter.convertMetricViewToOssie(mv, null)); + assertTrue(e.getMessage().contains("non-equi or unsupported")); + } + + @Test + public void importRejectsCrossJoin() { + String mv = + "version: '1.1'\n" + + "source: c.s.orders\n" + + "joins:\n" + + "- name: customer\n" + + " source: c.s.customer\n"; + OssieConverter.ConversionException e = assertThrows(OssieConverter.ConversionException.class, + () -> OssieConverter.convertMetricViewToOssie(mv, null)); + assertTrue(e.getMessage().contains("no join condition")); + } + + @Test + public void importRejectsUnsupportedVersion() { + OssieConverter.ConversionException e = assertThrows(OssieConverter.ConversionException.class, + () -> OssieConverter.convertMetricViewToOssie("version: '0.1'\nsource: c.s.t\n", null)); + assertTrue(e.getMessage().contains("Unsupported Metric View version")); + } + + @Test + public void mvToOssieToMvRoundTripsStash() { + // A view with MV-only features (filter/rely/format/window) must survive + // MV -> Ossie -> MV unchanged (the custom_extensions stash carries them). + String mv = + "version: '1.1'\n" + + "source: c.s.orders\n" + + "filter: o_orderstatus = 'F'\n" + + "joins:\n" + + "- name: customer\n" + + " source: c.s.customer\n" + + " on: source.o_custkey = customer.c_custkey\n" + + " rely:\n" + + " at_most_one_match: true\n" + + "dimensions:\n" + + "- name: net\n" + + " expr: o_totalprice\n" + + " format:\n" + + " type: currency\n" + + " currency_code: USD\n" + + "measures:\n" + + "- name: running\n" + + " expr: SUM(o_totalprice)\n" + + " window:\n" + + " - order: net\n" + + " semiadditive: last\n" + + " range: cumulative\n"; + String ossie = OssieConverter.convertMetricViewToOssie(mv, null).yaml; + String back = OssieConverter.convertOssieToMetricView(ossie, null).yaml; + assertEquals(OssieConverter.parseYaml(mv), OssieConverter.parseYaml(back)); + } + + // -- fixture comparisons --------------------------------------------------- + // The fixtures under src/test/resources/ossie_*.yaml pin the expected outputs + // checked into apache/ossie. Asserting the Java output parses equal to them (structure + // + stash blob STRINGS) is the real "one behavior, two implementations" guarantee -- + // it catches divergences like stash JSON spacing that a Java->Java round-trip misses. + + private static String loadFixture(String name) { + try (InputStream in = + OssieConverterSuite.class.getClassLoader().getResourceAsStream("ossie_" + name)) { + if (in == null) { + throw new IllegalStateException("fixture not found: ossie_" + name); + } + return new String(in.readAllBytes(), StandardCharsets.UTF_8); + } catch (java.io.IOException e) { + throw new RuntimeException(e); + } + } + + @Test + public void fixtureAExportMatchesFixture() { + String out = OssieConverter.convertOssieToMetricView(loadFixture("fixtureA_ossie.yaml"), null).yaml; + assertEquals(OssieConverter.parseYaml(loadFixture("fixtureA_metric_view.yaml")), + OssieConverter.parseYaml(out)); + } + + @Test + public void fixtureBImportMatchesFixture() { + // fixtureB exercises the custom_extensions stash (format/rely/filter) -- the parse + // includes the blob strings, so this is what pins the stash blob's exact spacing. + String out = OssieConverter.convertMetricViewToOssie(loadFixture("fixtureB_metric_view.yaml"), null).yaml; + assertEquals(OssieConverter.parseYaml(loadFixture("fixtureB_ossie.yaml")), + OssieConverter.parseYaml(out)); + } + + @Test + public void tpcdsExportMatchesFixture() { + String out = OssieConverter.convertOssieToMetricView(loadFixture("tpcds_ossie.yaml"), null).yaml; + assertEquals(OssieConverter.parseYaml(loadFixture("tpcds_metric_view.yaml")), + OssieConverter.parseYaml(out)); + } + + @Test + @SuppressWarnings("unchecked") + public void stashBlobUsesExpectedSpacing() { + // The strongest byte-level check: the emitted stash blob string (the `data` value of a + // custom_extensions entry) must use the stash format's separators + // (", " / ": "). Pull the blob out of the parsed model rather than substring-matching + // the outer YAML (where it appears escaped). + Object out = OssieConverter.parseYaml( + OssieConverter.convertMetricViewToOssie(loadFixture("fixtureB_metric_view.yaml"), null).yaml); + Map model = + (Map) ((List) ((Map) out).get("semantic_model")).get(0); + List exts = (List) model.get("custom_extensions"); + String blob = (String) ((Map) exts.get(0)).get("data"); + assertTrue(blob.startsWith("{\"_v\": 1, "), + "stash blob must use the spacing '{\"_v\": 1, ...}', got: " + blob); + } + + // -- parity edge cases found in review round 3 ---------------------------- + + @Test + public void pickExpressionFallsThroughEmptyDatabricksToAnsi() { + // An empty DATABRICKS dialect must fall through to ANSI_SQL, + // not be selected as the (empty) expression. + String osi = + "version: 0.2.0.dev0\n" + + "semantic_model:\n" + + "- name: m\n" + + " datasets:\n" + + " - name: f\n" + + " source: c.s.f\n" + + " fields:\n" + + " - name: d\n" + + " expression:\n" + + " dialects:\n" + + " - {dialect: DATABRICKS, expression: ''}\n" + + " - {dialect: ANSI_SQL, expression: ansi_col}\n" + + " metrics:\n" + + " - {name: n, expression: {dialects: [{dialect: DATABRICKS, expression: COUNT(*)}]}}\n"; + Object out = export(osi, null); + @SuppressWarnings("unchecked") + List dims = (List) ((Map) out).get("dimensions"); + @SuppressWarnings("unchecked") + Map dim = (Map) dims.get(0); + assertEquals("ansi_col", dim.get("expr")); + } + + @Test + public void pickExpressionRejectsNonStringExpression() { + // A non-string dialect expression (e.g. a YAML number) must raise, not be coerced. + String osi = + "version: 0.2.0.dev0\n" + + "semantic_model:\n" + + "- name: m\n" + + " datasets:\n" + + " - name: f\n" + + " source: c.s.f\n" + + " fields:\n" + + " - name: d\n" + + " expression:\n" + + " dialects:\n" + + " - {dialect: DATABRICKS, expression: 123}\n" + + " metrics:\n" + + " - {name: n, expression: {dialects: [{dialect: DATABRICKS, expression: COUNT(*)}]}}\n"; + OssieConverter.ConversionException e = assertThrows(OssieConverter.ConversionException.class, + () -> OssieConverter.convertOssieToMetricView(osi, null)); + assertTrue(e.getMessage().contains("expression must be a string")); + } + + @Test + public void bareOnOffValuesStayStringsNotBooleans() { + // YAML 1.1 would read a bare `on`/`off`/`yes`/`no` value as a boolean, silently losing + // a join condition or turning a synonym into `true`. Confirm the converter's parser + // keeps them as strings (the reader uses YAML 1.2 boolean semantics). + Object parsed = OssieConverter.parseYaml("a: on\nb: off\nc: yes\nd: no\n"); + @SuppressWarnings("unchecked") + Map m = (Map) parsed; + assertEquals("on", m.get("a")); + assertEquals("off", m.get("b")); + assertEquals("yes", m.get("c")); + assertEquals("no", m.get("d")); + } + + @Test + @SuppressWarnings("unchecked") + public void importDropsEmptyOptionalFields() { + // Optional fields are mapped only when non-empty, so an empty + // comment / empty synonyms list are omitted, not emitted as `description: ""` or an + // empty ai_context. The Java port must match (empty string / empty list are falsy). + String mv = + "version: '1.1'\n" + + "source: c.s.orders\n" + + "comment: ''\n" + + "dimensions:\n" + + "- {name: o_status, expr: o_orderstatus, comment: '', display_name: '', synonyms: []}\n" + + "measures:\n" + + "- {name: revenue, expr: SUM(o_totalprice), comment: '', synonyms: []}\n"; + Map out = (Map) importMv(mv); + List models = (List) out.get("semantic_model"); + Map model = (Map) models.get(0); + assertFalse(model.containsKey("description"), "empty comment must not become a description"); + Map ds = (Map) ((List) model.get("datasets")).get(0); + Map field = (Map) ((List) ds.get("fields")).get(0); + assertFalse(field.containsKey("description"), "empty comment must not map to description"); + assertFalse(field.containsKey("label"), "empty display_name must not map to label"); + assertFalse(field.containsKey("ai_context"), "empty synonyms must not map to ai_context"); + Map metric = (Map) ((List) model.get("metrics")).get(0); + assertFalse(metric.containsKey("description"), "empty comment must not map to description"); + assertFalse(metric.containsKey("ai_context"), "empty synonyms must not map to ai_context"); + } + + @Test + public void stashEscapesNonAsciiAsLowercaseHex() { + // The stash blob is pure ASCII: non-ASCII is escaped to \\uXXXX. The + // stash blob must be byte-identical, so a non-ASCII stashed value (here a `filter` + // literal) escapes rather than emitting raw UTF-8. + String mv = + "version: '1.1'\n" + + "source: c.s.orders\n" + + "filter: \"region = 'café'\"\n" + + "dimensions:\n" + + "- {name: o_status, expr: o_orderstatus}\n"; + String ossieYaml = OssieConverter.convertMetricViewToOssie(mv, null).yaml; + // Extract the stash blob and assert the exact expected bytes: + // the non-ASCII char is escaped as lowercase \\u00e9 (a single backslash + 5 chars), + // not emitted raw. (Comparing the parsed `data` string sidesteps YAML's own quoting.) + @SuppressWarnings("unchecked") + Map out = (Map) OssieConverter.parseYaml(ossieYaml); + @SuppressWarnings("unchecked") + List models = (List) out.get("semantic_model"); + Map model = (Map) models.get(0); + @SuppressWarnings("unchecked") + List exts = (List) model.get("custom_extensions"); + @SuppressWarnings("unchecked") + String blob = (String) ((Map) exts.get(0)).get("data"); + assertEquals("{\"_v\": 1, \"filter\": \"region = 'caf\\u00e9'\"}", blob, + "stash blob must use a lowercase \\u escape"); + // And it still round-trips back to the original view. + String back = OssieConverter.convertOssieToMetricView(ossieYaml, null).yaml; + assertEquals(OssieConverter.parseYaml(mv), OssieConverter.parseYaml(back)); + } + + @Test + public void stashPreservesAValueContainingALiteralUnicodeEscape() { + // A stashed value may itself contain the text of a unicode escape. Serialized, that is a + // DOUBLED backslash, so the hex-lowercasing pass must not treat it as a real escape -- + // doing so silently lowercases the value's own characters. + // Single-quoted YAML: a backslash is an ordinary character there, so `filter` really holds + // the six characters \ u A B C D rather than the character U+ABCD. + String mv = + "version: '1.1'\n" + + "source: c.s.orders\n" + + "filter: 'tag = \\uABCD'\n" + + "dimensions:\n" + + "- {name: o_status, expr: o_orderstatus}\n"; + @SuppressWarnings("unchecked") + Map parsedIn = (Map) OssieConverter.parseYaml(mv); + String original = (String) parsedIn.get("filter"); + // Guard the fixture itself: the value must contain a real backslash for this to be a test. + assertTrue(original.indexOf('\\') >= 0, + "test setup: filter must hold a literal backslash, got: " + original); + + String ossieYaml = OssieConverter.convertMetricViewToOssie(mv, null).yaml; + String back = OssieConverter.convertOssieToMetricView(ossieYaml, null).yaml; + @SuppressWarnings("unchecked") + Map restored = (Map) OssieConverter.parseYaml(back); + assertEquals(original, restored.get("filter"), + "a literal unicode-escape sequence in a stashed value must survive unchanged"); + } + + @Test + public void joinOnTakesPrecedenceOverUsing() { + // Metric View validation requires only that one of `on`/`using` is present, so both may be + // set. Databricks resolves the criteria from `on` when it is present, so the converter must + // decompose `on` and ignore `using` -- otherwise the relationship joins on other columns. + String mv = + "version: '1.1'\n" + + "source: c.s.orders\n" + + "joins:\n" + + "- name: cust\n" + + " source: c.s.customer\n" + + " on: source.o_custkey = cust.c_custkey\n" + + " using: [nation_key]\n" + + "dimensions:\n" + + "- {name: c_name, expr: cust.c_name}\n" + + "measures:\n" + + "- {name: cnt, expr: COUNT(1)}\n"; + @SuppressWarnings("unchecked") + Map out = (Map) OssieConverter.parseYaml( + OssieConverter.convertMetricViewToOssie(mv, null).yaml); + @SuppressWarnings("unchecked") + List models = (List) out.get("semantic_model"); + @SuppressWarnings("unchecked") + Map model = (Map) models.get(0); + @SuppressWarnings("unchecked") + List rels = (List) model.get("relationships"); + @SuppressWarnings("unchecked") + Map rel = (Map) rels.get(0); + assertEquals(List.of("o_custkey"), rel.get("from_columns"), + "`on` must win over `using`: expected the o_custkey/c_custkey pair"); + assertEquals(List.of("c_custkey"), rel.get("to_columns"), + "`on` must win over `using`: expected the o_custkey/c_custkey pair"); + } + + @Test + public void measureRewriteLeavesStringLiteralsAlone() { + // The fact qualifier is added/stripped by rewriting the measure expression. That rewrite must + // skip string literals: rewriting inside one changes the predicate and therefore the value. + String mv = + "version: '1.1'\n" + + "source: c.s.orders\n" + + "dimensions:\n" + + "- {name: o_status, expr: o_orderstatus}\n" + + "measures:\n" + + "- name: tagged\n" + + " expr: \"SUM(IF(source.region = 'source.us', 1, 0))\"\n"; + String ossieYaml = OssieConverter.convertMetricViewToOssie(mv, null).yaml; + assertTrue(ossieYaml.contains("'source.us'"), + "a literal mentioning the qualifier must not be rewritten, got:\n" + ossieYaml); + // The literal also survives the trip back. Note the *code* qualifier is normalized on the + // way through (`source.region` -> bare `region`, the Metric View idiom for fact columns); + // only the literal is required to come back byte-identical. + String back = OssieConverter.convertOssieToMetricView(ossieYaml, null).yaml; + assertTrue(back.contains("'source.us'"), + "the literal must survive the round trip, got:\n" + back); + } +} diff --git a/converters/databricks/tests/fixtures/fixtureA_metric_view.yaml b/converters/databricks/java/src/test/resources/ossie_fixtureA_metric_view.yaml similarity index 100% rename from converters/databricks/tests/fixtures/fixtureA_metric_view.yaml rename to converters/databricks/java/src/test/resources/ossie_fixtureA_metric_view.yaml diff --git a/converters/databricks/tests/fixtures/fixtureA_ossie.yaml b/converters/databricks/java/src/test/resources/ossie_fixtureA_ossie.yaml similarity index 100% rename from converters/databricks/tests/fixtures/fixtureA_ossie.yaml rename to converters/databricks/java/src/test/resources/ossie_fixtureA_ossie.yaml diff --git a/converters/databricks/java/src/test/resources/ossie_fixtureB_metric_view.yaml b/converters/databricks/java/src/test/resources/ossie_fixtureB_metric_view.yaml new file mode 100644 index 00000000..723bcec8 --- /dev/null +++ b/converters/databricks/java/src/test/resources/ossie_fixtureB_metric_view.yaml @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Fixture B -- stash round-trip (MV -> Apache Ossie -> MV, lossless). +# Exercises the stash at every placement level: model (filter), relationship +# (rely), dimension (format), measure (format, partition). Must parse under v1.1. +# +# `partition` has no Apache Ossie representation, so it round-trips through the +# DATABRICKS stash like `format`/`window`. It is attached to a measure with no +# window block, since a partition combined with a window is rejected by v1.1 +# validation, and `partition.include` names an existing dimension as required. + +version: '1.1' +source: samples.tpch.lineitem +filter: l_returnflag = 'N' +comment: Line item shipping metrics +joins: +- name: orders + source: samples.tpch.orders + on: source.l_orderkey = orders.o_orderkey + rely: + at_most_one_match: true +dimensions: +- name: line_number + expr: l_linenumber + format: + type: number + decimal_places: + type: exact + places: 0 +measures: +- name: revenue + expr: SUM(l_extendedprice * (1 - l_discount)) + comment: Net revenue + format: + type: currency + currency_code: USD + decimal_places: + type: exact + places: 2 +- name: order_count + expr: COUNT(DISTINCT l_orderkey) + partition: + include: + - line_number + outer_aggregate: SUM diff --git a/converters/databricks/java/src/test/resources/ossie_fixtureB_ossie.yaml b/converters/databricks/java/src/test/resources/ossie_fixtureB_ossie.yaml new file mode 100644 index 00000000..65abec77 --- /dev/null +++ b/converters/databricks/java/src/test/resources/ossie_fixtureB_ossie.yaml @@ -0,0 +1,75 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yaml-language-server: $schema=../../../../core-spec/osi-schema.json +# +# Fixture B -- expected Apache Ossie produced from fixtureB_metric_view.yaml. MV-only +# features are stashed in custom_extensions[DATABRICKS], keyed by their exact v1.1 +# field name. Exporting this back must reproduce fixtureB_metric_view.yaml. +# Model name is derived from the fact table (`lineitem`). + +version: "0.2.0.dev0" + +semantic_model: + - name: lineitem + description: Line item shipping metrics + datasets: + - name: lineitem + source: samples.tpch.lineitem + fields: + - name: line_number + expression: + dialects: + - dialect: DATABRICKS + expression: l_linenumber + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "format": {"type": "number", "decimal_places": {"type": "exact", "places": 0}}}' + - name: orders + source: samples.tpch.orders + unique_keys: + - [o_orderkey] + relationships: + - name: lineitem_to_orders + from: lineitem + to: orders + from_columns: [l_orderkey] + to_columns: [o_orderkey] + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "rely": {"at_most_one_match": true}}' + metrics: + - name: revenue + expression: + dialects: + - dialect: DATABRICKS + expression: SUM(l_extendedprice * (1 - l_discount)) + description: Net revenue + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "format": {"type": "currency", "currency_code": "USD", "decimal_places": {"type": "exact", "places": 2}}}' + - name: order_count + expression: + dialects: + - dialect: DATABRICKS + expression: COUNT(DISTINCT l_orderkey) + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "partition": {"include": ["line_number"], "outer_aggregate": "SUM"}}' + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "filter": "l_returnflag = ''N''"}' diff --git a/converters/databricks/tests/fixtures/tpcds_metric_view.yaml b/converters/databricks/java/src/test/resources/ossie_tpcds_metric_view.yaml similarity index 100% rename from converters/databricks/tests/fixtures/tpcds_metric_view.yaml rename to converters/databricks/java/src/test/resources/ossie_tpcds_metric_view.yaml diff --git a/converters/databricks/tests/fixtures/tpcds_ossie.yaml b/converters/databricks/java/src/test/resources/ossie_tpcds_ossie.yaml similarity index 100% rename from converters/databricks/tests/fixtures/tpcds_ossie.yaml rename to converters/databricks/java/src/test/resources/ossie_tpcds_ossie.yaml diff --git a/converters/databricks/python/README.md b/converters/databricks/python/README.md new file mode 100644 index 00000000..e2fe6ccb --- /dev/null +++ b/converters/databricks/python/README.md @@ -0,0 +1,124 @@ + + +# Apache Ossie Databricks Converter + +Bidirectional, offline conversion between an [Apache Ossie](https://github.com/apache/ossie) +semantic model and a Databricks +[Unity Catalog Metric View](https://docs.databricks.com/aws/en/metric-views/) (YAML +`1.1`). No Databricks connection required. + +- **Export** (`ossie-databricks export`): Apache Ossie -> Metric View (one fact + `source` with a nested `joins` tree and a flat `dimensions` list). +- **Import** (`ossie-databricks import`): Metric View -> Apache Ossie. Metric View features Apache Ossie has + no native field for are preserved in `custom_extensions[DATABRICKS]`, so + `MV -> Apache Ossie -> MV` is lossless. + +On **export** (Apache Ossie -> Metric View), Apache Ossie features with no Metric View slot -- relationship +`ai_context`, `dimension.is_time`, non-`DATABRICKS`/`ANSI_SQL` dialects, foreign-vendor +`custom_extensions` -- are **dropped with a warning**. On **import** (Metric View -> Apache Ossie), +Metric View only features (filter, window, format, rely, ...) are instead **preserved** in +`custom_extensions[DATABRICKS]`, so `MV -> Apache Ossie -> MV` is lossless. Any input that breaks a +[requirement](#requirements) **raises a `ConversionError`** -- the converter never +silently drops a field or produces an invalid result. + +## Installation + +```bash +pip install apache-ossie-databricks # once published to PyPI +# or, from a checkout of this directory: +pip install -e . +``` + +The only runtime dependency is `PyYAML`. Python 3.11+. + +## Usage + +### Command line + +```bash +ossie-databricks export -i model.yaml -o view.yaml [--source orders] # Apache Ossie -> Metric View +ossie-databricks import -i view.yaml -o model.yaml [--name my_model] # Metric View -> Apache Ossie +``` + +With no `-o`, output goes to stdout. `--source` (export) picks the fact/grain (default: +the FK-sink dataset; naming a coarser-grain dataset produces `one_to_many` joins); +`--name` (import) sets the Apache Ossie model name (default: the source's last identifier). + +### Python API + +```python +from ossie_databricks import convert_ossie_to_metric_view, convert_metric_view_to_ossie + +metric_view_yaml = convert_ossie_to_metric_view(ossie_yaml_str) # optionally choose the fact/grain, e.g. (ossie_yaml_str, source="orders") +ossie_yaml = convert_metric_view_to_ossie(metric_view_yaml_str, model_name="sales") +``` + +## Mapping + +Each row maps in both directions; the **Notes** flag where a behavior is specific to +**export** (Apache Ossie -> Metric View) or **import** (Metric View -> Apache Ossie). + +| Apache Ossie | Metric View (v1.1) | Notes | +|---|---|---| +| `semantic_model.description` | `comment` | Model-level description only. | +| root dataset | `source` | The fact/grain. | +| other `datasets` | nested `joins[]` | Export: the relationship graph is reassembled into the join tree; a dataset reached by two paths (a diamond) fans out into one aliased join per path. | +| `relationship` `from_columns`/`to_columns` | join `on` (differing names) / `using` (shared names) | Decomposed into columns on import; rebuilt into `on`/`using` on export. | +| `relationship.from`/`to` direction | join `cardinality` | Export: source on the many (`from`) side -> `many_to_one`; on the one (`to`) side -> `one_to_many`. | +| `dataset.primary_key` / `unique_keys` | join `rely.at_most_one_match` | Both directions: export sets `at_most_one_match` when a key covers the join columns; import recovers a `unique_keys` from it. | +| `dataset.fields[]` | `dimensions[]` | Export: fields flatten into one list and a joined column is qualified by its full join path (`customer.c_name`; `customer.region.r_name` when nested). | +| `field.expression.dialects[]` | `expr` | Export: prefer the `DATABRICKS` dialect, else `ANSI_SQL`. | +| `metrics[]` | `measures[]` | Export: fact columns are referenced bare (`SUM(amount)`). | +| `field.label` | `display_name` | | +| `field` / `metric` `description` | `comment` | | +| `ai_context.synonyms` | `synonyms` | | +| `custom_extensions[DATABRICKS]` | `filter`, `window`, `format`, `rely`, `materialization` | Import stashes Metric View only features here; export restores them -- keeping `MV -> Apache Ossie -> MV` lossless. | + +## Requirements + +Conversion raises a `ConversionError` (rather than guessing or emitting something +invalid) when an input breaks one of these: + +- the Metric View `version` is not `1.1`; +- a `source` is not a 3-part `catalog.schema.table` name or a `SELECT`/`WITH` subquery; +- the relationship graph is not acyclic and resolvable to a single fact -- a cycle, or + multiple candidate facts without `--source`, is rejected (a diamond is allowed and + fanned out); +- a join has no condition (a cross join has no Apache Ossie relationship form); +- a join condition is non-equi or otherwise can't be decomposed into equi-join columns + (Apache Ossie relationships are equi-joins, so the join has no Apache Ossie representation); +- the input YAML is malformed. + +## Development + +```bash +pip install -e ".[dev]" +python3 -m pytest tests/ +``` + +Example-based unit tests plus Hypothesis property-based round-trip tests +(`test_roundtrip_properties.py`, which skip if `hypothesis` is not installed). + +## Future effort + +Both the Apache Ossie specification and the Databricks Unity Catalog Metric View YAML are still +evolving. As either side adds or changes fields, this converter will be updated to track +them -- extending the mapping and coverage in both directions to keep the conversion +current and to support as much as each format allows over time. diff --git a/converters/databricks/pyproject.toml b/converters/databricks/python/pyproject.toml similarity index 77% rename from converters/databricks/pyproject.toml rename to converters/databricks/python/pyproject.toml index d4ed3511..769c68b1 100644 --- a/converters/databricks/pyproject.toml +++ b/converters/databricks/python/pyproject.toml @@ -19,34 +19,44 @@ requires = ["hatchling"] build-backend = "hatchling.build" +[dependency-groups] +dev = [ + "pytest>=8.0", + "hypothesis>=6.0", +] + [project] name = "apache-ossie-databricks" version = "0.2.0.dev0" description = "Databricks Unity Catalog Metric View <> Apache Ossie converter" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] requires-python = ">=3.11" -classifiers = [ - "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 3", +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "Databricks" ] dependencies = [ "PyYAML>=6.0", ] -[project.license] -text = "Apache-2.0" - -[project.optional-dependencies] -dev = [ - "pytest>=8.0", - "hypothesis>=6.0", -] - [project.scripts] ossie-databricks = "ossie_databricks.cli:main" +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" + [tool.hatch.build.targets.wheel] packages = ["src/ossie_databricks"] [tool.pytest.ini_options] testpaths = ["tests"] -pythonpath = ["src"] + +[tool.uv] +required-version = ">=0.9.0" +default-groups = [ + "dev" +] \ No newline at end of file diff --git a/converters/databricks/src/ossie_databricks/__init__.py b/converters/databricks/python/src/ossie_databricks/__init__.py similarity index 100% rename from converters/databricks/src/ossie_databricks/__init__.py rename to converters/databricks/python/src/ossie_databricks/__init__.py diff --git a/converters/databricks/src/ossie_databricks/_common.py b/converters/databricks/python/src/ossie_databricks/_common.py similarity index 100% rename from converters/databricks/src/ossie_databricks/_common.py rename to converters/databricks/python/src/ossie_databricks/_common.py diff --git a/converters/databricks/src/ossie_databricks/cli.py b/converters/databricks/python/src/ossie_databricks/cli.py similarity index 100% rename from converters/databricks/src/ossie_databricks/cli.py rename to converters/databricks/python/src/ossie_databricks/cli.py diff --git a/converters/databricks/src/ossie_databricks/metric_view_to_ossie.py b/converters/databricks/python/src/ossie_databricks/metric_view_to_ossie.py similarity index 100% rename from converters/databricks/src/ossie_databricks/metric_view_to_ossie.py rename to converters/databricks/python/src/ossie_databricks/metric_view_to_ossie.py diff --git a/converters/databricks/src/ossie_databricks/ossie_to_metric_view.py b/converters/databricks/python/src/ossie_databricks/ossie_to_metric_view.py similarity index 100% rename from converters/databricks/src/ossie_databricks/ossie_to_metric_view.py rename to converters/databricks/python/src/ossie_databricks/ossie_to_metric_view.py diff --git a/converters/databricks/tests/_roundtrip_helpers.py b/converters/databricks/python/tests/_roundtrip_helpers.py similarity index 100% rename from converters/databricks/tests/_roundtrip_helpers.py rename to converters/databricks/python/tests/_roundtrip_helpers.py diff --git a/converters/databricks/tests/_util.py b/converters/databricks/python/tests/_util.py similarity index 100% rename from converters/databricks/tests/_util.py rename to converters/databricks/python/tests/_util.py diff --git a/converters/databricks/tests/conftest.py b/converters/databricks/python/tests/conftest.py similarity index 100% rename from converters/databricks/tests/conftest.py rename to converters/databricks/python/tests/conftest.py diff --git a/converters/databricks/python/tests/fixtures/fixtureA_metric_view.yaml b/converters/databricks/python/tests/fixtures/fixtureA_metric_view.yaml new file mode 100644 index 00000000..4ffe6819 --- /dev/null +++ b/converters/databricks/python/tests/fixtures/fixtureA_metric_view.yaml @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Fixture A -- expected UC Metric View (v1.1, single-source) produced from +# fixtureA_ossie.yaml. Must parse under the v1.1 strict schema. + +version: '1.1' +source: samples.tpch.orders +comment: Sales orders with customer attributes +joins: +- name: customer + source: samples.tpch.customer + on: source.o_custkey = customer.c_custkey + rely: + at_most_one_match: true +dimensions: +- name: o_orderkey + expr: o_orderkey + comment: Order identifier +- name: o_orderdate + expr: o_orderdate + display_name: Order Date + synonyms: + - order date + - date +- name: c_name + expr: customer.c_name + comment: Customer name +measures: +- name: total_revenue + expr: SUM(o_totalprice) + comment: Total order revenue + synonyms: + - revenue + - total revenue + - sales +- name: order_count + expr: COUNT(*) + comment: Number of orders diff --git a/converters/databricks/python/tests/fixtures/fixtureA_ossie.yaml b/converters/databricks/python/tests/fixtures/fixtureA_ossie.yaml new file mode 100644 index 00000000..a53942f3 --- /dev/null +++ b/converters/databricks/python/tests/fixtures/fixtureA_ossie.yaml @@ -0,0 +1,79 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# yaml-language-server: $schema=../../../../core-spec/osi-schema.json +# +# Fixture A -- all-native round-trip (Apache Ossie -> MV -> Apache Ossie). +# Star schema; every field maps to a native MV field. Documented losses on the +# Apache Ossie -> MV -> Apache Ossie trip: the model name (MV carries none) and primary_key. + +version: "0.2.0.dev0" + +semantic_model: + - name: sales + description: Sales orders with customer attributes + datasets: + - name: orders # fact: no incoming relationship -> becomes `source` + source: samples.tpch.orders + primary_key: [o_orderkey] # dropped on export (Apache Ossie-only) + description: One row per order + fields: + - name: o_orderkey + expression: + dialects: + - dialect: DATABRICKS + expression: o_orderkey + description: Order identifier + - name: o_orderdate + expression: + dialects: + - dialect: DATABRICKS + expression: o_orderdate + label: Order Date + ai_context: + synonyms: [order date, date] + - name: customer + source: samples.tpch.customer + primary_key: [c_custkey] + fields: + - name: c_name + expression: + dialects: + - dialect: DATABRICKS + expression: c_name + description: Customer name + relationships: + - name: orders_to_customer + from: orders + to: customer + from_columns: [o_custkey] + to_columns: [c_custkey] + metrics: + - name: total_revenue + expression: + dialects: + - dialect: DATABRICKS + expression: SUM(o_totalprice) # fact columns are bare in measures + description: Total order revenue + ai_context: + synonyms: [revenue, total revenue, sales] + - name: order_count + expression: + dialects: + - dialect: DATABRICKS + expression: COUNT(*) + description: Number of orders diff --git a/converters/databricks/tests/fixtures/fixtureB_metric_view.yaml b/converters/databricks/python/tests/fixtures/fixtureB_metric_view.yaml similarity index 100% rename from converters/databricks/tests/fixtures/fixtureB_metric_view.yaml rename to converters/databricks/python/tests/fixtures/fixtureB_metric_view.yaml diff --git a/converters/databricks/tests/fixtures/fixtureB_ossie.yaml b/converters/databricks/python/tests/fixtures/fixtureB_ossie.yaml similarity index 100% rename from converters/databricks/tests/fixtures/fixtureB_ossie.yaml rename to converters/databricks/python/tests/fixtures/fixtureB_ossie.yaml diff --git a/converters/databricks/python/tests/fixtures/tpcds_metric_view.yaml b/converters/databricks/python/tests/fixtures/tpcds_metric_view.yaml new file mode 100644 index 00000000..b22fc698 --- /dev/null +++ b/converters/databricks/python/tests/fixtures/tpcds_metric_view.yaml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: '1.1' +source: tpcds.public.store_sales +comment: Store sales enriched with date, item, and customer dimensions +filter: ss_net_profit > 0 +joins: +- name: date_dim + source: tpcds.public.date_dim + on: source.ss_sold_date_sk = date_dim.d_date_sk + rely: + at_most_one_match: true +- name: item + source: tpcds.public.item + on: source.ss_item_sk = item.i_item_sk + rely: + at_most_one_match: true +- name: customer + source: tpcds.public.customer + on: source.ss_customer_sk = customer.c_customer_sk + rely: + at_most_one_match: true +dimensions: +- name: ticket_number + expr: ss_ticket_number +- name: sold_year + expr: date_dim.d_year + display_name: Year + synonyms: + - year + - yr +- name: sold_date + expr: date_dim.d_date +- name: item_category + expr: item.i_category + synonyms: + - category + - product type +- name: item_brand + expr: item.i_brand +- name: birth_country + expr: customer.c_birth_country +measures: +- name: total_sales + expr: SUM(ss_ext_sales_price) + comment: Total sales revenue + format: + type: currency + currency_code: USD +- name: total_quantity + expr: SUM(ss_quantity) + comment: Total units sold diff --git a/converters/databricks/python/tests/fixtures/tpcds_ossie.yaml b/converters/databricks/python/tests/fixtures/tpcds_ossie.yaml new file mode 100644 index 00000000..e055eab1 --- /dev/null +++ b/converters/databricks/python/tests/fixtures/tpcds_ossie.yaml @@ -0,0 +1,89 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: "0.2.0.dev0" +semantic_model: + - name: tpcds_store_sales + description: Store sales enriched with date, item, and customer dimensions + datasets: + - name: store_sales + source: tpcds.public.store_sales + fields: + - name: ticket_number + expression: + dialects: [{dialect: DATABRICKS, expression: ss_ticket_number}] + - name: date_dim + source: tpcds.public.date_dim + primary_key: [d_date_sk] + fields: + - name: sold_year + expression: + dialects: [{dialect: DATABRICKS, expression: d_year}] + label: Year + ai_context: {synonyms: [year, yr]} + - name: sold_date + expression: + dialects: [{dialect: DATABRICKS, expression: d_date}] + - name: item + source: tpcds.public.item + primary_key: [i_item_sk] + fields: + - name: item_category + expression: + dialects: [{dialect: DATABRICKS, expression: i_category}] + ai_context: {synonyms: [category, product type]} + - name: item_brand + expression: + dialects: [{dialect: DATABRICKS, expression: i_brand}] + - name: customer + source: tpcds.public.customer + primary_key: [c_customer_sk] + fields: + - name: birth_country + expression: + dialects: [{dialect: DATABRICKS, expression: c_birth_country}] + relationships: + - name: store_sales_to_date_dim + from: store_sales + to: date_dim + from_columns: [ss_sold_date_sk] + to_columns: [d_date_sk] + - name: store_sales_to_item + from: store_sales + to: item + from_columns: [ss_item_sk] + to_columns: [i_item_sk] + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: [ss_customer_sk] + to_columns: [c_customer_sk] + metrics: + - name: total_sales + expression: + dialects: [{dialect: DATABRICKS, expression: SUM(ss_ext_sales_price)}] + description: Total sales revenue + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "format": {"type": "currency", "currency_code": "USD"}}' + - name: total_quantity + expression: + dialects: [{dialect: DATABRICKS, expression: SUM(ss_quantity)}] + description: Total units sold + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "filter": "ss_net_profit > 0"}' diff --git a/converters/databricks/tests/test_metric_view_to_ossie.py b/converters/databricks/python/tests/test_metric_view_to_ossie.py similarity index 100% rename from converters/databricks/tests/test_metric_view_to_ossie.py rename to converters/databricks/python/tests/test_metric_view_to_ossie.py diff --git a/converters/databricks/tests/test_ossie_to_metric_view.py b/converters/databricks/python/tests/test_ossie_to_metric_view.py similarity index 100% rename from converters/databricks/tests/test_ossie_to_metric_view.py rename to converters/databricks/python/tests/test_ossie_to_metric_view.py diff --git a/converters/databricks/tests/test_roundtrip.py b/converters/databricks/python/tests/test_roundtrip.py similarity index 100% rename from converters/databricks/tests/test_roundtrip.py rename to converters/databricks/python/tests/test_roundtrip.py diff --git a/converters/databricks/tests/test_roundtrip_properties.py b/converters/databricks/python/tests/test_roundtrip_properties.py similarity index 100% rename from converters/databricks/tests/test_roundtrip_properties.py rename to converters/databricks/python/tests/test_roundtrip_properties.py diff --git a/converters/databricks/python/uv.lock b/converters/databricks/python/uv.lock new file mode 100644 index 00000000..9a4c8c11 --- /dev/null +++ b/converters/databricks/python/uv.lock @@ -0,0 +1,214 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "apache-ossie-databricks" +version = "0.2.0.dev0" +source = { editable = "." } +dependencies = [ + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "hypothesis" }, + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [{ name = "pyyaml", specifier = ">=6.0" }] + +[package.metadata.requires-dev] +dev = [ + { name = "hypothesis", specifier = ">=6.0" }, + { name = "pytest", specifier = ">=8.0" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "hypothesis" +version = "6.161.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f7/5b/98162d7cf48866e18b59d12e975229af411575a20d9a75c303345cc3380b/hypothesis-6.161.2.tar.gz", hash = "sha256:e25f1e6f8a2ea4cadfeaeba18cb8676e5510f52fefd5c9bc165e3eb81e1ef497", size = 486148, upload-time = "2026-07-24T06:43:23.774Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/8f/a565e6ab67de327eee310ef306d715d8f5143e729835450715dde121e912/hypothesis-6.161.2-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c61b868484dbcd9d2d6d25662f60fa2cee464d18061315efd997efe721250f14", size = 766523, upload-time = "2026-07-24T06:42:57.381Z" }, + { url = "https://files.pythonhosted.org/packages/92/d7/5deb1e38c8c253c879bec75a95fe0ff01d60366d7bd33c59012a8d23a8a1/hypothesis-6.161.2-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:b22012a92b8d2f70f7e7a6a4761166b920edf92900de458d6c9d272057dd48d8", size = 762160, upload-time = "2026-07-24T06:42:21.945Z" }, + { url = "https://files.pythonhosted.org/packages/dd/21/1f03b88df69b9b22429ad608c7a4778e01e9bb10656142e0426dbf0865eb/hypothesis-6.161.2-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d59687ed88b290e450505d71a5950422d39791dbaf48a4159b876634976e7898", size = 1091358, upload-time = "2026-07-24T06:42:03.947Z" }, + { url = "https://files.pythonhosted.org/packages/70/9f/5c761fdb60ed5c547b6803620822bca14486fda32d785a31416f1bbab970/hypothesis-6.161.2-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7af888ad7fe38c33112fde21756621b0d8f69167bbb01a5d66569aa90d6c692c", size = 1140836, upload-time = "2026-07-24T06:43:22.104Z" }, + { url = "https://files.pythonhosted.org/packages/1e/d5/1eb85756989ea885c325442ef2ebe5bd7347cbd4f99a907c384d10b343e5/hypothesis-6.161.2-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:9bd07a407addd5a9318305bc3cf78215662c6970b699a2df610955bf41750abc", size = 1096169, upload-time = "2026-07-24T06:42:09.992Z" }, + { url = "https://files.pythonhosted.org/packages/08/0e/c65f8c76f4d9d54203cbe5df6cf33e9b918629150b526d5df0ca5350b9fe/hypothesis-6.161.2-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:140e154d7317e8fcb538a8ff1c7b8ee52b67c9ab53647078e11cb950d2264c11", size = 1132927, upload-time = "2026-07-24T06:42:49.536Z" }, + { url = "https://files.pythonhosted.org/packages/37/da/be099e47ec288a8ce7a9030c47882166694b62c0fcd5d468f46559333848/hypothesis-6.161.2-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:febed60ec1ce744040f8f1e94a994a567a61b3f4bd82be4c3883b93ac76ec58a", size = 1265174, upload-time = "2026-07-24T06:42:34.36Z" }, + { url = "https://files.pythonhosted.org/packages/fa/7c/e591135729eee9dd84cf68a539612ee9571b449f7a32ed9070802ccae4c4/hypothesis-6.161.2-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:d01840d49772e3a29117b936dd5b71c13a9a82b39a739ac49f228af9601c40e8", size = 1265785, upload-time = "2026-07-24T06:42:18.135Z" }, + { url = "https://files.pythonhosted.org/packages/d1/e3/67a5f389fa1be4f4a68b3c65c15f3273d4fb5910cbc4c99754260ae52112/hypothesis-6.161.2-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:4179f4db70daec64dc43d3fa9b23851d78791ca5d84605b3794df2dfc3150c61", size = 1307846, upload-time = "2026-07-24T06:43:05.511Z" }, + { url = "https://files.pythonhosted.org/packages/4c/d8/45ffe9a9f069853fabb7b91fe72d46f9b2f08508550f87f3567fd1d46775/hypothesis-6.161.2-cp310-abi3-win32.whl", hash = "sha256:fabc7595dd1e0c66936e9777843dbef3c2e3f31983a669bda6b635909878176e", size = 652400, upload-time = "2026-07-24T06:42:54.175Z" }, + { url = "https://files.pythonhosted.org/packages/6a/14/81af65ce429671ee4e4b9fb6924a02564dd9e430c543da9ac9449dd07042/hypothesis-6.161.2-cp310-abi3-win_amd64.whl", hash = "sha256:425a35e9240761a2e71743424b193fd799dfa3117bad21982bbe8110637256b2", size = 658534, upload-time = "2026-07-24T06:43:10.538Z" }, + { url = "https://files.pythonhosted.org/packages/2b/4c/672a3fa3400696beac41097191d356b6e1f959a138ad00b5072f17dacb28/hypothesis-6.161.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:727733b1f334d9e9b8fc1bb62cd5175e9eb6ed37eae04db2c686fcd4859084ed", size = 767019, upload-time = "2026-07-24T06:43:12.062Z" }, + { url = "https://files.pythonhosted.org/packages/71/47/0f2b2df0cc2e54e2edefbb0fdb43c2b7a6032b1e0cc53eda640bb6767f20/hypothesis-6.161.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4d986a62ff90383eb4e37f60fb90740adcde5b1637f8f35012149248dac15aea", size = 762793, upload-time = "2026-07-24T06:43:02.276Z" }, + { url = "https://files.pythonhosted.org/packages/bf/c5/cd3a6638e7f4884a5475ad385d8a4baedd20f59c5d739f79a5adb493120b/hypothesis-6.161.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d383a57238e30fec0f22343ce1133ccca0152274460a36c7f37b5c45264d44d1", size = 1091718, upload-time = "2026-07-24T06:42:14.027Z" }, + { url = "https://files.pythonhosted.org/packages/a8/61/fa67d44e15639cc7ca1a8f0728ce0cf0bf9ecefd3f5c5da5e26b9f8f5f6c/hypothesis-6.161.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4d468008e8f571844ca8d4aa5940305b844e1833ad2f4a5636d5d04e0427a379", size = 1141153, upload-time = "2026-07-24T06:42:38.79Z" }, + { url = "https://files.pythonhosted.org/packages/7b/7c/3a1edd529505585bc34c7252d51e57047563561b194699491c740127ee13/hypothesis-6.161.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:71a273502fd251651ad2a134dc51703096c89f79177e91a1c96d1a2b33fd8c1c", size = 1265535, upload-time = "2026-07-24T06:42:37.382Z" }, + { url = "https://files.pythonhosted.org/packages/ca/05/ad2a3460a460942ca9f1197decf8fd7b22f917a3d6063fce600569286664/hypothesis-6.161.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:75eb708b2e194dc0a7e23276d1cbb676040b4198d3822505d400cebbcfb7cfe2", size = 1308120, upload-time = "2026-07-24T06:42:12.663Z" }, + { url = "https://files.pythonhosted.org/packages/a6/29/a5e65328ddc7f00a525cbf4d31231e617f6b58c64dcd91e70c9ce78327f3/hypothesis-6.161.2-cp311-cp311-win_amd64.whl", hash = "sha256:cc099b96454ff0047b7fc9ee973064162f2a64f4876d307d05be3c1ffd6cc152", size = 658254, upload-time = "2026-07-24T06:42:31.552Z" }, + { url = "https://files.pythonhosted.org/packages/b4/c7/26500d97f8dbe4327a8d80beebc9a030527a1e9bcce35a22b0ef578d4040/hypothesis-6.161.2-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:517ba8831f083d25f961eb980d8f59f030a8fb685da27d6c61988c3ff6e89607", size = 768100, upload-time = "2026-07-24T06:43:18.552Z" }, + { url = "https://files.pythonhosted.org/packages/4e/c2/ad5778532eb8e1a3f876f5a5a2e9014847747d9f8eba54333f14a45a7062/hypothesis-6.161.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0e6157e70f8e9f53a4025e932b6fa1c8068ff7b62063647985d3a193bc198994", size = 759776, upload-time = "2026-07-24T06:41:58.322Z" }, + { url = "https://files.pythonhosted.org/packages/e7/3c/b9566d91b3cbca5bb3d840b695a2585e1649c8e1890f4bc8722ce3236b23/hypothesis-6.161.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c13df0507a19fbe8fa358b55322b2b20a9f8a3c11885f3a3c621f52354b6a875", size = 1090169, upload-time = "2026-07-24T06:43:13.643Z" }, + { url = "https://files.pythonhosted.org/packages/2f/53/beebfbc9df7bfcb6da3d51d5d3af02839e245a0a8d93ea7ce2d281f5283f/hypothesis-6.161.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:08af49b8c5e39f235f1bac97559fa6df5854ef1ed2aeb5974e71d1efa06757f4", size = 1140197, upload-time = "2026-07-24T06:42:28.941Z" }, + { url = "https://files.pythonhosted.org/packages/1d/9a/df90f88ecef33c5025a747dcc7b00bd680ae6b34679ebbdf7e47eec71195/hypothesis-6.161.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7bc2f9d873821f9af1f4578bd5ab9c8a3f152dce2390a9d9c64c49ff9322154f", size = 1262977, upload-time = "2026-07-24T06:43:07.274Z" }, + { url = "https://files.pythonhosted.org/packages/20/12/9e548564138eddd26faa5fcce01d242d65671f1a83c064262f62fee7fc9c/hypothesis-6.161.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ca250fc04c44816001551041d2ddf718a45bbc1d9c6ae480c98a558964a057cd", size = 1307184, upload-time = "2026-07-24T06:42:35.874Z" }, + { url = "https://files.pythonhosted.org/packages/55/9c/71e4eec244b9d76a29e9429c878fcb91a165fad3dcee8b935a31e527b1fd/hypothesis-6.161.2-cp312-cp312-win_amd64.whl", hash = "sha256:cae4dc03b2830ca2d4fee2fc5d8edc9ea72ee1c0db6b2abbb9fd59ef9961a45f", size = 655685, upload-time = "2026-07-24T06:42:30.285Z" }, + { url = "https://files.pythonhosted.org/packages/8e/98/3d4849d78b7eeafac335d55a8cbc647602eca41cce9fe5b99f28025d6f3b/hypothesis-6.161.2-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:c77580949be61ce4b946f8dfba38ed652ce28deb6302167550f85720f08c1864", size = 767990, upload-time = "2026-07-24T06:42:43.432Z" }, + { url = "https://files.pythonhosted.org/packages/50/eb/096d58e24d727a69e3ad3fb3887f4b3750d0e41287bb3a6ebfb9f20b6b8d/hypothesis-6.161.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:ed59ea708743da8e91f975d3848d2ed0a46548d833c8703b53fa854bbb8f3ebd", size = 759677, upload-time = "2026-07-24T06:42:07.612Z" }, + { url = "https://files.pythonhosted.org/packages/e3/11/511ff00654fdc9a52316e1a772cecc6ef0a0c24f1138a20aeab2f632ae5b/hypothesis-6.161.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:73fa1136acb5f554c6b958f8a9d33b2d7f6816909ccf07b22e17351df9bec146", size = 1090072, upload-time = "2026-07-24T06:43:03.917Z" }, + { url = "https://files.pythonhosted.org/packages/99/b8/2c833554fb3ba22d2841888b3409b634d6059420d1f5269f7cf3294358ed/hypothesis-6.161.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:584b1d91b5f3fa200b35bc0533e30e9fea81c852bb150eedbf0a7fc46d7639be", size = 1140009, upload-time = "2026-07-24T06:42:58.975Z" }, + { url = "https://files.pythonhosted.org/packages/35/36/c351c14155225b6290ce94fbd5610a690231312d5aa2121ad436888e3dfe/hypothesis-6.161.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e1f3d2f1b8324259cc1a149286b8856ed07447f9f28f1338452eec73708cd7e5", size = 1263030, upload-time = "2026-07-24T06:42:23.475Z" }, + { url = "https://files.pythonhosted.org/packages/13/f7/1dcbafa23a2fc07aede597e0c9ea2093378a1c05a858dfbbb3627a1a7e11/hypothesis-6.161.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c1c6c7d30c9aaa11b3e79c40a9a7ac1f559fce44853292d5383c0b1d11623658", size = 1306906, upload-time = "2026-07-24T06:42:06.2Z" }, + { url = "https://files.pythonhosted.org/packages/26/93/371966030a92748fcdef978f4ecfd69aa7b0d92e4bcdc398f5938941ef2c/hypothesis-6.161.2-cp313-cp313-win_amd64.whl", hash = "sha256:176d4f1a58f808891eec2a448889a24522002cb952df99c6fe4ab150f81a0eae", size = 655650, upload-time = "2026-07-24T06:42:51.08Z" }, + { url = "https://files.pythonhosted.org/packages/96/bf/fc502b4e92361e0ded96f77f3f40bcc6b16d0ab0511a54afb1110133d18a/hypothesis-6.161.2-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:0101112993070b1f3ba00b44ea3913e06d7b14c0c658f61e19f196aa7a36c2ce", size = 768219, upload-time = "2026-07-24T06:43:20.437Z" }, + { url = "https://files.pythonhosted.org/packages/33/f0/aa051525002a853914fd5e14ea0921ccdd02081d715cb52473f35fcb2325/hypothesis-6.161.2-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a8323ffde6b3c01103c69af857e085dc4d74cbde07e8f8dac297d81870f0b8dc", size = 759825, upload-time = "2026-07-24T06:42:44.901Z" }, + { url = "https://files.pythonhosted.org/packages/32/24/88b22af1f34b773542eb8c465d3609bc6e35ddf22e4598fbed9b3aa12956/hypothesis-6.161.2-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d05e9343fbf172dc409fbf2b5c8894d2fde6d0748a852b9b8ef2366292240ff", size = 1090594, upload-time = "2026-07-24T06:42:47.918Z" }, + { url = "https://files.pythonhosted.org/packages/bc/4d/b800afacd75cb89e9663178e22393e9b69289ad43ef1cc868a8d0a243482/hypothesis-6.161.2-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84f5f15959b16356534007ba4a0cc95184576639b839a311eb953b07947c9e8c", size = 1140179, upload-time = "2026-07-24T06:42:41.644Z" }, + { url = "https://files.pythonhosted.org/packages/56/89/c33695781141b78b84ce888beb6a0e6865f0fc45b1395fa4d7039d31e29a/hypothesis-6.161.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:2c5e898aaf65476e4c3e2953dfb71b16112537704c9cb083e7920c2525a96d97", size = 1263351, upload-time = "2026-07-24T06:42:26.369Z" }, + { url = "https://files.pythonhosted.org/packages/98/9e/8e508005a821a46e230704c69790ab6631d6e94404c7275d5bf7f34aa8d7/hypothesis-6.161.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:f733deeeae110c490610eba1c72adbf2b56f358bed3750e1a07d4c42a5ddaffa", size = 1307209, upload-time = "2026-07-24T06:42:19.493Z" }, + { url = "https://files.pythonhosted.org/packages/23/03/62301eab71ce45cf7e9c84e52a4cfcd2d51abcc81a9f733e89a171474395/hypothesis-6.161.2-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:7870df5cc41377aa76a42650be68b1cab87fd7b7a1ae1a7a83b06fc49809b6f1", size = 599729, upload-time = "2026-07-24T06:41:59.905Z" }, + { url = "https://files.pythonhosted.org/packages/ee/a8/4e1479e2136b051adc2a141b2c26b8c8a18deda9d7c1713e816eac6b374b/hypothesis-6.161.2-cp314-cp314-win_amd64.whl", hash = "sha256:cc6bc6e1c6228ac32e7c22110eb663dc5abb844ed83eb2199f00fac004327d6d", size = 655563, upload-time = "2026-07-24T06:42:05.089Z" }, + { url = "https://files.pythonhosted.org/packages/22/8c/9c0f53d4d244cb8ad243a5dbfd1e733eaf5617f1e867c45bac15dfaa88c7/hypothesis-6.161.2-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f91ffa32ef1597158da5696e68f1d99ef0c23dbe286a6d4ea5d94e4f856614d5", size = 766796, upload-time = "2026-07-24T06:42:16.803Z" }, + { url = "https://files.pythonhosted.org/packages/ce/0c/3aa7877ebb5f333ff864e1465d579cbad85eb87ab45e9d532093ebd88dd0/hypothesis-6.161.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:825feec7f819c44e02a047b7c1c6672793e584b7a659cf2fad107fa031722515", size = 758288, upload-time = "2026-07-24T06:43:16.998Z" }, + { url = "https://files.pythonhosted.org/packages/05/f0/452289f386654e22f11d9cef03a8d2931fede39f028e679ebd40b2dd9513/hypothesis-6.161.2-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8aeecadd8d0df819ccde1d00d8f2f8e5cd5d0ac196967d959b789b111be54cb1", size = 1089175, upload-time = "2026-07-24T06:42:32.859Z" }, + { url = "https://files.pythonhosted.org/packages/85/d9/9e70c36b84392b0e13ead6601032e06fdbac6898d67515140a123242a081/hypothesis-6.161.2-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:526663659adf7408a2e550440d16020a2f49e4e3e69ccea0a6aa23ed5ff6f6b8", size = 1139093, upload-time = "2026-07-24T06:42:08.771Z" }, + { url = "https://files.pythonhosted.org/packages/98/e5/464b33a0eab6cd8df301ec45bea759c6f66a331f59f4a2d9c600c8e0a07f/hypothesis-6.161.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:96d3609972a89cb28794014a67aab4354dfb30e65c3ea92ce5318044d69f0f47", size = 1261590, upload-time = "2026-07-24T06:42:24.876Z" }, + { url = "https://files.pythonhosted.org/packages/58/b3/e37438facec5f3c3021e2d49610d803ccbf88b594090194fc4fa7f89c49b/hypothesis-6.161.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:adc49e00f4e3245ae5372c602bab929fec10be922fdb5766c73f936a616c0948", size = 1305967, upload-time = "2026-07-24T06:42:15.575Z" }, + { url = "https://files.pythonhosted.org/packages/5b/43/6ac7a850101db2ddaf351ee9e2d649a1a2a10bdc9ce1d83a60cbb0ddc565/hypothesis-6.161.2-cp314-cp314t-win_amd64.whl", hash = "sha256:e368ebca6616dd17cd4d00ea1e03705aac850bcc4a8de322c4942a1d63a80c88", size = 655715, upload-time = "2026-07-24T06:42:40.252Z" }, + { url = "https://files.pythonhosted.org/packages/4b/4a/88e654fdae4bc01274d14b62d228aa77145137f7fb479d7be7310b50cfbc/hypothesis-6.161.2-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:9c8c9df201a925bee3793dd199bc0abe36fbc190a96f6716fffa796a4e56e82c", size = 767908, upload-time = "2026-07-24T06:42:11.249Z" }, + { url = "https://files.pythonhosted.org/packages/69/3b/6ac1c7fc7bf9d9a87fab40d388aa40cf7a1e34586e0dedb2818023daeba9/hypothesis-6.161.2-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:776b8e1d5e282b8075aa51d3f09568a39414354ccf3260519e6cba179d66772e", size = 763858, upload-time = "2026-07-24T06:42:46.516Z" }, + { url = "https://files.pythonhosted.org/packages/91/dc/a5527f7bf0a227c46c537b07916b3df25b39113f053dec9d1245dca00595/hypothesis-6.161.2-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea6a7e512c51eff629b0aa9535bfb05a10537b9d63655793bddfc09780dba881", size = 1092675, upload-time = "2026-07-24T06:43:15.251Z" }, + { url = "https://files.pythonhosted.org/packages/68/40/3cab36af4784a9384f18cb3a1c3a3c704f71abcecb526d76bcb23317b634/hypothesis-6.161.2-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:39939bff7bbe422325f3b570b44902e4103f49cd35cb688eb64fb65b058b7e91", size = 1142459, upload-time = "2026-07-24T06:42:01.028Z" }, + { url = "https://files.pythonhosted.org/packages/fd/d3/10b9f92388434c09ea50364779a7f847975ec77348ed93a11f887e754639/hypothesis-6.161.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:0a3083599bddfa9a5ce1f401233e9ac0f67ef3d3ef9fe88a43907ca317fa509c", size = 659353, upload-time = "2026-07-24T06:42:52.677Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] From 3c1645627fa1d7456872761e52df2563f365d30d Mon Sep 17 00:00:00 2001 From: Haoran Li Date: Wed, 19 Aug 2026 17:11:15 +0000 Subject: [PATCH 3/3] [OSSIE] Mark the Python converter as deprecated in favor of Java Replace the Python README's forward-looking 'Future effort' section with a deprecation note: the Java converter under java/ is the maintained implementation; the Python copy is kept for reference and no longer actively extended. Signed-off-by: Haoran Li --- converters/databricks/python/README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/converters/databricks/python/README.md b/converters/databricks/python/README.md index e2fe6ccb..6ef4ea8b 100644 --- a/converters/databricks/python/README.md +++ b/converters/databricks/python/README.md @@ -116,9 +116,8 @@ python3 -m pytest tests/ Example-based unit tests plus Hypothesis property-based round-trip tests (`test_roundtrip_properties.py`, which skip if `hypothesis` is not installed). -## Future effort +## Status -Both the Apache Ossie specification and the Databricks Unity Catalog Metric View YAML are still -evolving. As either side adds or changes fields, this converter will be updated to track -them -- extending the mapping and coverage in both directions to keep the conversion -current and to support as much as each format allows over time. +This Python implementation is the original reference and is **deprecated** in favor of the Java +converter under [`../java/`](../java/), which is the maintained implementation. New behavior and +fixes land in Java; this copy is kept for reference and is no longer actively extended.