diff --git a/converters/databricks/src/ossie_databricks/_common.py b/converters/databricks/src/ossie_databricks/_common.py index 4f17a5b3..b77b6428 100644 --- a/converters/databricks/src/ossie_databricks/_common.py +++ b/converters/databricks/src/ossie_databricks/_common.py @@ -261,9 +261,16 @@ def validate_source(source, dataset_name): Accepts a 3-part `catalog.schema.table` identifier or a `SELECT`/`WITH` subquery. Raises ConversionError otherwise. """ - if not source or not str(source).strip(): + if not source: raise ConversionError(f"Dataset '{dataset_name}': missing/empty 'source'") - s = str(source).strip() + if not isinstance(source, str): + kind = source.get("kind") if isinstance(source, dict) else type(source).__name__ + raise ConversionError( + f"Dataset '{dataset_name}': structured source kind {kind!r} is not supported by the Databricks converter" + ) + if not source.strip(): + raise ConversionError(f"Dataset '{dataset_name}': missing/empty 'source'") + s = source.strip() # A SELECT/WITH subquery source. `\b` after the keyword matches `WITH(...)` (no # space) too, but not an identifier like `WITHHELD`. if re.match(r"(?i)(select|with)\b", s): diff --git a/converters/databricks/tests/test_ossie_to_metric_view.py b/converters/databricks/tests/test_ossie_to_metric_view.py index 8b8808b2..33d37386 100644 --- a/converters/databricks/tests/test_ossie_to_metric_view.py +++ b/converters/databricks/tests/test_ossie_to_metric_view.py @@ -26,6 +26,13 @@ from _util import canon, load_fixture, parse +def test_structured_dataset_source_is_rejected(): + source = {"kind": "file", "format": "parquet", "locations": ["s3://bucket/orders.parquet"]} + + with pytest.raises(ConversionError, match="structured source kind.*file.*not supported"): + exporter.validate_source(source, "orders") + + def test_fixtureA_export_matches_expected(): out = exporter.convert_ossie_to_metric_view(load_fixture("fixtureA_ossie.yaml")) assert parse(out) == parse(load_fixture("fixtureA_metric_view.yaml")) diff --git a/converters/dbt/src/ossie_dbt/osi_to_msi.py b/converters/dbt/src/ossie_dbt/osi_to_msi.py index c8f155fc..f5659b7d 100644 --- a/converters/dbt/src/ossie_dbt/osi_to_msi.py +++ b/converters/dbt/src/ossie_dbt/osi_to_msi.py @@ -25,6 +25,7 @@ OSIExpression, OSIField, OSISemanticModel, + OSISource, ) from ossie_dbt.converter_issues import ConverterResult from ossie_dbt.expression_utils import ( @@ -395,8 +396,13 @@ def _get_expression(self, osi_expr: OSIExpression) -> str: return osi_expr.dialects[0].expression if osi_expr.dialects else "" @staticmethod - def _parse_source(source: str) -> PydanticNodeRelation: - """Parse `schema.table` or `db.schema.table` into a PydanticNodeRelation.""" + def _parse_source(source: OSISource) -> PydanticNodeRelation: + """Parse a legacy string source into a PydanticNodeRelation.""" + if not isinstance(source, str): + kind = getattr(source, "kind", type(source).__name__) + raise TypeError( + f"Structured dataset source kind {kind!r} is not supported by the dbt converter" + ) parts = source.split(".") if len(parts) >= 3: database, schema, alias = parts[0], parts[1], ".".join(parts[2:]) diff --git a/converters/dbt/tests/test_osi_to_msi.py b/converters/dbt/tests/test_osi_to_msi.py index dfed1e42..ccf84aa1 100644 --- a/converters/dbt/tests/test_osi_to_msi.py +++ b/converters/dbt/tests/test_osi_to_msi.py @@ -20,7 +20,7 @@ import pytest from syrupy.assertion import SnapshotAssertion -from ossie import OSIDataType, OSIDimension +from ossie import OSIDataType, OSIDimension, OSIFileSource from ossie_dbt.msi_to_osi import MSIToOSIConverter from ossie_dbt.osi_to_msi import OSIToMSIConverter from metricflow_semantic_interfaces.type_enums import ( @@ -37,6 +37,13 @@ ) +def test_structured_dataset_source_is_rejected() -> None: + source = OSIFileSource(kind="file", format="parquet", locations=["s3://bucket/orders.parquet"]) + + with pytest.raises(TypeError, match="Structured dataset source kind.*file.*not supported"): + OSIToMSIConverter._parse_source(source) + + class TestOSIToMSIBasicConversion: def test_empty_document_produces_empty_manifest(self) -> None: result = OSIToMSIConverter().convert(_osi_doc()).output diff --git a/converters/gooddata/src/ossie_gooddata/osi_to_gooddata.py b/converters/gooddata/src/ossie_gooddata/osi_to_gooddata.py index a8dbac1c..2f9ffc7a 100644 --- a/converters/gooddata/src/ossie_gooddata/osi_to_gooddata.py +++ b/converters/gooddata/src/ossie_gooddata/osi_to_gooddata.py @@ -357,8 +357,13 @@ def _is_multivalue(rel: dict[str, Any]) -> bool: return bool(gd_ext and gd_ext.get("multivalue")) -def _parse_source_to_table_id(source: str, data_source_id: str) -> GdDataSourceTableId: - """Parse an Ossie source string into a GoodData DataSourceTableId.""" +def _parse_source_to_table_id(source: object, data_source_id: str) -> GdDataSourceTableId: + """Parse a legacy Ossie string source into a GoodData DataSourceTableId.""" + if not isinstance(source, str): + kind = source.get("kind") if isinstance(source, dict) else type(source).__name__ + raise TypeError( + f"Structured dataset source kind {kind!r} is not supported by the GoodData converter" + ) parts = source.split(".") if len(parts) >= 3: # source_id.schema.table or more diff --git a/converters/gooddata/tests/test_osi_to_gooddata.py b/converters/gooddata/tests/test_osi_to_gooddata.py index 7848eec4..dab6dcc9 100644 --- a/converters/gooddata/tests/test_osi_to_gooddata.py +++ b/converters/gooddata/tests/test_osi_to_gooddata.py @@ -26,10 +26,18 @@ from ossie_gooddata.osi_to_gooddata import ( _convert_to_attribute, _convert_to_fact, + _parse_source_to_table_id, osi_to_gooddata, ) +def test_structured_dataset_source_is_rejected() -> None: + source = {"kind": "file", "format": "parquet", "locations": ["s3://bucket/orders.parquet"]} + + with pytest.raises(TypeError, match="Structured dataset source kind.*file.*not supported"): + _parse_source_to_table_id(source, "default") + + def _field_with_extension(datatype: str | None, extension_type: str) -> dict: field = { "name": "value", diff --git a/converters/gsf/src/ossie_gsf/native_converter.py b/converters/gsf/src/ossie_gsf/native_converter.py index f2c4fd7e..269aa4a6 100644 --- a/converters/gsf/src/ossie_gsf/native_converter.py +++ b/converters/gsf/src/ossie_gsf/native_converter.py @@ -1923,6 +1923,10 @@ def _parse_source( default_database: str | None, ) -> dict[str, str | None]: if isinstance(source, dict): + if "kind" in source: + raise GSFConversionError( + f"Structured dataset source kind {source.get('kind')!r} is not supported by the NVIDIA GSF converter" + ) database = source.get("database") or default_database schema = source.get("schema") table = source.get("table") diff --git a/converters/gsf/tests/test_converter.py b/converters/gsf/tests/test_converter.py index 29ac624d..715e0b2a 100644 --- a/converters/gsf/tests/test_converter.py +++ b/converters/gsf/tests/test_converter.py @@ -50,6 +50,13 @@ SCHEMA = Path(__file__).resolve().parents[3] / "core-spec" / "osi-schema.json" +def test_structured_dataset_source_is_rejected() -> None: + source = {"kind": "file", "format": "parquet", "locations": ["s3://bucket/orders.parquet"]} + + with pytest.raises(GSFConversionError, match="Structured dataset source kind.*file.*not supported"): + _parse_source(source, "analytics") + + def _ossie_yaml() -> str: return (FIXTURES / "sales.ossie.yaml").read_text(encoding="utf-8") diff --git a/converters/honeydew/src/honeydew_osi/converter.py b/converters/honeydew/src/honeydew_osi/converter.py index e3f78f4f..bc4fe8b8 100644 --- a/converters/honeydew/src/honeydew_osi/converter.py +++ b/converters/honeydew/src/honeydew_osi/converter.py @@ -453,7 +453,12 @@ def _is_simple_identifier(expr: str) -> bool: return bool(re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", expr.strip())) -def _parse_osi_source(source: str) -> tuple[str, str]: +def _parse_osi_source(source: object) -> tuple[str, str]: + if not isinstance(source, str): + kind = source.get("kind") if isinstance(source, dict) else type(source).__name__ + raise HoneydewConversionError( + f"Structured dataset source kind {kind!r} is not supported by the Honeydew converter" + ) source = (source or "").strip() if not source: return ("", "table") diff --git a/converters/honeydew/tests/test_honeydew_osi_converter.py b/converters/honeydew/tests/test_honeydew_osi_converter.py index c3aaa75c..375e448d 100644 --- a/converters/honeydew/tests/test_honeydew_osi_converter.py +++ b/converters/honeydew/tests/test_honeydew_osi_converter.py @@ -50,6 +50,13 @@ OSI_VERSION = "0.2.0.dev0" +def test_structured_dataset_source_is_rejected() -> None: + source = {"kind": "file", "format": "parquet", "locations": ["s3://bucket/orders.parquet"]} + + with pytest.raises(HoneydewConversionError, match="Structured dataset source kind.*file.*not supported"): + _parse_osi_source(source) + + def _osi(model_dict): return yaml.dump( {"version": OSI_VERSION, "semantic_model": [model_dict]}, diff --git a/converters/omni/src/osi_omni/_common.py b/converters/omni/src/osi_omni/_common.py index ad799bca..5d3836da 100644 --- a/converters/omni/src/osi_omni/_common.py +++ b/converters/omni/src/osi_omni/_common.py @@ -328,9 +328,16 @@ def parse_source(source, dataset_name): (parts may be double-quoted: `"Omni Views".channel_info`). Omni views require a `schema`, so a bare 1-part table name is rejected. """ - if not source or not str(source).strip(): + if not source: raise ConversionError(f"Dataset '{dataset_name}': missing/empty 'source'") - s = str(source).strip() + if not isinstance(source, str): + kind = source.get("kind") if isinstance(source, dict) else type(source).__name__ + raise ConversionError( + f"Dataset '{dataset_name}': structured source kind {kind!r} is not supported by the Omni converter" + ) + if not source.strip(): + raise ConversionError(f"Dataset '{dataset_name}': missing/empty 'source'") + s = source.strip() if re.match(r"(?i)(select|with)\b", s): return ("sql", s) if not _SOURCE_PARTS_RE.match(s): diff --git a/converters/omni/tests/test_osi_to_omni.py b/converters/omni/tests/test_osi_to_omni.py index e6d1d0a7..09010960 100644 --- a/converters/omni/tests/test_osi_to_omni.py +++ b/converters/omni/tests/test_osi_to_omni.py @@ -22,10 +22,17 @@ import pytest from osi_omni import ConversionError, convert_osi_to_omni -from osi_omni._common import dump_yaml +from osi_omni._common import dump_yaml, parse_source from _util import REPO_ROOT, load_fixture, load_fixture_dir, parse, parse_files +def test_structured_dataset_source_is_rejected(): + source = {"kind": "file", "format": "parquet", "locations": ["s3://bucket/orders.parquet"]} + + with pytest.raises(ConversionError, match="structured source kind.*file.*not supported"): + parse_source(source, "orders") + + def export(osi_yaml, **kwargs): with warnings.catch_warnings(): warnings.simplefilter("ignore") diff --git a/converters/orionbelt/src/ossie_orionbelt/osi_to_obml.py b/converters/orionbelt/src/ossie_orionbelt/osi_to_obml.py index 6ca635dd..66ca63cb 100644 --- a/converters/orionbelt/src/ossie_orionbelt/osi_to_obml.py +++ b/converters/orionbelt/src/ossie_orionbelt/osi_to_obml.py @@ -226,8 +226,14 @@ def _carry_foreign_extensions(osi_exts: list[dict] | None, obml_target: dict[str {"vendor": vendor, "data": ext.get("data", "")} ) - def _parse_source(self, source: str) -> tuple[str, str, str]: - """Parse 'database.schema.table' into parts.""" + def _parse_source(self, source: object) -> tuple[str, str, str]: + """Parse a legacy string source into database/schema/table parts.""" + if not isinstance(source, str): + kind = source.get("kind") if isinstance(source, dict) else type(source).__name__ + raise TypeError( + f"Structured dataset source kind {kind!r} is not supported " + "by the OrionBelt converter" + ) parts = source.split(".") if len(parts) == 3: return parts[0], parts[1], parts[2] diff --git a/converters/orionbelt/tests/test_osi_v02_compat.py b/converters/orionbelt/tests/test_osi_v02_compat.py index 144b2b13..e2cdb951 100644 --- a/converters/orionbelt/tests/test_osi_v02_compat.py +++ b/converters/orionbelt/tests/test_osi_v02_compat.py @@ -43,6 +43,14 @@ # --------------------------------------------------------------------------- +def test_structured_dataset_source_is_rejected() -> None: + converter = conv.OSItoOBML({"version": "0.2.0.dev0", "semantic_model": []}) + source = {"kind": "file", "format": "parquet", "locations": ["s3://bucket/orders.parquet"]} + + with pytest.raises(TypeError, match="Structured dataset source kind.*file.*not supported"): + converter._parse_source(source) + + @pytest.fixture(scope="module") def schema_validator() -> Any: """Draft 2020-12 validator pinned to the resolved OSI v0.2 core schema diff --git a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiModelParser.java b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiModelParser.java index 593452b6..8b7b5407 100644 --- a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiModelParser.java +++ b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OsiModelParser.java @@ -112,7 +112,13 @@ private SemanticModel parseSemanticModel(Map map) { private Dataset parseDataset(Map map) { Dataset ds = new Dataset(); ds.setName((String) map.get("name")); - ds.setSource((String) map.get("source")); + Object source = map.get("source"); + if (source != null && !(source instanceof String)) { + Object kind = source instanceof Map sourceMap ? sourceMap.get("kind") : source.getClass().getSimpleName(); + throw new IllegalArgumentException( + "Structured dataset source kind '" + kind + "' is not supported by the Polaris converter"); + } + ds.setSource((String) source); ds.setDescription((String) map.get("description")); List pk = (List) map.get("primary_key"); diff --git a/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/OsiPolarisConverterTest.java b/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/OsiPolarisConverterTest.java index 456bdb2c..58ee02e8 100644 --- a/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/OsiPolarisConverterTest.java +++ b/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/OsiPolarisConverterTest.java @@ -120,6 +120,27 @@ void testParseMinimalModel() { assertEquals(1, sm.getMetrics().size()); } + @Test + void testStructuredDatasetSourceIsRejected() { + String structuredSourceModel = + "version: \"0.2.0.dev0\"\n" + + "semantic_model:\n" + + " - name: test_model\n" + + " datasets:\n" + + " - name: orders\n" + + " source:\n" + + " kind: file\n" + + " format: parquet\n" + + " locations: [s3://bucket/orders.parquet]\n"; + + IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, + () -> new OsiModelParser().parse( + new ByteArrayInputStream(structuredSourceModel.getBytes(StandardCharsets.UTF_8)))); + + assertTrue(error.getMessage().contains("Structured dataset source kind 'file' is not supported")); + } + @Test void testParseDatasetFields() { OsiModelParser parser = new OsiModelParser(); diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java index 787b7af8..35eb0d40 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java @@ -64,6 +64,7 @@ public enum Level { public static final String API_NAME = "apiName"; public static final String LABEL = "label"; public static final String DESCRIPTION = "description"; + public static final String SOURCE = "source"; public static final String DATA_TYPE = "dataType"; public static final String OSI_DATATYPE = "datatype"; public static final String AI_CONTEXT = "ai_context"; diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/DatasetMappingHandler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/DatasetMappingHandler.java index 5163f5db..b68f4989 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/DatasetMappingHandler.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/DatasetMappingHandler.java @@ -69,6 +69,21 @@ public void execute(Map sourceData, Map outputDa private void mapOsiToSalesforce( Map sourceData, Map outputData, Map mappings) { + List osiDatasets = getList(sourceData, DATASETS); + if (osiDatasets != null) { + streamMaps(osiDatasets).forEach(dataset -> { + Object source = dataset.get(SOURCE); + if (source != null && !(source instanceof String)) { + Object kind = source instanceof Map sourceMap + ? sourceMap.get("kind") + : source.getClass().getSimpleName(); + throw new org.apache.ossie.exception.ConversionException( + "Structured dataset source kind '" + kind + + "' is not supported by the Salesforce converter"); + } + }); + } + Map datasetMappings = MappingUtils.filterMappingsByPrefix(mappings, DATASETS); var mappedData = GenericMappingEngine.applyMappings(sourceData, datasetMappings); diff --git a/converters/salesforce/src/test/java/org/apache/ossie/OsiToSalesforceConverterTest.java b/converters/salesforce/src/test/java/org/apache/ossie/OsiToSalesforceConverterTest.java index c298df6b..6bd51092 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/OsiToSalesforceConverterTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/OsiToSalesforceConverterTest.java @@ -25,6 +25,7 @@ import org.apache.ossie.converter.ConverterFactory; import org.apache.ossie.converter.ConversionDirection; import org.apache.ossie.converter.CustomExtensionHandler; +import org.apache.ossie.exception.ConversionException; import org.apache.ossie.validator.SchemaValidator; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -94,6 +95,23 @@ void setUp() throws IOException { osiYaml = osiYamlAnsiSql; } + @Test + void testStructuredDatasetSourceIsRejected() { + String structuredSourceModel = + "version: \"0.2.0.dev0\"\n" + + "semantic_model:\n" + + " - name: test_model\n" + + " datasets:\n" + + " - name: orders\n" + + " source:\n" + + " kind: file\n" + + " format: parquet\n" + + " locations: [s3://bucket/orders.parquet]\n"; + + ConversionException error = assertThrows(ConversionException.class, () -> converter.convert(structuredSourceModel)); + assertTrue(error.getMessage().contains("Structured dataset source kind 'file' is not supported")); + } + @Test void testCompleteConversion() throws Exception { List results = converter.convert(osiYaml); diff --git a/converters/snowflake/src/ossie_snowflake/converter.py b/converters/snowflake/src/ossie_snowflake/converter.py index 6dcd77f3..1eb1b333 100644 --- a/converters/snowflake/src/ossie_snowflake/converter.py +++ b/converters/snowflake/src/ossie_snowflake/converter.py @@ -446,8 +446,13 @@ def _parse_source(source): """ if not source: return None + if not isinstance(source, str): + kind = source.get("kind") if isinstance(source, dict) else type(source).__name__ + raise OsiConversionError( + f"Structured dataset source kind {kind!r} is not supported by the Snowflake converter" + ) - source_stripped = str(source).strip() + source_stripped = source.strip() if not source_stripped: return None diff --git a/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py b/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py index 738fdf1f..b331cffe 100644 --- a/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py +++ b/converters/snowflake/tests/test_osi_to_snowflake_yaml_converter.py @@ -38,6 +38,13 @@ ) +def test_structured_dataset_source_is_rejected(): + source = {"kind": "file", "format": "parquet", "locations": ["s3://bucket/orders.parquet"]} + + with pytest.raises(OsiConversionError, match="Structured dataset source kind.*file.*not supported"): + _parse_source(source) + + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- diff --git a/converters/wisdom/src/ossie_wisdom/osi_to_wisdom.py b/converters/wisdom/src/ossie_wisdom/osi_to_wisdom.py index 0ae111fa..413f0f10 100644 --- a/converters/wisdom/src/ossie_wisdom/osi_to_wisdom.py +++ b/converters/wisdom/src/ossie_wisdom/osi_to_wisdom.py @@ -40,6 +40,7 @@ OSIDocument, OSIExpression, OSISemanticModel, + OSISource, ) from ossie_wisdom.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult @@ -270,7 +271,12 @@ def _ai_context_text(self, ai_context, element_name: str, issues: List[Converter return ai_context.instructions or "" return ai_context - def _split_source(self, source: str) -> Tuple[str, str, str]: + def _split_source(self, source: OSISource) -> tuple[str, str, str]: + if not isinstance(source, str): + kind = getattr(source, "kind", type(source).__name__) + raise TypeError( + f"Structured dataset source kind {kind!r} is not supported by the Wisdom converter" + ) parts = source.split(".") if len(parts) >= 3: return parts[0], parts[1], ".".join(parts[2:]) diff --git a/converters/wisdom/tests/test_osi_to_wisdom.py b/converters/wisdom/tests/test_osi_to_wisdom.py index 1ee17173..c2d7da67 100644 --- a/converters/wisdom/tests/test_osi_to_wisdom.py +++ b/converters/wisdom/tests/test_osi_to_wisdom.py @@ -27,6 +27,7 @@ OSIDocument, OSIExpression, OSIField, + OSIFileSource, OSIRelationship, OSISemanticModel, ) @@ -35,6 +36,13 @@ FIXTURE = Path(__file__).parent / "fixtures" / "sample_export.json" +def test_structured_dataset_source_is_rejected() -> None: + source = OSIFileSource(kind="file", format="parquet", locations=["s3://bucket/orders.parquet"]) + + with pytest.raises(TypeError, match="Structured dataset source kind.*file.*not supported"): + OSIToWisdomConverter()._split_source(source) + + def _snowflake(expression): return OSIExpression(dialects=[OSIDialectExpression(dialect=OSIDialect.SNOWFLAKE, expression=expression)]) diff --git a/core-spec/osi-schema.json b/core-spec/osi-schema.json index f24e45f1..4ff232a9 100644 --- a/core-spec/osi-schema.json +++ b/core-spec/osi-schema.json @@ -173,6 +173,40 @@ "required": ["name", "expression"], "additionalProperties": false }, + "FileSource": { + "type": "object", + "description": "Structured descriptor for a file-backed dataset source.", + "properties": { + "kind": { + "type": "string", + "const": "file", + "description": "Discriminator identifying a file-backed source." + }, + "format": { + "type": "string", + "minLength": 1, + "description": "Physical file format, for example parquet." + }, + "locations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1, + "description": "One or more file, object-store, or HTTPS locations." + } + }, + "required": ["kind", "format", "locations"], + "additionalProperties": false + }, + "Source": { + "description": "Dataset source. Legacy string sources remain supported for backward compatibility.", + "oneOf": [ + {"type": "string"}, + {"$ref": "#/$defs/FileSource"} + ] + }, "Dataset": { "type": "object", "description": "Logical dataset representing a business entity (fact or dimension table)", @@ -182,8 +216,7 @@ "description": "Unique identifier for the dataset" }, "source": { - "type": "string", - "description": "Reference to underlying physical table/view (database.schema.table) or query" + "$ref": "#/$defs/Source" }, "primary_key": { "type": "array", diff --git a/core-spec/spec.md b/core-spec/spec.md index 156cb1db..f6f57ddd 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -124,7 +124,7 @@ Logical datasets represent business entities or concepts (fact and dimension tab | Field | Type | Required | Description | |-------|------|----------|-------------| | `name` | string | Yes | Unique identifier for the dataset | -| `source` | string | Yes | Reference to underlying physical table/view (e.g., `database.schema.table`) or query | +| `source` | string/object | Yes | Legacy table/view/query string, or a structured file-backed source descriptor | | `primary_key` | array | No | Primary key columns that uniquely identify rows (single or composite) | | `unique_keys` | array of arrays | No | Array of unique key definitions (each can be single or composite) | | `description` | string | No | Human-readable description | @@ -132,6 +132,26 @@ Logical datasets represent business entities or concepts (fact and dimension tab | `fields` | array | No | Row-level attributes for grouping, filtering, and metric expressions | | `custom_extensions` | array | No | Vendor-specific attributes | +### Source Forms + +Existing string sources remain valid and preserve current behavior: + +```yaml +source: sales.public.orders +``` + +File-backed datasets may use an explicit structured form. `kind` is the discriminator; the initial portable file form requires a physical format and at least one location: + +```yaml +source: + kind: file + format: parquet + locations: + - s3://analytics-data/orders/*.parquet +``` + +Locations are metadata references only. The Ossie document does not imply that every converter or consumer can read the referenced storage system; unsupported source kinds must be handled explicitly rather than silently reinterpreted as table names. + ### Primary Key Examples ```yaml diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 32fbb3e1..b6ad290e 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -96,9 +96,17 @@ datasets: # Required: Unique identifier for the logical dataset - name: string - # Required: Reference to the underlying physical table/view or query - # Format should be either database_name.schema_name.table_name or query - source: string + # Required: Dataset source. Existing table/view/query strings remain valid. + # Legacy form: + # source: database_name.schema_name.table_name + # + # Structured file-backed form: + # source: + # kind: file + # format: parquet + # locations: + # - s3://bucket/path/*.parquet + source: string | object # Optional: Primary key definition that uniquely identifies rows in this dataset # Can be a single column or a composite of multiple columns diff --git a/docs/index.md b/docs/index.md index 738c5724..191d2a80 100644 --- a/docs/index.md +++ b/docs/index.md @@ -52,7 +52,7 @@ The Ossie core specification (current version: **0.2.0.dev0**, latest released: | Construct | Description | |-----------|-------------| | **Semantic Model** | The top-level container representing a complete semantic model, including datasets, relationships, and metrics. | -| **Datasets** | Logical datasets representing business entities (fact and dimension tables), with fields, primary keys, and unique keys. | +| **Datasets** | Logical datasets representing business entities, with fields, keys, and a `source` that can be a legacy table/view/query string or a structured file-backed descriptor. | | **Fields** | Row-level attributes for grouping, filtering, and metric expressions. Fields support multiple SQL dialects for cross-platform compatibility. | | **Relationships** | Foreign key connections between datasets, supporting both simple and composite keys. | | **Metrics** | Quantitative measures (sums, averages, ratios, etc.) defined at the model level, capable of spanning multiple datasets. | @@ -61,7 +61,7 @@ The Ossie core specification (current version: **0.2.0.dev0**, latest released: The specification supports multiple SQL dialects (`ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`, `MDX`, `TABLEAU`) so that expressions can be tailored to each platform while maintaining a common model structure. -For the full specification, see [core-spec/spec.md](../core-spec/spec.md). For validation tooling, see [validation/validate.py](../validation/validate.py). For a complete example, see the [TPC-DS semantic model](../examples/tpcds_semantic_model.yaml). +For the full specification, see [core-spec/spec.md](../core-spec/spec.md). For validation tooling, see [validation/validate.py](../validation/validate.py). For examples, see the [TPC-DS semantic model](../examples/tpcds_semantic_model.yaml) and the [file-backed semantic model](../examples/file_backed_semantic_model.yaml). ### Participating Organizations diff --git a/examples/file_backed_semantic_model.yaml b/examples/file_backed_semantic_model.yaml new file mode 100644 index 00000000..b87d5233 --- /dev/null +++ b/examples/file_backed_semantic_model.yaml @@ -0,0 +1,37 @@ +# yaml-language-server: $schema=../core-spec/osi-schema.json +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# File-backed Dataset Example +# Demonstrates a portable structured file source without requiring object-store access. + +version: 0.2.0.dev0 +semantic_model: + - name: file_backed_example + datasets: + - name: events + source: + kind: file + format: parquet + locations: + - s3://analytics-data/events/*.parquet + fields: + - name: event_id + expression: + dialects: + - dialect: ANSI_SQL + expression: event_id diff --git a/python/src/ossie/__init__.py b/python/src/ossie/__init__.py index c7b33604..4b357253 100644 --- a/python/src/ossie/__init__.py +++ b/python/src/ossie/__init__.py @@ -26,10 +26,12 @@ OSIDimension, OSIDocument, OSIExpression, + OSIFileSource, OSIField, OSIMetric, OSIRelationship, OSISemanticModel, + OSISource, OSIVendor, ) @@ -44,9 +46,11 @@ "OSIDimension", "OSIDocument", "OSIExpression", + "OSIFileSource", "OSIField", "OSIMetric", "OSIRelationship", "OSISemanticModel", + "OSISource", "OSIVendor", ] diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index 5406a743..fa4029f6 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -16,7 +16,7 @@ # under the License. from enum import Enum -from typing import Any, Optional, Union +from typing import Annotated, Any, Literal, Optional, Union import yaml from pydantic import BaseModel, ConfigDict, Field @@ -147,13 +147,26 @@ def is_time_dimension(self) -> bool: return self.datatype in _TEMPORAL_DATA_TYPES +class OSIFileSource(BaseModel): + """Structured descriptor for a file-backed dataset source.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + kind: Literal["file"] + format: str = Field(min_length=1) + locations: list[Annotated[str, Field(min_length=1)]] = Field(min_length=1) + + +OSISource = Union[str, OSIFileSource] + + class OSIDataset(BaseModel): """Logical dataset representing a business entity (fact or dimension table).""" model_config = ConfigDict(frozen=True) name: str - source: str + source: OSISource primary_key: Optional[list[str]] = None unique_keys: Optional[list[list[str]]] = None description: Optional[str] = None diff --git a/python/tests/test_models.py b/python/tests/test_models.py index 749a226c..86516d56 100644 --- a/python/tests/test_models.py +++ b/python/tests/test_models.py @@ -27,6 +27,7 @@ OSIDimension, OSIDocument, OSIExpression, + OSIFileSource, OSIField, ) @@ -135,3 +136,81 @@ def test_effective_time_dimension_role( ) assert field.is_time_dimension() is expected + + +def test_legacy_dataset_source_remains_a_string() -> None: + document = OSIDocument.model_validate(_document()) + assert document.semantic_model[0].datasets[0].source == "catalog.schema.events" + + +def test_file_dataset_source_survives_serialization() -> None: + data = _document() + data["semantic_model"][0]["datasets"][0]["source"] = { + "kind": "file", + "format": "parquet", + "locations": ["s3://analytics/events/*.parquet"], + } + + document = OSIDocument.model_validate(data) + source = document.semantic_model[0].datasets[0].source + assert isinstance(source, OSIFileSource) + assert source.kind == "file" + assert source.format == "parquet" + assert source.locations == ["s3://analytics/events/*.parquet"] + + for serialized in (json.loads(document.to_osi_json()), yaml.safe_load(document.to_osi_yaml())): + source_data = serialized["semantic_model"][0]["datasets"][0]["source"] + assert source_data == { + "kind": "file", + "format": "parquet", + "locations": ["s3://analytics/events/*.parquet"], + } + + +def test_file_dataset_source_requires_at_least_one_location() -> None: + data = _document() + data["semantic_model"][0]["datasets"][0]["source"] = { + "kind": "file", + "format": "parquet", + "locations": [], + } + + with pytest.raises(ValidationError): + OSIDocument.model_validate(data) + + +def test_file_source_definition_matches_core_schema() -> None: + schema_path = Path(__file__).parents[2] / "core-spec" / "osi-schema.json" + schema = json.loads(schema_path.read_text()) + + assert schema["$defs"]["Source"]["oneOf"] == [ + {"type": "string"}, + {"$ref": "#/$defs/FileSource"}, + ] + assert schema["$defs"]["FileSource"]["required"] == [ + "kind", + "format", + "locations", + ] + + +@pytest.mark.parametrize( + "source", + [ + {"kind": "file", "format": "", "locations": ["s3://bucket/events.parquet"]}, + {"kind": "file", "format": "parquet", "locations": [""]}, + {"kind": "table", "format": "parquet", "locations": ["s3://bucket/events.parquet"]}, + { + "kind": "file", + "format": "parquet", + "locations": ["s3://bucket/events.parquet"], + "unknown": True, + }, + ], +) +def test_invalid_file_dataset_source_is_rejected(source: dict) -> None: + data = _document() + data["semantic_model"][0]["datasets"][0]["source"] = source + + with pytest.raises(ValidationError): + OSIDocument.model_validate(data)