diff --git a/contrib/dbt_factory/.gitignore b/contrib/dbt_factory/.gitignore new file mode 100644 index 00000000..0ea8c968 --- /dev/null +++ b/contrib/dbt_factory/.gitignore @@ -0,0 +1,13 @@ +.venv/ +__pycache__/ +*.pyc +.databricks/ +logs/ +dbt_packages/ +dbt_profiles/.user.yml +uv.lock + +# dbt build artifacts, but keep the committed manifest that resources/__init__.py reads at +# deploy time. Regenerate it with `make manifest`. +target/* +!target/manifest.json diff --git a/contrib/dbt_factory/Makefile b/contrib/dbt_factory/Makefile new file mode 100644 index 00000000..d26bc3b3 --- /dev/null +++ b/contrib/dbt_factory/Makefile @@ -0,0 +1,27 @@ +.PHONY: setup deps manifest validate deploy run test + +# Install dependencies into the .venv used by the bundle (databricks.yml -> python.venv_path). +setup: + uv sync --dev + +# Install dbt package dependencies declared in packages.yml / dependencies.yml (if any). +deps: + uv run dbt deps + +# Regenerate the dbt manifest that resources/__init__.py reads at deploy time. +# `dbt parse` does not connect to a warehouse; it only reads the project files. +manifest: deps + uv run dbt parse --profiles-dir dbt_profiles + +validate: + databricks bundle validate + +# Regenerate the manifest and deploy the generated job to the dev target. +deploy: manifest + databricks bundle deploy --target dev + +run: + databricks bundle run dbt_factory_job + +test: + uv run pytest tests diff --git a/contrib/dbt_factory/NOTICE b/contrib/dbt_factory/NOTICE new file mode 100644 index 00000000..703b9022 --- /dev/null +++ b/contrib/dbt_factory/NOTICE @@ -0,0 +1,33 @@ +This project includes code adapted from the "databricks-dbt-factory" library. + + Source: https://github.com/mwojtyczka/databricks-dbt-factory + Full history: https://github.com/mwojtyczka/databricks-dbt-factory/commits/main + Adapted from: commit e767a9d865581226e4f144fb17b7a822df1ea1f4 (v0.2.1) + Location: src/databricks_dbt_factory/ + +The code under src/databricks_dbt_factory/ originates from that repository (reformatted to +this repository's style; otherwise functionally unchanged) and is provided under the MIT +license reproduced below, which this NOTICE preserves for attribution. All other files are +part of the bundle-examples repository and are covered by that repository's license. + +-------------------------------------------------------------------------------- + +MIT License + +Copyright (c) 2024-present mwojtyczka + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/contrib/dbt_factory/README.md b/contrib/dbt_factory/README.md new file mode 100644 index 00000000..805a26c5 --- /dev/null +++ b/contrib/dbt_factory/README.md @@ -0,0 +1,212 @@ +# dbt_factory + +This example runs a [dbt](https://docs.getdbt.com/) project on Databricks as a +**Databricks Workflow with one task per dbt object** (model, seed, snapshot, test) instead of +running the whole project as a single opaque task. + +It does this by combining two pieces: + +* **[databricks-dbt-factory](https://github.com/mwojtyczka/databricks-dbt-factory)** — a small + library that reads a dbt `manifest.json` and expands it into Databricks job tasks, wiring up + the dependencies between them. Its source is included under `src/databricks_dbt_factory/` + (see [`NOTICE`](NOTICE) for attribution and license). +* **[PyDABs](https://docs.databricks.com/dev-tools/bundles/python)** — the Databricks Asset + Bundle Python resources hook. At `databricks bundle deploy` time the Databricks CLI calls + `load_resources` in [`resources/__init__.py`](resources/__init__.py), which runs the factory + against the manifest and returns the generated job. + +The result: **no per-model job YAML is checked in**. The task graph is generated on the fly from +the dbt manifest each time you deploy. + +## Why one task per dbt object? + +By default dbt's integration with Databricks Workflows treats the whole project as a single +task — a black box. Expanding it into one task per object gives: + +* **Faster execution** — independent models run in parallel, and the notebook task type keeps + dbt's dependencies pre-cached in the serverless environment, avoiding a cold start on every task. +* **Visibility & simplified troubleshooting** — pinpoint and fix issues at the model level right + in the Databricks Workflows UI. +* **Enhanced logging & notifications** — per-task logs and precise, model-level error alerts. +* **Improved retriability** — retry only the failed model tasks without rerunning the whole project. +* **Seamless testing** — dbt data tests run as their own tasks right after each model finishes, + for faster validation and feedback. + +This example uses **serverless compute** and the **notebook task type** (each task triggers dbt +through a small runner notebook using the `dbtRunner` Python API) for the fastest task start +times. See the [databricks-dbt-factory README](https://github.com/mwojtyczka/databricks-dbt-factory#benefits) +for more. + +## How it works + +The [`dbt-factory` template](../templates/dbt-factory) scaffolds a self-contained project. +From then on, each `databricks bundle deploy` regenerates the Workflow from your current dbt +manifest — add or remove a model and the task graph follows on the next deploy, with no per-model +YAML to maintain. + +```mermaid +flowchart TD + subgraph setup["One-time setup"] + T["dbt-factory bundle template"] -->|databricks bundle init| B["Scaffolded project:
dbt project + PyDABs hook + factory code"] + X["Existing dbt project
(optional)"] -.->|move models/seeds/... into src/| B + end + subgraph deploy["Every deploy"] + C["make manifest
(dbt parse)"] --> D["target/manifest.json"] + D --> E["databricks bundle deploy"] + E --> F["PyDABs load_resources reads the
manifest and generates the job"] + end + subgraph runtime["At run time — serverless"] + G["Databricks Workflow:
one task per model / seed / snapshot / test"] --> H["Each task triggers dbt
via the runner notebook"] + H --> I[("SQL warehouse")] + end + B --> C + F --> G + + classDef optional stroke:#999,stroke-dasharray:5 4,color:#888; + class X optional; +``` + +## Project structure + +``` +dbt_factory/ +├── databricks.yml # Bundle definition; wires up the PyDABs `load_resources` hook +├── dbt_project.yml # dbt project (models under src/models, etc.) +├── dbt_profiles/profiles.yml # dbt profiles for the deployed job (dev / prod targets) +├── profile_template.yml # prompts for `dbt init` (local development) +├── resources/__init__.py # PyDABs glue: manifest -> generated job (the only integration code) +├── src/ +│ ├── models/ # your dbt models (example: orders_raw, orders_daily) +│ └── databricks_dbt_factory/ # vendored factory library (unchanged; see NOTICE) +├── target/manifest.json # committed dbt manifest, read at deploy time (regenerate with `make manifest`) +├── tests/ # tests for the vendored factory + the PyDABs integration +├── pyproject.toml # dependencies (installed into .venv via `uv sync`) +└── Makefile # convenience targets: setup, manifest, validate, deploy, run, test +``` + +## Setup + +1. Install the [Databricks CLI](https://docs.databricks.com/dev-tools/cli/databricks-cli.html) + and the [uv](https://docs.astral.sh/uv/) package manager. + +2. Authenticate to your Databricks workspace: + ``` + $ databricks configure + ``` + +3. Install dependencies into the `.venv` the bundle uses: + ``` + $ make setup # == uv sync --dev + ``` + +4. Edit `dbt_profiles/profiles.yml` and set your SQL warehouse `http_path`, `catalog`, and + `schema`. Set the workspace host in `databricks.yml` (and the prod `root_path` / permissions). + +## The dbt manifest + +`resources/__init__.py` reads `target/manifest.json` at deploy time to build the task graph. A +manifest is committed so the bundle deploys out of the box. **After you change your models, +regenerate it:** + +``` +$ make manifest # == uv run dbt deps && uv run dbt parse +``` + +`dbt parse` only reads your project files; it does not connect to a warehouse. The manifest +location is configurable — point at a different file via the `DBT_MANIFEST_PATH` environment +variable or by editing `MANIFEST_PATH` in `resources/__init__.py`. + +> **Large projects with many parallel tasks.** At runtime each task runs dbt from the shared +> project directory and writes dbt's artifacts (`target/`, `logs/`) there, which can contend +> under high parallelism. To avoid this, generate a `target/partial_parse.msgpack` (a local +> `dbt parse` produces it next to the manifest) and ship it with the bundle — it's `.gitignore`d +> by default, so force-add it (`git add -f target/partial_parse.msgpack`). The runner notebook +> then routes each task's artifacts to a private temp dir and skips re-parsing. See the +> databricks-dbt-factory README, "Faster parsing on large projects". + +## Deploy and run + +``` +$ databricks bundle deploy --target dev # or: make deploy +$ databricks bundle run dbt_factory_job # or: make run +``` + +Open the run URL the CLI prints to watch the generated per-model task graph execute. Deploying +in `dev` mode prefixes resources with `[dev your_name]` and pauses the daily schedule; deploy to +`prod` with `--target prod`. + +## Configuring the generated job + +A few knobs are exposed as constants at the top of `resources/__init__.py`: + +* `BUNDLE_TESTS` — when `True`, single-model tests are bundled into one `dbt test` task per + resource (fewer task startups; faster for test-heavy projects). Default `False` (one task per + test node, for maximum per-test visibility). +* `ENVIRONMENT_KEY` — the serverless environment key (default `Default`). +* `EXTRA_DBT_COMMAND_OPTIONS` — extra options appended to every generated dbt command. + +The dbt target, warehouse, catalog, and schema are configured in `dbt_profiles/profiles.yml` +and selected per bundle target via `--target ${bundle.target}`. + +## Migrating an existing dbt project + +Bring your own dbt project by **generating a fresh project from the template and moving your dbt +files into it.** You don't touch dependencies, the vendored factory, or any paths — the generated +project already ships all of that. + +1. Generate a new project (or copy this `dbt_factory` example): + + ``` + $ databricks bundle init https://github.com/databricks/bundle-examples --template-dir contrib/templates/dbt-factory + ``` + +2. Remove the starter models and copy your dbt sources into the matching `src/` subdirectories: + + ``` + $ rm -r src/models/example + # Copy whichever of these your project has (skip the ones you don't use): + $ cp -R /path/to/your/dbt/models/* src/models/ + $ cp -R /path/to/your/dbt/seeds/* src/seeds/ + $ cp -R /path/to/your/dbt/snapshots/* src/snapshots/ + $ cp -R /path/to/your/dbt/macros/* src/macros/ + $ cp -R /path/to/your/dbt/tests/* src/tests/ + ``` + + The generated `dbt_project.yml` already points `model-paths`, `seed-paths`, etc. at these + `src/` folders, so your files are picked up as-is. Merge any model/seed configuration from your + own `dbt_project.yml` into the generated one (keep the generated `name`/`profile`), and remove + the leftover `models: dbt_factory: example:` block that referenced the deleted starter models — + otherwise `dbt parse` warns that those config paths don't apply to any resource. If you use dbt + packages, copy your `packages.yml` to the project root too. + +3. Point `dbt_profiles/profiles.yml` at your warehouse (`http_path`, `catalog`, `schema`). Leave + the `host`/`token` lines as they are — the runner notebook sets those at runtime. + +4. Generate the manifest and deploy: + + ``` + $ make setup + $ make manifest # dbt parse -> target/manifest.json + $ databricks bundle deploy --target dev + ``` + +That's the whole migration: no dependency wrangling and no path edits, because your project keeps +the generated layout (dbt project at the bundle root, factory under `src/`). If you'd rather keep +your project's existing directory structure instead of `src/`, edit the `*-paths` in +`dbt_project.yml` to point at your folders — nothing else changes. + +## Tests + +``` +$ make test # == uv run pytest tests +``` + +This runs the vendored factory's own test suite (proving the vendored core is intact) plus an +offline test that exercises the PyDABs integration against the committed manifest — no workspace +required. + +## Local development with dbt + +You can still develop the dbt project locally with the dbt CLI. Initialize your own profile with +`dbt init` (see `profile_template.yml`), then use `dbt run`, `dbt test`, etc. as usual. See the +[`dbt_sql`](../../dbt_sql) example for a more detailed local-dbt walkthrough. diff --git a/contrib/dbt_factory/databricks.yml b/contrib/dbt_factory/databricks.yml new file mode 100644 index 00000000..51c69a96 --- /dev/null +++ b/contrib/dbt_factory/databricks.yml @@ -0,0 +1,39 @@ +# This is a Databricks Asset Bundle definition for dbt_factory. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +# +# The job for this bundle is NOT defined in YAML. Instead it is generated at deploy time +# from the dbt manifest by resources/__init__.py (see the `python.resources` hook below), +# producing one Databricks task per dbt object (model / seed / snapshot / test). +bundle: + name: dbt_factory + uuid: 19ecc815-cff0-449c-91c1-e68239d49ccb + +# PyDABs: the Databricks CLI calls `load_resources` during `bundle deploy` to build resources +# defined in Python. See resources/__init__.py. +python: + venv_path: .venv + resources: + - "resources:load_resources" + +# Deployment targets. The dbt target is selected via `--target ${bundle.target}`, so these +# names match the dbt outputs in dbt_profiles/profiles.yml. +targets: + dev: + # The default target uses 'mode: development' to create a development copy. + # - Deployed resources get prefixed with '[dev my_user_name]' + # - Any job schedules and triggers are paused by default. + # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. + mode: development + default: true + workspace: + host: https://company.databricks.com + + prod: + mode: production + workspace: + host: https://company.databricks.com + # We explicitly deploy to /Workspace/Users/user@company.com to make sure we only have a single copy. + root_path: /Workspace/Users/user@company.com/.bundle/${bundle.name}/${bundle.target} + permissions: + - user_name: user@company.com + level: CAN_MANAGE diff --git a/contrib/dbt_factory/dbt_profiles/profiles.yml b/contrib/dbt_factory/dbt_profiles/profiles.yml new file mode 100644 index 00000000..46e604f1 --- /dev/null +++ b/contrib/dbt_factory/dbt_profiles/profiles.yml @@ -0,0 +1,37 @@ + +# This file defines dbt profiles for deployed dbt jobs. +# The generated Databricks job selects the target via `--target ${bundle.target}`, +# so the dbt target names below (dev / prod) match the bundle targets in databricks.yml. +dbt_factory: + target: dev # default target + outputs: + + # Doing local development with the dbt CLI? + # Then you should create your own profile in your .dbt/profiles.yml using 'dbt init' + # (See README.md) + + # The default target when deployed with the Databricks CLI + dev: + type: databricks + method: http + catalog: catalog + schema: default + + http_path: /sql/1.0/warehouses/abcdef1234567890 + + # The workspace host / token are provided by the runner notebook at runtime + # (see src/databricks_dbt_factory/notebook/run_dbt_command.py). + host: "{{ env_var('DBT_HOST', '') }}" + token: "{{ env_var('DBT_ACCESS_TOKEN', '') }}" + + # The production target when deployed with the Databricks CLI + prod: + type: databricks + method: http + catalog: catalog + schema: default + + http_path: /sql/1.0/warehouses/abcdef1234567890 + + host: "{{ env_var('DBT_HOST', '') }}" + token: "{{ env_var('DBT_ACCESS_TOKEN', '') }}" diff --git a/contrib/dbt_factory/dbt_project.yml b/contrib/dbt_factory/dbt_project.yml new file mode 100644 index 00000000..63de848b --- /dev/null +++ b/contrib/dbt_factory/dbt_project.yml @@ -0,0 +1,28 @@ +name: 'dbt_factory' +version: '1.0.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. +profile: 'dbt_factory' + +# These configurations specify where dbt should look for different types of files. +# Everything dbt-related lives under src/ so the project can also hold non-dbt resources +# (such as the vendored databricks_dbt_factory library under src/databricks_dbt_factory). +model-paths: ["src/models"] +analysis-paths: ["src/analyses"] +test-paths: ["src/tests"] +seed-paths: ["src/seeds"] +macro-paths: ["src/macros"] +snapshot-paths: ["src/snapshots"] + +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + +# In this example config, we tell dbt to build all models in the example/ +# directory as views by default. These settings can be overridden in the +# individual model files using the `{{ config(...) }}` macro. +models: + dbt_factory: + example: + +materialized: view diff --git a/contrib/dbt_factory/profile_template.yml b/contrib/dbt_factory/profile_template.yml new file mode 100644 index 00000000..f26ec0b3 --- /dev/null +++ b/contrib/dbt_factory/profile_template.yml @@ -0,0 +1,23 @@ +# This file defines prompts with defaults for dbt initialization. +# It is used when the `dbt init` command is invoked. +# +fixed: + type: databricks +prompts: + host: + default: company.databricks.com + token: + hint: 'personal access token to use, dapiXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' + hide_input: true + http_path: + hint: 'HTTP path of SQL warehouse to use' + default: /sql/1.0/warehouses/abcdef1234567890 + catalog: + hint: 'initial catalog' + default: catalog + schema: + hint: 'personal schema where dbt will build objects during development, example: user_name' + threads: + hint: 'threads to use during development, 1 or more' + type: 'int' + default: 4 diff --git a/contrib/dbt_factory/pyproject.toml b/contrib/dbt_factory/pyproject.toml new file mode 100644 index 00000000..ea8e586a --- /dev/null +++ b/contrib/dbt_factory/pyproject.toml @@ -0,0 +1,31 @@ +[project] +name = "dbt_factory" +version = "0.0.1" +requires-python = ">=3.10,<3.13" +# Runtime dependency of the vendored databricks_dbt_factory core (read by load_resources +# during `bundle deploy`). +dependencies = [ + "PyYAML~=6.0.1", +] + +[dependency-groups] +dev = [ + # Required by the PyDABs hook (resources/__init__.py) at `bundle deploy` time. + "databricks-bundles==0.295.0", + # Used locally to regenerate the dbt manifest (`make manifest`). At runtime dbt is + # installed into the serverless environment (see resources/__init__.py). + "dbt-databricks>=1.8.0,<2.0.0", + "pytest", + "ruff", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +# Only the vendored library is packaged; the dbt project (src/models, ...) is not a package. +[tool.hatch.build.targets.wheel] +packages = ["src/databricks_dbt_factory"] + +[tool.ruff] +line-length = 120 diff --git a/contrib/dbt_factory/resources/__init__.py b/contrib/dbt_factory/resources/__init__.py new file mode 100644 index 00000000..450fecdf --- /dev/null +++ b/contrib/dbt_factory/resources/__init__.py @@ -0,0 +1,122 @@ +""" +PyDABs integration for databricks-dbt-factory. + +At ``databricks bundle deploy`` time the Databricks CLI calls :func:`load_resources`, which +reads the dbt manifest and expands it into a Databricks job with one task per dbt object +(model / seed / snapshot / test) using the vendored ``databricks_dbt_factory`` core. No +per-model YAML is generated or checked in — the job graph is built on the fly from the manifest. + +This module is the only integration glue; the vendored core under ``src/databricks_dbt_factory`` +is unchanged. It mirrors the wiring in ``databricks_dbt_factory.main`` but is fixed to the +options this example exposes (serverless + notebook task type) and returns a ``Resources`` +object instead of writing a job-spec YAML file. +""" + +import os + +from databricks.bundles.core import Bundle, Resources +from databricks.bundles.jobs import Job + +from databricks_dbt_factory.DbtFactory import DbtFactory +from databricks_dbt_factory.SpecsHandler import SpecsHandler +from databricks_dbt_factory.DbtTask import DbtTaskOptions, TaskType +from databricks_dbt_factory.TaskFactory import ( + DbtDependencyResolver, + ModelTaskFactory, + SnapshotTaskFactory, + SeedTaskFactory, + TestTaskFactory, +) + +# --- Configuration ------------------------------------------------------------------------- + +# Path to the dbt manifest, read at deploy time. Regenerate with `make manifest` (dbt parse). +# Override the location without editing this file via the DBT_MANIFEST_PATH env var. +MANIFEST_PATH = os.environ.get("DBT_MANIFEST_PATH", "target/manifest.json") + +# Name of the generated Databricks job. +JOB_NAME = "dbt_factory_job" + +# Key of the serverless environment (defined on the job below). +ENVIRONMENT_KEY = "Default" + +# Bundle single-model tests per resource into one `dbt test` task (fewer task startups, +# faster end-to-end runtime for projects with many tests). See the databricks-dbt-factory +# README "DBT Tests handling" section for the trade-offs. +BUNDLE_TESTS = False + +# Extra options appended to every generated dbt command (e.g. "--vars '{...}'"). Empty by default. +EXTRA_DBT_COMMAND_OPTIONS = "" + +# dbt-databricks version installed into the serverless environment at runtime. +DBT_DATABRICKS_DEPENDENCY = "dbt-databricks>=1.8.0,<2.0.0" + +# The runner notebook shipped with the core, referenced in place (path relative to the bundle +# root). `PROJECT_DIRECTORY` is the path from the notebook's own directory back up to the dbt +# project root (== bundle root), where the runner changes directory to before running dbt. It is +# derived from the notebook path so the two stay consistent if the layout ever changes. +RUNNER_NOTEBOOK_PATH = "src/databricks_dbt_factory/notebook/run_dbt_command.py" +PROJECT_DIRECTORY = os.path.relpath(".", os.path.dirname(RUNNER_NOTEBOOK_PATH)) +PROFILES_DIRECTORY = "dbt_profiles" + + +def _build_tasks(target: str) -> list[dict]: + """Reads the dbt manifest and returns the list of Databricks task dicts (one per dbt node).""" + resolver = DbtDependencyResolver() + task_options = DbtTaskOptions( + task_type=TaskType.NOTEBOOK, + environment_key=ENVIRONMENT_KEY, + notebook_path=RUNNER_NOTEBOOK_PATH, + project_directory=PROJECT_DIRECTORY, + profiles_directory=PROFILES_DIRECTORY, + ) + dbt_options = f"--target {target} {EXTRA_DBT_COMMAND_OPTIONS}".strip() + + task_factories = { + "model": ModelTaskFactory(resolver, task_options, dbt_options), + "snapshot": SnapshotTaskFactory(resolver, task_options, dbt_options), + "seed": SeedTaskFactory(resolver, task_options, dbt_options), + "test": TestTaskFactory(resolver, task_options, dbt_options), + } + + handler = SpecsHandler() + factory = DbtFactory(handler, task_factories, bundle_tests=BUNDLE_TESTS) + manifest = handler.read_dbt_manifest(MANIFEST_PATH) + return factory.create_tasks(manifest) + + +def build_job(target: str) -> Job: + """Builds the Databricks job that runs the dbt project on serverless via notebook tasks.""" + return Job.from_dict( + { + "name": JOB_NAME, + "queue": {"enabled": True}, + "trigger": { + # Run this job every day, exactly one day from the last run; + # see https://docs.databricks.com/api/workspace/jobs/create#trigger + "periodic": {"interval": 1, "unit": "DAYS"}, + }, + "tasks": _build_tasks(target), + "environments": [ + { + "environment_key": ENVIRONMENT_KEY, + "spec": { + "environment_version": "4", + "dependencies": [DBT_DATABRICKS_DEPENDENCY], + }, + } + ], + } + ) + + +def load_resources(bundle: Bundle) -> Resources: + """ + Called by the Databricks CLI during `bundle deploy` to load resources defined in Python. + + Reads the dbt manifest and registers the generated job. After deployment this function is + not used. + """ + resources = Resources() + resources.add_job(JOB_NAME, build_job(bundle.target)) + return resources diff --git a/contrib/dbt_factory/src/analyses/.gitkeep b/contrib/dbt_factory/src/analyses/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/dbt_factory/src/databricks_dbt_factory/DbtFactory.py b/contrib/dbt_factory/src/databricks_dbt_factory/DbtFactory.py new file mode 100644 index 00000000..5d7e2d20 --- /dev/null +++ b/contrib/dbt_factory/src/databricks_dbt_factory/DbtFactory.py @@ -0,0 +1,322 @@ +from dataclasses import replace + +from databricks_dbt_factory import TaskFactory +from databricks_dbt_factory.SpecsHandler import SpecsHandler +from databricks_dbt_factory.DbtTask import DbtTask +from databricks_dbt_factory.Utils import generate_task_key + + +class DbtFactory: + """A factory for generating Databricks job definitions from dbt manifests.""" + + def __init__( + self, + file_handler: SpecsHandler, + task_factories: dict[str, TaskFactory], + bundle_tests: bool = False, + ): + """ + Initializes the dbt factory. + + Args: + file_handler (SpecsHandler): Handles reading the dbt manifest and writing the job spec. + task_factories (dict[str, TaskFactory]): Maps dbt resource types (`model`, `seed`, + `snapshot`, `test`) to their respective `TaskFactory` instances. Omitting `test` + disables test-task generation entirely. + bundle_tests (bool): When True, emit one `tests_` task per tested resource + and rewire downstream models/seeds/snapshots to depend on the upstream's + `tests_` task so failing tests halt the DAG. When False, emit one task + per dbt test node. + """ + self.file_handler = file_handler + self.task_factories = task_factories + self.bundle_tests = bundle_tests + + def create_tasks_and_update_job_spec( + self, + dbt_manifest_path: str, + input_job_spec_path: str, + target_job_spec_path: str, + new_job_name: str | None = None, + dry_run: bool = False, + ): + """ + Generates tasks from a dbt manifest and writes them into a Databricks job spec. + + Args: + dbt_manifest_path (str): Path to the dbt `manifest.json`. + input_job_spec_path (str): Path to the input (template) job spec YAML. + target_job_spec_path (str): Path to write the updated job spec YAML. + new_job_name (str | None): Optional replacement job name. If provided, overrides the + name in the input spec. + dry_run (bool): When True, print the generated tasks instead of writing to disk. + """ + manifest = self.file_handler.read_dbt_manifest(dbt_manifest_path) + tasks = self.create_tasks(manifest) + if dry_run: + print(tasks) + else: + self.file_handler.replace_tasks_in_job_spec(input_job_spec_path, tasks, target_job_spec_path, new_job_name) + + def create_tasks(self, dbt_manifest: dict) -> list[dict]: + """ + Generates the Databricks task dictionaries from a dbt manifest. + + Args: + dbt_manifest (dict): Parsed dbt manifest content. + + Returns: + list[dict]: Task dictionaries ready to be injected into the `tasks` list of a + Databricks job spec. + """ + tasks = self._create_tasks(dbt_manifest) + return [task.to_dict() for task in tasks] + + _GATEABLE_TYPES = frozenset({"model", "seed", "snapshot"}) + _DBT_TEST_TARGET_PREFIXES = ("model.", "seed.", "snapshot.", "source.") + + def _create_tasks(self, dbt_manifest: dict) -> list[DbtTask]: + """ + Builds `DbtTask` instances from the manifest, applying the bundling and gating policies. + + Args: + dbt_manifest (dict): Parsed dbt manifest content. + + Returns: + list[DbtTask]: `DbtTask` instances (not yet rendered to dicts). + """ + dbt_nodes = dbt_manifest.get("nodes", {}) + dbt_sources = dbt_manifest.get("sources", {}) + + bundle = "test" in self.task_factories and self.bundle_tests + single_model_tested: set[str] = set() + standalone_tests: list[tuple[str, dict]] = [] + tests_by_resource: dict[str, list[tuple[str, frozenset[str]]]] = {} + ancestors: dict[str, set[str]] = {} + if bundle: + single_model_tested, standalone_tests = self._classify_tests(dbt_nodes, dbt_sources) + elif "test" in self.task_factories: + tests_by_resource = self._index_tests_by_resource(dbt_nodes, dbt_sources) + ancestors = self._compute_ancestors(dbt_nodes, dbt_sources) + task_keys_with_tests = {generate_task_key(fn) for fn in single_model_tested} + + tasks = self._build_resource_tasks(dbt_nodes, bundle, task_keys_with_tests, tests_by_resource, ancestors) + + if bundle: + tasks.extend(self._build_bundled_test_tasks(dbt_nodes, dbt_sources, single_model_tested)) + tasks.extend(self._build_standalone_test_tasks(standalone_tests)) + + return tasks + + def _compute_ancestors(self, dbt_nodes: dict, dbt_sources: dict) -> dict[str, set[str]]: + """ + Maps each testable resource's full name to the set of resources it transitively depends + on (not including itself). Used in per-test mode to decide whether a test can safely + gate a downstream node: a test `T` with refs `R` is only safe to add to node `N`'s + deps if `R ⊆ ancestors(N)` — i.e. `N` already waits for all of `T`'s endpoints, + transitively. Otherwise adding `T` would create a cycle (since `T` depends on each + ref, and some ref might depend on `N`). + """ + ancestors: dict[str, set[str]] = {} + + def visit(full_name: str) -> set[str]: + cached = ancestors.get(full_name) + if cached is not None: + return cached + result: set[str] = set() + info = dbt_nodes.get(full_name) or dbt_sources.get(full_name) + if info is not None: + for dep in info.get("depends_on", {}).get("nodes", []): + if dep in dbt_nodes or dep in dbt_sources: + result.add(dep) + result.update(visit(dep)) + ancestors[full_name] = result + return result + + for full_name in list(dbt_nodes.keys()) + list(dbt_sources.keys()): + visit(full_name) + return ancestors + + def _index_tests_by_resource( + self, dbt_nodes: dict, dbt_sources: dict + ) -> dict[str, list[tuple[str, frozenset[str]]]]: + """ + Maps each testable resource's full name to a list of (test_task_key, test_refs) pairs + for tests whose `severity` is `error` (the default). Warn-severity tests still run but + are NOT indexed here, so they do not appear in any downstream model's `depends_on` — + their job is to surface findings, not halt the DAG. This matches `dbt build` semantics: + dbt itself exits 0 on warn-severity failures, so even if we did gate on them the + Databricks task would succeed and downstream would run; keeping warn tests out of the + dep graph just avoids the extra DAG clutter. + + The refs set is carried alongside each test so `_extend_deps_with_upstream_tests` can + avoid cycles: a test with refs that aren't all ancestors of a candidate node would + create a cycle if added as that node's dep. + """ + index: dict[str, list[tuple[str, frozenset[str]]]] = {} + for node_full_name, node_info in dbt_nodes.items(): + if node_info["resource_type"] != "test": + continue + if self._test_severity(node_info) != "error": + continue + test_task_key = generate_task_key(node_full_name) + refs: set[str] = set() + for dep in node_info.get("depends_on", {}).get("nodes", []): + if dep.startswith(self._DBT_TEST_TARGET_PREFIXES) and (dep in dbt_nodes or dep in dbt_sources): + refs.add(dep) + frozen_refs = frozenset(refs) + for resource_full in refs: + index.setdefault(resource_full, []).append((test_task_key, frozen_refs)) + return index + + @staticmethod + def _test_severity(test_node_info: dict) -> str: + """Reads the test's severity from the manifest, defaulting to `error` when unset.""" + config = test_node_info.get("config") or {} + severity = config.get("severity") + if isinstance(severity, str): + return severity.lower() + return "error" + + @staticmethod + def _extend_deps_with_upstream_tests( + node_full_name: str, + existing_deps: list[str] | None, + tests_by_resource: dict[str, list[tuple[str, frozenset[str]]]], + ancestors_by_node: dict[str, set[str]], + ) -> list[str]: + """ + Appends task keys of tests that safely gate this node — i.e. tests whose refs are all + ancestors of the current node. This prevents both direct and transitive cycles: a test + `T` with refs `R` is added to node `N`'s deps only if `N` transitively depends on every + resource in `R`. If any ref of `T` is downstream of (or equal to) `N`, adding `T` would + cycle because `T` already depends on that ref, and the ref depends on `N`. + """ + extended: list[str] = list(existing_deps or []) + seen = set(extended) + node_ancestors = ancestors_by_node.get(node_full_name, set()) + for ancestor in node_ancestors: + for test_key, test_refs in tests_by_resource.get(ancestor, []): + if test_key in seen: + continue + if test_refs <= node_ancestors: + extended.append(test_key) + seen.add(test_key) + return extended + + def _classify_tests(self, dbt_nodes: dict, dbt_sources: dict) -> tuple[set[str], list[tuple[str, dict]]]: + """ + Classifies test nodes for bundled mode so that no test is silently dropped. + + - Tests with exactly 1 testable dep: will be covered by their resource's bundled + `tests_` task under `--indirect-selection cautious`. + - Tests with >1 testable deps (cross-model, e.g. `relationships`): emitted as their own + tasks with multi-resource deps — `cautious` filters them out of bundles. + - Tests with 0 testable deps (singular/custom tests that don't `ref()` or `source()` + any resource): also emitted as their own tasks, since no bundle would pick them up. + + Returns: + (single_model_tested, standalone_tests): + - `single_model_tested`: full names of resources with at least one single-model + test — these become `tests_` bundled tasks. + - `standalone_tests`: list of `(test_full_name, test_node_info)` for tests + that must run as individual tasks (cross-model or zero-dep). + """ + single_model_tested: set[str] = set() + standalone_tests: list[tuple[str, dict]] = [] + for node_full_name, node_info in dbt_nodes.items(): + if node_info["resource_type"] != "test": + continue + testable_deps: list[str] = [] + for dep in node_info.get("depends_on", {}).get("nodes", []): + if dep.startswith(self._DBT_TEST_TARGET_PREFIXES) and (dep in dbt_nodes or dep in dbt_sources): + testable_deps.append(dep) + if len(testable_deps) == 1: + single_model_tested.add(testable_deps[0]) + else: + standalone_tests.append((node_full_name, node_info)) + return single_model_tested, standalone_tests + + def _build_resource_tasks( + self, + dbt_nodes: dict, + bundle: bool, + task_keys_with_tests: set[str], + tests_by_resource: dict[str, list[tuple[str, frozenset[str]]]], + ancestors_by_node: dict[str, set[str]], + ) -> list[DbtTask]: + """Builds tasks for every non-test resource (plus per-test tasks when not bundling).""" + tasks: list[DbtTask] = [] + for node_full_name, node_info in dbt_nodes.items(): + resource_type = node_info["resource_type"] + if resource_type not in self.task_factories: + continue + if bundle and resource_type == "test": + continue + + task_key = generate_task_key(node_full_name) + factory = self.task_factories[resource_type] + task = factory.create_task(node_info["name"], node_info, task_key) + + if resource_type in self._GATEABLE_TYPES: + if bundle: + task = replace(task, depends_on=self._rewire_deps(task.depends_on, task_keys_with_tests)) + elif tests_by_resource: + task = replace( + task, + depends_on=self._extend_deps_with_upstream_tests( + node_full_name, task.depends_on, tests_by_resource, ancestors_by_node + ), + ) + + tasks.append(task) + return tasks + + @staticmethod + def _rewire_deps(deps: list[str] | None, task_keys_with_tests: set[str]) -> list[str]: + """Rewrites dependencies that point at a tested resource to its `tests_` gating task.""" + rewired: list[str] = [] + for dep_key in deps or []: + rewired.append(f"tests_{dep_key}" if dep_key in task_keys_with_tests else dep_key) + return rewired + + def _build_bundled_test_tasks( + self, + dbt_nodes: dict, + dbt_sources: dict, + nodes_with_tests: set[str], + ) -> list[DbtTask]: + """Emits one `tests_` task per tested resource using `TestTaskFactory.create_bundled_task`.""" + test_factory = self.task_factories["test"] + tasks: list[DbtTask] = [] + for full_name in sorted(nodes_with_tests): + is_source = full_name.startswith("source.") + info = dbt_sources[full_name] if is_source else dbt_nodes[full_name] + resource_task_key = generate_task_key(full_name) + bare_name = info["name"] + qualified = f"{info['package_name']}.{bare_name}" + select = f"source:{info['package_name']}.{info['source_name']}.{bare_name}" if is_source else qualified + tasks.append( + test_factory.create_bundled_task( + task_key=f"tests_{resource_task_key}", + select=select, + deps_command_name=bare_name, + depends_on=[] if is_source else [resource_task_key], + ) + ) + return tasks + + def _build_standalone_test_tasks( + self, + standalone_tests: list[tuple[str, dict]], + ) -> list[DbtTask]: + """ + Emits one task per standalone test — cross-model tests (e.g. `relationships`) gated on + every referenced resource, plus any zero-dep singular tests that bundles can't cover. + """ + test_factory = self.task_factories["test"] + tasks: list[DbtTask] = [] + for test_full_name, test_info in sorted(standalone_tests, key=lambda item: item[0]): + test_task_key = generate_task_key(test_full_name) + tasks.append(test_factory.create_task(test_info["name"], test_info, test_task_key)) + return tasks diff --git a/contrib/dbt_factory/src/databricks_dbt_factory/DbtTask.py b/contrib/dbt_factory/src/databricks_dbt_factory/DbtTask.py new file mode 100644 index 00000000..c0523720 --- /dev/null +++ b/contrib/dbt_factory/src/databricks_dbt_factory/DbtTask.py @@ -0,0 +1,150 @@ +import json +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + + +class TaskType(Enum): + """Supported task types for generated Databricks tasks.""" + + DBT = "dbt" + NOTEBOOK = "notebook" + + +@dataclass(frozen=True) +class DbtTaskOptions: + environment_key: str = "Default" # serverless env + """The key of an environment. It has to be unique within a job.""" + + catalog: str | None = None + """Optional name of the catalog to use. The value is the top level in the 3-level namespace of + Unity Catalog (catalog / schema / relation). The catalog value can only be specified if a + warehouse_id is specified. Requires dbt-databricks >= 1.1.1.""" + + profiles_directory: str | None = None + """Optional (relative) path to the profiles directory. Can only be specified if no warehouse_id is + specified. If no warehouse_id is specified and this folder is unset, the root directory is used.""" + + project_directory: str | None = None + """Path to the project directory. Optional for Git sourced tasks, in which case if no value is + provided, the root of the Git repository is used.""" + + schema: str | None = None + """Optional schema to write to. This parameter is only used when a warehouse_id is also provided. + If not provided, the `default` schema is used.""" + + source: str | None = None + """Optional location type of the project directory. When set to `WORKSPACE`, the project will be + retrieved from the local Databricks workspace. When set to `GIT`, the project will be retrieved + from a Git repository defined in `git_source`. If the value is empty, the task will use `GIT` if + `git_source` is defined and `WORKSPACE` otherwise. + + * `WORKSPACE`: Project is located in Databricks workspace. * `GIT`: Project is located in cloud + Git provider.""" + + warehouse_id: str | None = None + """ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the + profile and connection details to dbt. It can be overridden on a per-command basis by using the + `--profiles-dir` command line argument.""" + + dbt_deps_enabled: bool = False + """Optional flag to enable dbt deps to be run before each task. Defaults to False.""" + + dbt_tasks_deps: list[str] = field(default_factory=list) + """Optional comma separated list of tasks that requires dbt debs. Only in effect if dbt_deps_enabled is enabled.""" + + task_type: TaskType = TaskType.DBT + """Task type to generate: `TaskType.DBT` for native dbt_task, `TaskType.NOTEBOOK` for notebook_task + wrapper. Strings are accepted and coerced — any value outside the enum raises `ValueError`. + Notebook mode enables base environment support and environment variables on serverless.""" + + notebook_path: str | None = None + """Path to the dbt runner notebook. Required when task_type is `TaskType.NOTEBOOK`.""" + + job_cluster_key: str | None = None + """Job cluster key for running tasks on job compute instead of serverless.""" + + def __post_init__(self): + if not isinstance(self.task_type, TaskType): + object.__setattr__(self, "task_type", TaskType(self.task_type)) + if self.task_type is TaskType.NOTEBOOK: + unsupported = [] + for name, value in ( + ("warehouse_id", self.warehouse_id), + ("schema", self.schema), + ("catalog", self.catalog), + ): + if value: + unsupported.append(name) + if unsupported: + raise ValueError( + f"{', '.join(unsupported)} cannot be set with task_type=NOTEBOOK; " + "notebook tasks connect via profiles.yml." + ) + + +@dataclass(frozen=True) +class DbtTask: + """Represents a dbt task in the Databricks job definition.""" + + task_key: str + commands: list[str] + options: DbtTaskOptions + depends_on: list[str] | None = None + + def to_dict(self) -> dict: + """Converts the Task to a dictionary suitable for the job definition.""" + if self.options.task_type is TaskType.NOTEBOOK: + return self._to_notebook_dict() + return self._to_dbt_dict() + + def _base_spec(self) -> dict[str, Any]: + spec: dict[str, Any] = { + "task_key": self.task_key, + "depends_on": [{"task_key": dep} for dep in (self.depends_on or [])], + } + if self.options.job_cluster_key: + spec["job_cluster_key"] = self.options.job_cluster_key + else: + spec["environment_key"] = self.options.environment_key + return spec + + def _to_dbt_dict(self) -> dict[str, Any]: + spec = self._base_spec() + dbt_task: dict[str, Any] = {"commands": self.commands} + + if self.options.source: + dbt_task["source"] = self.options.source + if self.options.project_directory: + dbt_task["project_directory"] = self.options.project_directory + if self.options.schema: + dbt_task["schema"] = self.options.schema + if self.options.warehouse_id: + dbt_task["warehouse_id"] = self.options.warehouse_id + if self.options.catalog: + dbt_task["catalog"] = self.options.catalog + if self.options.profiles_directory: + dbt_task["profiles_directory"] = self.options.profiles_directory + + spec["dbt_task"] = dbt_task + return spec + + def _to_notebook_dict(self) -> dict[str, Any]: + base_parameters: dict[str, str] = { + "dbt_commands": json.dumps(self.commands), + } + if self.options.project_directory: + base_parameters["project_directory"] = self.options.project_directory + if self.options.profiles_directory: + base_parameters["profiles_directory"] = self.options.profiles_directory + + notebook_task: dict[str, Any] = { + "notebook_path": self.options.notebook_path, + "base_parameters": base_parameters, + } + if self.options.source: + notebook_task["source"] = self.options.source + + spec = self._base_spec() + spec["notebook_task"] = notebook_task + return spec diff --git a/contrib/dbt_factory/src/databricks_dbt_factory/SpecsHandler.py b/contrib/dbt_factory/src/databricks_dbt_factory/SpecsHandler.py new file mode 100644 index 00000000..fd9e72a4 --- /dev/null +++ b/contrib/dbt_factory/src/databricks_dbt_factory/SpecsHandler.py @@ -0,0 +1,68 @@ +import json +import yaml + + +class SpecsHandler: + """Handles reading and writing files for dbt manifests and databricks job definitions.""" + + @staticmethod + def read_dbt_manifest(path: str) -> dict: + """ + Reads a JSON manifest file and returns its content as a dictionary. + + Args: + path (str): Path to the manifest file. + + Returns: + dict: Parsed content of the manifest file. + + Raises: + FileNotFoundError: If the file does not exist. + ValueError: If the file is not a valid manifest file. + """ + try: + with open(path, "r", encoding="utf-8") as file: + return json.load(file) + except FileNotFoundError as e: + raise FileNotFoundError(f"Manifest file not found: {path}. Details: {e}") from e + except json.JSONDecodeError as e: + raise ValueError(f"Error parsing JSON from manifest file: {path}. Details: {e}") from e + + @staticmethod + def replace_tasks_in_job_spec( + input_job_spec_path: str, + new_tasks: list[dict], + target_job_spec_path: str, + new_job_name: str | None = None, + ) -> None: + """Replace the tasks field in a Databricks job definition YAML file. The first job is only updated. + Args: + input_job_spec_path (str): Path to the job definition YAML file. + new_tasks (dict): New tasks to replace the existing tasks in the job definition file. + target_job_spec_path (str): Path to save the updated job definition file. + new_job_name (str, optional): The name of the job to update. Defaults to None. + + Raises: + KeyError: If no jobs are found in the provided YAML file. + """ + with open(input_job_spec_path, "r", encoding="utf-8") as file: + job_definition = yaml.safe_load(file) + + jobs = job_definition.get("resources", {}).get("jobs", {}) + + if jobs is None: + raise KeyError("No jobs found in the provided YAML file.") + + # replaces the first job only! + first_job_key = next(iter(jobs)) + if new_job_name: + jobs[new_job_name] = jobs.pop(first_job_key) + first_job_key = new_job_name + + first_job = jobs[first_job_key] + if new_job_name: + first_job["name"] = new_job_name + first_job["tasks"] = new_tasks # Replace tasks field + + with open(target_job_spec_path, "w", encoding="utf-8") as file: + yaml.dump(job_definition, file, sort_keys=False, width=1000) diff --git a/contrib/dbt_factory/src/databricks_dbt_factory/TaskFactory.py b/contrib/dbt_factory/src/databricks_dbt_factory/TaskFactory.py new file mode 100644 index 00000000..7f4378fb --- /dev/null +++ b/contrib/dbt_factory/src/databricks_dbt_factory/TaskFactory.py @@ -0,0 +1,216 @@ +from abc import ABC, abstractmethod +from enum import Enum + +from databricks_dbt_factory.DbtTask import DbtTask, DbtTaskOptions +from databricks_dbt_factory.Utils import generate_task_key + + +class DbtNodeTypes(Enum): + """Enum class to represent dbt node types.""" + + MODEL = "model" + TEST = "test" + SEED = "seed" + SNAPSHOT = "snapshot" + + +class DbtDependencyResolver: + @staticmethod + def resolve(node_info: dict, valid_deps_types: list[str]) -> list[str]: + """ + Resolves dependencies for a given DBT node. + + Args: + node_info (dict): Information about the DBT node. + valid_deps_types (list[str]): List of valid DBT dependency types for the node. + + Returns: + list[str]: List of resolved dependencies. + """ + deps = node_info.get("depends_on", {}).get("nodes", []) + resolved_deps = [] + for node_full_name in deps: + if any(node_full_name.startswith(dbt_type + ".") for dbt_type in valid_deps_types): + resolved_deps.append(generate_task_key(node_full_name)) + return resolved_deps + + +class TaskFactory(ABC): + """Abstract base class for creating tasks.""" + + def __init__(self, resolver: DbtDependencyResolver, task_options: DbtTaskOptions, dbt_options: str = ""): + """ + Initializes the TaskFactory. + + Args: + resolver (DbtDependencyResolver): An instance of DbtDependencyResolver to resolve dependencies. + task_options (DbtTaskOptions): Options for the task. + dbt_options (str, optional): Additional DBT options. Defaults to "". + """ + self.resolver = resolver + self.task_options = task_options + self.dbt_options = dbt_options + + @abstractmethod + def create_task(self, dbt_node_name: str, dbt_node_info: dict, task_key: str) -> DbtTask: + """ + Abstract method to create a task. + + Args: + dbt_node_name (str): Name of the DBT node. + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + + Returns: + DbtTask: An instance of Task. + """ + + def get_dbt_deps_command(self, dbt_task_name: str) -> str | None: + """Adds the dbt deps command if enabled and applicable. + Only return the command if enabled, and available in the task deps or not specific tasks provided. + + Args: + dbt_task_name (str): Name of the DBT task. + """ + if self.task_options.dbt_deps_enabled and ( + not self.task_options.dbt_tasks_deps or dbt_task_name in self.task_options.dbt_tasks_deps + ): + return f"dbt deps {self.dbt_options}" + return None + + +class ModelTaskFactory(TaskFactory): + """Factory for creating model tasks.""" + + def create_task(self, dbt_node_name: str, dbt_node_info: dict, task_key: str) -> DbtTask: + """ + Creates a model task. + + Args: + dbt_node_name (str): Name of the DBT node. + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + + Returns: + DbtTask: An instance of Task. + """ + valid_dbt_deps_types: list[str] = [ + DbtNodeTypes.MODEL.value, + DbtNodeTypes.SEED.value, + DbtNodeTypes.SNAPSHOT.value, + DbtNodeTypes.TEST.value, + ] + depends_on = self.resolver.resolve(dbt_node_info, valid_dbt_deps_types) + + dbt_deps = self.get_dbt_deps_command(dbt_node_name) + commands = [dbt_deps] if dbt_deps else [] + commands.append(f"dbt run --select {dbt_node_name}" + (f" {self.dbt_options}" if self.dbt_options else "")) + + return DbtTask(task_key, commands, self.task_options, depends_on) + + +class SnapshotTaskFactory(TaskFactory): + """Factory for creating snapshot tasks.""" + + def create_task(self, dbt_node_name: str, dbt_node_info: dict, task_key: str) -> DbtTask: + """ + Creates a snapshot task. + + Args: + dbt_node_name (str): Name of the DBT node. + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + + Returns: + DbtTask: An instance of Task. + """ + valid_dbt_deps_types: list[str] = [DbtNodeTypes.MODEL.value] + depends_on = self.resolver.resolve(dbt_node_info, valid_dbt_deps_types) + + dbt_deps = self.get_dbt_deps_command(dbt_node_name) + commands = [dbt_deps] if dbt_deps else [] + commands.append(f"dbt snapshot --select {dbt_node_name}" + (f" {self.dbt_options}" if self.dbt_options else "")) + + return DbtTask(task_key, commands, self.task_options, depends_on) + + +class SeedTaskFactory(TaskFactory): + """Factory for creating seed tasks.""" + + def create_task(self, dbt_node_name: str, dbt_node_info: dict, task_key: str) -> DbtTask: + """ + Creates a seed task. + + Args: + dbt_node_name (str): Name of the DBT node. + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + + Returns: + DbtTask: An instance of Task. + """ + valid_dbt_deps_types: list[str] = [] # Seeds don't have dependencies + + depends_on = self.resolver.resolve(dbt_node_info, valid_dbt_deps_types) + + dbt_deps = self.get_dbt_deps_command(dbt_node_name) + commands = [dbt_deps] if dbt_deps else [] + commands.append(f"dbt seed --select {dbt_node_name}" + (f" {self.dbt_options}" if self.dbt_options else "")) + + return DbtTask(task_key, commands, self.task_options, depends_on) + + +class TestTaskFactory(TaskFactory): + """Factory for creating test tasks.""" + + def create_task(self, dbt_node_name: str, dbt_node_info: dict, task_key: str) -> DbtTask: + """ + Creates a test task for a single dbt test node. + + Args: + dbt_node_name (str): Name of the DBT node. + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + + Returns: + DbtTask: An instance of Task. + """ + valid_dbt_deps_types: list[str] = [ + DbtNodeTypes.MODEL.value, + DbtNodeTypes.SEED.value, + DbtNodeTypes.SNAPSHOT.value, + ] + + depends_on = self.resolver.resolve(dbt_node_info, valid_dbt_deps_types) + + dbt_deps = self.get_dbt_deps_command(dbt_node_name) + commands = [dbt_deps] if dbt_deps else [] + commands.append(f"dbt test --select {dbt_node_name}" + (f" {self.dbt_options}" if self.dbt_options else "")) + + return DbtTask(task_key, commands, self.task_options, depends_on) + + def create_bundled_task(self, task_key: str, select: str, deps_command_name: str, depends_on: list[str]) -> DbtTask: + """ + Creates a single test task that runs the single-model tests for a given resource via + `dbt test --select --indirect-selection cautious`. The cautious selector + ensures only tests whose referenced resources are entirely within this bundle are + included; cross-model tests (e.g. `relationships`) are excluded and handled separately. + + Args: + task_key (str): Key for the bundled task. + select (str): Pre-computed dbt `--select` argument (qualified model name, or + `source:..` for sources). + deps_command_name (str): Name used by `get_dbt_deps_command` to decide whether to prepend `dbt deps`. + depends_on (list[str]): Upstream task keys this bundled task should gate on. + + Returns: + DbtTask: An instance of Task. + """ + dbt_deps = self.get_dbt_deps_command(deps_command_name) + commands = [dbt_deps] if dbt_deps else [] + commands.append( + f"dbt test --select {select} --indirect-selection cautious" + + (f" {self.dbt_options}" if self.dbt_options else "") + ) + + return DbtTask(task_key, commands, self.task_options, depends_on) diff --git a/contrib/dbt_factory/src/databricks_dbt_factory/Utils.py b/contrib/dbt_factory/src/databricks_dbt_factory/Utils.py new file mode 100644 index 00000000..a29cffdd --- /dev/null +++ b/contrib/dbt_factory/src/databricks_dbt_factory/Utils.py @@ -0,0 +1,11 @@ +def generate_task_key(dbt_node_full_name: str) -> str: + """ + Generates a task key from a dbt node's fully qualified name by replacing dots with underscores. + + Args: + dbt_node_full_name (str): Fully qualified dbt node name, e.g. `model.my_project.customers`. + + Returns: + str: The generated task key (e.g. `model_my_project_customers`). + """ + return dbt_node_full_name.replace(".", "_") diff --git a/contrib/dbt_factory/src/databricks_dbt_factory/__about__.py b/contrib/dbt_factory/src/databricks_dbt_factory/__about__.py new file mode 100644 index 00000000..3ced3581 --- /dev/null +++ b/contrib/dbt_factory/src/databricks_dbt_factory/__about__.py @@ -0,0 +1 @@ +__version__ = "0.2.1" diff --git a/contrib/dbt_factory/src/databricks_dbt_factory/__init__.py b/contrib/dbt_factory/src/databricks_dbt_factory/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/contrib/dbt_factory/src/databricks_dbt_factory/main.py b/contrib/dbt_factory/src/databricks_dbt_factory/main.py new file mode 100644 index 00000000..884ad55d --- /dev/null +++ b/contrib/dbt_factory/src/databricks_dbt_factory/main.py @@ -0,0 +1,273 @@ +import argparse +import os +import shutil +from importlib import resources +from pathlib import Path + +from databricks_dbt_factory.DbtFactory import DbtFactory +from databricks_dbt_factory.SpecsHandler import SpecsHandler +from databricks_dbt_factory.DbtTask import DbtTaskOptions +from databricks_dbt_factory.TaskFactory import ( + ModelTaskFactory, + SnapshotTaskFactory, + SeedTaskFactory, + TestTaskFactory, + DbtDependencyResolver, +) + +_RUNNER_NOTEBOOK_FILENAME = "run_dbt_command.py" + + +def main(): + args = parse_args() + + file_handler = SpecsHandler() + resolver = DbtDependencyResolver() + + dbt_options = build_dbt_options(args) + dbt_tasks_deps = ( + [item.strip() for item in args.dbt_tasks_deps.split(",") if item.strip()] if args.dbt_tasks_deps else [] + ) + + notebook_path = args.notebook_path + effective_project_directory = args.project_directory + if args.task_type == "notebook" and notebook_path is None: + notebook_path, notebook_at_project_root = _copy_runner_notebook( + args.target_job_spec_path, args.project_directory + ) + # If the runner landed at the project root, CWD at task runtime already equals the + # project root. Pass `.` explicitly so the generated spec is self-documenting (and + # immune to any future change in dbt's default); the user's original `../` would + # resolve one level too high and is no longer correct. + if notebook_at_project_root: + effective_project_directory = "." + + task_options = DbtTaskOptions( + environment_key=args.environment_key if args.environment_key is not None else "Default", + warehouse_id=args.warehouse_id, + catalog=args.catalog, + schema=args.schema, + profiles_directory=args.profiles_directory, + project_directory=effective_project_directory, + source=args.source, + dbt_deps_enabled=args.enable_dbt_deps, + dbt_tasks_deps=dbt_tasks_deps, + task_type=args.task_type, + notebook_path=notebook_path, + job_cluster_key=args.job_cluster_key, + ) + task_factories = { + "model": ModelTaskFactory(resolver, task_options, dbt_options), + "snapshot": SnapshotTaskFactory(resolver, task_options, dbt_options), + "seed": SeedTaskFactory(resolver, task_options, dbt_options), + } + if args.run_tests: + task_factories["test"] = TestTaskFactory(resolver, task_options, dbt_options) + + factory = DbtFactory(file_handler, task_factories, bundle_tests=args.bundle_tests) + factory.create_tasks_and_update_job_spec( + args.dbt_manifest_path, args.input_job_spec_path, args.target_job_spec_path, args.new_job_name, args.dry_run + ) + + +def _copy_runner_notebook(target_job_spec_path: str, project_directory: str | None) -> tuple[str, bool]: + """ + Copies the packaged dbt runner notebook into the bundle so `databricks bundle deploy` + uploads it automatically. + + When `project_directory` is a relative path (the common case, e.g. `../` when the spec + lives in a subdirectory), the notebook is placed at the computed project root. This way + the notebook sits next to `dbt_project.yml` / `profiles.yml`, and at task runtime CWD = + project root — dbt finds everything without any path gymnastics in the runner. + + When `project_directory` is absolute (typical for `--source WORKSPACE` with a pinned + workspace path we can't write to from local CLI) or missing, falls back to copying the + notebook next to the generated job spec. + + Returns `(notebook_path, notebook_at_project_root)`: + - `notebook_path`: relative path from the spec's directory to the copied notebook, + which DAB resolves at deploy time. + - `notebook_at_project_root`: True when the runner landed at the computed project root + (so the caller knows CWD = project root at runtime and can drop `--project-dir`). + + Overwrites any existing file at the destination. + """ + source = resources.files("databricks_dbt_factory") / "notebook" / _RUNNER_NOTEBOOK_FILENAME + spec_dir = Path(target_job_spec_path).resolve().parent + + if project_directory and not Path(project_directory).is_absolute(): + notebook_at_project_root = True + dest_dir = (spec_dir / project_directory).resolve() + else: + notebook_at_project_root = False + dest_dir = spec_dir + + dest = dest_dir / _RUNNER_NOTEBOOK_FILENAME + dest_dir.mkdir(parents=True, exist_ok=True) + with resources.as_file(source) as src_path: + shutil.copyfile(src_path, dest) + + relative = Path(os.path.relpath(dest, start=spec_dir)).as_posix() + notebook_path = relative if relative.startswith("..") else f"./{relative}" + return notebook_path, notebook_at_project_root + + +def build_dbt_options(args): + """Builds the dbt command options based on the provided arguments.""" + dbt_options = "" + + if args.target: + dbt_options += f"--target {args.target}" + + if args.extra_dbt_command_options: + dbt_options += f" {args.extra_dbt_command_options}" + + return dbt_options + + +def parse_args(): + parser = argparse.ArgumentParser(description="Generate Databricks job definition from dbt manifest.") + parser.add_argument( + "--new-job-name", + type=str, + help="Optional job name. If provided the existing job name in job spec is updated", + required=False, + default=None, + ) + parser.add_argument("--dbt-manifest-path", type=str, help="Path to the manifest file", required=True) + parser.add_argument("--input-job-spec-path", type=str, help="Path to the input job spec file", required=True) + parser.add_argument( + "--target-job-spec-path", + type=str, + help="Path to the target job spec file.", + required=True, + ) + parser.add_argument("--target", type=str, help="Optional dbt target to use.", required=False) + parser.add_argument( + "--source", + type=str, + help="Optional project source. If not provided WORKSPACE will be used.", + required=False, + default=None, + ) + parser.add_argument( + "--warehouse_id", type=str, help="Optional SQL Warehouse to run dbt models on", required=False, default=None + ) + parser.add_argument("--schema", type=str, help="Optional schema to write to.", required=False, default=None) + parser.add_argument("--catalog", type=str, help="Optional catalog to write to.", required=False, default=None) + parser.add_argument( + "--profiles-directory", + type=str, + help="Optional (relative) path to the profiles directory.", + required=False, + default=None, + ) + parser.add_argument( + "--project-directory", + type=str, + help="Optional (relative) path to the project directory.", + required=False, + default=None, + ) + parser.add_argument( + "--environment-key", + type=str, + help="Optional (relative) key of an environment. Defaults to 'Default' when unset.", + required=False, + default=None, + ) + parser.add_argument( + "--extra-dbt-command-options", + type=str, + help="Optional additional dbt command options", + required=False, + default="", + ) + parser.add_argument( + "--no-run-tests", + action="store_false", + dest="run_tests", + help="Skip generating dbt test tasks. Tests are included by default.", + ) + parser.add_argument( + "--bundle-tests", + action="store_true", + help=( + "Bundle single-model tests for a given resource into one " + "`dbt test --select . --indirect-selection cautious` task (default: " + "one task per test node). Cross-model tests (e.g. `relationships`) are detected " + "from the manifest and emitted as their own tasks gated on every referenced " + "resource, so no tests are silently dropped. Trade-off: fewer tasks and a smaller " + "DAG, but per-test failures show up as a single red `_tests` task — drill " + "into the logs to see which assertion failed." + ), + ) + parser.add_argument( + "--enable-dbt-deps", + action="store_true", + help="Run `dbt deps` before each task.", + ) + parser.add_argument( + "--dbt-tasks-deps", + type=str, + help="Optional list of tasks that require dbt deps. Only in effect if `--enable-dbt-deps` is enabled.", + required=False, + default=None, + ) + parser.add_argument( + "--task-type", + type=str, + help="Task type to generate: 'dbt' for native dbt_task (default), 'notebook' for notebook_task wrapper.", + required=False, + default="dbt", + choices=["dbt", "notebook"], + ) + parser.add_argument( + "--notebook-path", + type=str, + help=( + "Path to the dbt runner notebook (used when --task-type is 'notebook'). If omitted, " + "the factory copies the packaged runner notebook next to the generated job spec and " + "references it relatively, so `databricks bundle deploy` uploads it automatically. " + "Pass an explicit path to pin the notebook elsewhere and manage it yourself." + ), + required=False, + default=None, + ) + parser.add_argument( + "--job-cluster-key", + type=str, + help="Job cluster key for running tasks on job compute instead of serverless. Mutually exclusive with --environment-key.", + required=False, + default=None, + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print generated tasks without updating the job spec file.", + ) + args = parser.parse_args() + + if args.job_cluster_key and args.environment_key is not None: + parser.error("--job-cluster-key and --environment-key are mutually exclusive") + + if args.task_type == "notebook": + conflicting = [] + for flag, value in ( + ("--warehouse_id", args.warehouse_id), + ("--schema", args.schema), + ("--catalog", args.catalog), + ): + if value: + conflicting.append(flag) + if conflicting: + parser.error( + f"{', '.join(conflicting)} cannot be used with --task-type notebook; " + "notebook tasks connect via profiles.yml." + ) + + return args + + +if __name__ == "__main__": + main() diff --git a/contrib/dbt_factory/src/databricks_dbt_factory/notebook/run_dbt_command.py b/contrib/dbt_factory/src/databricks_dbt_factory/notebook/run_dbt_command.py new file mode 100644 index 00000000..a0feb0a2 --- /dev/null +++ b/contrib/dbt_factory/src/databricks_dbt_factory/notebook/run_dbt_command.py @@ -0,0 +1,106 @@ +# Databricks notebook source + +import json +import os +import shlex +import shutil +import tempfile + +from dbt.cli.main import dbtRunner + +# COMMAND ---------- + +dbutils.widgets.text("dbt_commands", "") +dbutils.widgets.text("project_directory", "") +dbutils.widgets.text("profiles_directory", "") + +dbt_commands = dbutils.widgets.get("dbt_commands") +project_directory = dbutils.widgets.get("project_directory") +profiles_directory = dbutils.widgets.get("profiles_directory") + +if not dbt_commands: + raise ValueError("dbt_commands parameter is required") + +# COMMAND ---------- + +ctx = dbutils.notebook.entry_point.getDbutils().notebook().getContext() +os.environ["DBT_ACCESS_TOKEN"] = ctx.apiToken().get() +os.environ["DBT_HOST"] = ctx.apiUrl().get() + +# chdir to the dbt project so dbt runs from inside it. Relative `project_directory` is +# resolved against this notebook's own workspace location — the same anchor native +# `dbt_task` uses. Auto-copy mode sends `.` (resolves to the notebook's dir, which is +# project root by construction); user-pinned `--notebook-path` with relative +# `--project-directory` resolves against wherever the user placed the notebook; absolute +# `project_directory` is used as-is. +if project_directory: + notebook_dir = os.path.dirname("/Workspace" + ctx.notebookPath().get()) + target_dir = ( + project_directory + if os.path.isabs(project_directory) + else os.path.normpath(os.path.join(notebook_dir, project_directory)) + ) + os.chdir(target_dir) + +# dbt writes `logs/dbt.log` and `target/` inside CWD on every run. DAB sync only uploads +# files, not empty directories — pre-create them (idempotent). +os.makedirs("logs", exist_ok=True) +os.makedirs("target", exist_ok=True) + +# If a pre-built msgpack sits next to the project, deserialize it into a manifest and inject it into +# dbtRunner to skip dbt's parse phase (re-reading/hashing every file + DAG rebuild) on each task. Each +# task then writes artifacts to a private local dir (DBT_TARGET_PATH/DBT_LOG_PATH) to avoid contention +# on the shared workspace `target/`. Falls back to a normal parse if the msgpack is absent or unusable. +manifest = None +local_dir = None +prebuilt_manifest_path = os.path.join("target", "partial_parse.msgpack") +if os.path.exists(prebuilt_manifest_path): + try: + from dbt.contracts.graph.manifest import Manifest + + with open(prebuilt_manifest_path, "rb") as f: + manifest = Manifest.from_msgpack(f.read()) + manifest.build_flat_graph() + local_dir = tempfile.mkdtemp(prefix="dbt_local_") + os.environ["DBT_TARGET_PATH"] = local_dir + os.environ["DBT_LOG_PATH"] = local_dir + print(f"[dbt-factory] injecting pre-built manifest from {prebuilt_manifest_path} (skipping dbt parse)") + except Exception as e: + print(f"[dbt-factory] manifest injection unavailable, falling back to dbt parse: {e}") + manifest = None + +try: + runner = dbtRunner(manifest=manifest) + + for command_str in json.loads(dbt_commands): + command_str = command_str.strip() + if not command_str: + continue + + if command_str.startswith("dbt "): + command_str = command_str[4:] + + args = shlex.split(command_str) + + if profiles_directory: + args.extend(["--profiles-dir", profiles_directory]) + + print(f"Running: dbt {' '.join(args)}") + print("-" * 60) + + result = runner.invoke(args) + + if not result.success: + detail = result.exception or result.result or "(no further details)" + raise RuntimeError(f"dbt command failed: dbt {' '.join(args)}\n{detail}") + + print(f"Completed successfully: dbt {' '.join(args)}") +finally: + os.environ.pop("DBT_ACCESS_TOKEN", None) + os.environ.pop("DBT_HOST", None) + os.environ.pop("DBT_TARGET_PATH", None) + os.environ.pop("DBT_LOG_PATH", None) + # Remove the private per-task target/log dir; on reused (all-purpose) clusters these + # would otherwise accumulate under the system temp dir for the life of the cluster. + if local_dir: + shutil.rmtree(local_dir, ignore_errors=True) diff --git a/contrib/dbt_factory/src/macros/.gitkeep b/contrib/dbt_factory/src/macros/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/dbt_factory/src/models/example/orders_daily.sql b/contrib/dbt_factory/src/models/example/orders_daily.sql new file mode 100644 index 00000000..e32736ce --- /dev/null +++ b/contrib/dbt_factory/src/models/example/orders_daily.sql @@ -0,0 +1,17 @@ + +-- This model file defines a materialized view called 'orders_daily' +-- +-- Read more about materialized at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables +-- Current limitation: a "full refresh" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561. +{{ config(materialized = 'materialized_view') }} + +select order_date, count(*) AS number_of_orders + +from {{ ref('orders_raw') }} + +-- During development, only process a smaller range of data +{% if target.name != 'prod' %} +where order_date >= '2019-08-01' and order_date < '2019-09-01' +{% endif %} + +group by order_date diff --git a/contrib/dbt_factory/src/models/example/orders_raw.sql b/contrib/dbt_factory/src/models/example/orders_raw.sql new file mode 100644 index 00000000..8faf8f38 --- /dev/null +++ b/contrib/dbt_factory/src/models/example/orders_raw.sql @@ -0,0 +1,16 @@ +-- This model file defines a streaming table called 'orders_raw' +-- +-- The streaming table below ingests all JSON files in /databricks-datasets/retail-org/sales_orders/ +-- Read more about streaming tables at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables +-- Current limitation: a "full refresh" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561. +{{ config(materialized = 'streaming_table') }} + +select + customer_name, + date(timestamp(from_unixtime(try_cast(order_datetime as bigint)))) as order_date, + order_number +from stream read_files( + "/databricks-datasets/retail-org/sales_orders/", + format => "json", + header => true +) diff --git a/contrib/dbt_factory/src/models/example/schema.yml b/contrib/dbt_factory/src/models/example/schema.yml new file mode 100644 index 00000000..c17c810e --- /dev/null +++ b/contrib/dbt_factory/src/models/example/schema.yml @@ -0,0 +1,22 @@ + +version: 2 + +models: + - name: orders_raw + description: "Raw ingested orders" + columns: + - name: customer_name + description: "The name of a customer" + # customer_name repeats across orders, so only a not_null test applies here. + data_tests: + - not_null + + - name: orders_daily + description: "Number of orders by day" + columns: + - name: order_date + description: "The date on which orders took place" + # orders_daily groups by order_date, so it is unique and not null. + data_tests: + - unique + - not_null diff --git a/contrib/dbt_factory/src/seeds/.gitkeep b/contrib/dbt_factory/src/seeds/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/dbt_factory/src/snapshots/.gitkeep b/contrib/dbt_factory/src/snapshots/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/dbt_factory/src/tests/.gitkeep b/contrib/dbt_factory/src/tests/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/dbt_factory/target/manifest.json b/contrib/dbt_factory/target/manifest.json new file mode 100644 index 00000000..21822ec5 --- /dev/null +++ b/contrib/dbt_factory/target/manifest.json @@ -0,0 +1 @@ +{"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v12.json", "dbt_version": "1.11.11", "generated_at": "2026-07-02T17:03:13.876639Z", "invocation_id": "04710f75-f17a-49ce-b19e-779d81afacd1", "invocation_started_at": "2026-07-02T17:03:12.678402Z", "env": {}, "project_name": "dbt_factory", "project_id": "7085a6babca105474633a8d7cb1f5b8d", "user_id": "0aecafb9-78fd-4aa4-aba3-0e8995898cd4", "send_anonymous_usage_stats": true, "adapter_type": "databricks", "quoting": {"database": true, "schema": true, "identifier": true, "column": null}, "run_started_at": "2026-07-02T17:03:12.678594+00:00"}, "nodes": {"model.dbt_factory.orders_raw": {"database": "catalog", "schema": "default", "name": "orders_raw", "resource_type": "model", "package_name": "dbt_factory", "path": "example/orders_raw.sql", "original_file_path": "src/models/example/orders_raw.sql", "unique_id": "model.dbt_factory.orders_raw", "fqn": ["dbt_factory", "example", "orders_raw"], "alias": "orders_raw", "checksum": {"name": "sha256", "checksum": "057e5b384cf1b4d8a5e29453f900cbd74609cbfb070aee02baa0c691f5bd2179"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "static_analysis": null, "materialized": "streaming_table", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "event_time": null, "concurrent_batches": null, "access": "protected", "freshness": null}, "tags": [], "description": "Raw ingested orders", "columns": {"customer_name": {"name": "customer_name", "description": "The name of a customer", "meta": {}, "data_type": null, "constraints": [], "quote": null, "config": {"meta": {}, "tags": []}, "tags": [], "granularity": null, "doc_blocks": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_factory://src/models/example/schema.yml", "build_path": null, "unrendered_config": {"materialized": "streaming_table"}, "created_at": 1783011794.5405571, "relation_name": "`catalog`.`default`.`orders_raw`", "raw_code": "-- This model file defines a streaming table called 'orders_raw'\n--\n-- The streaming table below ingests all JSON files in /databricks-datasets/retail-org/sales_orders/\n-- Read more about streaming tables at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables\n-- Current limitation: a \"full refresh\" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561.\n{{ config(materialized = 'streaming_table') }}\n\nselect\n customer_name,\n date(timestamp(from_unixtime(try_cast(order_datetime as bigint)))) as order_date,\n order_number\nfrom stream read_files(\n \"/databricks-datasets/retail-org/sales_orders/\",\n format => \"json\",\n header => true\n)", "doc_blocks": [], "language": "sql", "refs": [], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null, "primary_key": [], "time_spine": null}, "model.dbt_factory.orders_daily": {"database": "catalog", "schema": "default", "name": "orders_daily", "resource_type": "model", "package_name": "dbt_factory", "path": "example/orders_daily.sql", "original_file_path": "src/models/example/orders_daily.sql", "unique_id": "model.dbt_factory.orders_daily", "fqn": ["dbt_factory", "example", "orders_daily"], "alias": "orders_daily", "checksum": {"name": "sha256", "checksum": "5ec9641ef90bdd9f6011aea792cb9042e1e21b7ece23f55ca9b2eddb5bff5630"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "static_analysis": null, "materialized": "materialized_view", "incremental_strategy": null, "batch_size": null, "lookback": 1, "begin": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "event_time": null, "concurrent_batches": null, "access": "protected", "freshness": null}, "tags": [], "description": "Number of orders by day", "columns": {"order_date": {"name": "order_date", "description": "The date on which orders took place", "meta": {}, "data_type": null, "constraints": [], "quote": null, "config": {"meta": {}, "tags": []}, "tags": [], "granularity": null, "doc_blocks": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_factory://src/models/example/schema.yml", "build_path": null, "unrendered_config": {"materialized": "materialized_view"}, "created_at": 1783011794.540988, "relation_name": "`catalog`.`default`.`orders_daily`", "raw_code": "-- This model file defines a materialized view called 'orders_daily'\n--\n-- Read more about materialized at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables\n-- Current limitation: a \"full refresh\" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561.\n{{ config(materialized = 'materialized_view') }}\n\nselect order_date, count(*) AS number_of_orders\n\nfrom {{ ref('orders_raw') }}\n\n-- During development, only process a smaller range of data\n{% if target.name != 'prod' %}\nwhere order_date >= '2019-08-01' and order_date < '2019-09-01'\n{% endif %}\n\ngroup by order_date", "doc_blocks": [], "language": "sql", "refs": [{"name": "orders_raw", "package": null, "version": null}], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": [], "nodes": ["model.dbt_factory.orders_raw"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null, "primary_key": ["order_date"], "time_spine": null}, "test.dbt_factory.not_null_orders_raw_customer_name.f1166f8d91": {"database": "catalog", "schema": "default_dbt_test__audit", "name": "not_null_orders_raw_customer_name", "resource_type": "test", "package_name": "dbt_factory", "path": "not_null_orders_raw_customer_name.sql", "original_file_path": "src/models/example/schema.yml", "unique_id": "test.dbt_factory.not_null_orders_raw_customer_name.f1166f8d91", "fqn": ["dbt_factory", "models", "example", "not_null_orders_raw_customer_name"], "alias": "not_null_orders_raw_customer_name", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "static_analysis": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1783011794.567219, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "doc_blocks": [], "language": "sql", "refs": [{"name": "orders_raw", "package": null, "version": null}], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_factory.orders_raw"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "customer_name", "file_key_name": "models.orders_raw", "attached_node": "model.dbt_factory.orders_raw", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "customer_name", "model": "{{ get_where_subquery(ref('orders_raw')) }}"}, "namespace": null}}, "test.dbt_factory.unique_orders_daily_order_date.8b58585cb3": {"database": "catalog", "schema": "default_dbt_test__audit", "name": "unique_orders_daily_order_date", "resource_type": "test", "package_name": "dbt_factory", "path": "unique_orders_daily_order_date.sql", "original_file_path": "src/models/example/schema.yml", "unique_id": "test.dbt_factory.unique_orders_daily_order_date.8b58585cb3", "fqn": ["dbt_factory", "models", "example", "unique_orders_daily_order_date"], "alias": "unique_orders_daily_order_date", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "static_analysis": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1783011794.568104, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "doc_blocks": [], "language": "sql", "refs": [{"name": "orders_daily", "package": null, "version": null}], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_factory.orders_daily"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "order_date", "file_key_name": "models.orders_daily", "attached_node": "model.dbt_factory.orders_daily", "test_metadata": {"name": "unique", "kwargs": {"column_name": "order_date", "model": "{{ get_where_subquery(ref('orders_daily')) }}"}, "namespace": null}}, "test.dbt_factory.not_null_orders_daily_order_date.e8076bc258": {"database": "catalog", "schema": "default_dbt_test__audit", "name": "not_null_orders_daily_order_date", "resource_type": "test", "package_name": "dbt_factory", "path": "not_null_orders_daily_order_date.sql", "original_file_path": "src/models/example/schema.yml", "unique_id": "test.dbt_factory.not_null_orders_daily_order_date.e8076bc258", "fqn": ["dbt_factory", "models", "example", "not_null_orders_daily_order_date"], "alias": "not_null_orders_daily_order_date", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "static_analysis": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1783011794.5687559, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "doc_blocks": [], "language": "sql", "refs": [{"name": "orders_daily", "package": null, "version": null}], "sources": [], "metrics": [], "functions": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_factory.orders_daily"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "order_date", "file_key_name": "models.orders_daily", "attached_node": "model.dbt_factory.orders_daily", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "order_date", "model": "{{ get_where_subquery(ref('orders_daily')) }}"}, "namespace": null}}}, "sources": {}, "macros": {"macro.dbt_databricks.materialization_materialized_view_databricks": {"name": "materialization_materialized_view_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialization_materialized_view_databricks", "macro_sql": "{% materialization materialized_view, adapter = 'databricks' %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation) %}\n\n {% if build_sql == '' %}\n {{ execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.run_hooks", "macro.dbt_databricks.materialized_view_get_build_sql", "macro.dbt_databricks.execute_no_op", "macro.dbt_databricks.materialized_view_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.962945, "supported_languages": ["sql"]}, "macro.dbt_databricks.materialized_view_get_build_sql": {"name": "materialized_view_get_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialized_view_get_build_sql", "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation) %}\n {% set sql = adapter.clean_sql(sql) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_configuration_changes(existing_relation) %}\n\n {# Skip manual REFRESH on no-op re-runs for auto-refreshed modes. #}\n {% if configuration_changes is none %}\n {%- set refresh = adapter.get_config_from_model(config.model).config[\"refresh\"] -%}\n {%- if refresh.auto_refreshed -%}\n {% set build_sql = '' %}\n {%- else -%}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n {%- endif -%}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, None, None) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = \"\" %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_configuration_changes", "macro.dbt.refresh_materialized_view", "macro.dbt_databricks.get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9637198, "supported_languages": null}, "macro.dbt_databricks.materialized_view_execute_build_sql": {"name": "materialized_view_execute_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialized_view_execute_build_sql", "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n {% set tags = config.get('databricks_tags') %}\n\n {{ execute_multiple_statements(build_sql) }}\n\n {%- do apply_tags(target_relation, tags) -%}\n\n {% set column_tags = adapter.get_column_tags_from_model(config.model) %}\n {% if column_tags %}\n {{ apply_column_tags(target_relation, column_tags) }}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt_databricks.execute_multiple_statements", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_column_tags", "macro.dbt.should_revoke", "macro.dbt.apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.96409, "supported_languages": null}, "macro.dbt_databricks.materialization_view_databricks": {"name": "materialization_view_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "unique_id": "macro.dbt_databricks.materialization_view_databricks", "macro_sql": "{% materialization view, adapter='databricks' -%}\n {{ log(\"MATERIALIZING VIEW\") }}\n {%- set existing_relation = load_relation_with_metadata(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {% set grant_config = config.get('grants') %}\n {% set tags = config.get('databricks_tags') %}\n {% set sql = adapter.clean_sql(sql) %}\n\n {% if adapter.get_behavior_flag_no_warn('use_materialization_v2') %}\n {{ run_pre_hooks() }}\n {% if existing_relation %}\n {% if relation_should_be_altered(existing_relation) %}\n {% set configuration_changes = get_configuration_changes(existing_relation) %}\n {% if configuration_changes and configuration_changes.changes %}\n {% if configuration_changes.requires_full_refresh %}\n {{ log('Using replace_with_view') }}\n {{ replace_with_view(existing_relation, target_relation) }}\n {% else %}\n {{ log('Using alter_view') }}\n {{ log(configuration_changes.changes) }}\n {{ alter_view(target_relation, configuration_changes.changes) }}\n {% endif %}\n {% else %}\n {{ execute_no_op(target_relation) }}\n {% endif %}\n {% else %}\n {{ replace_with_view(existing_relation, target_relation) }}\n {% endif %}\n {% else %}\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n {{ apply_tags(target_relation, tags) }}\n {% set column_tags = adapter.get_column_tags_from_model(config.model) %}\n {% if column_tags and column_tags.set_column_tags %}\n {{ apply_column_tags(target_relation, column_tags) }}\n {% endif %}\n {% endif %}\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=True) %}\n\n {{ run_post_hooks() }}\n\n {% else %}\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if existing_relation is not none and not existing_relation.is_view -%}\n {{ handle_existing_table(should_full_refresh(), existing_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=True) %}\n\n {%- do apply_tags(target_relation, tags) -%}\n\n {% set column_tags = adapter.get_column_tags_from_model(config.model) %}\n {% if column_tags and column_tags.set_column_tags %}\n {{ apply_column_tags(target_relation, column_tags) }}\n {% endif %}\n\n {{ run_hooks(post_hooks) }}\n {% endif %}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.load_relation_with_metadata", "macro.dbt_databricks.run_pre_hooks", "macro.dbt_databricks.relation_should_be_altered", "macro.dbt_databricks.get_configuration_changes", "macro.dbt_databricks.replace_with_view", "macro.dbt_databricks.alter_view", "macro.dbt_databricks.execute_no_op", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_column_tags", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.run_post_hooks", "macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.965732, "supported_languages": ["sql"]}, "macro.dbt_databricks.replace_with_view": {"name": "replace_with_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "unique_id": "macro.dbt_databricks.replace_with_view", "macro_sql": "{% macro replace_with_view(existing_relation, target_relation) %}\n {% set sql = adapter.clean_sql(sql) %}\n {% set tags = config.get('databricks_tags') %}\n {{ execute_multiple_statements(get_replace_sql(existing_relation, target_relation, sql)) }}\n {%- do apply_tags(target_relation, tags) -%}\n \n {% set column_tags = adapter.get_column_tags_from_model(config.model) %}\n {% if column_tags and column_tags.set_column_tags %}\n {{ apply_column_tags(target_relation, column_tags) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.execute_multiple_statements", "macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_column_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.966026, "supported_languages": null}, "macro.dbt_databricks.relation_should_be_altered": {"name": "relation_should_be_altered", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "unique_id": "macro.dbt_databricks.relation_should_be_altered", "macro_sql": "{% macro relation_should_be_altered(existing_relation) %}\n {% set update_via_alter = config.get('view_update_via_alter', False) | as_bool %}\n {% if (existing_relation.is_view or existing_relation.is_metric_view) and update_via_alter %}\n {% if existing_relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Cannot update a view in the Hive metastore via ALTER VIEW. Please set `view_update_via_alter: false` in your model configuration.\") }}\n {% endif %}\n {{ return(True) }}\n {% endif %}\n {{ return(False) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.966244, "supported_languages": null}, "macro.dbt_databricks.materialization_table_databricks": {"name": "materialization_table_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_databricks.materialization_table_databricks", "macro_sql": "{% materialization table, adapter = 'databricks', supported_languages=['sql', 'python'] %}\n {{ log(\"MATERIALIZING TABLE\") }}\n {%- set language = model['language'] -%}\n {%- set identifier = model['alias'] -%}\n {%- set grant_config = config.get('grants') -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- set tags = config.get('databricks_tags') -%}\n {%- set safe_create = config.get('use_safer_relation_operations', False) %}\n {% set existing_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier, needs_information=True) %}\n {% set target_relation = this.incorporate(type='table') %}\n {% set compiled_code = adapter.clean_sql(compiled_code) %}\n\n {% if adapter.get_behavior_flag_no_warn('use_materialization_v2') %}\n {% set intermediate_relation = make_intermediate_relation(target_relation) %}\n {% set staging_relation = make_staging_relation(target_relation) %}\n\n {{ run_pre_hooks() }}\n\n {% call statement('main', language=language) %}\n {{ get_create_intermediate_table(intermediate_relation, compiled_code, language) }}\n {% endcall %}\n {% if not existing_relation %}\n {{ create_table_at(target_relation, intermediate_relation, compiled_code) }}\n {% else %}\n {% if safe_create and existing_relation.can_be_renamed %}\n {{ safe_relation_replace(existing_relation, staging_relation, intermediate_relation, compiled_code) }}\n {% else %}\n {% if existing_relation and (existing_relation.type != 'table' or not (existing_relation.can_be_replaced and adapter.resolve_file_format(config) in ('delta', 'iceberg'))) -%}\n {{ adapter.drop_relation(existing_relation) }}\n {%- endif %}\n {{ create_table_at(target_relation, intermediate_relation, compiled_code) }}\n {% endif %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {{ apply_grants(target_relation, grant_config, should_revoke) }}\n {% do optimize(target_relation) %}\n\n {% if language == 'python' %}\n {{ drop_relation_if_exists(intermediate_relation) }}\n {% endif %}\n \n {{ run_post_hooks() }}\n {% else %}\n {{ run_hooks(pre_hooks) }}\n -- setup: if the target relation already exists, drop it\n -- in case if the existing and future table is delta or iceberg, we want to do a\n -- create or replace table instead of dropping, so we don't have the table unavailable\n {% if existing_relation and (existing_relation.type != 'table' or not (existing_relation.can_be_replaced and adapter.resolve_file_format(config) in ('delta', 'iceberg'))) -%}\n {{ adapter.drop_relation(existing_relation) }}\n {%- endif %}\n\n -- build model\n\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n {% if language==\"python\" %}\n {% do apply_tblproperties(target_relation, tblproperties) %}\n {% endif %}\n {%- do apply_tags(target_relation, tags) -%}\n\n {% set column_tags = adapter.get_column_tags_from_model(config.model) %}\n {% if column_tags and column_tags.set_column_tags %}\n {{ apply_column_tags(target_relation, column_tags) }}\n {% endif %}\n\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {% do optimize(target_relation) %}\n\n {{ run_hooks(post_hooks) }}\n\n {% endif %}\n {{ return({'relations': [target_relation]})}}\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt_databricks.make_staging_relation", "macro.dbt_databricks.run_pre_hooks", "macro.dbt.statement", "macro.dbt_databricks.get_create_intermediate_table", "macro.dbt_databricks.create_table_at", "macro.dbt_databricks.safe_relation_replace", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.optimize", "macro.dbt.drop_relation_if_exists", "macro.dbt_databricks.run_post_hooks", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt_databricks.apply_tblproperties", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_column_tags", "macro.dbt.persist_docs", "macro.dbt_databricks.persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9679391, "supported_languages": ["sql", "python"]}, "macro.dbt_databricks.run_pre_hooks": {"name": "run_pre_hooks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt_databricks.run_pre_hooks", "macro_sql": "{% macro run_pre_hooks() %}\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.968092, "supported_languages": null}, "macro.dbt_databricks.run_post_hooks": {"name": "run_post_hooks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt_databricks.run_post_hooks", "macro_sql": "{% macro run_post_hooks() %}\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.968184, "supported_languages": null}, "macro.dbt_databricks.materialization_snapshot_databricks": {"name": "materialization_snapshot_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_databricks.materialization_snapshot_databricks", "macro_sql": "{% materialization snapshot, adapter='databricks' %}\n {%- set target_table = model.get('alias', model.get('name')) -%}\n {% set sql = adapter.clean_sql(sql) %}\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n {%- set file_format = adapter.resolve_file_format(config) -%}\n {%- set grant_config = config.get('grants') -%}\n {%- set tags = config.get('databricks_tags') -%}\n\n {% set target_relation_exists, target_relation = databricks__get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table',\n needs_information=True) -%}\n\n {%- if file_format not in ['delta', 'hudi'] -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n Snapshot functionality requires file_format be set to 'delta' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n\n {%- if target_relation_exists -%}\n {%- if not target_relation.is_delta and not target_relation.is_hudi -%}\n {% set invalid_format_msg -%}\n The existing table {{ model.schema }}.{{ target_table }} is in another format than 'delta' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n {% endif %}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", model['config'], target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, adapter.clean_sql(model['compiled_code'])) %}\n {% set build_or_select_sql = build_sql %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {{ adapter.assert_valid_snapshot_target_given_strategy(target_relation, columns, strategy) }}\n\n {% set build_or_select_sql = snapshot_staging_table(strategy, sql, target_relation) %}\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set remove_columns = ['dbt_change_type', 'DBT_CHANGE_TYPE', 'dbt_unique_key', 'DBT_UNIQUE_KEY'] %}\n {% if unique_key | is_list %}\n {% for key in strategy.unique_key %}\n {{ remove_columns.append('dbt_unique_key_' + loop.index|string) }}\n {{ remove_columns.append('DBT_UNIQUE_KEY_' + loop.index|string) }}\n {% endfor %}\n {% endif %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'in', remove_columns)\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'in', remove_columns)\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(column.quoted) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n\n {{ check_time_data_types(build_or_select_sql) }}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n \n {%- if tags -%}\n {%- do apply_tags(target_relation, tags) -%}\n {%- endif -%}\n\n {% set column_tags = adapter.get_column_tags_from_model(config.model) %}\n {% if column_tags and column_tags.set_column_tags %}\n {{ apply_column_tags(target_relation, column_tags) }}\n {% endif %}\n\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {% do optimize(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_staging_table", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.check_time_data_types", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_column_tags", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot", "macro.dbt_databricks.persist_constraints", "macro.dbt_databricks.optimize"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.970536, "supported_languages": ["sql"]}, "macro.dbt_databricks.materialization_streaming_table_databricks": {"name": "materialization_streaming_table_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.materialization_streaming_table_databricks", "macro_sql": "{% materialization streaming_table, adapter='databricks' %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.StreamingTable) %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% set build_sql = streaming_table_get_build_sql(existing_relation, target_relation) %}\n\n {% if build_sql == '' %}\n {{ execute_no_op(target_relation) }}\n {% else %}\n {{ streaming_table_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.run_hooks", "macro.dbt_databricks.streaming_table_get_build_sql", "macro.dbt_databricks.execute_no_op", "macro.dbt_databricks.streaming_table_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.971122, "supported_languages": ["sql"]}, "macro.dbt_databricks.streaming_table_get_build_sql": {"name": "streaming_table_get_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.streaming_table_get_build_sql", "macro_sql": "{% macro streaming_table_get_build_sql(existing_relation, target_relation) %}\n {% set sql = adapter.clean_sql(sql) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_streaming_table_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_streaming_table %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_configuration_changes(existing_relation) %}\n {# Skip manual REFRESH on no-op re-runs for auto-refreshed modes. #}\n {% if configuration_changes is none %}\n {%- set refresh = adapter.get_config_from_model(config.model).config[\"refresh\"] -%}\n {%- if refresh.auto_refreshed -%}\n {% set build_sql = '' %}\n {%- else -%}\n {{ log(\"REFRESHING STREAMING TABLE: \" ~ target_relation) }}\n {% set build_sql = refresh_streaming_table(target_relation, sql) %}\n {%- endif -%}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_streaming_table_as_sql(target_relation, configuration_changes, sql, existing_relation, None, None) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = \"\" %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt_databricks.get_create_streaming_table_as_sql", "macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_configuration_changes", "macro.dbt_databricks.refresh_streaming_table", "macro.dbt_databricks.get_alter_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.971827, "supported_languages": null}, "macro.dbt_databricks.streaming_table_execute_build_sql": {"name": "streaming_table_execute_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.streaming_table_execute_build_sql", "macro_sql": "{% macro streaming_table_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n {% set tags = config.get('databricks_tags') %}\n\n {{ execute_multiple_statements(build_sql) }}\n\n {%- do apply_tags(target_relation, tags) -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% set column_tags = adapter.get_column_tags_from_model(config.model) %}\n {% if column_tags and column_tags.set_column_tags %}\n {{ apply_column_tags(target_relation, column_tags) }}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt_databricks.execute_multiple_statements", "macro.dbt_databricks.apply_tags", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.apply_column_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.972262, "supported_languages": null}, "macro.dbt_databricks.materialization_metric_view_databricks": {"name": "materialization_metric_view_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/metric_view.sql", "original_file_path": "macros/materializations/metric_view.sql", "unique_id": "macro.dbt_databricks.materialization_metric_view_databricks", "macro_sql": "{% materialization metric_view, adapter='databricks' -%}\n {%- set existing_relation = load_relation_with_metadata(this) -%}\n {%- set target_relation = this.incorporate(type='metric_view') -%}\n {% set grant_config = config.get('grants') %}\n {% set tags = config.get('databricks_tags') %}\n {% set sql = adapter.clean_sql(sql) %}\n\n {{ run_pre_hooks() }}\n\n {% if existing_relation %}\n {#- Only use alter path if existing relation is actually a metric_view -#}\n {% if existing_relation.is_metric_view and relation_should_be_altered(existing_relation) %}\n {% set configuration_changes = get_configuration_changes(existing_relation) %}\n {% if configuration_changes and configuration_changes.changes %}\n {% if configuration_changes.requires_full_refresh %}\n {{ replace_with_metric_view(existing_relation, target_relation) }}\n {% else %}\n {{ alter_metric_view(target_relation, configuration_changes.changes) }}\n {% endif %}\n {% else %}\n {# No changes detected - run a no-op statement for dbt tracking #}\n {% call statement('main') %}\n select 1\n {% endcall %}\n {% endif %}\n {% else %}\n {{ replace_with_metric_view(existing_relation, target_relation) }}\n {% endif %}\n {% else %}\n {% call statement('main') -%}\n {{ get_create_metric_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n {{ apply_tags(target_relation, tags) }}\n {% endif %}\n\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke(existing_relation, full_refresh_mode=True)) %}\n\n {{ run_post_hooks() }}\n\n {{ return({'relations': [target_relation]}) }}\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.load_relation_with_metadata", "macro.dbt_databricks.run_pre_hooks", "macro.dbt_databricks.relation_should_be_altered", "macro.dbt_databricks.get_configuration_changes", "macro.dbt_databricks.replace_with_metric_view", "macro.dbt_databricks.alter_metric_view", "macro.dbt.statement", "macro.dbt_databricks.get_create_metric_view_as_sql", "macro.dbt_databricks.apply_tags", "macro.dbt.apply_grants", "macro.dbt.should_revoke", "macro.dbt_databricks.run_post_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.973301, "supported_languages": ["sql"]}, "macro.dbt_databricks.databricks__snapshot_merge_sql": {"name": "databricks__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot_helpers.sql", "original_file_path": "macros/materializations/snapshot_helpers.sql", "unique_id": "macro.dbt_databricks.databricks__snapshot_merge_sql", "macro_sql": "{% macro databricks__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n {# Get the hard_deletes configuration from config #}\n {%- set hard_deletes = config.get('hard_deletes', 'ignore') -%}\n {%- set invalidate_hard_deletes = (hard_deletes == 'invalidate') -%}\n\n {# Get column names configuration #}\n {%- set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n {%- if target.is_iceberg %}\n {# create view only supports a name (no catalog, or schema) #}\n using {{ source.identifier }} as DBT_INTERNAL_SOURCE\n {%- else %}\n using {{ source }} as DBT_INTERNAL_SOURCE\n {%- endif %}\n on DBT_INTERNAL_SOURCE.{{ adapter.quote(columns.dbt_scd_id) }} = DBT_INTERNAL_DEST.{{ adapter.quote(columns.dbt_scd_id) }}\n when matched\n {%- if config.get(\"dbt_valid_to_current\") %}\n and ( DBT_INTERNAL_DEST.{{ adapter.quote(columns.dbt_valid_to) }} = {{ config.get('dbt_valid_to_current') }} or\n DBT_INTERNAL_DEST.{{ adapter.quote(columns.dbt_valid_to) }} is null )\n {%- else %}\n and DBT_INTERNAL_DEST.{{ adapter.quote(columns.dbt_valid_to) }} is null\n {%- endif %}\n and DBT_INTERNAL_SOURCE.{{ adapter.quote('dbt_change_type') }} in ('update', 'delete')\n then update\n set {{ adapter.quote(columns.dbt_valid_to) }} = DBT_INTERNAL_SOURCE.{{ adapter.quote(columns.dbt_valid_to) }}\n\n when not matched\n and DBT_INTERNAL_SOURCE.{{ adapter.quote('dbt_change_type') }} = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n ;\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.974571, "supported_languages": null}, "macro.dbt_databricks.databricks__create_columns": {"name": "databricks__create_columns", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot_helpers.sql", "original_file_path": "macros/materializations/snapshot_helpers.sql", "unique_id": "macro.dbt_databricks.databricks__create_columns", "macro_sql": "{% macro databricks__create_columns(relation, columns) %}\n {%- if columns|length > 0 %}\n {%- call statement() %}\n alter table {{ relation }} add columns (\n {%- for column in columns %}\n {{ adapter.quote(column.name) }} {{ column.data_type }} {{- ',' if not loop.last -}}\n {%- endfor %}\n );\n {%- endcall %}\n {%- endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.974782, "supported_languages": null}, "macro.dbt_databricks.databricks__build_snapshot_table": {"name": "databricks__build_snapshot_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot_helpers.sql", "original_file_path": "macros/materializations/snapshot_helpers.sql", "unique_id": "macro.dbt_databricks.databricks__build_snapshot_table", "macro_sql": "{% macro databricks__build_snapshot_table(strategy, sql) %}\n {%- set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() -%}\n {%- set hard_deletes = strategy.hard_deletes -%}\n\n select *,\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ get_dbt_valid_to_current(strategy, columns) }}\n {%- if hard_deletes == 'new_record' -%}\n , false as {{ columns.dbt_is_deleted }}\n {%- endif %}\n from (\n {{ sql }}\n ) sbq\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.get_dbt_valid_to_current"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.975029, "supported_languages": null}, "macro.dbt_databricks.databricks__snapshot_staging_table": {"name": "databricks__snapshot_staging_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot_helpers.sql", "original_file_path": "macros/materializations/snapshot_helpers.sql", "unique_id": "macro.dbt_databricks.databricks__snapshot_staging_table", "macro_sql": "{% macro databricks__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n {% if strategy.hard_deletes == 'new_record' %}\n {% set new_scd_id = snapshot_hash_arguments([columns.dbt_scd_id, snapshot_get_time()]) %}\n {% endif %}\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }}\n from {{ target_relation }}\n where\n {% if config.get('dbt_valid_to_current') %}\n {% set source_unique_key = columns.dbt_valid_to | trim %}\n {% set target_unique_key = config.get('dbt_valid_to_current') | trim %}\n\n {# The exact equals semantics between NULL values depends on the current behavior flag set. Also, update records if the source field is null #}\n ( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null )\n {% else %}\n {{ columns.dbt_valid_to }} is null\n {% endif %}\n\n ),\n\n insertions_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ get_dbt_valid_to_current(strategy, columns) }},\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_to }}\n\n from snapshot_query\n ),\n\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n\n deletes_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }}\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n {%- if strategy.hard_deletes == 'new_record' -%}\n , false as {{ columns.dbt_is_deleted }}\n {%- endif %}\n\n from insertions_source_data as source_data\n left outer join snapshotted_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"snapshotted_data\") }}\n or ({{ unique_key_is_not_null(strategy.unique_key, \"snapshotted_data\") }} and (\n {{ strategy.row_changed }} {%- if strategy.hard_deletes == 'new_record' -%} or snapshotted_data.{{ columns.dbt_is_deleted }} = true {% endif %}\n )\n\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.{{ columns.dbt_scd_id }}\n {%- if strategy.hard_deletes == 'new_record' -%}\n , snapshotted_data.{{ columns.dbt_is_deleted }}\n {%- endif %}\n\n from updates_source_data as source_data\n join snapshotted_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where (\n {{ strategy.row_changed }} {%- if strategy.hard_deletes == 'new_record' -%} or snapshotted_data.{{ columns.dbt_is_deleted }} = true {% endif %}\n )\n )\n\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n ,\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},\n {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_to }},\n snapshotted_data.{{ columns.dbt_scd_id }}\n {%- if strategy.hard_deletes == 'new_record' -%}\n , snapshotted_data.{{ columns.dbt_is_deleted }}\n {%- endif %}\n from snapshotted_data\n left join deletes_source_data as source_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"source_data\") }}\n\n {%- if strategy.hard_deletes == 'new_record' %}\n and not (\n -- avoid updating the record's valid_to if the latest entry is marked as deleted\n snapshotted_data.{{ columns.dbt_is_deleted }} = true\n and\n {% if config.get('dbt_valid_to_current') -%}\n snapshotted_data.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }}\n {%- else -%}\n snapshotted_data.{{ columns.dbt_valid_to }} is null\n {%- endif %}\n )\n {%- endif %}\n )\n {%- endif %}\n\n {%- if strategy.hard_deletes == 'new_record' %}\n {# Databricks-specific: Extract column names from agate.Row tuples #}\n {% set target_columns_raw = get_columns_in_relation(target_relation) %}\n {% set snapshotted_cols = [] %}\n {% for row in target_columns_raw %}\n {# agate.Row is a tuple: (col_name, data_type, comment) #}\n {# Filter out Databricks metadata rows (starting with # or empty) #}\n {% set col_name = row[0] %}\n {% if col_name and not col_name.startswith('#') %}\n {% do snapshotted_cols.append(col_name) %}\n {% endif %}\n {% endfor %}\n {% set source_sql_cols = get_column_schema_from_query(source_sql) %}\n ,\n deletion_records as (\n\n select\n 'insert' as dbt_change_type,\n {#\n If a column has been added to the source it won't yet exist in the\n snapshotted table so we insert a null value as a placeholder for the column.\n #}\n {%- for col in source_sql_cols -%}\n {%- if col.name in snapshotted_cols -%}\n snapshotted_data.{{ adapter.quote(col.column) }},\n {%- else -%}\n NULL as {{ adapter.quote(col.column) }},\n {%- endif -%}\n {% endfor -%}\n {%- if strategy.unique_key | is_list -%}\n {%- for key in strategy.unique_key -%}\n snapshotted_data.{{ key }} as dbt_unique_key_{{ loop.index }},\n {% endfor -%}\n {%- else -%}\n snapshotted_data.dbt_unique_key as dbt_unique_key,\n {% endif -%}\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},\n {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},\n snapshotted_data.{{ columns.dbt_valid_to }} as {{ columns.dbt_valid_to }},\n {{ new_scd_id }} as {{ columns.dbt_scd_id }},\n true as {{ columns.dbt_is_deleted }}\n from snapshotted_data\n left join deletes_source_data as source_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"source_data\") }}\n and not (\n -- avoid inserting a new record if the latest one is marked as deleted\n snapshotted_data.{{ columns.dbt_is_deleted }} = true\n and\n {% if config.get('dbt_valid_to_current') -%}\n snapshotted_data.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }}\n {%- else -%}\n snapshotted_data.{{ columns.dbt_valid_to }} is null\n {%- endif %}\n )\n\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n union all\n select * from deletes\n {%- endif %}\n {%- if strategy.hard_deletes == 'new_record' %}\n union all\n select * from deletion_records\n {%- endif %}\n\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_hash_arguments", "macro.dbt.snapshot_get_time", "macro.dbt.unique_key_fields", "macro.dbt.equals", "macro.dbt.get_dbt_valid_to_current", "macro.dbt.unique_key_join_on", "macro.dbt.unique_key_is_null", "macro.dbt.unique_key_is_not_null", "macro.dbt.get_columns_in_relation", "macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9774451, "supported_languages": null}, "macro.dbt_databricks.dbt_databricks_validate_get_file_format": {"name": "dbt_databricks_validate_get_file_format", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_databricks.dbt_databricks_validate_get_file_format", "macro_sql": "{% macro dbt_databricks_validate_get_file_format(raw_file_format) %}\n {#-- Validate the file format #}\n\n {% set accepted_formats = ['text', 'csv', 'json', 'jdbc', 'parquet', 'orc', 'hive', 'delta', 'libsvm', 'hudi'] %}\n\n {% set invalid_file_format_msg -%}\n Invalid file format provided: {{ raw_file_format }}\n Expected one of: {{ accepted_formats | join(', ') }}\n {%- endset %}\n\n {% if raw_file_format not in accepted_formats %}\n {% do exceptions.raise_compiler_error(invalid_file_format_msg) %}\n {% endif %}\n\n {% do return(raw_file_format) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9778838, "supported_languages": null}, "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy": {"name": "dbt_databricks_validate_get_incremental_strategy", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy", "macro_sql": "{% macro dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format) %}\n {#-- Validate the incremental strategy #}\n\n {% set invalid_delta_only_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You can only choose this strategy when file_format is set to 'delta'\n {%- endset %}\n\n {% set invalid_insert_overwrite_endpoint_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You cannot use this strategy when connecting via warehouse\n Use the 'merge' or 'replace_where' strategy instead\n {%- endset %}\n\n {% set missing_unique_key_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n This strategy requires 'unique_key' to be configured\n {%- endset %}\n\n {% if raw_strategy not in adapter.valid_incremental_strategies() %}\n {{ log(\"WARNING - You are using an unsupported incremental strategy: \" ~ raw_strategy) }}\n {{ log(\"You can ignore this warning if you are using a custom incremental strategy\") }}\n {%-else %}\n {% if raw_strategy == 'merge' and file_format not in ['delta', 'hudi'] %}\n {% do exceptions.raise_compiler_error(invalid_delta_only_msg) %}\n {% endif %}\n {% if raw_strategy in ('replace_where', 'microbatch', 'delete+insert') and file_format not in ['delta'] %}\n {% do exceptions.raise_compiler_error(invalid_delta_only_msg) %}\n {% endif %}\n {% if raw_strategy == 'delete+insert' and config.get('unique_key') is none %}\n {% do exceptions.raise_compiler_error(missing_unique_key_msg) %}\n {% endif %}\n {% endif %}\n\n {% do return(raw_strategy) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.97838, "supported_languages": null}, "macro.dbt_databricks.get_incremental_strategy": {"name": "get_incremental_strategy", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_incremental_strategy", "macro_sql": "{% macro get_incremental_strategy(file_format) %}\n {% set raw_strategy = config.get('incremental_strategy') or 'merge' %}\n {% do return(dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.979957, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_default_sql": {"name": "databricks__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_default_sql", "macro_sql": "{% macro databricks__get_incremental_default_sql(arg_dict) %}\n {{ return(get_incremental_merge_sql(arg_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.980026, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_append_sql": {"name": "databricks__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_append_sql", "macro_sql": "{% macro databricks__get_incremental_append_sql(arg_dict) %}\n {% do return(get_insert_into_sql(arg_dict[\"temp_relation\"], arg_dict[\"target_relation\"])) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.980118, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_replace_where_sql": {"name": "databricks__get_incremental_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_replace_where_sql", "macro_sql": "{% macro databricks__get_incremental_replace_where_sql(arg_dict) %}\n {% do return(get_replace_where_sql(arg_dict)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_where_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.980194, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_delete_insert_sql": {"name": "databricks__get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_delete_insert_sql", "macro_sql": "{% macro databricks__get_incremental_delete_insert_sql(arg_dict) %}\n {% do return(get_delete_insert_sql(arg_dict)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_delete_insert_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.980268, "supported_languages": null}, "macro.dbt_databricks.get_incremental_replace_where_sql": {"name": "get_incremental_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_incremental_replace_where_sql", "macro_sql": "{% macro get_incremental_replace_where_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_replace_where_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_replace_where_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9803588, "supported_languages": null}, "macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql": {"name": "databricks__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql", "macro_sql": "{% macro databricks__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) %}\n {{ return(get_insert_overwrite_sql(source, target)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.980452, "supported_languages": null}, "macro.dbt_databricks.get_insert_overwrite_sql": {"name": "get_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_insert_overwrite_sql", "macro_sql": "{% macro get_insert_overwrite_sql(source_relation, target_relation) %}\n {%- if adapter.is_cluster() -%}\n {#-- Clusters: Check DBR capability for REPLACE ON (DBR 17.1+) --#}\n {%- if adapter.has_dbr_capability('replace_on') -%}\n {{ get_insert_replace_on_sql(source_relation, target_relation) }}\n {%- else -%}\n {%- if adapter.behavior.use_replace_on_for_insert_overwrite -%}\n {{ exceptions.warn(\"insert_overwrite: use_replace_on_for_insert_overwrite is enabled but this cluster's DBR version does not support REPLACE ON (requires DBR 17.1+). Falling back to legacy INSERT OVERWRITE.\") }}\n {%- endif -%}\n {%- set has_insert_by_name = adapter.has_dbr_capability('insert_by_name') -%}\n insert overwrite table {{ target_relation }}\n {{ partition_cols(label=\"partition\") }}\n {%- if has_insert_by_name %} by name{% endif %}\n select * from {{ source_relation }}\n {%- endif -%}\n {%- else -%}\n {%- if adapter.behavior.use_replace_on_for_insert_overwrite -%}\n {{ get_insert_replace_on_sql(source_relation, target_relation) }}\n {%- else -%}\n {#-- Behavior flag disabled, use legacy DPO INSERT OVERWRITE --#}\n {%- set has_insert_by_name = adapter.has_dbr_capability('insert_by_name') -%}\n insert overwrite table {{ target_relation }}\n {{ partition_cols(label=\"partition\") }}\n {%- if has_insert_by_name %} by name{% endif %}\n select * from {{ source_relation }}\n {%- endif -%}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_insert_replace_on_sql", "macro.dbt_spark.partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.980876, "supported_languages": null}, "macro.dbt_databricks.add_columns_to_list": {"name": "add_columns_to_list", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.add_columns_to_list", "macro_sql": "{% macro add_columns_to_list(target_list, columns) %}\n {#-- Local helper to add columns to the target list, converting string to list if needed --#}\n {%- if columns -%}\n {%- if columns is string -%}\n {%- do target_list.append(columns) -%}\n {%- else -%}\n {%- for col in columns -%}\n {%- do target_list.append(col) -%}\n {%- endfor -%}\n {%- endif -%}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.981042, "supported_languages": null}, "macro.dbt_databricks.get_insert_replace_on_sql": {"name": "get_insert_replace_on_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_insert_replace_on_sql", "macro_sql": "{% macro get_insert_replace_on_sql(source_relation, target_relation) %}\n {%- set partition_by = config.get('partition_by') -%}\n {%- set liquid_clustered_by = config.get('liquid_clustered_by') -%}\n {%- set replace_columns = [] -%}\n \n {#-- If both partition_by and liquid_clustered_by are defined, it will fail before this point with a SPECIFY_CLUSTER_BY_WITH_PARTITIONED_BY_IS_NOT_ALLOWED error from Databricks --#}\n {%- do add_columns_to_list(replace_columns, partition_by) -%}\n {%- do add_columns_to_list(replace_columns, liquid_clustered_by) -%}\n \n {%- if replace_columns -%}\n {%- set replace_conditions = [] -%}\n {%- for col in replace_columns -%}\n {%- do replace_conditions.append('t.' ~ col ~ ' <=> s.' ~ col) -%}\n {%- endfor -%}\n {%- set replace_conditions_csv = replace_conditions | join(' AND ') -%}\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) | map(attribute=\"name\") | list -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) | map(attribute=\"name\") | list -%}\n {%- set source_cols_lower = source_columns | map('lower') | list -%}\n {%- set select_columns = [] -%}\n {%- for dest_col in dest_columns -%}\n {%- set dest_col_lower = dest_col | lower -%}\n {%- set matched_col = namespace(value=none) -%}\n {%- for src_col in source_columns -%}\n {%- if src_col | lower == dest_col_lower and matched_col.value is none -%}\n {%- set matched_col.value = src_col -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if matched_col.value is not none -%}\n {%- do select_columns.append(matched_col.value) -%}\n {%- else -%}\n {%- do select_columns.append('NULL as ' ~ dest_col) -%}\n {%- endif -%}\n {%- endfor -%}\n insert into table {{ target_relation }} AS t\n replace on ({{ replace_conditions_csv }})\n (select {{ select_columns | join(', ') }} from {{ source_relation }}) AS s\n {%- else -%}\n {#-- Fallback to regular insert overwrite if no partitioning nor liquid clustering defined --#}\n {%- set has_insert_by_name = adapter.has_dbr_capability('insert_by_name') -%}\n insert overwrite table {{ target_relation }}\n {%- if has_insert_by_name %} by name{% endif %}\n select * from {{ source_relation }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.add_columns_to_list"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.981857, "supported_languages": null}, "macro.dbt_databricks.get_replace_where_sql": {"name": "get_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_replace_where_sql", "macro_sql": "{% macro get_replace_where_sql(args_dict) -%}\n {%- set predicates = args_dict['incremental_predicates'] -%}\n {%- set target_relation = args_dict['target_relation'] -%}\n {%- set temp_relation = args_dict['temp_relation'] -%}\n {%- set has_insert_by_name = adapter.has_dbr_capability('insert_by_name') -%}\nINSERT INTO {{ target_relation.render() }}\n{%- if has_insert_by_name %} BY NAME{% endif %}\n{%- if predicates %}\n {%- if predicates is sequence and predicates is not string %}\n REPLACE WHERE {{ predicates | join(' and ') }}\n {%- else %}\n REPLACE WHERE {{ predicates }}\n {%- endif %}\n{%- endif %}\n TABLE {{ temp_relation.render() }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9822028, "supported_languages": null}, "macro.dbt_databricks.get_delete_insert_sql": {"name": "get_delete_insert_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_delete_insert_sql", "macro_sql": "{% macro get_delete_insert_sql(arg_dict) -%}\n {%- set source_relation = arg_dict.get('temp_relation') -%}\n {%- set target_relation = arg_dict.get('target_relation') -%}\n {%- set incremental_predicates = config.get('incremental_predicates') -%}\n {%- set target_columns = (adapter.get_columns_in_relation(target_relation) | map(attribute='quoted') | list) -%}\n {%- set unique_key = config.require('unique_key') -%}\n {% do return(delete_insert_sql_impl(source_relation, target_relation, target_columns, unique_key, incremental_predicates)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.delete_insert_sql_impl"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.98253, "supported_languages": null}, "macro.dbt_databricks.delete_insert_sql_impl": {"name": "delete_insert_sql_impl", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.delete_insert_sql_impl", "macro_sql": "{% macro delete_insert_sql_impl(source_relation, target_relation, target_columns, unique_key, incremental_predicates) %}\n {# we use target_columns to select from source to avoid potential column order issues during insert #}\n {%- set target_cols_csv = target_columns | join(', ') -%}\n {%- set predicates -%}\n {%- if incremental_predicates is sequence and incremental_predicates is not string -%}\n where {{ incremental_predicates | join(' and ') }}\n {%- elif incremental_predicates is string and incremental_predicates is not none -%}\n where {{ incremental_predicates }}\n {%- endif -%}\n {%- endset -%}\n {%- set unique_keys = unique_key if unique_key is sequence and unique_key is not string else [unique_key] -%}\n \n {%- if adapter.has_dbr_capability('replace_on') -%}\n {#-- DBR 17.1+: Use efficient REPLACE ON syntax --#}\n {%- set replace_on_expr = [] -%}\n {%- for key in unique_keys -%}\n {%- do replace_on_expr.append('target.' ~ key ~ ' <=> temp.' ~ key) -%}\n {%- endfor -%}\n {%- set replace_on_expr = replace_on_expr | join(' and ') -%}\n insert into table {{ target_relation }} as target\nreplace on ({{ replace_on_expr }})\n(select {{ target_cols_csv }}\n from {{ source_relation }} {{ predicates }}) as temp\n {%- else -%}\n {#-- DBR < 17.1: Fallback to DELETE FROM + INSERT INTO --#}\n {% do return(delete_insert_legacy_sql(source_relation, target_relation, target_columns, unique_keys, incremental_predicates)) %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.delete_insert_legacy_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.983084, "supported_languages": null}, "macro.dbt_databricks.delete_insert_legacy_sql": {"name": "delete_insert_legacy_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.delete_insert_legacy_sql", "macro_sql": "{% macro delete_insert_legacy_sql(source_relation, target_relation, target_columns, unique_keys, incremental_predicates) %}\n {#-- Legacy implementation for DBR < 17.1 using DELETE FROM + INSERT INTO --#}\n {#-- Returns a list of SQL statements to be executed via execute_multiple_statements --#}\n {%- set target_cols_csv = target_columns | join(', ') -%}\n {%- set statements = [] -%}\n \n {#-- Build WHERE clause for DELETE statement --#}\n {%- set delete_conditions = [] -%}\n {%- for key in unique_keys -%}\n {%- do delete_conditions.append(target_relation ~ '.' ~ key ~ ' IN (SELECT ' ~ key ~ ' FROM ' ~ source_relation ~ ')') -%}\n {%- endfor -%}\n \n {#-- Add incremental predicates to DELETE if specified --#}\n {%- if incremental_predicates is sequence and incremental_predicates is not string -%}\n {%- for predicate in incremental_predicates -%}\n {%- do delete_conditions.append(predicate) -%}\n {%- endfor -%}\n {%- elif incremental_predicates is string and incremental_predicates is not none -%}\n {%- do delete_conditions.append(incremental_predicates) -%}\n {%- endif -%}\n \n {#-- Step 1: DELETE matching rows --#}\n {%- set delete_sql -%}\ndelete from {{ target_relation }}\nwhere {{ delete_conditions | join('\\n and ') }}\n {%- endset -%}\n {%- do statements.append(delete_sql) -%}\n\n {#-- Step 2: INSERT new rows --#}\n {%- set has_insert_by_name = adapter.has_dbr_capability('insert_by_name') -%}\n {%- set insert_sql -%}\ninsert into {{ target_relation }}{% if has_insert_by_name %} by name{% endif %}\n\nselect {{ target_cols_csv }}\nfrom {{ source_relation }}\n {%- if incremental_predicates is sequence and incremental_predicates is not string %}\nwhere {{ incremental_predicates | join(' and ') }}\n {%- elif incremental_predicates is string and incremental_predicates is not none %}\nwhere {{ incremental_predicates }}\n {%- endif %}\n {%- endset -%}\n {%- do statements.append(insert_sql) -%}\n \n {% do return(statements) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.983878, "supported_languages": null}, "macro.dbt_databricks.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(source_relation, target_relation) %}\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) | map(attribute=\"name\") | list -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) | map(attribute=\"name\") | list -%}\n {{ insert_into_sql_impl(target_relation, dest_columns, source_relation, source_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.insert_into_sql_impl"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.984077, "supported_languages": null}, "macro.dbt_databricks.insert_into_sql_impl": {"name": "insert_into_sql_impl", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.insert_into_sql_impl", "macro_sql": "{% macro insert_into_sql_impl(target_relation, dest_columns, source_relation, source_columns) %}\n {%- set dest_cols_lower = dest_columns | map('lower') | list -%}\n {%- set source_cols_lower = source_columns | map('lower') | list -%}\n {%- set has_insert_by_name = adapter.has_dbr_capability('insert_by_name') -%}\n\n {%- if dest_cols_lower | sort == source_cols_lower | sort -%}\n {#-- All columns match (case-insensitive) --#}\n {%- if has_insert_by_name -%}\n {#-- DBR 12.2+: Use INSERT BY NAME for name-based matching --#}\n insert into {{ target_relation }} by name\n select * from {{ source_relation }}\n {%- else -%}\n {#-- DBR < 12.2: Use positional INSERT (columns must be in same order) --#}\n insert into {{ target_relation }}\n select * from {{ source_relation }}\n {%- endif -%}\n {%- else -%}\n {#-- Columns don't match, select only columns that exist in target --#}\n {#-- Note: Cannot use BY NAME with explicit column list, so use traditional syntax --#}\n {%- set common_columns = [] -%}\n {%- for dest_col in dest_columns -%}\n {%- if dest_col | lower in source_cols_lower -%}\n {%- do common_columns.append(dest_col) -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if common_columns | length > 0 -%}\n insert into {{ target_relation }} ({{ common_columns | join(', ') }})\n select {{ common_columns | join(', ') }} from {{ source_relation }}\n {%- else -%}\n {#-- No common columns, this shouldn't happen but handle it gracefully --#}\n {%- if has_insert_by_name -%}\n insert into {{ target_relation }} by name\n select * from {{ source_relation }}\n {%- else -%}\n insert into {{ target_relation }}\n select * from {{ source_relation }}\n {%- endif -%}\n {%- endif -%}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.984592, "supported_languages": null}, "macro.dbt_databricks.databricks__get_merge_sql": {"name": "databricks__get_merge_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_merge_sql", "macro_sql": "{% macro databricks__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %}\n {# need dest_columns for merge_exclude_columns, default to use \"*\" #}\n\n {%- set target_alias = config.get('target_alias', 'DBT_INTERNAL_DEST') -%}\n {%- set source_alias = config.get('source_alias', 'DBT_INTERNAL_SOURCE') -%}\n\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target) -%}\n {%- set source_columns = (adapter.get_columns_in_relation(source) | map(attribute='name') | list)-%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set merge_with_schema_evolution = (config.get('merge_with_schema_evolution') | lower == 'true') -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set skip_matched_step = (config.get('skip_matched_step') | lower == 'true') -%}\n {%- set skip_not_matched_step = (config.get('skip_not_matched_step') | lower == 'true') -%}\n\n {%- set matched_condition = config.get('matched_condition') -%}\n {%- set not_matched_condition = config.get('not_matched_condition') -%}\n\n {%- set not_matched_by_source_action = config.get('not_matched_by_source_action') -%}\n {%- set not_matched_by_source_condition = config.get('not_matched_by_source_condition') -%}\n\n {%- set not_matched_by_source_action_trimmed = not_matched_by_source_action | lower | trim(' \\n\\t') %}\n {%- set not_matched_by_source_action_is_set = (\n not_matched_by_source_action_trimmed == 'delete'\n or not_matched_by_source_action_trimmed.startswith('update')\n )\n %}\n \n \n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n {{ source_alias }}.{{ adapter.quote(key) }} <=> {{ target_alias }}.{{ adapter.quote(key) }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n {{ source_alias }}.{{ adapter.quote(unique_key) }} <=> {{ target_alias }}.{{ adapter.quote(unique_key) }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n merge\n {%- if merge_with_schema_evolution %}\n with schema evolution\n {%- endif %}\n into\n {{ target }} as {{ target_alias }}\n using\n {{ source }} as {{ source_alias }}\n on\n {{ predicates | join('\\n and ') }}\n {%- if not skip_matched_step %}\n when matched\n {%- if matched_condition %}\n and ({{ matched_condition }})\n {%- endif %}\n then update set\n {{ get_merge_update_set(update_columns, on_schema_change, source_columns, source_alias) }}\n {%- endif %}\n {%- if not skip_not_matched_step %}\n when not matched\n {%- if not_matched_condition %}\n and ({{ not_matched_condition }})\n {%- endif %}\n then insert\n {{ get_merge_insert(on_schema_change, source_columns, source_alias) }}\n {%- endif %}\n {%- if not_matched_by_source_action_is_set %}\n when not matched by source\n {%- if not_matched_by_source_condition %}\n and ({{ not_matched_by_source_condition }})\n {%- endif %}\n then {{ not_matched_by_source_action }}\n {%- endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.incremental_validate_on_schema_change", "macro.dbt.get_merge_update_columns", "macro.dbt_databricks.get_merge_update_set", "macro.dbt_databricks.get_merge_insert"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.985947, "supported_languages": null}, "macro.dbt_databricks.get_merge_update_set": {"name": "get_merge_update_set", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_merge_update_set", "macro_sql": "{% macro get_merge_update_set(update_columns, on_schema_change, source_columns, source_alias='DBT_INTERNAL_SOURCE') %}\n {%- if update_columns -%}\n {%- for column_name in update_columns -%}\n {{ adapter.quote(column_name) }} = {{ source_alias }}.{{ adapter.quote(column_name) }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n {%- elif on_schema_change == 'ignore' -%}\n *\n {%- else -%}\n {%- for column in source_columns -%}\n {{ adapter.quote(column) }} = {{ source_alias }}.{{ adapter.quote(column) }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9862661, "supported_languages": null}, "macro.dbt_databricks.get_merge_insert": {"name": "get_merge_insert", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_merge_insert", "macro_sql": "{% macro get_merge_insert(on_schema_change, source_columns, source_alias='DBT_INTERNAL_SOURCE') %}\n {%- if on_schema_change == 'ignore' -%}\n *\n {%- else -%}\n ({% for column in source_columns %}{{ adapter.quote(column) }}{% if not loop.last %}, {% endif %}{% endfor %}) VALUES (\n {%- for column in source_columns -%}\n {{ source_alias }}.{{ adapter.quote(column) }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %})\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.986496, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_microbatch_sql": {"name": "databricks__get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_microbatch_sql", "macro_sql": "{% macro databricks__get_incremental_microbatch_sql(arg_dict) %}\n {%- set incremental_predicates = [] if arg_dict.get('incremental_predicates') is none else arg_dict.get('incremental_predicates') -%}\n {%- set event_time = model.config.event_time -%}\n {%- set start_time = config.get(\"__dbt_internal_microbatch_event_time_start\") -%}\n {%- set end_time = config.get(\"__dbt_internal_microbatch_event_time_end\") -%}\n {%- if start_time -%}\n {%- do incremental_predicates.append(\"cast(\" ~ event_time ~ \" as TIMESTAMP) >= '\" ~ start_time ~ \"'\") -%}\n {%- endif -%}\n {%- if end_time -%}\n {%- do incremental_predicates.append(\"cast(\" ~ event_time ~ \" as TIMESTAMP) < '\" ~ end_time ~ \"'\") -%}\n {%- endif -%}\n {%- do arg_dict.update({'incremental_predicates': incremental_predicates}) -%}\n {{ return(get_replace_where_sql(arg_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_where_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.986871, "supported_languages": null}, "macro.dbt_databricks.databricks__get_merge_update_columns": {"name": "databricks__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_merge_update_columns", "macro_sql": "{% macro databricks__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = None -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {# Don't quote here - columns will be quoted in get_merge_update_set #}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.column) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.987186, "supported_languages": null}, "macro.dbt_databricks.materialization_incremental_databricks": {"name": "materialization_incremental_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_databricks.materialization_incremental_databricks", "macro_sql": "{% materialization incremental, adapter='databricks', supported_languages=['sql', 'python'] -%}\n {{ log(\"MATERIALIZING INCREMENTAL\") }}\n\n {%- set catalog_relation = adapter.build_catalog_relation(config.model) -%}\n\n {% set existing_relation = load_relation_with_metadata(this) %}\n {% set target_relation = this.incorporate(type='table') %}\n {% set incremental_strategy = get_incremental_strategy(catalog_relation.file_format) %}\n {% set grant_config = config.get('grants') %}\n {% set full_refresh = should_full_refresh() %}\n {% set partition_by = config.get('partition_by') %}\n {% set language = model['language'] %}\n {% set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') %}\n {% set is_delta = (catalog_relation.file_format == 'delta' and existing_relation.is_delta) %}\n {% set is_iceberg = (catalog_relation.file_format == 'iceberg' and existing_relation.is_iceberg) %}\n {% set is_replaceable_format = is_delta or is_iceberg %}\n {% set compiled_code = adapter.clean_sql(model['compiled_code']) %}\n\n {% if adapter.get_behavior_flag_no_warn('use_materialization_v2') %}\n {{ log(\"USING V2 MATERIALIZATION\") }}\n {#-- Set vars --#}\n {% set safe_create = config.get('use_safer_relation_operations', False) | as_bool %}\n {{ log(\"Safe create: \" ~ safe_create) }}\n {% set should_replace = existing_relation.is_dlt or existing_relation.is_view or full_refresh %}\n {% set is_replaceable = existing_relation.can_be_replaced and is_replaceable_format %}\n\n {% set intermediate_relation = make_intermediate_relation(target_relation) %}\n {% set staging_relation = make_staging_relation(target_relation) %}\n\n {{ run_pre_hooks() }}\n\n {% call statement('main', language=language) %}\n {{ get_create_intermediate_table(intermediate_relation, compiled_code, language) }}\n {% endcall %}\n\n {#-- Incremental run logic --#}\n {%- if existing_relation is none -%}\n {{ log(\"No existing relation found\") }}\n {{ create_table_at(target_relation, intermediate_relation, compiled_code) }}\n {%- elif should_replace -%}\n {{ log(\"Existing relation found that requires replacement\") }}\n {% if safe_create and existing_relation.can_be_renamed %}\n {{ log(\"Safe create enabled and relation can be renamed\") }}\n {{ safe_relation_replace(existing_relation, staging_relation, intermediate_relation, compiled_code) }}\n {% else %}\n {#-- Relation must be dropped & recreated --#}\n {% if not is_replaceable %} {#-- If Delta, we will `create or replace` below, so no need to drop --#}\n {{ log(\"Dropping existing relation, as it is not replaceable\") }}\n {% do adapter.drop_relation(existing_relation) %}\n {% endif %}\n {{ log(\"Replacing target relation\") }}\n {{ create_table_at(target_relation, intermediate_relation, compiled_code) }}\n {% endif %}\n {%- else -%}\n {{ log(\"Existing relation found, proceeding with incremental work\")}}\n {#-- Set Overwrite Mode to DYNAMIC for subsequent incremental operations --#}\n {%- if incremental_strategy == 'insert_overwrite' and partition_by -%}\n {{ set_overwrite_mode('DYNAMIC') }}\n {%- endif -%}\n {#-- Relation must be merged --#}\n {%- do process_schema_changes(on_schema_change, intermediate_relation, existing_relation) -%}\n {{ process_config_changes(target_relation) }}\n {% set build_sql = get_build_sql(incremental_strategy, target_relation, intermediate_relation) %}\n {%- if language == 'sql' -%}\n {#-- Check if build_sql is a list (multi-statement strategy) or a string (single statement) --#}\n {%- if build_sql is sequence and build_sql is not string -%}\n {%- for sql_statement in build_sql -%}\n {%- call statement('main') -%}\n {{ sql_statement }}\n {%- endcall -%}\n {%- endfor -%}\n {%- else -%}\n {%- call statement('main') -%}\n {{ build_sql }}\n {%- endcall -%}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {%- call statement_with_staging_table('main', intermediate_relation) -%}\n {{ build_sql }}\n {%- endcall -%}\n {%- endif -%}\n {%- endif -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n {% do optimize(target_relation) %}\n\n {% if language == 'python' %}\n {{ drop_relation_if_exists(intermediate_relation) }}\n {% endif %}\n\n {{ run_post_hooks() }}\n\n {% else %}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- set tags = config.get('databricks_tags') -%}\n {% set temp_relation = make_temp_relation(target_relation) %}\n {% set incremental_predicates = config.get('predicates') or config.get('incremental_predicates') %}\n {%- set unique_key = config.get('unique_key') -%}\n\n {#-- Run pre-hooks --#}\n {{ run_hooks(pre_hooks) }}\n {#-- Incremental run logic --#}\n {%- if existing_relation is none -%}\n {#-- Relation must be created --#}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {% do apply_tags(target_relation, tags) %}\n {%- if language == 'python' -%}\n {%- do apply_tblproperties(target_relation, tblproperties) %}\n {%- endif -%}\n\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n {%- elif existing_relation.is_view or existing_relation.is_materialized_view or existing_relation.is_streaming_table or should_full_refresh() -%}\n {#-- Relation must be dropped & recreated --#}\n {% if not is_replaceable_format %} {#-- If Delta or Iceberg, we will `create or replace` below, so no need to drop --#}\n {% do adapter.drop_relation(existing_relation) %}\n {% endif %}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% if not existing_relation.is_view %}\n {% do persist_constraints(target_relation, model) %}\n {% endif %}\n {% do apply_tags(target_relation, tags) %}\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n {%- else -%}\n {#-- Set Overwrite Mode to DYNAMIC for subsequent incremental operations --#}\n {%- if incremental_strategy == 'insert_overwrite' and partition_by -%}\n {{ set_overwrite_mode('DYNAMIC') }}\n {%- endif -%}\n {#-- Relation must be merged --#}\n {%- set model_config = adapter.get_config_from_model(config.model) -%}\n {%- set _existing_config = adapter.get_relation_config(existing_relation, model_config) -%}\n {%- set _configuration_changes = model_config.get_changeset(_existing_config) -%}\n {%- call statement('create_temp_relation', language=language) -%}\n {{ create_table_as(True, temp_relation, compiled_code, language) }}\n {%- endcall -%}\n {%- do process_schema_changes(on_schema_change, temp_relation, existing_relation) -%}\n {%- set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) -%}\n {%- set strategy_arg_dict = ({\n 'target_relation': target_relation,\n 'temp_relation': temp_relation,\n 'unique_key': unique_key,\n 'dest_columns': none,\n 'incremental_predicates': incremental_predicates}) -%}\n {%- set build_sql = strategy_sql_macro_func(strategy_arg_dict) -%}\n {%- if language == 'sql' -%}\n {#-- Check if build_sql is a list (multi-statement strategy) or a string (single statement) --#}\n {%- if build_sql is sequence and build_sql is not string -%}\n {%- for sql_statement in build_sql -%}\n {%- call statement('main') -%}\n {{ sql_statement }}\n {%- endcall -%}\n {%- endfor -%}\n {%- else -%}\n {%- call statement('main') -%}\n {{ build_sql }}\n {%- endcall -%}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {%- call statement_with_staging_table('main', temp_relation) -%}\n {{ build_sql }}\n {%- endcall -%}\n {#--\n This is yucky.\n See note in dbt-spark/dbt/include/spark/macros/adapters.sql\n re: python models and temporary views.\n\n Also, why does not either drop_relation or adapter.drop_relation work here?!\n --#}\n {%- endif -%}\n {% if _configuration_changes is not none %}\n {% set tags = _configuration_changes.changes.get(\"tags\", None) %}\n {% set tblproperties = _configuration_changes.changes.get(\"tblproperties\", None) %}\n {% set liquid_clustering = _configuration_changes.changes.get(\"liquid_clustering\") %}\n {% set row_filter = _configuration_changes.changes.get(\"row_filter\") %}\n {% set constraints = _configuration_changes.changes.get(\"constraints\") %}\n {% if tags is not none %}\n {% do apply_tags(target_relation, tags.set_tags) %}\n {%- endif -%}\n {% if tblproperties is not none %}\n {% do apply_tblproperties(target_relation, tblproperties.tblproperties) %}\n {%- endif -%}\n {% if liquid_clustering is not none %}\n {% do apply_liquid_clustered_cols(target_relation, liquid_clustering) %}\n {% endif %}\n {% if row_filter is not none %}\n {{ apply_row_filter(target_relation, row_filter) }}\n {% endif %}\n {#- Incremental constraint application requires information_schema access (see fetch_*_constraints macros) -#}\n {% set contract_config = config.get('contract') %}\n {% if constraints and contract_config and contract_config.enforced and not target_relation.is_hive_metastore() %}\n {{ apply_constraints(target_relation, constraints) }}\n {% endif %}\n {%- endif -%}\n {% do persist_docs(target_relation, model, for_relation=True) %}\n {%- endif -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n {% do optimize(target_relation) %}\n\n {{ run_hooks(post_hooks) }}\n {%- endif -%}\n\n {%- if incremental_strategy == 'insert_overwrite' and not full_refresh -%}\n {{ set_overwrite_mode('STATIC') }}\n {%- endif -%}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.load_relation_with_metadata", "macro.dbt_databricks.get_incremental_strategy", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.make_intermediate_relation", "macro.dbt_databricks.make_staging_relation", "macro.dbt_databricks.run_pre_hooks", "macro.dbt.statement", "macro.dbt_databricks.get_create_intermediate_table", "macro.dbt_databricks.create_table_at", "macro.dbt_databricks.safe_relation_replace", "macro.dbt_databricks.set_overwrite_mode", "macro.dbt.process_schema_changes", "macro.dbt_databricks.process_config_changes", "macro.dbt_databricks.get_build_sql", "macro.dbt_databricks.statement_with_staging_table", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.optimize", "macro.dbt.drop_relation_if_exists", "macro.dbt_databricks.run_post_hooks", "macro.dbt.make_temp_relation", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt_databricks.persist_constraints", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_tblproperties", "macro.dbt.persist_docs", "macro.dbt_databricks.apply_liquid_clustered_cols", "macro.dbt_databricks.apply_row_filter", "macro.dbt_databricks.apply_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9923232, "supported_languages": ["sql", "python"]}, "macro.dbt_databricks.set_overwrite_mode": {"name": "set_overwrite_mode", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_databricks.set_overwrite_mode", "macro_sql": "{% macro set_overwrite_mode(value) %}\n {% if adapter.is_cluster() %}\n {%- call statement('Setting partitionOverwriteMode: ' ~ value) -%}\n set spark.sql.sources.partitionOverwriteMode = {{ value }}\n {%- endcall -%}\n {% else %}\n {{ exceptions.warn(\"insert_overwrite is supported on SQL warehouses with DBR 17.1+. On older DBR versions, this strategy would be equivalent to using the table materialization.\") }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9925241, "supported_languages": null}, "macro.dbt_databricks.get_build_sql": {"name": "get_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_databricks.get_build_sql", "macro_sql": "{% macro get_build_sql(incremental_strategy, target_relation, intermediate_relation) %}\n {%- set unique_key = config.get('unique_key') -%}\n {%- set incremental_predicates = config.get('predicates') or config.get('incremental_predicates') -%}\n {%- set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) -%}\n {%- set strategy_arg_dict = ({\n 'target_relation': target_relation,\n 'temp_relation': intermediate_relation,\n 'unique_key': unique_key,\n 'dest_columns': none,\n 'incremental_predicates': incremental_predicates}) -%}\n {% do return(strategy_sql_macro_func(strategy_arg_dict)) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9928372, "supported_languages": null}, "macro.dbt_databricks.process_config_changes": {"name": "process_config_changes", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_databricks.process_config_changes", "macro_sql": "{% macro process_config_changes(target_relation) %}\n {% set apply_config_changes = config.get('incremental_apply_config_changes', True) | as_bool %}\n {% if apply_config_changes %}\n {%- set model_config = adapter.get_config_from_model(config.model) -%}\n {%- set existing_config = adapter.get_relation_config(target_relation, model_config) -%}\n {%- set configuration_changes = model_config.get_changeset(existing_config) -%}\n {{ apply_config_changeset(target_relation, model, configuration_changes) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.apply_config_changeset"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.993077, "supported_languages": null}, "macro.dbt_databricks.databricks__create_or_replace_clone": {"name": "databricks__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone/strategies.sql", "original_file_path": "macros/materializations/clone/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__create_or_replace_clone", "macro_sql": "{% macro databricks__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace\n table {{ this_relation.render() }}\n shallow clone {{ defer_relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.993233, "supported_languages": null}, "macro.dbt_databricks.create_or_replace_clone_external": {"name": "create_or_replace_clone_external", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone/strategies.sql", "original_file_path": "macros/materializations/clone/strategies.sql", "unique_id": "macro.dbt_databricks.create_or_replace_clone_external", "macro_sql": "{% macro create_or_replace_clone_external(this_relation, defer_relation) %}\n\n {%- set catalog_relation = adapter.build_catalog_relation(config.model) -%}\n\n create or replace\n table {{ this_relation.render() }}\n shallow clone {{ defer_relation.render() }}\n {{ location_clause(catalog_relation) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.location_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.993377, "supported_languages": null}, "macro.dbt_databricks.databricks__can_clone_table": {"name": "databricks__can_clone_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone/clone.sql", "original_file_path": "macros/materializations/clone/clone.sql", "unique_id": "macro.dbt_databricks.databricks__can_clone_table", "macro_sql": "{% macro databricks__can_clone_table() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.993664, "supported_languages": null}, "macro.dbt_databricks.materialization_clone_databricks": {"name": "materialization_clone_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone/clone.sql", "original_file_path": "macros/materializations/clone/clone.sql", "unique_id": "macro.dbt_databricks.materialization_clone_databricks", "macro_sql": "\n\n\n{%- materialization clone, adapter='databricks' -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n {%- set file_format = adapter.resolve_file_format(config) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% if other_existing_relation.is_external_table %}\n {% call statement('main') %}\n {{ create_or_replace_clone_external(target_relation, defer_relation) }}\n {% endcall %}\n {% else %}\n {% call statement('main') %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endcall %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n {% endif %}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt_databricks.create_or_replace_clone_external", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.994726, "supported_languages": ["sql"]}, "macro.dbt_databricks.databricks__get_binding_char": {"name": "databricks__get_binding_char", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__get_binding_char", "macro_sql": "{% macro databricks__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.99523, "supported_languages": null}, "macro.dbt_databricks.databricks__load_csv_rows": {"name": "databricks__load_csv_rows", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__load_csv_rows", "macro_sql": "{% macro databricks__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n {% set column_override = model['config'].get('column_types', {}) %}\n {% set must_cast = model['config'].get('file_format', 'delta') == 'parquet' %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert {% if loop.index0 == 0 -%} overwrite {% else -%} into {% endif -%} {{ this.render() }} values\n {% for row in chunk -%}\n ({%- for col_name in agate_table.column_names -%}\n {%- if must_cast -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n cast({{ get_binding_char() }} as {{type}})\n {%- else -%}\n {{ get_binding_char() }}\n {%- endif -%}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True, close_cursor=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.995886, "supported_languages": null}, "macro.dbt_databricks.databricks__reset_csv_table": {"name": "databricks__reset_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__reset_csv_table", "macro_sql": "{% macro databricks__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% if old_relation %}\n {% if old_relation.can_be_replaced and adapter.resolve_file_format(config) in ('delta', 'iceberg') %}\n {% set sql = create_or_replace_csv_table(model, agate_table, True) %}\n {% else %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% endif %}\n {% else %}\n {% set sql = create_csv_table(model, agate_table) %}\n {% endif %}\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.create_or_replace_csv_table", "macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.996158, "supported_languages": null}, "macro.dbt_databricks.create_or_replace_csv_table": {"name": "create_or_replace_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.create_or_replace_csv_table", "macro_sql": "{% macro create_or_replace_csv_table(model, agate_table, replace=False) %}\n\n {%- set catalog_relation = adapter.build_catalog_relation(config.model) -%}\n\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {%- set column_comment = config.persist_column_docs() and model.columns %}\n {%- set identifier = model['alias'] -%}\n {%- set relation = api.Relation.create(database=database, schema=schema, identifier=identifier, type='table') -%}\n {%- set replace_clause = \"\" -%}\n {%- if replace -%}\n {%- set replace_clause = \"or replace\" -%}\n {%- endif -%}\n\n {% set sql %}\n create {{replace_clause}} table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {%- set column_comment_clause = \"\" -%}\n {%- if column_comment and col_name in model.columns.keys() -%} \n {%- set comment = model.columns[col_name]['description'] | replace(\"'\", \"\\\\'\") -%}\n {%- if comment and comment != \"\" -%}\n {%- set column_comment_clause = \"comment '\" ~ comment ~ \"'\" -%}\n {%- endif -%}\n {%- endif -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {{ column_comment_clause }}{%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {{ file_format_clause(catalog_relation) }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause(catalog_relation) }}\n {{ comment_clause() }}\n {{ tblproperties_clause() }}\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.file_format_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_databricks.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_databricks.tblproperties_clause", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9971032, "supported_languages": null}, "macro.dbt_databricks.databricks__create_csv_table": {"name": "databricks__create_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__create_csv_table", "macro_sql": "{% macro databricks__create_csv_table(model, agate_table) %}\n {{ return(create_or_replace_csv_table(model, agate_table)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.create_or_replace_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9971871, "supported_languages": null}, "macro.dbt_databricks.log_seed_operation": {"name": "log_seed_operation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.log_seed_operation", "macro_sql": "{% macro log_seed_operation(agate_table, full_refresh_mode, create_table_sql, sql) %}\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.noop_statement", "macro.dbt.get_csv_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.99737, "supported_languages": null}, "macro.dbt_databricks.materialization_seed_databricks": {"name": "materialization_seed_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/seeds.sql", "original_file_path": "macros/materializations/seeds/seeds.sql", "unique_id": "macro.dbt_databricks.materialization_seed_databricks", "macro_sql": "{% materialization seed, adapter='databricks' %}\n {% set target_relation = this.incorporate(type='table') %}\n\n {% if adapter.get_behavior_flag_no_warn('use_materialization_v2') %}\n {{ create_seed_v2(target_relation) }}\n {% else %}\n {{ create_seed_v1(target_relation) }}\n {% endif %}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.create_seed_v2", "macro.dbt_databricks.create_seed_v1"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9978728, "supported_languages": ["sql"]}, "macro.dbt_databricks.create_seed_v2": {"name": "create_seed_v2", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/seeds.sql", "original_file_path": "macros/materializations/seeds/seeds.sql", "unique_id": "macro.dbt_databricks.create_seed_v2", "macro_sql": "{% macro create_seed_v2(target_relation) %}\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier, needs_information=True) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and (old_relation.is_view or old_relation.is_materialized_view)) -%}\n {%- set exists_as_streaming_table = (old_relation is not none and old_relation.is_streaming_table) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparision later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_pre_hooks() }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view or a materialized view\".format(old_relation)) }}\n {% elif exists_as_streaming_table %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a streaming table\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {{ log_seed_operation(agate_table, full_refresh_mode, create_table_sql, sql) }}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n -- No need to persist docs, already handled in seed create\n\n {{ run_post_hooks() }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt_databricks.run_pre_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt_databricks.log_seed_operation", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.run_post_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.998687, "supported_languages": null}, "macro.dbt_databricks.create_seed_v1": {"name": "create_seed_v1", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/seeds.sql", "original_file_path": "macros/materializations/seeds/seeds.sql", "unique_id": "macro.dbt_databricks.create_seed_v1", "macro_sql": "{% macro create_seed_v1(target_relation) %}\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier, needs_information=True) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and (old_relation.is_view or old_relation.is_materialized_view)) -%}\n {%- set exists_as_streaming_table = (old_relation is not none and old_relation.is_streaming_table) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparision later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view or a materialized view\".format(old_relation)) }}\n {% elif exists_as_streaming_table %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a streaming table\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n -- No need to persist docs, already handled in seed create\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9997, "supported_languages": null}, "macro.dbt_databricks.databricks__scalar_function_create_replace_signature_sql": {"name": "databricks__scalar_function_create_replace_signature_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt_databricks.databricks__scalar_function_create_replace_signature_sql", "macro_sql": "{% macro databricks__scalar_function_create_replace_signature_sql(target_relation) %}\n CREATE OR REPLACE FUNCTION {{ target_relation.render() }} ({{ formatted_scalar_function_args_sql()}})\n RETURNS {{ model.returns.data_type }}\n LANGUAGE SQL\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.formatted_scalar_function_args_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.9999201, "supported_languages": null}, "macro.dbt_databricks.databricks__scalar_function_body_sql": {"name": "databricks__scalar_function_body_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt_databricks.databricks__scalar_function_body_sql", "macro_sql": "{% macro databricks__scalar_function_body_sql() %}\n RETURN\n {{ model.compiled_code }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011793.99997, "supported_languages": null}, "macro.dbt_databricks.databricks__scalar_function_create_replace_signature_python": {"name": "databricks__scalar_function_create_replace_signature_python", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt_databricks.databricks__scalar_function_create_replace_signature_python", "macro_sql": "{% macro databricks__scalar_function_create_replace_signature_python(target_relation) %}\n CREATE OR REPLACE FUNCTION {{ target_relation.render() }} ({{ formatted_scalar_function_args_sql() }})\n RETURNS {{ model.returns.data_type }}\n LANGUAGE PYTHON\n AS\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.formatted_scalar_function_args_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0000541, "supported_languages": null}, "macro.dbt_databricks.databricks__scalar_function_body_python": {"name": "databricks__scalar_function_body_python", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt_databricks.databricks__scalar_function_body_python", "macro_sql": "{% macro databricks__scalar_function_body_python() %}\n$$\n{{ model.compiled_code }}\n$$\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0001018, "supported_languages": null}, "macro.dbt_databricks.databricks__scalar_function_python": {"name": "databricks__scalar_function_python", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt_databricks.databricks__scalar_function_python", "macro_sql": "{% macro databricks__scalar_function_python(target_relation) %}\n {#- Warn if user explicitly provided no-op config fields -#}\n {%- if model.config.get('runtime_version') -%}\n {{ exceptions.warn(\"'runtime_version' is accepted for compatibility but has no effect on Databricks Python UDFs. Databricks manages the Python runtime internally.\") }}\n {%- endif -%}\n {%- if model.config.get('entry_point') -%}\n {{ exceptions.warn(\"'entry_point' is accepted for compatibility but has no effect on Databricks Python UDFs. The function body is used directly.\") }}\n {%- endif -%}\n {{ databricks__scalar_function_create_replace_signature_python(target_relation) }}\n {{ databricks__scalar_function_body_python() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__scalar_function_create_replace_signature_python", "macro.dbt_databricks.databricks__scalar_function_body_python"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.000284, "supported_languages": null}, "macro.dbt_databricks.databricks__generate_database_name": {"name": "databricks__generate_database_name", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt_databricks.databricks__generate_database_name", "macro_sql": "{% macro databricks__generate_database_name(custom_database_name=none, node=none) -%}\n {%- if custom_database_name is none -%}\n {%- if node is not none and node|attr('database') -%}\n {%- set catalog_relation = adapter.build_catalog_relation(node) -%}\n {{ return(catalog_relation.catalog_name) }}\n {%- elif 'config' in target -%}\n {%- set catalog_relation = adapter.build_catalog_relation(target) -%}\n {{ return(catalog_relation.catalog_name) }}\n {%- else -%}\n {{ return(target.database) }}\n {%- endif -%}\n {%- else -%}\n {{ return(custom_database_name) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.000664, "supported_languages": null}, "macro.dbt_databricks.databricks__get_drop_sql": {"name": "databricks__get_drop_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt_databricks.databricks__get_drop_sql", "macro_sql": "{% macro databricks__get_drop_sql(relation) -%}\n {%- if relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n {%- elif relation.is_streaming_table-%}\n {{ drop_streaming_table(relation) }}\n {%- elif relation.is_view or relation.is_metric_view -%}\n {{ drop_view(relation) }}\n {%- else -%}\n {{ drop_table(relation) }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.drop_materialized_view", "macro.dbt_databricks.drop_streaming_table", "macro.dbt.drop_view", "macro.dbt.drop_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.000881, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_relation": {"name": "databricks__drop_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_relation", "macro_sql": "{% macro databricks__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {{ get_drop_sql(relation) }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0009742, "supported_languages": null}, "macro.dbt_databricks.get_replace_sql": {"name": "get_replace_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.get_replace_sql", "macro_sql": "{% macro get_replace_sql(existing_relation, target_relation, sql) %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {% do return(adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.001388, "supported_languages": null}, "macro.dbt_databricks.databricks__get_replace_sql": {"name": "databricks__get_replace_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.databricks__get_replace_sql", "macro_sql": "{% macro databricks__get_replace_sql(existing_relation, target_relation, sql) %}\n {# /* if safe_relation_replace, prefer renaming */ #}\n {% if target_relation.type == \"table\" %}\n {{ exceptions.raise_not_implemented('get_replace_sql not implemented for target of table') }}\n {% endif %}\n\n {#- Metric views always support CREATE OR REPLACE (no delta/file_format dependency) -#}\n {#- Note: existing relation is typed as VIEW from DB, so check target for metric_view -#}\n {% if target_relation.is_metric_view %}\n {{ return(get_replace_metric_view_sql(target_relation, sql)) }}\n {% endif %}\n\n {% set safe_replace = config.get('use_safer_relation_operations', False) | as_bool %}\n {% set file_format = adapter.resolve_file_format(config) %}\n {% set is_replaceable = existing_relation.type == target_relation.type and existing_relation.can_be_replaced and file_format == \"delta\" %}\n\n {% if not safe_replace %}\n {# Prioritize 'create or replace' for speed #}\n {% if is_replaceable and existing_relation.is_view %}\n {{ return(get_replace_view_sql(target_relation, sql)) }}\n {% elif is_replaceable and existing_relation.is_table %}\n {{ return(get_replace_table_sql(target_relation, sql)) }}\n {% elif is_replaceable and existing_relation.is_materialized_view %}\n {{ return(get_replace_materialized_view_sql(target_relation, sql)) }}\n {% endif %}\n {% endif %}\n\n {# If safe_replace, then we know that anything that would have been caught above is instead caught here #}\n {% if target_relation.can_be_renamed and existing_relation.can_be_renamed %}\n {{ return(safely_replace(existing_relation, target_relation, sql)) }}\n {% elif target_relation.can_be_renamed %}\n {{ return(stage_then_replace(existing_relation, target_relation, sql)) }}\n {% elif existing_relation.can_be_renamed %}\n {{ return(backup_and_create_in_place(existing_relation, target_relation, sql)) }}\n {% else %}\n {{ return(drop_and_create(existing_relation, target_relation, sql)) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_metric_view_sql", "macro.dbt.get_replace_view_sql", "macro.dbt.get_replace_table_sql", "macro.dbt.get_replace_materialized_view_sql", "macro.dbt_databricks.safely_replace", "macro.dbt_databricks.stage_then_replace", "macro.dbt_databricks.backup_and_create_in_place", "macro.dbt_databricks.drop_and_create"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.002152, "supported_languages": null}, "macro.dbt_databricks.safely_replace": {"name": "safely_replace", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.safely_replace", "macro_sql": "{% macro safely_replace(existing_relation, target_relation, sql) %}\n {{ log('Using safely_replace') }}\n {% set staging_relation = make_staging_relation(target_relation, type='view') %}\n {{ drop_relation_if_exists(staging_relation) }}\n {% call statement(name=\"main\") %}\n {{ get_create_sql(staging_relation, sql) }}\n {% endcall %}\n {{ create_backup(existing_relation) }}\n {{ return([\n get_rename_sql(staging_relation, existing_relation.render()),\n get_drop_backup_sql(existing_relation)\n ]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.make_staging_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.get_create_sql", "macro.dbt_databricks.create_backup", "macro.dbt.get_rename_sql", "macro.dbt.get_drop_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.002402, "supported_languages": null}, "macro.dbt_databricks.stage_then_replace": {"name": "stage_then_replace", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.stage_then_replace", "macro_sql": "{% macro stage_then_replace(existing_relation, target_relation, sql) %}\n {{ log('Using stage_then_replace') }}\n {% set staging_relation = make_staging_relation(target_relation, type='view') %}\n {{ drop_relation_if_exists(staging_relation) }}\n {% call statement(name=\"main\") %}\n {{ get_create_sql(staging_relation, sql) }}\n {% endcall %}\n\n {{ return([\n get_drop_sql(existing_relation),\n get_rename_sql(staging_relation, existing_relation.render()),\n ]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.make_staging_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.get_create_sql", "macro.dbt.get_drop_sql", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0026188, "supported_languages": null}, "macro.dbt_databricks.backup_and_create_in_place": {"name": "backup_and_create_in_place", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.backup_and_create_in_place", "macro_sql": "{% macro backup_and_create_in_place(existing_relation, target_relation, sql) %}\n {{ log('Using backup_and_create_in_place') }}\n {{ create_backup(existing_relation) }}\n {{ return([\n get_create_sql(target_relation, sql),\n get_drop_backup_sql(existing_relation)\n ]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.create_backup", "macro.dbt.get_create_sql", "macro.dbt.get_drop_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.00275, "supported_languages": null}, "macro.dbt_databricks.drop_and_create": {"name": "drop_and_create", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.drop_and_create", "macro_sql": "{% macro drop_and_create(existing_relation, target_relation, sql) %}\n {{ log('Using drop_and_create') }}\n {{ return([\n get_drop_sql(existing_relation),\n get_create_sql(target_relation, sql)\n ]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.002859, "supported_languages": null}, "macro.dbt_databricks.databricks__comment_clause": {"name": "databricks__comment_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/comment.sql", "original_file_path": "macros/relations/comment.sql", "unique_id": "macro.dbt_databricks.databricks__comment_clause", "macro_sql": "{% macro databricks__comment_clause() %}\n {%- set raw_persist_docs = config.get('persist_docs', {}) -%}\n {%- if raw_persist_docs is mapping -%}\n {%- set raw_relation = raw_persist_docs.get('relation', false) -%}\n {%- if raw_relation and model.description -%}\n comment '{{ model.description | replace(\"'\", \"\\\\'\") }}'\n {%- endif -%}\n {%- elif raw_persist_docs -%}\n {{ exceptions.raise_compiler_error(\"Invalid value provided for 'persist_docs'. Expected dict but got value: \" ~ raw_persist_docs) }}\n {% endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0031428, "supported_languages": null}, "macro.dbt_databricks.fetch_tags": {"name": "fetch_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.fetch_tags", "macro_sql": "{% macro fetch_tags(relation) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Tags are only supported for Unity Catalog\") }}\n {%- endif %}\n {% call statement('list_tags', fetch_result=True) -%}\n {{ fetch_tags_sql(relation) }}\n {% endcall %}\n {% do return(load_result('list_tags').table) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.fetch_tags_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.003464, "supported_languages": null}, "macro.dbt_databricks.fetch_tags_sql": {"name": "fetch_tags_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.fetch_tags_sql", "macro_sql": "{% macro fetch_tags_sql(relation) -%}\n SELECT tag_name, tag_value\n FROM `system`.`information_schema`.`table_tags`\n WHERE catalog_name = '{{ relation.database|lower }}' \n AND schema_name = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}'\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0035589, "supported_languages": null}, "macro.dbt_databricks.apply_tags": {"name": "apply_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.apply_tags", "macro_sql": "{% macro apply_tags(relation, set_tags) -%}\n {{ log(\"Applying tags to relation \" ~ set_tags) }}\n {%- if set_tags and relation.is_hive_metastore() -%}\n {{ exceptions.raise_compiler_error(\"Tags are only supported for Unity Catalog\") }}\n {%- endif -%}\n {%- if set_tags and set_tags != [] %}\n {%- call statement('main') -%}\n {{ alter_set_tags(relation, set_tags) }}\n {%- endcall -%}\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.alter_set_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0037482, "supported_languages": null}, "macro.dbt_databricks.alter_set_tags": {"name": "alter_set_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.alter_set_tags", "macro_sql": "{% macro alter_set_tags(relation, tags) -%}\n ALTER {{ relation.type.render_for_alter() }} {{ relation.render() }} SET TAGS (\n {% for tag in tags -%}\n '{{ tag }}' = '{{ tags[tag] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n )\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0039198, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_intermediate_sql": {"name": "databricks__get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_intermediate_sql", "macro_sql": "{%- macro databricks__get_create_intermediate_sql(relation, sql) -%}\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n -- drop any pre-existing intermediate\n {{ drop_relation(intermediate_relation) }}\n\n {{ return(get_create_sql(intermediate_relation, sql)) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.drop_relation", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.004071, "supported_languages": null}, "macro.dbt_databricks.location_clause": {"name": "location_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/location.sql", "original_file_path": "macros/relations/location.sql", "unique_id": "macro.dbt_databricks.location_clause", "macro_sql": "{% macro location_clause(relation) %}\n {#--\n Moving forward, `relation` should be a `CatalogRelation`, which is covered by the first condition.\n However, there could be existing macros that are still passing in a `BaseRelation`, including user macros.\n Hence, we need to support the old code still, which is covered by the second condition.\n --#}\n {%- if relation.catalog_type is not none -%}\n\n {%- if relation.location is not none -%}\n location '{{ relation.location }}{% if is_incremental() %}_tmp{% endif %}'\n {%- endif -%}\n\n {%- else -%}\n\n {%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n {%- set file_format = adapter.resolve_file_format(config) -%}\n {%- set identifier = model['alias'] -%}\n {%- if location_root is not none %}\n {%- set model_path = adapter.compute_external_path(config, model, is_incremental()) %}\n location '{{ model_path }}'\n {%- elif (not relation.is_hive_metastore()) and file_format != 'delta' -%}\n {{ exceptions.raise_compiler_error(\n 'Incompatible configuration: `location_root` must be set when using a non-delta file format with Unity Catalog'\n ) }}\n {%- endif %}\n\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.is_incremental"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.004601, "supported_languages": null}, "macro.dbt_databricks.optimize": {"name": "optimize", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.optimize", "macro_sql": "{% macro optimize(relation) %}\n {{ return(adapter.dispatch('optimize', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__optimize"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.004975, "supported_languages": null}, "macro.dbt_databricks.databricks__optimize": {"name": "databricks__optimize", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.databricks__optimize", "macro_sql": "\n\n{%- macro databricks__optimize(relation) -%}\n {%- if var('DATABRICKS_SKIP_OPTIMIZE', 'false')|lower != 'true' and\n var('databricks_skip_optimize', 'false')|lower != 'true' and\n adapter.resolve_file_format(config) == 'delta' -%}\n {%- if (config.get('zorder', False) or config.get('liquid_clustered_by', False)) or config.get('auto_liquid_cluster', False) -%}\n {%- call statement('run_optimize_stmt') -%}\n {{ get_optimize_sql(relation) }}\n {%- endcall -%}\n {%- endif -%}\n {%- endif -%}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.get_optimize_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.005306, "supported_languages": null}, "macro.dbt_databricks.get_optimize_sql": {"name": "get_optimize_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.get_optimize_sql", "macro_sql": "{%- macro get_optimize_sql(relation) %}\n optimize {{ relation.render() }}\n {%- if config.get('zorder', False) and adapter.resolve_file_format(config) == 'delta' %}\n {%- if config.get('liquid_clustered_by', False) or config.get('auto_liquid_cluster', False) %}\n {{ exceptions.warn(\"Both zorder and liquid_clustering are set but they are incompatible. zorder will be ignored.\") }}\n {%- else %}\n {%- set zorder = config.get('zorder', none) %}\n {# TODO: predicates here? WHERE ... #}\n {%- if zorder is sequence and zorder is not string %}\n zorder by (\n {%- for col in zorder %}\n {{ col }}{% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- else %}\n zorder by ({{zorder}})\n {%- endif %}\n {%- endif %}\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0058408, "supported_languages": null}, "macro.dbt_databricks.file_format_clause": {"name": "file_format_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/file_format.sql", "original_file_path": "macros/relations/file_format.sql", "unique_id": "macro.dbt_databricks.file_format_clause", "macro_sql": "{% macro file_format_clause(catalog_relation=none) %}\n {#--\n Moving forward, this macro should require a `catalog_relation`, which is covered by the first condition.\n However, there could be existing macros that is still passing no arguments, including user macros.\n Hence, we need to support the old code still, which is covered by the second condition.\n --#}\n {% if catalog_relation is not none %}\n {%- set table_format = catalog_relation.table_format -%}\n {%- set file_format = catalog_relation.file_format -%}\n {% else %}\n {%- set table_format = config.get('table_format', default='default') -%}\n {%- set file_format = adapter.resolve_file_format(config) -%}\n {% endif %}\n \n {#-- Use managed Iceberg if behavior flag is enabled and table_format is iceberg --#}\n {% if table_format == 'iceberg' and adapter.behavior.use_managed_iceberg %}\n using iceberg\n {% else %}\n using {{ file_format }}\n {% endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.006356, "supported_languages": null}, "macro.dbt_databricks.get_file_format": {"name": "get_file_format", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/file_format.sql", "original_file_path": "macros/relations/file_format.sql", "unique_id": "macro.dbt_databricks.get_file_format", "macro_sql": "{% macro get_file_format(catalog_relation=none) %}\n {#-\n Moving forward, this macro should require a `catalog_relation`, which is covered by the first condition.\n However, there could be existing macros that is still passing no arguments, including user macros.\n Hence, we need to support the old code still, which is covered by the second condition.\n -#}\n {% if catalog_relation is not none %}\n {%- set raw_file_format = catalog_relation.file_format -%}\n {% else %}\n {%- set raw_file_format = adapter.resolve_file_format(config) -%}\n {% endif %}\n {% do return(dbt_databricks_validate_get_file_format(raw_file_format)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.dbt_databricks_validate_get_file_format"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.006557, "supported_languages": null}, "macro.dbt_databricks.liquid_clustered_cols": {"name": "liquid_clustered_cols", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/liquid_clustering.sql", "original_file_path": "macros/relations/liquid_clustering.sql", "unique_id": "macro.dbt_databricks.liquid_clustered_cols", "macro_sql": "{% macro liquid_clustered_cols() -%}\n {%- set cols = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}\n {%- set auto_cluster = config.get('auto_liquid_cluster', validator=validation.any[boolean]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n CLUSTER BY ({{ cols | join(', ') }})\n {%- elif auto_cluster -%}\n CLUSTER BY AUTO\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0070348, "supported_languages": null}, "macro.dbt_databricks.apply_liquid_clustered_cols": {"name": "apply_liquid_clustered_cols", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/liquid_clustering.sql", "original_file_path": "macros/relations/liquid_clustering.sql", "unique_id": "macro.dbt_databricks.apply_liquid_clustered_cols", "macro_sql": "{% macro apply_liquid_clustered_cols(target_relation, liquid_clustering) -%}\n {%- set cols = liquid_clustering.cluster_by -%}\n {%- set auto_cluster = liquid_clustering.auto_cluster -%}\n {%- if cols and cols != [] %}\n {%- call statement('set_cluster_by_columns') -%}\n ALTER {{ target_relation.type.render() }} {{ target_relation.render() }} CLUSTER BY ({{ cols | join(', ') }})\n {%- endcall -%}\n {%- elif auto_cluster -%}\n {%- call statement('set_cluster_by_auto') -%}\n ALTER {{ target_relation.type.render() }} {{ target_relation.render() }} CLUSTER BY AUTO\n {%- endcall -%}\n {% else %}\n {%- call statement('unset_cluster_by') -%}\n ALTER {{ target_relation.type.render() }} {{ target_relation.render() }} CLUSTER BY NONE\n {%- endcall -%}\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.007434, "supported_languages": null}, "macro.dbt_databricks.tblproperties_clause": {"name": "tblproperties_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.tblproperties_clause", "macro_sql": "{% macro tblproperties_clause() -%}\n {{ return(adapter.dispatch('tblproperties_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.007676, "supported_languages": null}, "macro.dbt_databricks.databricks__tblproperties_clause": {"name": "databricks__tblproperties_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.databricks__tblproperties_clause", "macro_sql": "{% macro databricks__tblproperties_clause(tblproperties=None) -%}\n {%- if adapter.is_uniform(config) -%}\n {%- set tblproperties = adapter.update_tblproperties_for_uniform_iceberg(config, tblproperties) -%}\n {%- else -%}\n {%- set tblproperties = tblproperties or config.get(\"tblproperties\", {}) -%}\n {%- endif -%}\n {%- if tblproperties != {} %}\n tblproperties (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}' {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.008002, "supported_languages": null}, "macro.dbt_databricks.apply_tblproperties": {"name": "apply_tblproperties", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.apply_tblproperties", "macro_sql": "{% macro apply_tblproperties(relation, tblproperties) -%}\n {% set tblproperty_statment = databricks__tblproperties_clause(tblproperties) %}\n {% if tblproperty_statment %}\n {%- call statement('main') -%}\n ALTER {{ relation.type.render_for_alter() }} {{ relation.render() }} SET {{ tblproperty_statment}}\n {%- endcall -%}\n {% endif %}\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__tblproperties_clause", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.008196, "supported_languages": null}, "macro.dbt_databricks.execute_no_op": {"name": "execute_no_op", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/execute.sql", "original_file_path": "macros/relations/execute.sql", "unique_id": "macro.dbt_databricks.execute_no_op", "macro_sql": "{% macro execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.008372, "supported_languages": null}, "macro.dbt_databricks.persist_constraints": {"name": "persist_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.persist_constraints", "macro_sql": "{% macro persist_constraints(relation, model) %}\n {{ return(adapter.dispatch('persist_constraints', 'dbt')(relation, model)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.009697, "supported_languages": null}, "macro.dbt_databricks.databricks__persist_constraints": {"name": "databricks__persist_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__persist_constraints", "macro_sql": "{% macro databricks__persist_constraints(relation, model) %}\n {%- set contract_config = config.get('contract') -%}\n {% set has_model_contract = contract_config and contract_config.enforced %}\n {% set has_databricks_constraints = config.get('persist_constraints', False) %}\n\n {% if (has_model_contract or has_databricks_constraints) %}\n {% if adapter.resolve_file_format(config) != 'delta' %}\n {# Constraints are only supported for delta tables #}\n {{ exceptions.warn(\"Constraints not supported for file format: \" ~ adapter.resolve_file_format(config)) }}\n {% elif relation.is_view %}\n {# Constraints are not supported for views. This point in the code should not have been reached. #}\n {{ exceptions.raise_compiler_error(\"Constraints not supported for views.\") }}\n {% elif is_incremental() %}\n {# Constraints are not applied for incremental updates. This point in the code should not have been reached #}\n {{ exceptions.raise_compiler_error(\"Constraints are not applied for incremental updates. Full refresh is required to update constraints.\") }}\n {% else %}\n {% do alter_column_set_constraints(relation, model) %}\n {% do alter_table_add_constraints(relation, model) %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.is_incremental", "macro.dbt_databricks.alter_column_set_constraints", "macro.dbt_databricks.alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.010078, "supported_languages": null}, "macro.dbt_databricks.apply_alter_constraints": {"name": "apply_alter_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.apply_alter_constraints", "macro_sql": "{% macro apply_alter_constraints(relation) %}\n {%- for constraint in relation.alter_constraints -%}\n {% call statement('add constraint') %}\n ALTER TABLE {{ relation.render() }} ADD {{ constraint.render() }}\n {% endcall %}\n {%- endfor -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.010201, "supported_languages": null}, "macro.dbt_databricks.alter_table_add_constraints": {"name": "alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.alter_table_add_constraints", "macro_sql": "{% macro alter_table_add_constraints(relation, constraints) %}\n {{ return(adapter.dispatch('alter_table_add_constraints', 'dbt')(relation, constraints)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.010293, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_table_add_constraints": {"name": "databricks__alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__alter_table_add_constraints", "macro_sql": "{% macro databricks__alter_table_add_constraints(relation, model) %}\n {% set constraints = get_model_constraints(model) %}\n {% set statements = get_constraints_sql(relation, constraints, model) %}\n {% for stmt in statements %}\n {% call statement() %}\n {{ stmt }}\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_model_constraints", "macro.dbt_databricks.get_constraints_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.010447, "supported_languages": null}, "macro.dbt_databricks.get_model_constraints": {"name": "get_model_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_model_constraints", "macro_sql": "{% macro get_model_constraints(model) %}\n {% set constraints = model.get('constraints', []) %}\n {% if config.get('persist_constraints', False) and model.get('meta', {}).get('constraints') is sequence %}\n {# Databricks constraints implementation. Constraints are in the meta property. #}\n {% set db_constraints = model.get('meta', {}).get('constraints', []) %}\n {% set constraints = databricks_constraints_to_dbt(db_constraints) %}\n {% endif %}\n {{ return(constraints) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks_constraints_to_dbt"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.010711, "supported_languages": null}, "macro.dbt_databricks.get_column_constraints": {"name": "get_column_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_column_constraints", "macro_sql": "{% macro get_column_constraints(column) %}\n {% set constraints = column.get('constraints', []) %}\n {% if config.get('persist_constraints', False) and column.get('meta', {}).get('constraint') %}\n {# Databricks constraints implementation. Constraint is in the meta property. #}\n {% set db_constraints = [column.get('meta', {}).get('constraint')] %}\n {% set constraints = databricks_constraints_to_dbt(db_constraints, column) %}\n {% endif %}\n {{ return(constraints) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks_constraints_to_dbt"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.010987, "supported_languages": null}, "macro.dbt_databricks.alter_column_set_constraints": {"name": "alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.alter_column_set_constraints", "macro_sql": "{% macro alter_column_set_constraints(relation, column_dict) %}\n {{ return(adapter.dispatch('alter_column_set_constraints', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0110788, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_column_set_constraints": {"name": "databricks__alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__alter_column_set_constraints", "macro_sql": "{% macro databricks__alter_column_set_constraints(relation, model) %}\n {% set column_dict = model.columns %}\n {% for column_name in column_dict %}\n {% set column = column_dict[column_name] %}\n {% set constraints = get_column_constraints(column) %}\n {% set statements = get_constraints_sql(relation, constraints, model, column) %}\n {% for stmt in statements %}\n {% call statement() %}\n {{ stmt }}\n {% endcall %}\n {% endfor %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_column_constraints", "macro.dbt_databricks.get_constraints_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0113149, "supported_languages": null}, "macro.dbt_databricks.get_constraints_sql": {"name": "get_constraints_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_constraints_sql", "macro_sql": "{% macro get_constraints_sql(relation, constraints, model, column={}) %}\n {% set statements = [] %}\n -- Hack so that not null constraints will be applied before other constraints\n {% for constraint in constraints|selectattr('type', 'eq', 'not_null') %}\n {% if constraint %}\n {% set constraint_statements = get_constraint_sql(relation, constraint, model, column) %}\n {% for statement in constraint_statements %}\n {% if statement %}\n {% do statements.append(statement) %}\n {% endif %}\n {% endfor %}\n {% endif %}\n {% endfor %}\n {% for constraint in constraints|rejectattr('type', 'eq', 'not_null') %}\n {% if constraint %}\n {% set constraint_statements = get_constraint_sql(relation, constraint, model, column) %}\n {% for statement in constraint_statements %}\n {% if statement %}\n {% do statements.append(statement) %}\n {% endif %}\n {% endfor %}\n {% endif %}\n {% endfor %}\n\n {{ return(statements) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_constraint_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0118499, "supported_languages": null}, "macro.dbt_databricks.get_constraint_sql": {"name": "get_constraint_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_constraint_sql", "macro_sql": "{% macro get_constraint_sql(relation, constraint, model, column={}) %}\n {% set statements = [] %}\n {% set type = constraint.get('type', '') %}\n\n {% if type == 'check' %}\n {% set expression = constraint.get('expression', '') %}\n {% if not expression %}\n {{ exceptions.raise_compiler_error('Invalid check constraint expression') }}\n {% endif %}\n\n {% set name = constraint.get('name') %}\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5 (relation.identifier ~ \";\" ~ column.get('name', '') ~ \";\" ~ expression ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %}\n {% endif %}\n {% set stmt = \"alter table \" ~ relation.render() ~ \" add constraint \" ~ name ~ \" check (\" ~ expression ~ \");\" %}\n {% do statements.append(stmt) %}\n {% elif type == 'not_null' %}\n {% set column_names = constraint.get('columns', []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if column %}\n {% set quoted_name = adapter.quote(column['name']) %}\n {% set stmt = \"alter table \" ~ relation.render() ~ \" change column \" ~ quoted_name ~ \" set not null \" ~ (constraint.expression or \"\") ~ \";\" %}\n {% do statements.append(stmt) %}\n {% else %}\n {{ exceptions.warn('not_null constraint on invalid column: ' ~ column_name) }}\n {% endif %}\n {% endfor %}\n {% elif type == 'primary_key' %}\n {% if constraint.get('warn_unenforced') %}\n {{ exceptions.warn(\"unenforced constraint type: \" ~ type)}}\n {% endif %}\n {% set column_names = constraint.get('columns', []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% set quoted_names = [] %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if not column %}\n {{ exceptions.warn('Invalid primary key column: ' ~ column_name) }}\n {% else %}\n {% set quoted_name = adapter.quote(column['name']) %}\n {% do quoted_names.append(quoted_name) %}\n {% endif %}\n {% endfor %}\n\n {% set joined_names = quoted_names|join(\", \") %}\n\n {% set name = constraint.get('name') %}\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"primary_key;\" ~ relation.identifier ~ \";\" ~ column_names ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %}\n {% endif %}\n {% set stmt = \"alter table \" ~ relation.render() ~ \" add constraint \" ~ name ~ \" primary key(\" ~ joined_names ~ \");\" %}\n {% do statements.append(stmt) %}\n {% elif type == 'foreign_key' %}\n\n {% if constraint.get('warn_unenforced') %}\n {{ exceptions.warn(\"unenforced constraint type: \" ~ constraint.type)}}\n {% endif %}\n\n {% set name = constraint.get('name') %}\n \n {% if constraint.get('expression') %}\n\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"foreign_key;\" ~ relation.identifier ~ \";\" ~ constraint.get('expression') ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %} \n {% endif %}\n\n {% set stmt = \"alter table \" ~ relation.render() ~ \" add constraint \" ~ name ~ \" foreign key\" ~ constraint.get('expression') %}\n {% else %}\n {% set column_names = constraint.get('columns', []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% set quoted_names = [] %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if not column %}\n {{ exceptions.warn('Invalid foreign key column: ' ~ column_name) }}\n {% else %}\n {% set quoted_name = adapter.quote(column['name']) %}\n {% do quoted_names.append(quoted_name) %}\n {% endif %}\n {% endfor %}\n\n {% set joined_names = quoted_names|join(\", \") %}\n\n {% set parent = constraint.get('to') %}\n {% if not parent %}\n {{ exceptions.raise_compiler_error('No parent table defined for foreign key: ' ~ expression) }}\n {% endif %}\n {% if not \".\" in parent %}\n {% set parent_relation = api.Relation.create(database=relation.database, schema=relation.schema, identifier=parent, type='table') %}\n {% set parent = parent_relation.render() %}\n {% endif %}\n\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"foreign_key;\" ~ relation.identifier ~ \";\" ~ column_names ~ \";\" ~ parent ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %} \n {% endif %}\n\n {% set stmt = \"alter table \" ~ relation.render() ~ \" add constraint \" ~ name ~ \" foreign key(\" ~ joined_names ~ \") references \" ~ parent %}\n {% set parent_columns = constraint.get('to_columns') %}\n {% if parent_columns %}\n {% set quoted_parent_columns = [] %}\n {% for parent_column in parent_columns %}\n {% do quoted_parent_columns.append(adapter.quote(parent_column)) %}\n {% endfor %}\n {% set stmt = stmt ~ \"(\" ~ quoted_parent_columns|join(\", \") ~ \")\"%}\n {% endif %}\n {% endif %}\n {% set stmt = stmt ~ \";\" %}\n {% do statements.append(stmt) %}\n {% elif type == 'custom' %}\n {% set expression = constraint.get('expression', '') %}\n {% if not expression %}\n {{ exceptions.raise_compiler_error('Missing custom constraint expression') }}\n {% endif %}\n\n {% set name = constraint.get('name') %}\n {% set expression = constraint.get('expression') %}\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5 (relation.identifier ~ \";\" ~ expression ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %}\n {% endif %}\n {% set stmt = \"alter table \" ~ relation.render() ~ \" add constraint \" ~ name ~ \" \" ~ expression ~ \";\" %}\n {% do statements.append(stmt) %}\n {% elif constraint.get('warn_unsupported') %}\n {{ exceptions.warn(\"unsupported constraint type: \" ~ constraint.type)}}\n {% endif %}\n\n {{ return(statements) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.01566, "supported_languages": null}, "macro.dbt_databricks.databricks_constraints_to_dbt": {"name": "databricks_constraints_to_dbt", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks_constraints_to_dbt", "macro_sql": "{% macro databricks_constraints_to_dbt(constraints, column) %}\n {# convert constraints defined using the original databricks format #}\n {% set dbt_constraints = [] %}\n {% for constraint in constraints %}\n {% if constraint.get and constraint.get('type') %}\n {# already in model contract format #}\n {% do dbt_constraints.append(constraint) %}\n {% else %}\n {% if column %}\n {% if constraint == \"not_null\" %}\n {% do dbt_constraints.append({\"type\": \"not_null\", \"columns\": [column.get('name')]}) %}\n {% else %}\n {{ exceptions.raise_compiler_error('Invalid constraint for column ' ~ column.get('name', \"\") ~ '. Only `not_null` is supported.') }}\n {% endif %}\n {% else %}\n {% set name = constraint['name'] %}\n {% if not name %}\n {{ exceptions.raise_compiler_error('Invalid check constraint name') }}\n {% endif %}\n {% set condition = constraint['condition'] %}\n {% if not condition %}\n {{ exceptions.raise_compiler_error('Invalid check constraint condition') }}\n {% endif %}\n {% do dbt_constraints.append({\"name\": name, \"type\": \"check\", \"expression\": condition}) %}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(dbt_constraints) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0166001, "supported_languages": null}, "macro.dbt_databricks.create_backup": {"name": "create_backup", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt_databricks.create_backup", "macro_sql": "{%- macro create_backup(relation) -%}\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n -- drop any pre-existing backup\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ adapter.rename_relation(relation, backup_relation) }}\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.drop_relation_if_exists"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.016763, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_sql": {"name": "databricks__get_create_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_sql", "macro_sql": "{%- macro databricks__get_create_sql(relation, sql) -%}\n {%- if relation.is_view -%}\n {{ get_create_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_table -%}\n {{ get_create_table_as_sql(False, relation, sql) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_create_materialized_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_streaming_table -%}\n {{ get_create_streaming_table_as_sql(relation, sql) }}\n\n {%- elif relation.is_metric_view -%}\n {{ get_create_metric_view_as_sql(relation, sql) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_create_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.get_create_view_as_sql", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt_databricks.get_create_streaming_table_as_sql", "macro.dbt_databricks.get_create_metric_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.017077, "supported_languages": null}, "macro.dbt_databricks.get_configuration_changes": {"name": "get_configuration_changes", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/config.sql", "original_file_path": "macros/relations/config.sql", "unique_id": "macro.dbt_databricks.get_configuration_changes", "macro_sql": "{%- macro get_configuration_changes(existing_relation) -%}\n {%- set model_config = adapter.get_config_from_model(config.model) -%}\n {%- set existing_config = adapter.get_relation_config(existing_relation, model_config) -%}\n {%- set configuration_changes = model_config.get_changeset(existing_config) -%}\n {% do return(configuration_changes) %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.017251, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_materialized_view": {"name": "databricks__drop_materialized_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_materialized_view", "macro_sql": "{% macro databricks__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation.render() }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.017326, "supported_languages": null}, "macro.dbt_databricks.databricks__get_replace_materialized_view_sql": {"name": "databricks__get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt_databricks.databricks__get_replace_materialized_view_sql", "macro_sql": "{% macro databricks__get_replace_materialized_view_sql(target_relation, sql) %}\n {{ get_create_materialized_view_as_sql(target_relation, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0174122, "supported_languages": null}, "macro.dbt_databricks.databricks__refresh_materialized_view": {"name": "databricks__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt_databricks.databricks__refresh_materialized_view", "macro_sql": "{% macro databricks__refresh_materialized_view(relation) -%}\n refresh materialized view {{ relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0174851, "supported_languages": null}, "macro.dbt_databricks.get_alter_materialized_view_as_sql": {"name": "get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_materialized_view_as_sql", "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {%- do return(adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n )) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.017803, "supported_languages": null}, "macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql": {"name": "databricks__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql", "macro_sql": "{% macro databricks__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n {#- CREATE OR REPLACE cannot change partition_by, so use DROP + CREATE when partition_by changes -#}\n {% if configuration_changes.changes[\"partition_by\"] %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {% do return(drop_and_create(existing_relation, relation, sql)) %}\n {% else %}\n {% do return(get_replace_sql(existing_relation, relation, sql)) %}\n {% endif %}\n\n -- otherwise apply individual changes as needed\n {% else %}\n {%- set alter_statement = get_alter_mv_internal(relation, configuration_changes) -%}\n {%- set return_statements = [] -%}\n {%- if alter_statement -%}\n {{ return_statements.append(alter_statement) }}\n {%- endif -%}\n {%- set tags = configuration_changes.changes[\"tags\"] -%}\n {%- if tags and tags.set_tags and tags.set_tags != [] -%}\n {{ return_statements.append(alter_set_tags(relation, tags.set_tags)) }}\n {%- endif -%}\n\n {#- Row filter handling - append SQL to list, don't execute -#}\n {#- is_change guard prevents false alters when row_filter is unchanged -#}\n {%- set row_filter = configuration_changes.changes.get(\"row_filter\") -%}\n {%- if row_filter and row_filter.is_change -%}\n {%- if row_filter.should_unset -%}\n {{ return_statements.append(alter_drop_row_filter(relation)) }}\n {%- elif row_filter.function -%}\n {{ return_statements.append(alter_set_row_filter(relation, row_filter)) }}\n {%- endif -%}\n {%- endif -%}\n\n {% do return(return_statements) %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.drop_and_create", "macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_alter_mv_internal", "macro.dbt_databricks.alter_set_tags", "macro.dbt_databricks.alter_drop_row_filter", "macro.dbt_databricks.alter_set_row_filter"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.01838, "supported_languages": null}, "macro.dbt_databricks.get_alter_mv_internal": {"name": "get_alter_mv_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_mv_internal", "macro_sql": "{% macro get_alter_mv_internal(relation, configuration_changes) %}\n {%- set refresh = configuration_changes.changes[\"refresh\"] -%}\n {%- if refresh -%}\n -- Currently only schedule can be altered\n ALTER MATERIALIZED VIEW {{ relation.render() }}\n {{ get_alter_sql_refresh_schedule(refresh) -}}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_alter_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.018514, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_materialized_view_as_sql": {"name": "databricks__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_materialized_view_as_sql", "macro_sql": "{% macro databricks__get_create_materialized_view_as_sql(relation, sql) -%}\n {# Column masks are supported in DBSQL, but not yet wired up to the adapter. Return a helpful error until supported. #}\n {% if column_mask_exists() %}\n {% do exceptions.raise_compiler_error(\"Column masks are not yet supported for materialized views.\") %}\n {% endif %}\n {%- set materialized_view = adapter.get_config_from_model(config.model) -%}\n {%- set partition_by = materialized_view.config[\"partition_by\"].partition_by -%}\n {%- set tblproperties = materialized_view.config[\"tblproperties\"].tblproperties -%}\n {%- set comment = materialized_view.config[\"comment\"].comment -%}\n {%- set refresh = materialized_view.config[\"refresh\"] -%}\n\n {#\n TODO: When DESCRIBE QUERY EXTENDED is supported, this implementation should be simplified\n to use that instead. For now, we work around this limitation by writing results to a\n temporary view and using DESCRIBE TABLE EXTENDED on the temporary view.\n #}\n {%- set temp_relation = make_temp_relation(relation) -%}\n {% call statement('create_temp_view') -%}\n {%- set sql_with_limit = sql.rstrip('; \\n\\t') ~ ' LIMIT 10' -%}\n {{ create_temporary_view(temp_relation, sql_with_limit) }}\n {%- endcall %}\n\n {%- set columns = adapter.get_columns_in_relation(temp_relation) -%}\n {%- set model_columns = model.get('columns', {}) -%}\n {%- set contract_config = config.get('contract') -%}\n {%- set contract_enforced = contract_config and contract_config.enforced -%}\n {%- if contract_enforced -%}\n {%- do exceptions.warn(\n \"contract.enforced=true on materialized_view '\" ~ model.name ~ \"': not supported by dbt (https://docs.getdbt.com/docs/mesh/govern/model-contracts). dbt-databricks provides best-effort support that may change without notice.\"\n ) -%}\n {%- set model_constraints = model.get('constraints', []) -%}\n {%- else -%}\n {%- set model_constraints = [] -%}\n {%- endif -%}\n {%- set columns_and_constraints = adapter.parse_columns_and_constraints(columns, model_columns, model_constraints, contract_enforced, model.name) -%}\n {%- set target_relation = relation.enrich(columns_and_constraints[1]) -%}\n\n create or replace materialized view {{ target_relation.render() }}\n {{ get_column_and_constraints_sql(target_relation, columns_and_constraints[0]) }}\n {{ get_create_row_filter_clause(target_relation) }}\n {{ get_create_sql_partition_by(partition_by) }}\n {{ liquid_clustered_cols() }}\n {{ get_create_sql_comment(comment) }}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {{ get_create_sql_refresh_schedule(refresh) }}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.column_mask_exists", "macro.dbt.make_temp_relation", "macro.dbt.statement", "macro.dbt_spark.create_temporary_view", "macro.dbt_databricks.get_column_and_constraints_sql", "macro.dbt_databricks.get_create_row_filter_clause", "macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.liquid_clustered_cols", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties", "macro.dbt_databricks.get_create_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.019477, "supported_languages": null}, "macro.dbt_databricks.drop_streaming_table": {"name": "drop_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/drop.sql", "original_file_path": "macros/relations/streaming_table/drop.sql", "unique_id": "macro.dbt_databricks.drop_streaming_table", "macro_sql": "{% macro drop_streaming_table(relation) -%}\n {{ return(adapter.dispatch('drop_streaming_table', 'dbt')(relation)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.default__drop_streaming_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.019594, "supported_languages": null}, "macro.dbt_databricks.default__drop_streaming_table": {"name": "default__drop_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/drop.sql", "original_file_path": "macros/relations/streaming_table/drop.sql", "unique_id": "macro.dbt_databricks.default__drop_streaming_table", "macro_sql": "{% macro default__drop_streaming_table(relation) -%}\n drop table if exists {{ relation.render() }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0196512, "supported_languages": null}, "macro.dbt_databricks.refresh_streaming_table": {"name": "refresh_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/refresh.sql", "original_file_path": "macros/relations/streaming_table/refresh.sql", "unique_id": "macro.dbt_databricks.refresh_streaming_table", "macro_sql": "{% macro refresh_streaming_table(relation, sql) -%}\n {{ adapter.dispatch('refresh_streaming_table', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__refresh_streaming_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.019776, "supported_languages": null}, "macro.dbt_databricks.databricks__refresh_streaming_table": {"name": "databricks__refresh_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/refresh.sql", "original_file_path": "macros/relations/streaming_table/refresh.sql", "unique_id": "macro.dbt_databricks.databricks__refresh_streaming_table", "macro_sql": "{% macro databricks__refresh_streaming_table(relation, sql) -%}\n refresh streaming table {{ relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0198421, "supported_languages": null}, "macro.dbt_databricks.get_alter_streaming_table_as_sql": {"name": "get_alter_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_streaming_table_as_sql", "macro_sql": "{% macro get_alter_streaming_table_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {%- do return(adapter.dispatch('get_alter_streaming_table_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n )) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.020233, "supported_languages": null}, "macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql": {"name": "databricks__get_alter_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql", "macro_sql": "{% macro databricks__get_alter_streaming_table_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n {% do return(get_replace_sql(existing_relation, relation, sql)) %}\n\n -- otherwise apply individual changes as needed\n {% else %}\n {%- set alter_statement = get_alter_st_internal(relation, configuration_changes) -%}\n {%- set create_statement = get_create_st_internal(relation, configuration_changes, sql) -%}\n {%- set return_statements = [] -%}\n {%- if create_statement -%}\n {{ return_statements.append(create_statement) }}\n {%- endif -%}\n {%- if alter_statement -%}\n {{ return_statements.append(alter_statement) }}\n {%- endif -%}\n {%- set tags = configuration_changes.changes[\"tags\"] -%}\n {%- if tags and tags.set_tags and tags.set_tags != [] -%}\n {{ return_statements.append(alter_set_tags(relation, tags.set_tags)) }}\n {%- endif -%}\n\n {#- Row filter handling - append SQL to list, don't execute -#}\n {#- is_change guard prevents false alters from `diff or value` fallback in streaming_table.py:56 -#}\n {%- set row_filter = configuration_changes.changes.get(\"row_filter\") -%}\n {%- if row_filter and row_filter.is_change -%}\n {%- if row_filter.should_unset -%}\n {{ return_statements.append(alter_drop_row_filter(relation)) }}\n {%- elif row_filter.function -%}\n {{ return_statements.append(alter_set_row_filter(relation, row_filter)) }}\n {%- endif -%}\n {%- endif -%}\n\n {% do return(return_statements) %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_alter_st_internal", "macro.dbt_databricks.get_create_st_internal", "macro.dbt_databricks.alter_set_tags", "macro.dbt_databricks.alter_drop_row_filter", "macro.dbt_databricks.alter_set_row_filter"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.020809, "supported_languages": null}, "macro.dbt_databricks.get_create_st_internal": {"name": "get_create_st_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_create_st_internal", "macro_sql": "{% macro get_create_st_internal(relation, configuration_changes, sql) %}\n {%- set partition_by = configuration_changes.changes[\"partition_by\"].partition_by -%}\n {%- set tblproperties = configuration_changes.changes[\"tblproperties\"].tblproperties -%}\n {%- set comment = configuration_changes.changes[\"comment\"].comment -%}\n CREATE OR REFRESH STREAMING TABLE {{ relation.render() }}\n {% if partition_by -%}\n {{ get_create_sql_partition_by(partition_by) }}\n {%- endif %}\n {{ liquid_clustered_cols() }}\n {% if comment -%}\n {{ get_create_sql_comment(comment) }}\n {%- endif %}\n {% if tblproperties -%}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {%- endif %}\n AS {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.liquid_clustered_cols", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.021146, "supported_languages": null}, "macro.dbt_databricks.get_alter_st_internal": {"name": "get_alter_st_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_st_internal", "macro_sql": "{% macro get_alter_st_internal(relation, configuration_changes) %}\n {%- set refresh = configuration_changes.changes[\"refresh\"] -%}\n {%- set is_scheduled = refresh and (refresh.cron or refresh.every or refresh.on_update) -%}\n {%- if is_scheduled -%}\n ALTER STREAMING TABLE {{ relation.render() }}\n ADD {{ get_create_sql_refresh_schedule(refresh) -}}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.021329, "supported_languages": null}, "macro.dbt_databricks.get_create_streaming_table_as_sql": {"name": "get_create_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/create.sql", "original_file_path": "macros/relations/streaming_table/create.sql", "unique_id": "macro.dbt_databricks.get_create_streaming_table_as_sql", "macro_sql": "{% macro get_create_streaming_table_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_streaming_table_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.021562, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_streaming_table_as_sql": {"name": "databricks__get_create_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/create.sql", "original_file_path": "macros/relations/streaming_table/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_streaming_table_as_sql", "macro_sql": "{% macro databricks__get_create_streaming_table_as_sql(relation, sql) -%}\n {%- set streaming_table = adapter.get_config_from_model(config.model) -%}\n {%- set partition_by = streaming_table.config[\"partition_by\"].partition_by -%}\n {%- set tblproperties = streaming_table.config[\"tblproperties\"].tblproperties -%}\n {%- set comment = streaming_table.config[\"comment\"].comment -%}\n {%- set refresh = streaming_table.config[\"refresh\"] -%}\n\n {%- set analysis_sql = sql | replace('STREAM ', '') | replace('stream ', '') -%}\n\n {#\n TODO: When DESCRIBE QUERY EXTENDED is supported, this implementation should be simplified\n to use that instead. For now, we work around this limitation by writing results to a\n temporary view and using DESCRIBE TABLE EXTENDED on the temporary view.\n #}\n {%- set temp_relation = make_temp_relation(relation) -%}\n {% call statement('create_temp_view') -%}\n {%- set view_sql = analysis_sql.rstrip('; \\n\\t') -%}\n {{ create_temporary_view(temp_relation, view_sql) }}\n {%- endcall %}\n\n {%- set columns = adapter.get_columns_in_relation(temp_relation) -%}\n {%- set model_columns = model.get('columns', {}) -%}\n {%- set contract_config = config.get('contract') -%}\n {%- set contract_enforced = contract_config and contract_config.enforced -%}\n {%- set columns_and_constraints = adapter.parse_columns_and_constraints(columns, model_columns, [], contract_enforced, model.name) -%}\n\n {#-- We don't enrich the relation with model constraints because they are not supported for streaming tables --#}\n CREATE STREAMING TABLE {{ relation.render() }}\n {{ get_column_and_constraints_sql(relation, columns_and_constraints[0]) }}\n {{ get_create_row_filter_clause(relation) }}\n {{ get_create_sql_partition_by(partition_by) }}\n {{ liquid_clustered_cols() }}\n {{ get_create_sql_comment(comment) }}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {{ get_create_sql_refresh_schedule(refresh) }}\n AS {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.statement", "macro.dbt_spark.create_temporary_view", "macro.dbt_databricks.get_column_and_constraints_sql", "macro.dbt_databricks.get_create_row_filter_clause", "macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.liquid_clustered_cols", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties", "macro.dbt_databricks.get_create_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.022214, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_comment": {"name": "get_create_sql_comment", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/comment.sql", "original_file_path": "macros/relations/components/comment.sql", "unique_id": "macro.dbt_databricks.get_create_sql_comment", "macro_sql": "{%- macro get_create_sql_comment(comment) -%}\n{% if comment is string -%}\n COMMENT '{{ comment }}'\n{%- endif -%}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.022325, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_refresh_schedule": {"name": "get_create_sql_refresh_schedule", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/refresh_schedule.sql", "original_file_path": "macros/relations/components/refresh_schedule.sql", "unique_id": "macro.dbt_databricks.get_create_sql_refresh_schedule", "macro_sql": "{% macro get_create_sql_refresh_schedule(refresh) %}\n {%- if refresh.cron -%}\n SCHEDULE CRON '{{ refresh.cron }}'{%- if refresh.time_zone_value %} AT TIME ZONE '{{ refresh.time_zone_value }}'{%- endif -%}\n {%- elif refresh.every -%}\n SCHEDULE EVERY {{ refresh.every }}\n {%- elif refresh.on_update -%}\n TRIGGER ON UPDATE{%- if refresh.at_most_every %} AT MOST EVERY INTERVAL {{ refresh.at_most_every }}{%- endif -%}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.022584, "supported_languages": null}, "macro.dbt_databricks.get_alter_sql_refresh_schedule": {"name": "get_alter_sql_refresh_schedule", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/refresh_schedule.sql", "original_file_path": "macros/relations/components/refresh_schedule.sql", "unique_id": "macro.dbt_databricks.get_alter_sql_refresh_schedule", "macro_sql": "{% macro get_alter_sql_refresh_schedule(refresh) %}\n {%- if not (refresh.cron or refresh.every or refresh.on_update) -%}\n DROP SCHEDULE\n {%- else -%}\n {{- 'ALTER ' if refresh.is_altered else 'ADD ' -}}{{- get_create_sql_refresh_schedule(refresh) -}}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.022727, "supported_languages": null}, "macro.dbt_databricks.alter_query": {"name": "alter_query", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/query.sql", "original_file_path": "macros/relations/components/query.sql", "unique_id": "macro.dbt_databricks.alter_query", "macro_sql": "{% macro alter_query(target_relation, query) %}\n {{ log(\"Altering query\") }}\n {% if query %}\n {% call statement('main') %}\n {{- get_alter_query_sql(target_relation, query) }}\n {% endcall %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.get_alter_query_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0229278, "supported_languages": null}, "macro.dbt_databricks.get_alter_query_sql": {"name": "get_alter_query_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/query.sql", "original_file_path": "macros/relations/components/query.sql", "unique_id": "macro.dbt_databricks.get_alter_query_sql", "macro_sql": "{% macro get_alter_query_sql(target_relation, query) -%}\n ALTER {{ target_relation.type.render_for_alter() }} {{ target_relation.render() }} AS (\n {{ query }}\n )\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.023026, "supported_languages": null}, "macro.dbt_databricks.fetch_row_filters": {"name": "fetch_row_filters", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.fetch_row_filters", "macro_sql": "\n\n{%- macro fetch_row_filters(relation) -%}\n {%- if relation.is_hive_metastore() -%}\n {{ exceptions.raise_compiler_error(\"Row filters are not supported for Hive Metastore\") }}\n {%- endif -%}\n {%- call statement('list_row_filters', fetch_result=True) -%}\n {{ fetch_row_filters_sql(relation) }}\n {%- endcall -%}\n {%- do return(load_result('list_row_filters').table) -%}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.fetch_row_filters_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.023744, "supported_languages": null}, "macro.dbt_databricks.fetch_row_filters_sql": {"name": "fetch_row_filters_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.fetch_row_filters_sql", "macro_sql": "{%- macro fetch_row_filters_sql(relation) -%}\n SELECT\n table_catalog,\n table_schema,\n table_name,\n filter_name,\n target_columns\n FROM `{{ relation.database }}`.`information_schema`.`row_filters`\n WHERE table_catalog = '{{ relation.database | lower }}'\n AND table_schema = '{{ relation.schema | lower }}'\n AND table_name = '{{ relation.identifier | lower }}'\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.023864, "supported_languages": null}, "macro.dbt_databricks.quote_row_filter_columns": {"name": "quote_row_filter_columns", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.quote_row_filter_columns", "macro_sql": "\n\n{%- macro quote_row_filter_columns(columns) -%}\n {#- Quote column names with backticks and join with comma.\n Shared by alter_set_row_filter and get_create_row_filter_clause. -#}\n {%- set quoted = [] -%}\n {%- for col in columns -%}\n {%- do quoted.append('`' ~ col ~ '`') -%}\n {%- endfor -%}\n {{- quoted | join(', ') -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0240052, "supported_languages": null}, "macro.dbt_databricks.quote_row_filter_function": {"name": "quote_row_filter_function", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.quote_row_filter_function", "macro_sql": "{%- macro quote_row_filter_function(function) -%}\n {#- Quote a fully qualified function name: catalog.schema.func -> `catalog`.`schema`.`func`\n Function names are stored raw internally; backticks are added at SQL generation time. -#}\n {%- set parts = function.split('.') -%}\n {%- if parts | length == 3 -%}\n `{{ parts[0] }}`.`{{ parts[1] }}`.`{{ parts[2] }}`\n {%- else -%}\n {#- Fallback for unexpected format -#}\n `{{ function }}`\n {%- endif -%}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.024173, "supported_languages": null}, "macro.dbt_databricks.alter_set_row_filter": {"name": "alter_set_row_filter", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.alter_set_row_filter", "macro_sql": "\n\n{%- macro alter_set_row_filter(relation, row_filter) -%}\n ALTER {{ relation.type.render() }} {{ relation.render() }}\n SET ROW FILTER {{ quote_row_filter_function(row_filter.function) }}\n ON ({{ quote_row_filter_columns(row_filter.columns) }})\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.quote_row_filter_function", "macro.dbt_databricks.quote_row_filter_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.024302, "supported_languages": null}, "macro.dbt_databricks.alter_drop_row_filter": {"name": "alter_drop_row_filter", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.alter_drop_row_filter", "macro_sql": "{%- macro alter_drop_row_filter(relation) -%}\n ALTER {{ relation.type.render() }} {{ relation.render() }}\n DROP ROW FILTER\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.024385, "supported_languages": null}, "macro.dbt_databricks.apply_row_filter": {"name": "apply_row_filter", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.apply_row_filter", "macro_sql": "\n\n{%- macro apply_row_filter(target_relation, row_filter) -%}\n {#- Executes SQL immediately via call statement('main') -#}\n {#- row_filter is a RowFilterConfig object with is_change, should_unset, function fields -#}\n {%- if target_relation.is_hive_metastore() -%}\n {{ exceptions.raise_compiler_error(\"Row filters are not supported for Hive Metastore\") }}\n {%- endif -%}\n\n {%- if row_filter.is_change -%}\n {%- if row_filter.should_unset -%}\n {%- call statement('main') -%}\n {{ alter_drop_row_filter(target_relation) }}\n {%- endcall -%}\n {%- elif row_filter.function -%}\n {%- call statement('main') -%}\n {{ alter_set_row_filter(target_relation, row_filter) }}\n {%- endcall -%}\n {%- endif -%}\n {%- endif -%}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.alter_drop_row_filter", "macro.dbt_databricks.alter_set_row_filter"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.024657, "supported_languages": null}, "macro.dbt_databricks.drop_row_filter_if_exists": {"name": "drop_row_filter_if_exists", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.drop_row_filter_if_exists", "macro_sql": "\n\n{%- macro drop_row_filter_if_exists(relation) -%}\n {#- Drops any existing row filter from a relation before CREATE OR REPLACE.\n Used by table materialization when the model has no row filter configured\n but the existing table might have one from a previous run. -#}\n {%- if not relation.is_hive_metastore() -%}\n {%- set existing_filters = fetch_row_filters(relation) -%}\n {%- if existing_filters | length > 0 -%}\n {%- call statement('drop_row_filter') -%}\n {{ alter_drop_row_filter(relation) }}\n {%- endcall -%}\n {%- endif -%}\n {%- endif -%}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.fetch_row_filters", "macro.dbt.statement", "macro.dbt_databricks.alter_drop_row_filter"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.024835, "supported_languages": null}, "macro.dbt_databricks.qualify_row_filter_function": {"name": "qualify_row_filter_function", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.qualify_row_filter_function", "macro_sql": "\n\n{%- macro qualify_row_filter_function(function, relation) -%}\n {#- Handle 1-part and 3-part, reject 2-part. -#}\n {#- IMPORTANT: Keep in sync with Python _qualify_function_name() -#}\n {%- set parts = function.replace('`', '').split('.') -%}\n\n {%- if parts | length == 1 -%}\n {#- Unqualified: fn -> catalog.schema.fn -#}\n {{ relation.database }}.{{ relation.schema }}.{{ parts[0] }}\n {%- elif parts | length == 2 -%}\n {#- Ambiguous: reject -#}\n {{ exceptions.raise_compiler_error(\n \"Row filter function '\" ~ function ~ \"' is ambiguous. \" ~\n \"Use either unqualified name (e.g., 'my_filter') or \" ~\n \"fully qualified name (e.g., 'catalog.schema.my_filter').\"\n ) }}\n {%- elif parts | length == 3 -%}\n {#- Fully qualified -#}\n {{ parts[0] }}.{{ parts[1] }}.{{ parts[2] }}\n {%- else -%}\n {{ exceptions.raise_compiler_error(\n \"Row filter function '\" ~ function ~ \"' has too many parts.\"\n ) }}\n {%- endif -%}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.replace"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.025177, "supported_languages": null}, "macro.dbt_databricks.get_create_row_filter_clause": {"name": "get_create_row_filter_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.get_create_row_filter_clause", "macro_sql": "{%- macro get_create_row_filter_clause(relation) -%}\n {%- set row_filter = config.get('row_filter') -%}\n\n {%- if not row_filter or not row_filter.get('function') -%}\n {#-- Model has no row filter - drop any existing filter from table --#}\n {{ drop_row_filter_if_exists(relation) }}\n {%- else -%}\n {#-- Model has row filter - generate the WITH ROW FILTER clause --#}\n {%- set columns = row_filter.get('columns', []) -%}\n\n {#- Normalize string to list -#}\n {%- if columns is string -%}\n {%- set columns = [columns] -%}\n {%- endif -%}\n\n {#- Validate columns is non-empty -#}\n {%- if not columns or columns | length == 0 -%}\n {{ exceptions.raise_compiler_error(\n \"Row filter function '\" ~ row_filter.get('function') ~ \"' requires a non-empty 'columns' value.\"\n ) }}\n {%- endif -%}\n\n {%- set function = qualify_row_filter_function(row_filter.get('function'), relation) -%}\n WITH ROW FILTER {{ quote_row_filter_function(function) }} ON ({{ quote_row_filter_columns(columns) }})\n {%- endif -%}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.drop_row_filter_if_exists", "macro.dbt_databricks.qualify_row_filter_function", "macro.dbt_databricks.quote_row_filter_function", "macro.dbt_databricks.quote_row_filter_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.025575, "supported_languages": null}, "macro.dbt_databricks.row_filter_exists": {"name": "row_filter_exists", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/row_filter.sql", "original_file_path": "macros/relations/components/row_filter.sql", "unique_id": "macro.dbt_databricks.row_filter_exists", "macro_sql": "{%- macro row_filter_exists() -%}\n {%- set row_filter = config.get('row_filter') -%}\n {%- set has_function = row_filter and row_filter.get('function') -%}\n {%- set columns = row_filter.get('columns') if row_filter else none -%}\n\n {#- Normalize string to list -#}\n {%- if columns is string -%}\n {%- set columns = [columns] -%}\n {%- endif -%}\n\n {%- set has_columns = columns and columns | length > 0 -%}\n {%- do return(has_function and has_columns) -%}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.025833, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_partition_by": {"name": "get_create_sql_partition_by", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/partitioning.sql", "original_file_path": "macros/relations/components/partitioning.sql", "unique_id": "macro.dbt_databricks.get_create_sql_partition_by", "macro_sql": "{% macro get_create_sql_partition_by(partition_by) -%}\n{%- if partition_by -%}\n PARTITIONED BY ({%- for col in partition_by -%}{{ col }}{% if not loop.last %}, {% endif %}{%- endfor %})\n{%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0259998, "supported_languages": null}, "macro.dbt_databricks.fetch_column_tags": {"name": "fetch_column_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_tags.sql", "original_file_path": "macros/relations/components/column_tags.sql", "unique_id": "macro.dbt_databricks.fetch_column_tags", "macro_sql": "{% macro fetch_column_tags(relation) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Column tags are only supported for Unity Catalog\") }}\n {%- endif %}\n {% call statement('list_column_tags', fetch_result=True) -%}\n {{ fetch_column_tags_sql(relation) }}\n {% endcall %}\n {% do return(load_result('list_column_tags').table) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.fetch_column_tags_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.026499, "supported_languages": null}, "macro.dbt_databricks.fetch_column_tags_sql": {"name": "fetch_column_tags_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_tags.sql", "original_file_path": "macros/relations/components/column_tags.sql", "unique_id": "macro.dbt_databricks.fetch_column_tags_sql", "macro_sql": "{% macro fetch_column_tags_sql(relation) -%}\n SELECT \n column_name,\n tag_name,\n tag_value\n FROM `system`.`information_schema`.`column_tags`\n WHERE catalog_name = '{{ relation.database|lower }}'\n AND schema_name = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}';\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.026617, "supported_languages": null}, "macro.dbt_databricks.apply_column_tags": {"name": "apply_column_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_tags.sql", "original_file_path": "macros/relations/components/column_tags.sql", "unique_id": "macro.dbt_databricks.apply_column_tags", "macro_sql": "{% macro apply_column_tags(relation, column_tags) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Column tags are only supported for Unity Catalog\") }}\n {%- endif %}\n {{ log(\"Applying column tags to relation \" ~ relation) }}\n {%- if column_tags.set_column_tags %}\n {%- for column, tags in column_tags.set_column_tags.items() -%}\n {%- call statement('main') -%}\n {{ alter_set_column_tags(relation, column, tags) }}\n {%- endcall -%}\n {%- endfor -%}\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.alter_set_column_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0269701, "supported_languages": null}, "macro.dbt_databricks.alter_set_column_tags": {"name": "alter_set_column_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_tags.sql", "original_file_path": "macros/relations/components/column_tags.sql", "unique_id": "macro.dbt_databricks.alter_set_column_tags", "macro_sql": "{% macro alter_set_column_tags(relation, column, tags) -%}\n {# ALTER VIEW does not support setting column tags, but ALTER TABLE works for views #}\n {%- if relation.type == 'view' -%}\n ALTER TABLE {{ relation.render() }}\n {%- else -%}\n ALTER {{ relation.type.render() }} {{ relation.render() }}\n {%- endif -%}\n ALTER COLUMN `{{ column }}`\n SET TAGS (\n {%- for tag_name, tag_value in tags.items() -%}\n '{{ tag_name }}' = '{{ tag_value }}'{%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.027207, "supported_languages": null}, "macro.dbt_databricks.column_tags_exist": {"name": "column_tags_exist", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_tags.sql", "original_file_path": "macros/relations/components/column_tags.sql", "unique_id": "macro.dbt_databricks.column_tags_exist", "macro_sql": "{% macro column_tags_exist() %}\n {% for column_name, column in model.columns.items() %}\n {% if column is mapping and column.get('databricks_tags') %}\n {{ return(true) }}\n {% endif %}\n {% endfor %}\n {{ return(false) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0273678, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_tblproperties": {"name": "get_create_sql_tblproperties", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/tblproperties.sql", "original_file_path": "macros/relations/components/tblproperties.sql", "unique_id": "macro.dbt_databricks.get_create_sql_tblproperties", "macro_sql": "{% macro get_create_sql_tblproperties(tblproperties) %}\n {{ databricks__tblproperties_clause(tblproperties)}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.027458, "supported_languages": null}, "macro.dbt_databricks.fetch_column_masks": {"name": "fetch_column_masks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_mask.sql", "original_file_path": "macros/relations/components/column_mask.sql", "unique_id": "macro.dbt_databricks.fetch_column_masks", "macro_sql": "{% macro fetch_column_masks(relation) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Column masks are not supported for Hive Metastore\") }}\n {%- endif %}\n {% call statement('list_column_masks', fetch_result=True) -%}\n {{ fetch_column_masks_sql(relation) }}\n {% endcall %}\n {% do return(load_result('list_column_masks').table) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.fetch_column_masks_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.027842, "supported_languages": null}, "macro.dbt_databricks.fetch_column_masks_sql": {"name": "fetch_column_masks_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_mask.sql", "original_file_path": "macros/relations/components/column_mask.sql", "unique_id": "macro.dbt_databricks.fetch_column_masks_sql", "macro_sql": "{% macro fetch_column_masks_sql(relation) -%}\n SELECT \n column_name,\n mask_name,\n using_columns\n FROM `system`.`information_schema`.`column_masks`\n WHERE table_catalog = '{{ relation.database|lower }}'\n AND table_schema = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}';\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.02794, "supported_languages": null}, "macro.dbt_databricks.apply_column_masks": {"name": "apply_column_masks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_mask.sql", "original_file_path": "macros/relations/components/column_mask.sql", "unique_id": "macro.dbt_databricks.apply_column_masks", "macro_sql": "{% macro apply_column_masks(relation, column_masks) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Column masks are not supported for Hive Metastore\") }}\n {%- endif %}\n {{ log(\"Applying column masks to relation \" ~ relation) }}\n {%- if column_masks.unset_column_masks %}\n {%- for column in column_masks.unset_column_masks -%}\n {%- call statement('main') -%}\n {{ alter_drop_column_mask(relation, column) }}\n {%- endcall -%}\n {%- endfor -%}\n {%- endif %}\n {%- if column_masks.set_column_masks %}\n {%- for column, mask in column_masks.set_column_masks.items() -%}\n {%- call statement('main') -%}\n {{ alter_set_column_mask(relation, column, mask) }}\n {%- endcall -%}\n {%- endfor -%}\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.alter_drop_column_mask", "macro.dbt_databricks.alter_set_column_mask"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.02824, "supported_languages": null}, "macro.dbt_databricks.alter_drop_column_mask": {"name": "alter_drop_column_mask", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_mask.sql", "original_file_path": "macros/relations/components/column_mask.sql", "unique_id": "macro.dbt_databricks.alter_drop_column_mask", "macro_sql": "{% macro alter_drop_column_mask(relation, column) -%}\n ALTER {{ relation.type.render() }} {{ relation.render() }}\n ALTER COLUMN `{{ column }}`\n DROP MASK;\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0283341, "supported_languages": null}, "macro.dbt_databricks.alter_set_column_mask": {"name": "alter_set_column_mask", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_mask.sql", "original_file_path": "macros/relations/components/column_mask.sql", "unique_id": "macro.dbt_databricks.alter_set_column_mask", "macro_sql": "{% macro alter_set_column_mask(relation, column, mask) -%}\n ALTER {{ relation.type.render() }} {{ relation.render() }}\n ALTER COLUMN `{{ column }}`\n SET MASK {{ mask.function }}\n {%- if mask.using_columns %}\n USING COLUMNS ({{ mask.using_columns }})\n {%- endif %};\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0284858, "supported_languages": null}, "macro.dbt_databricks.column_mask_exists": {"name": "column_mask_exists", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/column_mask.sql", "original_file_path": "macros/relations/components/column_mask.sql", "unique_id": "macro.dbt_databricks.column_mask_exists", "macro_sql": "{% macro column_mask_exists() %}\n {% for column_name, column in model.columns.items() %}\n {% if column is mapping and column.get('column_mask') %}\n {{ return(true) }}\n {% endif %}\n {% endfor %}\n {{ return(false) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.02866, "supported_languages": null}, "macro.dbt_databricks.fetch_non_null_constraint_columns": {"name": "fetch_non_null_constraint_columns", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.fetch_non_null_constraint_columns", "macro_sql": "{% macro fetch_non_null_constraint_columns(relation) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Incremental application of constraints is not supported for Hive Metastore\") }}\n {%- endif %}\n {% call statement('list_non_null_constraint_columns', fetch_result=True) -%}\n {{ fetch_non_null_constraint_columns_sql(relation) }}\n {% endcall %}\n {% do return(load_result('list_non_null_constraint_columns').table) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.fetch_non_null_constraint_columns_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.029232, "supported_languages": null}, "macro.dbt_databricks.fetch_non_null_constraint_columns_sql": {"name": "fetch_non_null_constraint_columns_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.fetch_non_null_constraint_columns_sql", "macro_sql": "{% macro fetch_non_null_constraint_columns_sql(relation) -%}\n SELECT column_name\n FROM `{{ relation.database|lower }}`.`information_schema`.`columns`\n WHERE table_catalog = '{{ relation.database|lower }}' \n AND table_schema = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}'\n AND is_nullable = 'NO';\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0293398, "supported_languages": null}, "macro.dbt_databricks.fetch_primary_key_constraints": {"name": "fetch_primary_key_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.fetch_primary_key_constraints", "macro_sql": "{% macro fetch_primary_key_constraints(relation) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Incremental application of constraints is not supported for Hive Metastore\") }}\n {%- endif %}\n {% call statement('list_primary_key_constraints', fetch_result=True) -%}\n {{ fetch_primary_key_constraints_sql(relation) }}\n {% endcall %}\n {% do return(load_result('list_primary_key_constraints').table) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.fetch_primary_key_constraints_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.029509, "supported_languages": null}, "macro.dbt_databricks.fetch_primary_key_constraints_sql": {"name": "fetch_primary_key_constraints_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.fetch_primary_key_constraints_sql", "macro_sql": "{% macro fetch_primary_key_constraints_sql(relation) -%}\n SELECT kcu.constraint_name, kcu.column_name\n FROM `{{ relation.database|lower }}`.information_schema.key_column_usage kcu\n WHERE kcu.table_catalog = '{{ relation.database|lower }}' \n AND kcu.table_schema = '{{ relation.schema|lower }}'\n AND kcu.table_name = '{{ relation.identifier|lower }}' \n AND kcu.constraint_name = (\n SELECT constraint_name\n FROM `{{ relation.database|lower }}`.information_schema.table_constraints\n WHERE table_catalog = '{{ relation.database|lower }}'\n AND table_schema = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}' \n AND constraint_type = 'PRIMARY KEY'\n )\n ORDER BY kcu.ordinal_position;\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0296829, "supported_languages": null}, "macro.dbt_databricks.fetch_foreign_key_constraints": {"name": "fetch_foreign_key_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.fetch_foreign_key_constraints", "macro_sql": "{% macro fetch_foreign_key_constraints(relation) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Incremental application of constraints is not supported for Hive Metastore\") }}\n {%- endif %}\n {% call statement('list_foreign_key_constraints', fetch_result=True) -%}\n {{ fetch_foreign_key_constraints_sql(relation) }}\n {% endcall %}\n {% do return(load_result('list_foreign_key_constraints').table) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.fetch_foreign_key_constraints_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.029862, "supported_languages": null}, "macro.dbt_databricks.fetch_foreign_key_constraints_sql": {"name": "fetch_foreign_key_constraints_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.fetch_foreign_key_constraints_sql", "macro_sql": "{% macro fetch_foreign_key_constraints_sql(relation) -%}\n SELECT\n kcu.constraint_name,\n kcu.column_name AS from_column,\n ukcu.table_catalog AS to_catalog,\n ukcu.table_schema AS to_schema,\n ukcu.table_name AS to_table,\n ukcu.column_name AS to_column\n FROM `{{ relation.database|lower }}`.information_schema.key_column_usage kcu\n JOIN `{{ relation.database|lower }}`.information_schema.referential_constraints rc\n ON kcu.constraint_name = rc.constraint_name\n JOIN `{{ relation.database|lower }}`.information_schema.key_column_usage ukcu\n ON rc.unique_constraint_name = ukcu.constraint_name\n AND kcu.ordinal_position = ukcu.ordinal_position\n WHERE kcu.table_catalog = '{{ relation.database|lower }}'\n AND kcu.table_schema = '{{ relation.schema|lower }}'\n AND kcu.table_name = '{{ relation.identifier|lower }}'\n AND kcu.constraint_name IN (\n SELECT constraint_name\n FROM `{{ relation.database|lower }}`.information_schema.table_constraints\n WHERE table_catalog = '{{ relation.database|lower }}'\n AND table_schema = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}'\n AND constraint_type = 'FOREIGN KEY'\n )\n ORDER BY kcu.ordinal_position;\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.030079, "supported_languages": null}, "macro.dbt_databricks.apply_constraints": {"name": "apply_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.apply_constraints", "macro_sql": "{% macro apply_constraints(relation, constraints) -%}\n {{ log(\"Applying constraints to relation \" ~ constraints) }}\n {%- if constraints and relation.is_hive_metastore() -%}\n {{ exceptions.raise_compiler_error(\"Constraints are only supported for Unity Catalog\") }}\n {%- endif -%}\n {# Order matters here because key constraints depend on non-null constraints #} \n {%- if constraints.unset_constraints %}\n {%- for constraint in constraints.unset_constraints -%}\n {%- call statement('main') -%}\n {{ alter_unset_constraint(relation, constraint) }}\n {%- endcall -%}\n {%- endfor -%}\n {%- endif %}\n {%- if constraints.unset_non_nulls %}\n {%- for column in constraints.unset_non_nulls -%}\n {%- call statement('main') -%}\n {{ alter_unset_non_null_constraint(relation, column) }}\n {%- endcall -%}\n {%- endfor -%}\n {%- endif %}\n {%- if constraints.set_non_nulls %}\n {%- for column in constraints.set_non_nulls -%}\n {%- call statement('main') -%}\n {{ alter_set_non_null_constraint(relation, column) }}\n {%- endcall -%}\n {%- endfor -%}\n {%- endif %}\n {%- if constraints.set_constraints %}\n {%- for constraint in constraints.set_constraints -%}\n {%- call statement('main') -%}\n {{ alter_set_constraint(relation, constraint) }}\n {%- endcall -%}\n {%- endfor -%}\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.alter_unset_constraint", "macro.dbt_databricks.alter_unset_non_null_constraint", "macro.dbt_databricks.alter_set_non_null_constraint", "macro.dbt_databricks.alter_set_constraint"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.03053, "supported_languages": null}, "macro.dbt_databricks.alter_set_non_null_constraint": {"name": "alter_set_non_null_constraint", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.alter_set_non_null_constraint", "macro_sql": "{% macro alter_set_non_null_constraint(relation, column) -%}\n ALTER {{ relation.type.render() }} {{ relation.render() }} ALTER COLUMN {{ adapter.quote(column) }} SET NOT NULL;\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0306408, "supported_languages": null}, "macro.dbt_databricks.alter_unset_non_null_constraint": {"name": "alter_unset_non_null_constraint", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.alter_unset_non_null_constraint", "macro_sql": "{% macro alter_unset_non_null_constraint(relation, column) -%}\n ALTER {{ relation.type.render() }} {{ relation.render() }} ALTER COLUMN {{ adapter.quote(column) }} DROP NOT NULL;\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0307462, "supported_languages": null}, "macro.dbt_databricks.alter_set_constraint": {"name": "alter_set_constraint", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.alter_set_constraint", "macro_sql": "{% macro alter_set_constraint(relation, constraint) -%}\n ALTER {{ relation.type.render() }} {{ relation.render() }} ADD {{ constraint.render() }};\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.030848, "supported_languages": null}, "macro.dbt_databricks.alter_unset_constraint": {"name": "alter_unset_constraint", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/constraints.sql", "original_file_path": "macros/relations/components/constraints.sql", "unique_id": "macro.dbt_databricks.alter_unset_constraint", "macro_sql": "{% macro alter_unset_constraint(relation, constraint) -%}\n {% set constraint_type = constraint.type %}\n {% if constraint_type == 'primary_key' %}\n {# Need to only add CASCADE to PK constraints because dropping check constraints break when adding CASCADE #}\n ALTER {{ relation.type.render() }} {{ relation.render() }} DROP CONSTRAINT {{ constraint.name }} CASCADE;\n {% else %}\n ALTER {{ relation.type.render() }} {{ relation.render() }} DROP CONSTRAINT IF EXISTS {{ constraint.name }};\n {% endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.031043, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_table": {"name": "databricks__drop_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_table", "macro_sql": "{% macro databricks__drop_table(relation) -%}\n drop table if exists {{ relation.render() }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.031122, "supported_languages": null}, "macro.dbt_databricks.safe_relation_replace": {"name": "safe_relation_replace", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt_databricks.safe_relation_replace", "macro_sql": "{% macro safe_relation_replace(existing_relation, staging_relation, intermediate_relation, compiled_code) %}\n \n {{ create_table_at(staging_relation, intermediate_relation, compiled_code) }}\n\n {{ create_backup(existing_relation) }}\n\n {{ adapter.rename_relation(staging_relation, existing_relation) }}\n\n \n\n {% call statement('main') %}\n {{ get_drop_backup_sql(existing_relation) }}\n {% endcall %}\n \n {{ adapter.cache_dropped(make_backup_relation(existing_relation, existing_relation.type)) }}\n\n {{ drop_relation_if_exists(intermediate_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.create_table_at", "macro.dbt_databricks.create_backup", "macro.dbt.statement", "macro.dbt.get_drop_backup_sql", "macro.dbt.make_backup_relation", "macro.dbt.drop_relation_if_exists"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.031401, "supported_languages": null}, "macro.dbt_databricks.databricks__get_rename_table_sql": {"name": "databricks__get_rename_table_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt_databricks.databricks__get_rename_table_sql", "macro_sql": "{% macro databricks__get_rename_table_sql(relation, new_name) %}\n ALTER TABLE {{ relation.render() }} RENAME TO `{{ new_name }}`\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.031498, "supported_languages": null}, "macro.dbt_databricks.apply_config_changeset": {"name": "apply_config_changeset", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/alter.sql", "original_file_path": "macros/relations/table/alter.sql", "unique_id": "macro.dbt_databricks.apply_config_changeset", "macro_sql": "{% macro apply_config_changeset(target_relation, model, configuration_changes) %}\n {{ log(\"Applying configuration changes to relation \" ~ target_relation) }}\n {% if configuration_changes %}\n {% set comment = configuration_changes.changes.get(\"comment\") %}\n {% set column_comments = configuration_changes.changes.get(\"column_comments\") %}\n {% set column_tags = configuration_changes.changes.get(\"column_tags\") %}\n {% set tags = configuration_changes.changes.get(\"tags\") %}\n {% set tblproperties = configuration_changes.changes.get(\"tblproperties\") %}\n {% set liquid_clustering = configuration_changes.changes.get(\"liquid_clustering\")%}\n {% set constraints = configuration_changes.changes.get(\"constraints\") %}\n {% set column_masks = configuration_changes.changes.get(\"column_masks\") %}\n {% set row_filter = configuration_changes.changes.get(\"row_filter\") %}\n {% if tags is not none %}\n {% do apply_tags(target_relation, tags.set_tags) %}\n {%- endif -%}\n {% if tblproperties is not none %}\n {% do apply_tblproperties(target_relation, tblproperties.tblproperties) %}\n {%- endif -%}\n {% if liquid_clustering is not none %}\n {% do apply_liquid_clustered_cols(target_relation, liquid_clustering) %}\n {%- endif -%}\n {% if comment %}\n {{ run_query_as(alter_relation_comment_sql(target_relation, comment.comment), 'alter_relation_comment', fetch_result=False) }}\n {% endif %}\n {% if column_comments %}\n {{ alter_column_comments(target_relation, column_comments.comments) }}\n {% endif %}\n {% if column_tags %}\n {{ apply_column_tags(target_relation, column_tags) }}\n {% endif %}\n {% if constraints %}\n {{ apply_constraints(target_relation, constraints) }}\n {% endif %}\n {% if column_masks %}\n {{ apply_column_masks(target_relation, column_masks) }}\n {% endif %}\n {% if row_filter %}\n {{ apply_row_filter(target_relation, row_filter) }}\n {% endif %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_tblproperties", "macro.dbt_databricks.apply_liquid_clustered_cols", "macro.dbt_databricks.run_query_as", "macro.dbt_databricks.alter_relation_comment_sql", "macro.dbt_databricks.alter_column_comments", "macro.dbt_databricks.apply_column_tags", "macro.dbt_databricks.apply_constraints", "macro.dbt_databricks.apply_column_masks", "macro.dbt_databricks.apply_row_filter"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.032541, "supported_languages": null}, "macro.dbt_databricks.create_table_at": {"name": "create_table_at", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.create_table_at", "macro_sql": "{% macro create_table_at(relation, intermediate_relation, compiled_code) %}\n {% set tags = config.get('databricks_tags') %}\n {% set model_columns = model.get('columns', []) %}\n {% set existing_columns = adapter.get_columns_in_relation(intermediate_relation) %}\n {% set contract_config = config.get('contract') %}\n {% set contract_enforced = contract_config and contract_config.enforced %}\n {% if contract_enforced %}\n {% set model_constraints = model.get('constraints', []) %}\n {% else %}\n {% set model_constraints = [] %}\n {% endif %}\n {% set columns_and_constraints = adapter.parse_columns_and_constraints(existing_columns, model_columns, model_constraints, contract_enforced, model.name) %}\n {% set target_relation = relation.enrich(columns_and_constraints[1]) %}\n \n {% call statement('main') %}\n {{ get_create_table_sql(target_relation, columns_and_constraints[0], compiled_code) }}\n {% endcall %}\n\n {{ apply_alter_constraints(target_relation) }}\n {{ apply_tags(target_relation, tags) }}\n {% set column_tags = adapter.get_column_tags_from_model(config.model) %}\n {% if column_tags and column_tags.set_column_tags %}\n {{ apply_column_tags(target_relation, column_tags) }}\n {% endif %}\n\n {% call statement('merge into target') %}\n insert into {{ target_relation }} by name select * from {{ intermediate_relation }}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.get_create_table_sql", "macro.dbt_databricks.apply_alter_constraints", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_column_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0335948, "supported_languages": null}, "macro.dbt_databricks.get_create_table_sql": {"name": "get_create_table_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.get_create_table_sql", "macro_sql": "{% macro get_create_table_sql(target_relation, columns, compiled_code) %}\n\n {%- set catalog_relation = adapter.build_catalog_relation(config.model) -%}\n\n {%- set contract = config.get('contract') -%}\n {%- set contract_enforced = contract and contract.enforced -%}\n {%- if contract_enforced -%}\n {{ get_assert_columns_equivalent(compiled_code) }}\n {%- endif -%}\n\n {%- if catalog_relation.file_format in ('delta', 'iceberg') %}\n create or replace table {{ target_relation.render() }}\n {% else %}\n create table {{ target_relation.render() }}\n {% endif -%}\n {{ get_column_and_constraints_sql(target_relation, columns) }}\n {{ file_format_clause(catalog_relation) }}\n {{ databricks__options_clause(catalog_relation) }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ get_create_row_filter_clause(target_relation) }}\n {{ liquid_clustered_cols() }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause(catalog_relation) }}\n {{ comment_clause() }}\n {{ tblproperties_clause() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent", "macro.dbt_databricks.get_column_and_constraints_sql", "macro.dbt_databricks.file_format_clause", "macro.dbt_databricks.databricks__options_clause", "macro.dbt_spark.partition_cols", "macro.dbt_databricks.get_create_row_filter_clause", "macro.dbt_databricks.liquid_clustered_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_databricks.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_databricks.tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.034013, "supported_languages": null}, "macro.dbt_databricks.databricks__create_table_as": {"name": "databricks__create_table_as", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.databricks__create_table_as", "macro_sql": "{% macro databricks__create_table_as(temporary, relation, compiled_code, language='sql') -%}\n\n {%- set catalog_relation = adapter.build_catalog_relation(config.model) -%}\n\n {%- if language == 'sql' -%}\n {%- if temporary -%}\n {{ create_temporary_view(relation, compiled_code) }}\n {%- else -%}\n {% if catalog_relation.file_format == 'delta' %}\n create or replace table {{ relation.render() }}\n {% else %}\n create table {{ relation.render() }}\n {% endif %}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config and contract_config.enforced %}\n {{ get_assert_columns_equivalent(compiled_code) }}\n {%- set compiled_code = get_select_subquery(compiled_code) %}\n {% endif %}\n {{ file_format_clause(catalog_relation) }}\n {{ databricks__options_clause(catalog_relation) }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ get_create_row_filter_clause(relation) }}\n {{ liquid_clustered_cols() }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause(catalog_relation) }}\n {{ comment_clause() }}\n {{ tblproperties_clause() }}\n as\n {{ compiled_code }}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {#--\n N.B. Python models _can_ write to temp views HOWEVER they use a different session\n and have already expired by the time they need to be used (I.E. in merges for incremental models)\n\n TODO: Deep dive into spark sessions to see if we can reuse a single session for an entire\n dbt invocation.\n --#}\n {{ databricks__py_write_table(compiled_code=compiled_code, target_relation=relation) }}\n {%- endif -%}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.create_temporary_view", "macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_select_subquery", "macro.dbt_databricks.file_format_clause", "macro.dbt_databricks.databricks__options_clause", "macro.dbt_spark.partition_cols", "macro.dbt_databricks.get_create_row_filter_clause", "macro.dbt_databricks.liquid_clustered_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_databricks.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_databricks.tblproperties_clause", "macro.dbt_databricks.databricks__py_write_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.034562, "supported_languages": null}, "macro.dbt_databricks.databricks__options_clause": {"name": "databricks__options_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.databricks__options_clause", "macro_sql": "{% macro databricks__options_clause(catalog_relation=none) -%}\n {#-\n Moving forward, this macro should require a `catalog_relation`, which is covered by the first condition.\n However, there could be existing macros that is still passing no arguments, including user macros.\n Hence, we need to support the old code still, which is covered by the second condition.\n Additionally, since this rolls up to `options_clause` in `dbt-spark`, which does not have any arguments,\n all calls to `options_clause` will take the second path. This macro needs to be called directly\n via `databricks__options_clause`.\n -#}\n {%- if catalog_relation is not none -%}\n {%- set file_format = catalog_relation.file_format -%}\n {%- else -%}\n {%- set file_format = adapter.resolve_file_format(config) -%}\n {%- endif -%}\n\n {%- set options = config.get('options') -%}\n {%- if file_format == 'hudi' -%}\n {%- set unique_key = config.get('unique_key') -%}\n {%- if unique_key is not none and options is none -%}\n {%- set options = {'primaryKey': config.get('unique_key')} -%}\n {%- elif unique_key is not none and options is not none and 'primaryKey' not in options -%}\n {%- set _ = options.update({'primaryKey': config.get('unique_key')}) -%}\n {%- elif options is not none and 'primaryKey' in options and options['primaryKey'] != unique_key -%}\n {{ exceptions.raise_compiler_error(\"unique_key and options('primaryKey') should be the same column(s).\") }}\n {%- endif %}\n {%- endif %}\n\n {%- if options is not none %}\n options (\n {%- for option in options -%}\n {{ option }} \"{{ options[option] }}\" {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.035139, "supported_languages": null}, "macro.dbt_databricks.get_create_intermediate_table": {"name": "get_create_intermediate_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.get_create_intermediate_table", "macro_sql": "{% macro get_create_intermediate_table(relation, compiled_code, language) %}\n {%- if language == 'sql' -%}\n {{ create_temporary_view(relation, compiled_code) }}\n {%- else -%}\n {{ create_python_intermediate_table(relation, compiled_code) }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.create_temporary_view", "macro.dbt_databricks.create_python_intermediate_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.035273, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_view": {"name": "databricks__drop_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_view", "macro_sql": "{% macro databricks__drop_view(relation) -%}\n DROP VIEW IF EXISTS {{ relation.render() }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.035372, "supported_languages": null}, "macro.dbt_databricks.databricks__get_replace_view_sql": {"name": "databricks__get_replace_view_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt_databricks.databricks__get_replace_view_sql", "macro_sql": "{% macro databricks__get_replace_view_sql(target_relation, sql) %}\n {{ create_view_as(target_relation, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.035466, "supported_languages": null}, "macro.dbt_databricks.databricks__get_rename_view_sql": {"name": "databricks__get_rename_view_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt_databricks.databricks__get_rename_view_sql", "macro_sql": "{% macro databricks__get_rename_view_sql(relation, new_name) %}\n ALTER VIEW {{ relation.render() }} RENAME TO {{ new_name }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.035554, "supported_languages": null}, "macro.dbt_databricks.alter_view": {"name": "alter_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/alter.sql", "original_file_path": "macros/relations/view/alter.sql", "unique_id": "macro.dbt_databricks.alter_view", "macro_sql": "{% macro alter_view(target_relation, changes) %}\n {{ log(\"Updating view via ALTER\") }}\n {{ adapter.dispatch('alter_view', 'dbt')(target_relation, changes) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.035757, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_view": {"name": "databricks__alter_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/alter.sql", "original_file_path": "macros/relations/view/alter.sql", "unique_id": "macro.dbt_databricks.databricks__alter_view", "macro_sql": "{% macro databricks__alter_view(target_relation, changes) %}\n {% set tags = changes.get(\"tags\") %}\n {% set tblproperties = changes.get(\"tblproperties\") %}\n {% set query = changes.get(\"query\") %}\n {% set column_comments = changes.get(\"column_comments\") %}\n {% if tags %}\n {{ apply_tags(target_relation, tags.set_tags) }}\n {% endif %}\n {% if tblproperties %}\n {{ apply_tblproperties(target_relation, tblproperties.tblproperties) }}\n {% endif %}\n {% if query %}\n {{ alter_query(target_relation, query.query) }}\n {% endif %}\n {% if column_comments %}\n {{ alter_column_comments(target_relation, column_comments.comments) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_tblproperties", "macro.dbt_databricks.alter_query", "macro.dbt_databricks.alter_column_comments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.036093, "supported_languages": null}, "macro.dbt_databricks.databricks__create_view_as": {"name": "databricks__create_view_as", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt_databricks.databricks__create_view_as", "macro_sql": "{% macro databricks__create_view_as(relation, sql) %}\n {% if column_mask_exists() %}\n {% do exceptions.raise_compiler_error(\"Column masks are not supported for views.\") %}\n {% endif %}\n {% if row_filter_exists() %}\n {% do exceptions.raise_compiler_error(\"Row filters are not supported for views.\") %}\n {% endif %}\n {{ log(\"Creating view \" ~ relation) }}\n create or replace view {{ relation.render() }}\n {%- if config.persist_column_docs() -%}\n {%- set model_columns = model.columns -%}\n {%- set query_columns = get_columns_in_query(sql) -%}\n {%- if query_columns %}\n (\n {{ get_persist_docs_column_list(model_columns, query_columns) }}\n )\n {%- endif -%}\n {%- endif %}\n {{ comment_clause() }}\n {%- set contract_config = config.get('contract') -%}\n {%- if contract_config and contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif -%}\n {{ tblproperties_clause() }}\n as (\n {{ sql }}\n )\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.column_mask_exists", "macro.dbt_databricks.row_filter_exists", "macro.dbt.get_columns_in_query", "macro.dbt_databricks.get_persist_docs_column_list", "macro.dbt_spark.comment_clause", "macro.dbt.get_assert_columns_equivalent", "macro.dbt_databricks.tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.037025, "supported_languages": null}, "macro.dbt_databricks.get_column_comment_sql": {"name": "get_column_comment_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt_databricks.get_column_comment_sql", "macro_sql": "{% macro get_column_comment_sql(column_name, column_dict) -%}\n {%- if column_name in column_dict and column_dict[column_name][\"description\"] -%}\n {%- set escaped_description = column_dict[column_name][\"description\"] | replace(\"'\", \"\\\\'\") -%}\n {%- set column_comment_clause = \"comment '\" ~ escaped_description ~ \"'\" -%}\n {{ adapter.quote(column_name) }} {{ column_comment_clause }}\n {%- else -%}\n {{ adapter.quote(column_name) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.037243, "supported_languages": null}, "macro.dbt_databricks.get_persist_docs_column_list": {"name": "get_persist_docs_column_list", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt_databricks.get_persist_docs_column_list", "macro_sql": "{% macro get_persist_docs_column_list(model_columns, query_columns) -%}\n {%- for column_name in query_columns -%}\n {{ get_column_comment_sql(column_name, model_columns) }}{{\",\\n\\t\" if not loop.last else \"\" }}\n {%- endfor -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_column_comment_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.037353, "supported_languages": null}, "macro.dbt_databricks.get_replace_metric_view_sql": {"name": "get_replace_metric_view_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/metric_view/replace.sql", "original_file_path": "macros/relations/metric_view/replace.sql", "unique_id": "macro.dbt_databricks.get_replace_metric_view_sql", "macro_sql": "{% macro get_replace_metric_view_sql(target_relation, sql) %}\n {{ adapter.dispatch('get_replace_metric_view_sql', 'dbt')(target_relation, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_metric_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0374742, "supported_languages": null}, "macro.dbt_databricks.databricks__get_replace_metric_view_sql": {"name": "databricks__get_replace_metric_view_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/metric_view/replace.sql", "original_file_path": "macros/relations/metric_view/replace.sql", "unique_id": "macro.dbt_databricks.databricks__get_replace_metric_view_sql", "macro_sql": "{% macro databricks__get_replace_metric_view_sql(target_relation, sql) %}\n {{ get_create_metric_view_as_sql(target_relation, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_metric_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0375402, "supported_languages": null}, "macro.dbt_databricks.alter_metric_view": {"name": "alter_metric_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/metric_view/alter.sql", "original_file_path": "macros/relations/metric_view/alter.sql", "unique_id": "macro.dbt_databricks.alter_metric_view", "macro_sql": "{% macro alter_metric_view(target_relation, changes) %}\n {{ log(\"Updating metric view via ALTER\") }}\n {{ adapter.dispatch('alter_metric_view', 'dbt')(target_relation, changes) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_metric_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.037801, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_metric_view": {"name": "databricks__alter_metric_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/metric_view/alter.sql", "original_file_path": "macros/relations/metric_view/alter.sql", "unique_id": "macro.dbt_databricks.databricks__alter_metric_view", "macro_sql": "{% macro databricks__alter_metric_view(target_relation, changes) %}\n {% set tags = changes.get(\"tags\") %}\n {% set tblproperties = changes.get(\"tblproperties\") %}\n {% set query = changes.get(\"query\") %}\n\n {# Handle YAML definition changes via ALTER VIEW AS #}\n {% if query %}\n {% call statement('main') %}\n {{ get_alter_metric_view_as_sql(target_relation, query.query) }}\n {% endcall %}\n {% else %}\n {# Ensure statement('main') is called for dbt to track the run #}\n {% call statement('main') %}\n select 1\n {% endcall %}\n {% endif %}\n\n {% if tags %}\n {{ apply_tags(target_relation, tags.set_tags) }}\n {% endif %}\n {% if tblproperties %}\n {{ apply_tblproperties(target_relation, tblproperties.tblproperties) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.get_alter_metric_view_as_sql", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_tblproperties"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.038118, "supported_languages": null}, "macro.dbt_databricks.get_alter_metric_view_as_sql": {"name": "get_alter_metric_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/metric_view/alter.sql", "original_file_path": "macros/relations/metric_view/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_metric_view_as_sql", "macro_sql": "{% macro get_alter_metric_view_as_sql(relation, yaml_content) -%}\n {{ adapter.dispatch('get_alter_metric_view_as_sql', 'dbt')(relation, yaml_content) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_metric_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.038201, "supported_languages": null}, "macro.dbt_databricks.databricks__get_alter_metric_view_as_sql": {"name": "databricks__get_alter_metric_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/metric_view/alter.sql", "original_file_path": "macros/relations/metric_view/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_alter_metric_view_as_sql", "macro_sql": "{% macro databricks__get_alter_metric_view_as_sql(relation, yaml_content) %}\n{%- set yaml_body = adapter.yaml_quote_backtick_values(yaml_content) -%}\nalter view {{ relation.render() }} as $$\n{{ yaml_body }}\n$$\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.038293, "supported_languages": null}, "macro.dbt_databricks.replace_with_metric_view": {"name": "replace_with_metric_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/metric_view/alter.sql", "original_file_path": "macros/relations/metric_view/alter.sql", "unique_id": "macro.dbt_databricks.replace_with_metric_view", "macro_sql": "{% macro replace_with_metric_view(existing_relation, target_relation) %}\n {% set sql = adapter.clean_sql(sql) %}\n {% set tags = config.get('databricks_tags') %}\n {% set tblproperties = config.get('tblproperties') %}\n {{ execute_multiple_statements(get_replace_sql(existing_relation, target_relation, sql)) }}\n {%- do apply_tags(target_relation, tags) -%}\n\n {% if tblproperties %}\n {{ apply_tblproperties(target_relation, tblproperties) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.execute_multiple_statements", "macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_tblproperties"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.038506, "supported_languages": null}, "macro.dbt_databricks.get_create_metric_view_as_sql": {"name": "get_create_metric_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/metric_view/create.sql", "original_file_path": "macros/relations/metric_view/create.sql", "unique_id": "macro.dbt_databricks.get_create_metric_view_as_sql", "macro_sql": "{% macro get_create_metric_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_metric_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_metric_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.038629, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_metric_view_as_sql": {"name": "databricks__get_create_metric_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/metric_view/create.sql", "original_file_path": "macros/relations/metric_view/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_metric_view_as_sql", "macro_sql": "{% macro databricks__get_create_metric_view_as_sql(relation, sql) %}\n{%- set yaml_body = adapter.yaml_quote_backtick_values(sql) -%}\ncreate or replace view {{ relation.render() }}\nwith metrics\nlanguage yaml\nas $$\n{{ yaml_body }}\n$$\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0387242, "supported_languages": null}, "macro.dbt_databricks.statement_with_staging_table": {"name": "statement_with_staging_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt_databricks.statement_with_staging_table", "macro_sql": "{% macro statement_with_staging_table(name=None, staging_table=None, fetch_result=False, auto_begin=True) -%}\n {%- if execute: -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- set res, table = adapter.execute(sql, auto_begin=auto_begin, fetch=fetch_result, staging_table=staging_table) -%}\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.039202, "supported_languages": null}, "macro.dbt_databricks.execute_multiple_statements": {"name": "execute_multiple_statements", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt_databricks.execute_multiple_statements", "macro_sql": "{% macro execute_multiple_statements(statements) %}\n {%- if statements is string %}\n {% call statement(name=\"main\") %}\n {{ statements }}\n {% endcall %}\n {%- else %}\n {%- for sql in statements %}\n {% call statement(name=\"main\") %}\n {{ sql }}\n {% endcall %}\n {% endfor %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.039372, "supported_languages": null}, "macro.dbt_databricks.run_query_as": {"name": "run_query_as", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt_databricks.run_query_as", "macro_sql": "{% macro run_query_as(sql, name, fetch_result=True) %}\n {% call statement(name, fetch_result, auto_begin=False) %}\n {{ sql }}\n {% endcall %}\n\n {% if fetch_result %}\n {{ return(load_result(name).table) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.039519, "supported_languages": null}, "macro.dbt_databricks.databricks__dateadd": {"name": "databricks__dateadd", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_databricks.databricks__dateadd", "macro_sql": "{% macro databricks__dateadd(datepart, interval, from_date_or_timestamp) %}\n {%- if adapter.has_dbr_capability('timestampdiff') -%}\n timestampadd({{datepart}}, {{interval}}, {{from_date_or_timestamp}})\n {%- else -%}\n {{ spark__dateadd(datepart, interval, from_date_or_timestamp) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.039696, "supported_languages": null}, "macro.dbt_databricks.databricks__datediff": {"name": "databricks__datediff", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_databricks.databricks__datediff", "macro_sql": "{% macro databricks__datediff(first_date, second_date, datepart) %}\n {%- if adapter.has_dbr_capability('timestampdiff') -%}\n timestampdiff({{datepart}}, {{date_trunc(datepart, first_date)}}, {{date_trunc(datepart, second_date)}})\n {%- else -%}\n {{ spark__datediff(first_date, second_date, datepart) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.date_trunc", "macro.dbt_spark.spark__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.039925, "supported_languages": null}, "macro.dbt_databricks.databricks__split_part": {"name": "databricks__split_part", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt_databricks.databricks__split_part", "macro_sql": "{% macro databricks__split_part(string_text, delimiter_text, part_number) %}\n\n {% set delimiter_expr %}\n\n -- escape if starts with a special character\n case when regexp_extract({{ delimiter_text }}, '([^A-Za-z0-9])(.*)', 1) != '_'\n then concat('\\\\', {{ delimiter_text }})\n else {{ delimiter_text }} end\n\n {% endset %}\n\n {% if part_number >= 0 %}\n\n {% set split_part_expr %}\n\n get(split(\n {{ string_text }},\n {{ delimiter_expr }}\n ), {{ part_number - 1 if part_number > 0 else part_number }})\n\n {% endset %}\n\n {% else %}\n\n {% set split_part_expr %}\n\n get(split(\n {{ string_text }},\n {{ delimiter_expr }}\n ), \n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 1 + {{ part_number }}\n )\n\n {% endset %}\n\n {% endif %}\n\n {{ return(split_part_expr) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0403001, "supported_languages": null}, "macro.dbt_databricks.databricks_copy_into": {"name": "databricks_copy_into", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/copy_into.sql", "original_file_path": "macros/adapters/copy_into.sql", "unique_id": "macro.dbt_databricks.databricks_copy_into", "macro_sql": "{% macro databricks_copy_into(\n target_table,\n source,\n file_format,\n expression_list=none,\n source_credential=none,\n source_encryption=none,\n validate=none,\n files=none,\n pattern=none,\n format_options=none,\n copy_options=none) -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=target.database,\n schema=target.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- set source_clause -%}\n {%- if expression_list -%}\n ( select {{ expression_list }} from '{{ source }}' )\n {%- else -%}\n '{{ source }}'\n {%- endif -%}\n {%- if source_credential or source_encryption %}\n WITH (\n {%- if source_credential %}\n credential (\n {%- for name in source_credential -%}\n '{{ name }}' = '{{ source_credential[name] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {%- if source_encryption %}\n encryption (\n {%- for name in source_encryption -%}\n '{{ name }}' = '{{ source_encryption[name] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n )\n {%- endif -%}\n {%- endset -%}\n\n {% set query %}\n copy into {{ target_relation }}\n from {{ source_clause }}\n fileformat = {{ file_format }}\n {% if validate -%} validate {{ validate }} {%- endif %}\n {% if files and pattern %}\n {{ exceptions.raise_compiler_error(\"You can only specify one of 'files' or 'pattern'\") }}\n {% endif %}\n {% if files -%}\n files = (\n {%- for file in files -%}\n '{{ file }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% if pattern -%}\n pattern = '{{ pattern }}'\n {%- endif %}\n {% if format_options -%}\n format_options (\n {%- for key in format_options -%}\n '{{ key }}' = '{{ format_options[key] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% if copy_options -%}\n copy_options (\n {%- for key in copy_options -%}\n '{{ key }}' = '{{ copy_options[key] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% endset %}\n\n {{ run_query_as(query, 'copy_into', fetch_result=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt_databricks.run_query_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.041465, "supported_languages": null}, "macro.dbt_databricks.make_staging_relation": {"name": "make_staging_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.make_staging_relation", "macro_sql": "{% macro make_staging_relation(base_relation, suffix='__dbt_stg', type='table') %}\n {% set unique_tmp_table_suffix = config.get('unique_tmp_table_suffix', False) | as_bool %}\n {% if unique_tmp_table_suffix %}\n {% set suffix = adapter.generate_unique_temporary_table_suffix(suffix) %}\n {% endif %}\n {% set stg_identifier = base_relation.identifier ~ suffix %}\n {% set stg_relation = api.Relation.create(database=base_relation.database, schema=base_relation.schema, identifier=stg_identifier, type=type) %}\n {% do return(stg_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.041954, "supported_languages": null}, "macro.dbt_databricks.databricks__make_intermediate_relation": {"name": "databricks__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.databricks__make_intermediate_relation", "macro_sql": "{% macro databricks__make_intermediate_relation(base_relation, suffix) %}\n {{ return(databricks__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.042033, "supported_languages": null}, "macro.dbt_databricks.databricks__make_temp_relation": {"name": "databricks__make_temp_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.databricks__make_temp_relation", "macro_sql": "{% macro databricks__make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {% set unique_tmp_table_suffix = config.get('unique_tmp_table_suffix', False) | as_bool %}\n\n {% if unique_tmp_table_suffix %}\n {% set suffix = adapter.generate_unique_temporary_table_suffix() %}\n {% endif %}\n \n {% if suffix == '__dbt_tmp' and model.batch %}\n {% set suffix = suffix ~ '_' ~ model.batch.id %}\n {% endif %}\n\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {% set language = model['language'] %}\n {%- if language == 'sql' -%}\n {% set temporary = not base_relation.is_hive_metastore() %}\n {% set tmp_relation = api.Relation.create(identifier=tmp_identifier, type='view', temporary=temporary) %}\n {%- else -%}\n {% set tmp_relation = api.Relation.create(database=base_relation.database, schema=base_relation.schema, identifier=tmp_identifier, type='table') %}\n {%- endif -%}\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.042485, "supported_languages": null}, "macro.dbt_databricks.databricks__get_or_create_relation": {"name": "databricks__get_or_create_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.databricks__get_or_create_relation", "macro_sql": "{% macro databricks__get_or_create_relation(database, schema, identifier, type, needs_information=False) %}\n {%- set target_relation = adapter.get_relation(\n database=database,\n schema=schema,\n identifier=identifier,\n needs_information=needs_information) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type,\n temporary=False\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.04276, "supported_languages": null}, "macro.dbt_databricks.get_column_and_constraints_sql": {"name": "get_column_and_constraints_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.get_column_and_constraints_sql", "macro_sql": "{% macro get_column_and_constraints_sql(relation, columns) %}\n (\n {% for column in columns %}\n {{ column.render_for_create() }}{% if not loop.last or relation.create_constraints %},{% endif %}\n {% endfor %}\n {% if relation.create_constraints %}\n {{ relation.render_constraints_for_create() }}\n {% endif %}\n )\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.042932, "supported_languages": null}, "macro.dbt_databricks.load_relation_with_metadata": {"name": "load_relation_with_metadata", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.load_relation_with_metadata", "macro_sql": "{% macro load_relation_with_metadata(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier,\n needs_information=True\n )) -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0430431, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog": {"name": "databricks__get_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog", "macro_sql": "{% macro databricks__get_catalog(information_schema, schemas) -%}\n {% set query %}\nWITH tables AS (\n {{ databricks__get_catalog_tables_sql(information_schema) }}\n {{ databricks__get_catalog_schemas_where_clause_sql(information_schema.database, schemas) }}\n),\ncolumns AS (\n {{ databricks__get_catalog_columns_sql(information_schema) }}\n {{ databricks__get_catalog_schemas_where_clause_sql(information_schema.database, schemas) }}\n)\n{{ databricks__get_catalog_results_sql() }}\n {%- endset -%}\n\n {{ return(run_query_as(query, 'get_catalog')) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql", "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro.dbt_databricks.run_query_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.043443, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_relations": {"name": "databricks__get_catalog_relations", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_relations", "macro_sql": "{% macro databricks__get_catalog_relations(information_schema, relations) -%}\n {% set query %}\nWITH tables AS (\n {{ databricks__get_catalog_tables_sql(information_schema) }}\n {{ databricks__get_catalog_relations_where_clause_sql(information_schema.database, relations) }}\n),\ncolumns AS (\n {{ databricks__get_catalog_columns_sql(information_schema) }}\n {{ databricks__get_catalog_relations_where_clause_sql(information_schema.database, relations) }}\n)\n{{ databricks__get_catalog_results_sql() }}\n {%- endset -%}\n\n {{ return(run_query_as(query, 'get_catalog_relations')) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql", "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro.dbt_databricks.run_query_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.043628, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_tables_sql": {"name": "databricks__get_catalog_tables_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro_sql": "{% macro databricks__get_catalog_tables_sql(information_schema) -%}\nSELECT\n table_catalog AS table_database,\n table_schema,\n table_name,\n lower(table_type) AS table_type,\n comment AS table_comment,\n table_owner,\n 'Last Modified' AS `stats:last_modified:label`,\n last_altered AS `stats:last_modified:value`,\n 'The timestamp for last update/change' AS `stats:last_modified:description`,\n (last_altered IS NOT NULL AND table_type NOT ILIKE '%VIEW%') AS `stats:last_modified:include`\nFROM `system`.`information_schema`.`tables`\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0436962, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_columns_sql": {"name": "databricks__get_catalog_columns_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro_sql": "{% macro databricks__get_catalog_columns_sql(information_schema) -%}\nSELECT\n table_catalog AS table_database,\n table_schema,\n table_name,\n column_name,\n ordinal_position AS column_index,\n lower(full_data_type) AS column_type,\n comment AS column_comment\nFROM `system`.`information_schema`.`columns`\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.043746, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_results_sql": {"name": "databricks__get_catalog_results_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro_sql": "{% macro databricks__get_catalog_results_sql() -%}\nSELECT *\nFROM tables\nJOIN columns USING (table_database, table_schema, table_name)\nORDER BY column_index\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.043786, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql": {"name": "databricks__get_catalog_schemas_where_clause_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql", "macro_sql": "{% macro databricks__get_catalog_schemas_where_clause_sql(catalog, schemas) -%}\nWHERE table_catalog = '{{ catalog|lower }}' AND (\n {%- for relation in schemas -%}\n table_schema = '{{ relation[1]|lower }}'{%- if not loop.last %} OR {% endif -%}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.043921, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql": {"name": "databricks__get_catalog_relations_where_clause_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql", "macro_sql": "{% macro databricks__get_catalog_relations_where_clause_sql(catalog, relations) -%}\nWHERE table_catalog = '{{ catalog|lower }}' AND (\n {%- for relation in relations -%}\n {%- if relation.schema and relation.identifier %}\n (\n table_schema = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}'\n )\n {%- elif relation.schema %}\n (\n table_schema = '{{ relation.schema|lower }}'\n )\n {% else %}\n {% do exceptions.raise_compiler_error(\n '`get_catalog_relations` requires a list of relations, each with a schema'\n ) %}\n {% endif %}\n {%- if not loop.last %} OR {% endif -%}\n {%- endfor -%}\n)\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.044159, "supported_languages": null}, "macro.dbt_databricks.current_catalog": {"name": "current_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.current_catalog", "macro_sql": "{% macro current_catalog() -%}\n {{ return(adapter.dispatch('current_catalog', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__current_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.044322, "supported_languages": null}, "macro.dbt_databricks.databricks__current_catalog": {"name": "databricks__current_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.databricks__current_catalog", "macro_sql": "{% macro databricks__current_catalog() -%}\n {{ return(run_query_as(current_catalog_sql(), 'current_catalog')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.current_catalog_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.044396, "supported_languages": null}, "macro.dbt_databricks.current_catalog_sql": {"name": "current_catalog_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.current_catalog_sql", "macro_sql": "{% macro current_catalog_sql() %}\nSELECT current_catalog()\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.044436, "supported_languages": null}, "macro.dbt_databricks.use_catalog": {"name": "use_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.use_catalog", "macro_sql": "{% macro use_catalog(catalog) -%}\n {{ adapter.dispatch('use_catalog', 'dbt')(catalog) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__use_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.04451, "supported_languages": null}, "macro.dbt_databricks.databricks__use_catalog": {"name": "databricks__use_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.databricks__use_catalog", "macro_sql": "{% macro databricks__use_catalog(catalog) -%}\n {{ run_query_as(use_catalog_sql(catalog), 'use_catalog', fetch_result=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.use_catalog_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.044729, "supported_languages": null}, "macro.dbt_databricks.use_catalog_sql": {"name": "use_catalog_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.use_catalog_sql", "macro_sql": "{% macro use_catalog_sql(catalog) %}\nUSE CATALOG {{ adapter.quote(catalog)|lower }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0448, "supported_languages": null}, "macro.dbt_databricks.databricks__py_write_table": {"name": "databricks__py_write_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.databricks__py_write_table", "macro_sql": "{% macro databricks__py_write_table(compiled_code, target_relation) %}\n{{ compiled_code }}\n# --- Autogenerated dbt materialization code. --- #\ndbt = dbtObj(spark.table)\ndf = model(dbt, spark)\n\nimport pyspark\n\n{{ py_try_import('pyspark.sql.connect.dataframe', 'newer_pyspark_available') }}\n{{ py_try_import('pandas', 'pandas_available') }}\n{{ py_try_import('pyspark.pandas', 'pyspark_pandas_api_available') }}\n{{ py_try_import('databricks.koalas', 'koalas_available') }}\n\n# preferentially convert pandas DataFrames to pandas-on-Spark or Koalas DataFrames first\n# since they know how to convert pandas DataFrames better than `spark.createDataFrame(df)`\n# and converting from pandas-on-Spark to Spark DataFrame has no overhead\n\nif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n if pyspark_pandas_api_available:\n try:\n df = pyspark.pandas.frame.DataFrame(df)\n except Exception as e:\n # If ANSI mode causes issues, fall back to spark.createDataFrame\n # This preserves the original pandas DataFrame for later conversion\n # Check for various ANSI mode related error messages\n error_str = str(e).lower()\n if any(ansi_error in error_str for ansi_error in [\n \"pandas_api_on_spark_fail_on_ansi_mode\",\n \"ansi mode\",\n \"ansimode\",\n \"sql_mode\",\n \"strict mode\"\n ]):\n pass # Will use spark.createDataFrame below\n else:\n raise e\n elif koalas_available:\n df = databricks.koalas.frame.DataFrame(df)\n\n# convert to pyspark.sql.dataframe.DataFrame\nif isinstance(df, pyspark.sql.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif newer_pyspark_available and isinstance(df, pyspark.sql.connect.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif pyspark_pandas_api_available and isinstance(df, pyspark.pandas.frame.DataFrame):\n df = df.to_spark()\nelif koalas_available and isinstance(df, databricks.koalas.frame.DataFrame):\n df = df.to_spark()\nelif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = spark.createDataFrame(df)\nelse:\n msg = f\"{type(df)} is not a supported type for dbt Python materialization\"\n raise Exception(msg)\n\nwriter = (\n df.write\n .mode(\"overwrite\")\n .option(\"overwriteSchema\", \"true\")\n{{ py_get_writer_options()|indent(8, True) }}\n)\n\nwriter.saveAsTable(\"{{ target_relation }}\")\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.py_try_import", "macro.dbt_databricks.py_get_writer_options"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.045419, "supported_languages": null}, "macro.dbt_databricks.py_get_writer_options": {"name": "py_get_writer_options", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.py_get_writer_options", "macro_sql": "\n\n{%- macro py_get_writer_options() -%}\n{%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n{%- set file_format = adapter.resolve_file_format(config) -%}\n{%- set partition_by = config.get('partition_by', validator=validation.any[list, basestring]) -%}\n{%- set liquid_clustered_by = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}\n{%- set clustered_by = config.get('clustered_by', validator=validation.any[list, basestring]) -%}\n{%- set buckets = config.get('buckets', validator=validation.any[int]) -%}\n.format(\"{{ file_format }}\")\n{%- if location_root is not none %}\n{%- set model_path = adapter.compute_external_path(config, model, is_incremental()) %}\n.option(\"path\", \"{{ model_path }}\")\n{%- endif -%}\n{%- if partition_by is not none -%}\n {%- if partition_by is string -%}\n {%- set partition_by = [partition_by] -%}\n {%- endif %}\n.partitionBy({{ partition_by }})\n{%- endif -%}\n{%- if liquid_clustered_by and not is_incremental() -%}\n {%- if liquid_clustered_by is string -%}\n {%- set liquid_clustered_by = [liquid_clustered_by] -%}\n {%- endif %}\n.clusterBy({{ liquid_clustered_by }})\n{%- endif -%}\n{%- if (clustered_by is not none) and (buckets is not none) -%}\n {%- if clustered_by is string -%}\n {%- set clustered_by = [clustered_by] -%}\n {%- endif %}\n.bucketBy({{ buckets }}, {{ clustered_by }})\n{%- endif -%}\n{% endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.is_incremental"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.046102, "supported_languages": null}, "macro.dbt_databricks.py_try_import": {"name": "py_try_import", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.py_try_import", "macro_sql": "{% macro py_try_import(library, var_name) -%}\n# make sure {{ library }} exists before using it\ntry:\n import {{ library }}\n {{ var_name }} = True\nexcept ImportError:\n {{ var_name }} = False\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0461938, "supported_languages": null}, "macro.dbt_databricks.create_python_intermediate_table": {"name": "create_python_intermediate_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.create_python_intermediate_table", "macro_sql": "{% macro create_python_intermediate_table(relation, compiled_code) %}\n{{ compiled_code }}\n\n{%- set file_format = adapter.resolve_file_format(config) -%}\n\n# --- Autogenerated dbt materialization code. --- #\ndbt = dbtObj(spark.table)\ndf = model(dbt, spark)\n\nimport pyspark\n\n{{ py_try_import('pyspark.sql.connect.dataframe', 'newer_pyspark_available') }}\n{{ py_try_import('pandas', 'pandas_available') }}\n{{ py_try_import('pyspark.pandas', 'pyspark_pandas_api_available') }}\n{{ py_try_import('databricks.koalas', 'koalas_available') }}\n\n# preferentially convert pandas DataFrames to pandas-on-Spark or Koalas DataFrames first\n# since they know how to convert pandas DataFrames better than `spark.createDataFrame(df)`\n# and converting from pandas-on-Spark to Spark DataFrame has no overhead\n\nif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n if pyspark_pandas_api_available:\n try:\n df = pyspark.pandas.frame.DataFrame(df)\n except Exception as e:\n # If ANSI mode causes issues, fall back to spark.createDataFrame\n # This preserves the original pandas DataFrame for later conversion\n # Check for various ANSI mode related error messages\n error_str = str(e).lower()\n if any(ansi_error in error_str for ansi_error in [\n \"pandas_api_on_spark_fail_on_ansi_mode\",\n \"ansi mode\",\n \"ansimode\",\n \"sql_mode\",\n \"strict mode\"\n ]):\n pass # Will use spark.createDataFrame below\n else:\n raise e\n elif koalas_available:\n df = databricks.koalas.frame.DataFrame(df)\n\n# convert to pyspark.sql.dataframe.DataFrame\nif isinstance(df, pyspark.sql.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif newer_pyspark_available and isinstance(df, pyspark.sql.connect.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif pyspark_pandas_api_available and isinstance(df, pyspark.pandas.frame.DataFrame):\n df = df.to_spark()\nelif koalas_available and isinstance(df, databricks.koalas.frame.DataFrame):\n df = df.to_spark()\nelif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = spark.createDataFrame(df)\nelse:\n msg = f\"{type(df)} is not a supported type for dbt Python materialization\"\n raise Exception(msg)\n\nwriter = (\n df.write\n .mode(\"overwrite\")\n .option(\"overwriteSchema\", \"true\")\n .format(\"{{ file_format }}\")\n)\n\nwriter.saveAsTable(\"{{ relation.render() }}\")\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.py_try_import"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.04663, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_column_comment": {"name": "databricks__alter_column_comment", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.databricks__alter_column_comment", "macro_sql": "{% macro databricks__alter_column_comment(relation, column_dict) %}\n {% set file_format = adapter.resolve_file_format(config) %}\n {% if file_format in ['delta', 'hudi'] %}\n {% for column in column_dict.values() %}\n {% set comment = column['description'] %}\n {% set escaped_comment = comment | replace('\\'', '\\\\\\'') %}\n {% set column_path = relation.render() ~ '.' ~ adapter.quote(column['name']) %}\n {{ run_query_as(comment_on_column_sql(column_path, escaped_comment), 'alter_column_comment', fetch_result=False) }}\n {% endfor %}\n {% else %}\n {{ log('WARNING - requested to update column comments, but file format ' ~ file_format ~ ' does not support that.') }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.comment_on_column_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0472262, "supported_languages": null}, "macro.dbt_databricks.comment_on_column_sql": {"name": "comment_on_column_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.comment_on_column_sql", "macro_sql": "{% macro comment_on_column_sql(column_path, escaped_comment) %}\n {%- if adapter.has_dbr_capability('comment_on_column') -%}\n COMMENT ON COLUMN {{ column_path }} IS '{{ escaped_comment }}'\n {%- else -%}\n {{ alter_table_change_column_comment_sql(column_path, escaped_comment) }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.alter_table_change_column_comment_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.04735, "supported_languages": null}, "macro.dbt_databricks.alter_table_change_column_comment_sql": {"name": "alter_table_change_column_comment_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.alter_table_change_column_comment_sql", "macro_sql": "{% macro alter_table_change_column_comment_sql(column_path, escaped_comment) %}\n {%- set parts = column_path.split('.') -%}\n {%- if parts|length >= 4 -%}\n {%- set table_path = parts[:-1] | join('.') -%}\n {%- set column_name = parts[-1] -%}\n ALTER TABLE {{ table_path }} ALTER COLUMN {{ column_name }} COMMENT '{{ escaped_comment }}'\n {%- else -%}\n {{ exceptions.raise_compiler_error(\"Invalid column path: \" ~ column_path ~ \". Expected format: database.schema.table.column\") }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.047612, "supported_languages": null}, "macro.dbt_databricks.databricks__persist_docs": {"name": "databricks__persist_docs", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.databricks__persist_docs", "macro_sql": "{% macro databricks__persist_docs(relation, model, for_relation, for_columns) -%}\n {%- if for_relation and config.persist_relation_docs() and model.description %}\n {{ run_query_as(alter_relation_comment_sql(relation, model.description), 'alter_relation_comment', fetch_result=False) }}\n {% endif %}\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {%- set existing_columns = adapter.get_columns_in_relation(relation) -%}\n {%- set columns_to_persist_docs = adapter.get_persist_doc_columns(existing_columns, model.columns) -%}\n {{ alter_column_comment(relation, columns_to_persist_docs) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.alter_relation_comment_sql", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0480251, "supported_languages": null}, "macro.dbt_databricks.alter_relation_comment_sql": {"name": "alter_relation_comment_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.alter_relation_comment_sql", "macro_sql": "{% macro alter_relation_comment_sql(relation, description) %}\nCOMMENT ON {{ relation.type.render().upper() }} {{ relation.render() }} IS '{{ description | replace(\"'\", \"\\\\'\") }}'\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.048165, "supported_languages": null}, "macro.dbt_databricks.alter_column_comments": {"name": "alter_column_comments", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.alter_column_comments", "macro_sql": "{% macro alter_column_comments(relation, column_dict) %}\n {% for column, comment in column_dict.items() %}\n {{ log('Updating comment for column ' ~ column ~ ' with comment ' ~ comment) }}\n {% set escaped_comment = comment | replace('\\'', '\\\\\\'') %}\n {% set column_path = relation.render() ~ '.' ~ adapter.quote(column) %}\n {{ run_query_as(comment_on_column_sql(column_path, escaped_comment), 'main', fetch_result=False) }}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.comment_on_column_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.048413, "supported_languages": null}, "macro.dbt_databricks.databricks__list_relations_without_caching": {"name": "databricks__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__list_relations_without_caching", "macro_sql": "{% macro databricks__list_relations_without_caching(schema_relation) %}\n {{ return(adapter.get_relations_without_caching(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.048851, "supported_languages": null}, "macro.dbt_databricks.show_table_extended": {"name": "show_table_extended", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_table_extended", "macro_sql": "{% macro show_table_extended(schema_relation) %}\n {{ return(adapter.dispatch('show_table_extended', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_table_extended"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.048933, "supported_languages": null}, "macro.dbt_databricks.databricks__show_table_extended": {"name": "databricks__show_table_extended", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_table_extended", "macro_sql": "{% macro databricks__show_table_extended(schema_relation) %}\n {{ return(run_query_as(show_table_extended_sql(schema_relation), 'show_table_extended')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.show_table_extended_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.049013, "supported_languages": null}, "macro.dbt_databricks.show_table_extended_sql": {"name": "show_table_extended_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_table_extended_sql", "macro_sql": "{% macro show_table_extended_sql(schema_relation) %}\nSHOW TABLE EXTENDED IN {{ schema_relation.without_identifier()|lower }} LIKE '{{ schema_relation.identifier|lower }}'\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.049095, "supported_languages": null}, "macro.dbt_databricks.show_tables": {"name": "show_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_tables", "macro_sql": "{% macro show_tables(relation) %}\n {{ return(adapter.dispatch('show_tables', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_tables"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.049176, "supported_languages": null}, "macro.dbt_databricks.databricks__show_tables": {"name": "databricks__show_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_tables", "macro_sql": "{% macro databricks__show_tables(relation) %}\n {{ return(run_query_as(show_tables_sql(relation), 'show_tables')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.show_tables_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.049251, "supported_languages": null}, "macro.dbt_databricks.show_tables_sql": {"name": "show_tables_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_tables_sql", "macro_sql": "{% macro show_tables_sql(relation) %}\n\nSHOW TABLES IN {{ relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0493052, "supported_languages": null}, "macro.dbt_databricks.databricks__list_schemas": {"name": "databricks__list_schemas", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__list_schemas", "macro_sql": "{% macro databricks__list_schemas(database) -%}\n {{ return(run_query_as(list_schemas_sql(database), 'list_schemas')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.list_schemas_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.049377, "supported_languages": null}, "macro.dbt_databricks.list_schemas_sql": {"name": "list_schemas_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.list_schemas_sql", "macro_sql": "{% macro list_schemas_sql(database) %}\n{% if database %}\n SHOW SCHEMAS IN {{ adapter.quote(database) }}\n{% else %}\n SHOW SCHEMAS\n{% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.049463, "supported_languages": null}, "macro.dbt_databricks.databricks__check_schema_exists": {"name": "databricks__check_schema_exists", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__check_schema_exists", "macro_sql": "{% macro databricks__check_schema_exists(database, schema) %}\n {{ return(run_query_as(check_schema_exists_sql(database, schema), 'check_schema_exists')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.check_schema_exists_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0495481, "supported_languages": null}, "macro.dbt_databricks.check_schema_exists_sql": {"name": "check_schema_exists_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.check_schema_exists_sql", "macro_sql": "{% macro check_schema_exists_sql(database, schema) %}\n SHOW SCHEMAS IN {{ adapter.quote(database) }} LIKE '{{ schema }}'\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.049621, "supported_languages": null}, "macro.dbt_databricks.show_views": {"name": "show_views", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_views", "macro_sql": "{% macro show_views(relation) %}\n {{ return(adapter.dispatch('show_views', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_views"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.04978, "supported_languages": null}, "macro.dbt_databricks.databricks__show_views": {"name": "databricks__show_views", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_views", "macro_sql": "{% macro databricks__show_views(relation) %}\n {{ return(run_query_as(show_views_sql(relation), 'show_views')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.show_views_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.049862, "supported_languages": null}, "macro.dbt_databricks.show_views_sql": {"name": "show_views_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_views_sql", "macro_sql": "{% macro show_views_sql(relation) %}\nSHOW VIEWS IN {{ relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0499241, "supported_languages": null}, "macro.dbt_databricks.databricks__get_relation_last_modified": {"name": "databricks__get_relation_last_modified", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__get_relation_last_modified", "macro_sql": "{% macro databricks__get_relation_last_modified(information_schema, relations) -%}\n {% call statement('last_modified', fetch_result=True) %}\n {{ get_relation_last_modified_sql(information_schema, relations) }}\n {% endcall %}\n {{ return(load_result('last_modified')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.get_relation_last_modified_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.05005, "supported_languages": null}, "macro.dbt_databricks.get_relation_last_modified_sql": {"name": "get_relation_last_modified_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_relation_last_modified_sql", "macro_sql": "{% macro get_relation_last_modified_sql(information_schema, relations) %}\n {% if information_schema.is_hive_metastore() %}\n {%- for relation in relations -%}\nSELECT\n '{{ relation.schema }}' AS schema,\n '{{ relation.identifier }}' AS identifier,\n max(timestamp) AS last_modified,\n {{ current_timestamp() }} AS snapshotted_at\n FROM (DESCRIBE HISTORY {{ relation.render() }})\n {% if not loop.last %}\nUNION ALL\n {% endif %}\n {%- endfor -%}\n {% else %}\nSELECT\n table_schema AS schema,\n table_name AS identifier,\n last_altered AS last_modified,\n {{ current_timestamp() }} AS snapshotted_at\nFROM `system`.`information_schema`.`tables`\nWHERE table_catalog = '{{ information_schema.database|lower }}'\n AND (\n {%- for relation in relations -%}\n (table_schema = '{{ relation.schema|lower }}' AND\n table_name = '{{ relation.identifier|lower }}'){%- if not loop.last %} OR {% endif -%}\n {%- endfor -%}\n )\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0503528, "supported_languages": null}, "macro.dbt_databricks.get_view_description": {"name": "get_view_description", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_view_description", "macro_sql": "{% macro get_view_description(relation) %}\n {{ return(run_query_as(get_view_description_sql(relation), 'get_view_description')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.get_view_description_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.050446, "supported_languages": null}, "macro.dbt_databricks.get_view_description_sql": {"name": "get_view_description_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_view_description_sql", "macro_sql": "{% macro get_view_description_sql(relation) %}\nSELECT *\nFROM `system`.`information_schema`.`views`\nWHERE table_catalog = '{{ relation.database|lower }}'\n AND table_schema = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}'\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0505338, "supported_languages": null}, "macro.dbt_databricks.get_uc_tables": {"name": "get_uc_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_uc_tables", "macro_sql": "{% macro get_uc_tables(relation) %}\n {{ return(run_query_as(get_uc_tables_sql(relation), 'get_uc_tables')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.get_uc_tables_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0506089, "supported_languages": null}, "macro.dbt_databricks.get_uc_tables_sql": {"name": "get_uc_tables_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_uc_tables_sql", "macro_sql": "{% macro get_uc_tables_sql(relation) %}\nSELECT\n table_name,\n if(table_type IN ('EXTERNAL', 'MANAGED', 'MANAGED_SHALLOW_CLONE', 'EXTERNAL_SHALLOW_CLONE'), 'table', lower(table_type)) AS table_type,\n lower(data_source_format) AS file_format,\n table_owner,\n if(\n table_type IN (\n 'EXTERNAL',\n 'MANAGED',\n 'MANAGED_SHALLOW_CLONE',\n 'EXTERNAL_SHALLOW_CLONE'\n ),\n lower(table_type),\n NULL\n ) AS databricks_table_type\nFROM `system`.`information_schema`.`tables`\nWHERE table_catalog = '{{ relation.database|lower }}'\n AND table_schema = '{{ relation.schema|lower }}'\n {%- if relation.identifier %}\n AND table_name = '{{ relation.identifier|lower }}'\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.050743, "supported_languages": null}, "macro.dbt_databricks.describe_table_extended_as_json": {"name": "describe_table_extended_as_json", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.describe_table_extended_as_json", "macro_sql": "{% macro describe_table_extended_as_json(relation) %}\n {{ return(run_query_as(describe_table_extended_as_json_sql(relation), 'describe_table_extended_as_json')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.describe_table_extended_as_json_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.050828, "supported_languages": null}, "macro.dbt_databricks.describe_table_extended_as_json_sql": {"name": "describe_table_extended_as_json_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.describe_table_extended_as_json_sql", "macro_sql": "{% macro describe_table_extended_as_json_sql(relation) %}\nDESCRIBE TABLE EXTENDED {{ relation.render() }} AS JSON\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.050889, "supported_languages": null}, "macro.dbt_databricks.get_columns_comments": {"name": "get_columns_comments", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt_databricks.get_columns_comments", "macro_sql": "{% macro get_columns_comments(relation) -%}\n {{ return(run_query_as(get_columns_comments_sql(relation), 'get_columns_comments')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.get_columns_comments_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.051106, "supported_languages": null}, "macro.dbt_databricks.get_columns_comments_sql": {"name": "get_columns_comments_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt_databricks.get_columns_comments_sql", "macro_sql": "{% macro get_columns_comments_sql(relation) %}\nDESCRIBE TABLE {{ relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.051162, "supported_languages": null}, "macro.dbt_databricks.get_columns_comments_as_json": {"name": "get_columns_comments_as_json", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt_databricks.get_columns_comments_as_json", "macro_sql": "{% macro get_columns_comments_as_json(relation) -%}\n {{ return(run_query_as(get_columns_comments_as_json_sql(relation), 'get_columns_comments_as_json')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.get_columns_comments_as_json_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0512462, "supported_languages": null}, "macro.dbt_databricks.get_columns_comments_as_json_sql": {"name": "get_columns_comments_as_json_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt_databricks.get_columns_comments_as_json_sql", "macro_sql": "{% macro get_columns_comments_as_json_sql(relation) %}\n DESCRIBE TABLE EXTENDED {{ relation.render() }} AS JSON\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.051308, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_relation_add_remove_columns": {"name": "databricks__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt_databricks.databricks__alter_relation_add_remove_columns", "macro_sql": "{% macro databricks__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n {% if remove_columns %}\n {{ run_query_as(drop_columns_sql(relation, remove_columns), 'alter_relation_remove_columns', fetch_result=False) }}\n {% endif %}\n\n {% if add_columns %}\n {{ run_query_as(add_columns_sql(relation, add_columns), 'alter_relation_add_columns', fetch_result=False) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.run_query_as", "macro.dbt_databricks.drop_columns_sql", "macro.dbt_databricks.add_columns_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0514932, "supported_languages": null}, "macro.dbt_databricks.drop_columns_sql": {"name": "drop_columns_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt_databricks.drop_columns_sql", "macro_sql": "{% macro drop_columns_sql(relation, remove_columns) %}\nALTER TABLE {{ relation.render() }} DROP COLUMNS ({{ api.Column.format_remove_column_list(remove_columns) }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.051583, "supported_languages": null}, "macro.dbt_databricks.add_columns_sql": {"name": "add_columns_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt_databricks.add_columns_sql", "macro_sql": "{% macro add_columns_sql(relation, add_columns) %}\nALTER TABLE {{ relation.render() }} ADD COLUMNS ({{ api.Column.format_add_column_list(add_columns) }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0516748, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_column_type": {"name": "databricks__alter_column_type", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt_databricks.databricks__alter_column_type", "macro_sql": "{% macro databricks__alter_column_type(relation, column_name, new_column_type) -%}\n {% call statement('alter_column_type') %}\n alter table {{ relation }} alter column {{ adapter.quote(column_name) }} type {{ new_column_type }};\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.051799, "supported_languages": null}, "macro.dbt_spark.tblproperties_clause": {"name": "tblproperties_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.tblproperties_clause", "macro_sql": "{% macro tblproperties_clause() %}\n {{ return(adapter.dispatch('tblproperties_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.053226, "supported_languages": null}, "macro.dbt_spark.spark__tblproperties_clause": {"name": "spark__tblproperties_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__tblproperties_clause", "macro_sql": "{% macro spark__tblproperties_clause() -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- if tblproperties is not none %}\n tblproperties (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}' {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.053417, "supported_languages": null}, "macro.dbt_spark.file_format_clause": {"name": "file_format_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.file_format_clause", "macro_sql": "{% macro file_format_clause() %}\n {{ return(adapter.dispatch('file_format_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__file_format_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.05351, "supported_languages": null}, "macro.dbt_spark.spark__file_format_clause": {"name": "spark__file_format_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__file_format_clause", "macro_sql": "{% macro spark__file_format_clause() %}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n {%- if file_format is not none %}\n using {{ file_format }}\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0536478, "supported_languages": null}, "macro.dbt_spark.location_clause": {"name": "location_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.location_clause", "macro_sql": "{% macro location_clause() %}\n {{ return(adapter.dispatch('location_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__location_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.053788, "supported_languages": null}, "macro.dbt_spark.spark__location_clause": {"name": "spark__location_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__location_clause", "macro_sql": "{% macro spark__location_clause() %}\n {%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n {%- set identifier = model['alias'] -%}\n {%- if location_root is not none %}\n location '{{ location_root }}/{{ identifier }}'\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.053953, "supported_languages": null}, "macro.dbt_spark.options_clause": {"name": "options_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.options_clause", "macro_sql": "{% macro options_clause() -%}\n {{ return(adapter.dispatch('options_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__options_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.054026, "supported_languages": null}, "macro.dbt_spark.spark__options_clause": {"name": "spark__options_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__options_clause", "macro_sql": "{% macro spark__options_clause() -%}\n {%- set options = config.get('options') -%}\n {%- if config.get('file_format') == 'hudi' -%}\n {%- set unique_key = config.get('unique_key') -%}\n {%- if unique_key is not none and options is none -%}\n {%- set options = {'primaryKey': config.get('unique_key')} -%}\n {%- elif unique_key is not none and options is not none and 'primaryKey' not in options -%}\n {%- set _ = options.update({'primaryKey': config.get('unique_key')}) -%}\n {%- elif options is not none and 'primaryKey' in options and options['primaryKey'] != unique_key -%}\n {{ exceptions.raise_compiler_error(\"unique_key and options('primaryKey') should be the same column(s).\") }}\n {%- endif %}\n {%- endif %}\n\n {%- if options is not none %}\n options (\n {%- for option in options -%}\n {{ option }} \"{{ options[option] }}\" {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.054511, "supported_languages": null}, "macro.dbt_spark.comment_clause": {"name": "comment_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.comment_clause", "macro_sql": "{% macro comment_clause() %}\n {{ return(adapter.dispatch('comment_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__comment_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.054594, "supported_languages": null}, "macro.dbt_spark.spark__comment_clause": {"name": "spark__comment_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__comment_clause", "macro_sql": "{% macro spark__comment_clause() %}\n {%- set raw_persist_docs = config.get('persist_docs', {}) -%}\n\n {%- if raw_persist_docs is mapping -%}\n {%- set raw_relation = raw_persist_docs.get('relation', false) -%}\n {%- if raw_relation -%}\n comment '{{ model.description | replace(\"'\", \"\\\\'\") }}'\n {% endif %}\n {%- elif raw_persist_docs -%}\n {{ exceptions.raise_compiler_error(\"Invalid value provided for 'persist_docs'. Expected dict but got value: \" ~ raw_persist_docs) }}\n {% endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.054822, "supported_languages": null}, "macro.dbt_spark.partition_cols": {"name": "partition_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.partition_cols", "macro_sql": "{% macro partition_cols(label, required=false) %}\n {{ return(adapter.dispatch('partition_cols', 'dbt')(label, required)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0549371, "supported_languages": null}, "macro.dbt_spark.spark__partition_cols": {"name": "spark__partition_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__partition_cols", "macro_sql": "{% macro spark__partition_cols(label, required=false) %}\n {%- set cols = config.get('partition_by', validator=validation.any[list, basestring]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {{ label }} (\n {%- for item in cols -%}\n {{ item }}\n {%- if not loop.last -%},{%- endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.055181, "supported_languages": null}, "macro.dbt_spark.clustered_cols": {"name": "clustered_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.clustered_cols", "macro_sql": "{% macro clustered_cols(label, required=false) %}\n {{ return(adapter.dispatch('clustered_cols', 'dbt')(label, required)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__clustered_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0552902, "supported_languages": null}, "macro.dbt_spark.spark__clustered_cols": {"name": "spark__clustered_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__clustered_cols", "macro_sql": "{% macro spark__clustered_cols(label, required=false) %}\n {%- set cols = config.get('clustered_by', validator=validation.any[list, basestring]) -%}\n {%- set buckets = config.get('buckets', validator=validation.any[int]) -%}\n {%- if (cols is not none) and (buckets is not none) %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {{ label }} (\n {%- for item in cols -%}\n {{ item }}\n {%- if not loop.last -%},{%- endif -%}\n {%- endfor -%}\n ) into {{ buckets }} buckets\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.055617, "supported_languages": null}, "macro.dbt_spark.fetch_tbl_properties": {"name": "fetch_tbl_properties", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.fetch_tbl_properties", "macro_sql": "{% macro fetch_tbl_properties(relation) -%}\n {% call statement('list_properties', fetch_result=True) -%}\n SHOW TBLPROPERTIES {{ relation }}\n {% endcall %}\n {% do return(load_result('list_properties').table) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.055747, "supported_languages": null}, "macro.dbt_spark.create_temporary_view": {"name": "create_temporary_view", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.create_temporary_view", "macro_sql": "{% macro create_temporary_view(relation, compiled_code) -%}\n {{ return(adapter.dispatch('create_temporary_view', 'dbt')(relation, compiled_code)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__create_temporary_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.055846, "supported_languages": null}, "macro.dbt_spark.spark__create_temporary_view": {"name": "spark__create_temporary_view", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_temporary_view", "macro_sql": "{% macro spark__create_temporary_view(relation, compiled_code) -%}\n create or replace temporary view {{ relation }} as\n {{ compiled_code }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.055918, "supported_languages": null}, "macro.dbt_spark.spark__create_table_as": {"name": "spark__create_table_as", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_table_as", "macro_sql": "{%- macro spark__create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {%- if language == 'sql' -%}\n {%- if temporary -%}\n {{ create_temporary_view(relation, compiled_code) }}\n {%- else -%}\n {% if config.get('file_format', validator=validation.any[basestring]) in ['delta', 'iceberg'] %}\n create or replace table {{ relation }}\n {% else %}\n create table {{ relation }}\n {% endif %}\n {%- set contract_config = config.get('contract') -%}\n {%- if contract_config.enforced -%}\n {{ get_assert_columns_equivalent(compiled_code) }}\n {%- set compiled_code = get_select_subquery(compiled_code) %}\n {% endif %}\n {{ file_format_clause() }}\n {{ options_clause() }}\n {{ tblproperties_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause() }}\n {{ comment_clause() }}\n\n as\n {{ compiled_code }}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {#--\n N.B. Python models _can_ write to temp views HOWEVER they use a different session\n and have already expired by the time they need to be used (I.E. in merges for incremental models)\n\n TODO: Deep dive into spark sessions to see if we can reuse a single session for an entire\n dbt invocation.\n --#}\n {{ py_write_table(compiled_code=compiled_code, target_relation=relation) }}\n {%- endif -%}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_spark.create_temporary_view", "macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_select_subquery", "macro.dbt_spark.file_format_clause", "macro.dbt_spark.options_clause", "macro.dbt_spark.tblproperties_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_spark.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_spark.py_write_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.056461, "supported_languages": null}, "macro.dbt_spark.persist_constraints": {"name": "persist_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.persist_constraints", "macro_sql": "{% macro persist_constraints(relation, model) %}\n {{ return(adapter.dispatch('persist_constraints', 'dbt')(relation, model)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.056577, "supported_languages": null}, "macro.dbt_spark.spark__persist_constraints": {"name": "spark__persist_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__persist_constraints", "macro_sql": "{% macro spark__persist_constraints(relation, model) %}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config.enforced and config.get('file_format', 'delta') == 'delta' %}\n {% do alter_table_add_constraints(relation, model.constraints) %}\n {% do alter_column_set_constraints(relation, model.columns) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.alter_table_add_constraints", "macro.dbt_spark.alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.056758, "supported_languages": null}, "macro.dbt_spark.alter_table_add_constraints": {"name": "alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.alter_table_add_constraints", "macro_sql": "{% macro alter_table_add_constraints(relation, constraints) %}\n {{ return(adapter.dispatch('alter_table_add_constraints', 'dbt')(relation, constraints)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0568519, "supported_languages": null}, "macro.dbt_spark.spark__alter_table_add_constraints": {"name": "spark__alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_table_add_constraints", "macro_sql": "{% macro spark__alter_table_add_constraints(relation, constraints) %}\n {% for constraint in constraints %}\n {% if constraint.type == 'check' and not is_incremental() %}\n {%- set constraint_hash = local_md5(column_name ~ \";\" ~ constraint.expression ~ \";\" ~ loop.index) -%}\n {% call statement() %}\n alter table {{ relation }} add constraint {{ constraint.name if constraint.name else constraint_hash }} check ({{ constraint.expression }});\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.is_incremental", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0570781, "supported_languages": null}, "macro.dbt_spark.alter_column_set_constraints": {"name": "alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.alter_column_set_constraints", "macro_sql": "{% macro alter_column_set_constraints(relation, column_dict) %}\n {{ return(adapter.dispatch('alter_column_set_constraints', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.057175, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_set_constraints": {"name": "spark__alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_set_constraints", "macro_sql": "{% macro spark__alter_column_set_constraints(relation, column_dict) %}\n {% for column_name in column_dict %}\n {% set constraints = column_dict[column_name]['constraints'] %}\n {% for constraint in constraints %}\n {% if constraint.type != 'not_null' %}\n {{ exceptions.warn('Invalid constraint for column ' ~ column_name ~ '. Only `not_null` is supported.') }}\n {% else %}\n {% set quoted_name = adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name %}\n {% call statement() %}\n alter table {{ relation }} change column {{ quoted_name }} set not null {{ constraint.expression or \"\" }};\n {% endcall %}\n {% endif %}\n {% endfor %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0574908, "supported_languages": null}, "macro.dbt_spark.get_column_comment_sql": {"name": "get_column_comment_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_column_comment_sql", "macro_sql": "{% macro get_column_comment_sql(column_name, column_dict) -%}\n {% if column_name in column_dict and column_dict[column_name][\"description\"] -%}\n {% set escaped_description = column_dict[column_name][\"description\"] | replace(\"'\", \"\\\\'\") %}\n {% set column_comment_clause = \"comment '\" ~ escaped_description ~ \"'\" %}\n {%- endif -%}\n {{ adapter.quote(column_name) }} {{ column_comment_clause }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.057767, "supported_languages": null}, "macro.dbt_spark.get_persist_docs_column_list": {"name": "get_persist_docs_column_list", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_persist_docs_column_list", "macro_sql": "{% macro get_persist_docs_column_list(model_columns, query_columns) %}\n {% for column_name in query_columns %}\n {{ get_column_comment_sql(column_name, model_columns) }}\n {{- \", \" if not loop.last else \"\" }}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.get_column_comment_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.057884, "supported_languages": null}, "macro.dbt_spark.spark__create_view_as": {"name": "spark__create_view_as", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_view_as", "macro_sql": "{% macro spark__create_view_as(relation, sql) -%}\n create or replace view {{ relation }}\n {% if config.persist_column_docs() -%}\n {% set model_columns = model.columns %}\n {% set query_columns = get_columns_in_query(sql) %}\n (\n {{ get_persist_docs_column_list(model_columns, query_columns) }}\n )\n {% endif %}\n {{ comment_clause() }}\n {%- set contract_config = config.get('contract') -%}\n {%- if contract_config.enforced -%}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt_spark.get_persist_docs_column_list", "macro.dbt_spark.comment_clause", "macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.05811, "supported_languages": null}, "macro.dbt_spark.spark__create_schema": {"name": "spark__create_schema", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_schema", "macro_sql": "{% macro spark__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{relation}}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.058201, "supported_languages": null}, "macro.dbt_spark.spark__drop_schema": {"name": "spark__drop_schema", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__drop_schema", "macro_sql": "{% macro spark__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation }} cascade\n {%- endcall -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.058292, "supported_languages": null}, "macro.dbt_spark.get_columns_in_relation_raw": {"name": "get_columns_in_relation_raw", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_columns_in_relation_raw", "macro_sql": "{% macro get_columns_in_relation_raw(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation_raw', 'dbt')(relation)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__get_columns_in_relation_raw"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.058371, "supported_languages": null}, "macro.dbt_spark.spark__get_columns_in_relation_raw": {"name": "spark__get_columns_in_relation_raw", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__get_columns_in_relation_raw", "macro_sql": "{% macro spark__get_columns_in_relation_raw(relation) -%}\n {% call statement('get_columns_in_relation_raw', fetch_result=True) %}\n describe extended {{ relation }}\n {% endcall %}\n {% do return(load_result('get_columns_in_relation_raw').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.058483, "supported_languages": null}, "macro.dbt_spark.spark__get_columns_in_relation": {"name": "spark__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__get_columns_in_relation", "macro_sql": "{% macro spark__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n describe extended {{ relation.include(schema=(schema is not none)) }}\n {% endcall %}\n {% do return(load_result('get_columns_in_relation').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.058639, "supported_languages": null}, "macro.dbt_spark.spark__list_relations_without_caching": {"name": "spark__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__list_relations_without_caching", "macro_sql": "{% macro spark__list_relations_without_caching(relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n show table extended in {{ relation.schema }} like '*'\n {% endcall %}\n\n {% do return(load_result('list_relations_without_caching').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.058775, "supported_languages": null}, "macro.dbt_spark.list_relations_show_tables_without_caching": {"name": "list_relations_show_tables_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.list_relations_show_tables_without_caching", "macro_sql": "{% macro list_relations_show_tables_without_caching(schema_relation) %}\n {#-- Spark with iceberg tables don't work with show table extended for #}\n {#-- V2 iceberg tables #}\n {#-- https://issues.apache.org/jira/browse/SPARK-33393 #}\n {% call statement('list_relations_without_caching_show_tables', fetch_result=True) -%}\n show tables in {{ schema_relation.schema }} like '*'\n {% endcall %}\n\n {% do return(load_result('list_relations_without_caching_show_tables').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.058906, "supported_languages": null}, "macro.dbt_spark.describe_table_extended_without_caching": {"name": "describe_table_extended_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.describe_table_extended_without_caching", "macro_sql": "{% macro describe_table_extended_without_caching(table_name) %}\n {#-- Spark with iceberg tables don't work with show table extended for #}\n {#-- V2 iceberg tables #}\n {#-- https://issues.apache.org/jira/browse/SPARK-33393 #}\n {% call statement('describe_table_extended_without_caching', fetch_result=True) -%}\n describe extended {{ table_name }}\n {% endcall %}\n {% do return(load_result('describe_table_extended_without_caching').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.059037, "supported_languages": null}, "macro.dbt_spark.spark__list_schemas": {"name": "spark__list_schemas", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__list_schemas", "macro_sql": "{% macro spark__list_schemas(database) -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n show databases\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0591629, "supported_languages": null}, "macro.dbt_spark.spark__rename_relation": {"name": "spark__rename_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__rename_relation", "macro_sql": "{% macro spark__rename_relation(from_relation, to_relation) -%}\n {% call statement('rename_relation') -%}\n {% if not from_relation.type %}\n {% do exceptions.raise_database_error(\"Cannot rename a relation with a blank type: \" ~ from_relation.identifier) %}\n {% elif from_relation.type in ('table') %}\n alter table {{ from_relation }} rename to {{ to_relation }}\n {% elif from_relation.type == 'view' %}\n alter view {{ from_relation }} rename to {{ to_relation }}\n {% else %}\n {% do exceptions.raise_database_error(\"Unknown type '\" ~ from_relation.type ~ \"' for relation: \" ~ from_relation.identifier) %}\n {% endif %}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0594292, "supported_languages": null}, "macro.dbt_spark.spark__drop_relation": {"name": "spark__drop_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__drop_relation", "macro_sql": "{% macro spark__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.059542, "supported_languages": null}, "macro.dbt_spark.spark__generate_database_name": {"name": "spark__generate_database_name", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__generate_database_name", "macro_sql": "{% macro spark__generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(None) %}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.059618, "supported_languages": null}, "macro.dbt_spark.spark__persist_docs": {"name": "spark__persist_docs", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__persist_docs", "macro_sql": "{% macro spark__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% set filtered_columns = validate_doc_columns(relation, model.columns, existing_columns) %}\n {% do alter_column_comment(relation, filtered_columns) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.validate_doc_columns", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.059833, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_comment": {"name": "spark__alter_column_comment", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_comment", "macro_sql": "{% macro spark__alter_column_comment(relation, column_dict) %}\n {% if config.get('file_format', validator=validation.any[basestring]) in ['delta', 'hudi', 'iceberg'] %}\n {% for column_name in column_dict %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = comment | replace('\\'', '\\\\\\'') %}\n {% set comment_query %}\n {% if relation.is_iceberg %}\n alter table {{ relation }} alter column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% else %}\n alter table {{ relation }} change column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% endif %}\n {% endset %}\n {% do run_query(comment_query) %}\n {% endfor %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.060274, "supported_languages": null}, "macro.dbt_spark.spark__make_temp_relation": {"name": "spark__make_temp_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__make_temp_relation", "macro_sql": "{% macro spark__make_temp_relation(base_relation, suffix) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {% set tmp_relation = base_relation.incorporate(path = {\n \"identifier\": tmp_identifier\n }) -%}\n\n {%- set tmp_relation = tmp_relation.include(database=false, schema=false) -%}\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.060457, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_type": {"name": "spark__alter_column_type", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_type", "macro_sql": "{% macro spark__alter_column_type(relation, column_name, new_column_type) -%}\n {% call statement('alter_column_type') %}\n alter table {{ relation }} alter column {{ column_name }} type {{ new_column_type }};\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.060568, "supported_languages": null}, "macro.dbt_spark.spark__alter_relation_add_remove_columns": {"name": "spark__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_relation_add_remove_columns", "macro_sql": "{% macro spark__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if remove_columns %}\n {% if relation.is_delta %}\n {% set platform_name = 'Delta Lake' %}\n {% elif relation.is_iceberg %}\n {% set platform_name = 'Iceberg' %}\n {% else %}\n {% set platform_name = 'Apache Spark' %}\n {% endif %}\n {{ exceptions.raise_compiler_error(platform_name + ' does not support dropping columns from tables') }}\n {% endif %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% if add_columns %} add columns {% endif %}\n {% for column in add_columns %}\n {{ column.quoted }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0609531, "supported_languages": null}, "macro.dbt_spark.spark__copy_grants": {"name": "spark__copy_grants", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__copy_grants", "macro_sql": "{% macro spark__copy_grants() %}\n\n {% if config.materialized == 'view' %}\n {#-- Spark views don't copy grants when they're replaced --#}\n {{ return(False) }}\n\n {% else %}\n {#-- This depends on how we're replacing the table, which depends on its file format\n -- Just play it safe by assuming that grants have been copied over, and need to be checked / possibly revoked\n -- We can make this more efficient in the future\n #}\n {{ return(True) }}\n\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0612109, "supported_languages": null}, "macro.dbt_spark.spark__get_grant_sql": {"name": "spark__get_grant_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__get_grant_sql", "macro_sql": "\n\n\n{%- macro spark__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ adapter.quote(grantees[0]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.061316, "supported_languages": null}, "macro.dbt_spark.spark__get_revoke_sql": {"name": "spark__get_revoke_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__get_revoke_sql", "macro_sql": "\n\n\n{%- macro spark__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ adapter.quote(grantees[0]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.061407, "supported_languages": null}, "macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement": {"name": "spark__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n\n{%- macro spark__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(False) }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.061465, "supported_languages": null}, "macro.dbt_spark.spark__call_dcl_statements": {"name": "spark__call_dcl_statements", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__call_dcl_statements", "macro_sql": "{% macro spark__call_dcl_statements(dcl_statement_list) %}\n {% for dcl_statement in dcl_statement_list %}\n {% call statement('grant_or_revoke') %}\n {{ dcl_statement }}\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0615768, "supported_languages": null}, "macro.dbt_spark.spark__get_binding_char": {"name": "spark__get_binding_char", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__get_binding_char", "macro_sql": "{% macro spark__get_binding_char() %}\n {{ return('?' if target.method == 'odbc' else '%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.062026, "supported_languages": null}, "macro.dbt_spark.spark__reset_csv_table": {"name": "spark__reset_csv_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__reset_csv_table", "macro_sql": "{% macro spark__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% if old_relation %}\n {{ adapter.drop_relation(old_relation) }}\n {% endif %}\n {% set sql = create_csv_table(model, agate_table) %}\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.06225, "supported_languages": null}, "macro.dbt_spark.spark__load_csv_rows": {"name": "spark__load_csv_rows", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__load_csv_rows", "macro_sql": "{% macro spark__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n {% set column_override = model['config'].get('column_types', {}) %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} values\n {% for row in chunk -%}\n ({%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n cast({{ get_binding_char() }} as {{type}})\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.062837, "supported_languages": null}, "macro.dbt_spark.spark__create_csv_table": {"name": "spark__create_csv_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__create_csv_table", "macro_sql": "{% macro spark__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {{ file_format_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause() }}\n {{ comment_clause() }}\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.file_format_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_spark.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.063313, "supported_languages": null}, "macro.dbt_spark.materialization_view_spark": {"name": "materialization_view_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "unique_id": "macro.dbt_spark.materialization_view_spark", "macro_sql": "{% materialization view, adapter='spark' -%}\n {{ return(create_or_replace_view()) }}\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.create_or_replace_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0634181, "supported_languages": ["sql"]}, "macro.dbt_spark.materialization_table_spark": {"name": "materialization_table_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.materialization_table_spark", "macro_sql": "{% materialization table, adapter = 'spark', supported_languages=['sql', 'python'] %}\n {%- set language = model['language'] -%}\n {%- set identifier = model['alias'] -%}\n {%- set grant_config = config.get('grants') -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n\n {{ run_hooks(pre_hooks) }}\n\n -- setup: if the target relation already exists, drop it\n -- in case if the existing and future table is delta or iceberg, we want to do a\n -- create or replace table instead of dropping, so we don't have the table unavailable\n {% if old_relation is not none %}\n {% set is_delta = (old_relation.is_delta and config.get('file_format', validator=validation.any[basestring]) == 'delta') %}\n {% set is_iceberg = (old_relation.is_iceberg and config.get('file_format', validator=validation.any[basestring]) == 'iceberg') %}\n {% set old_relation_type = old_relation.type %}\n {% else %}\n {% set is_delta = false %}\n {% set is_iceberg = false %}\n {% set old_relation_type = target_relation.type %}\n {% endif %}\n\n {% if not is_delta and not is_iceberg %}\n {% set existing_relation = target_relation %}\n {{ adapter.drop_relation(existing_relation.incorporate(type=old_relation_type)) }}\n {% endif %}\n\n -- build model\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]})}}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt_spark.persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.064458, "supported_languages": ["sql", "python"]}, "macro.dbt_spark.py_write_table": {"name": "py_write_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.py_write_table", "macro_sql": "{% macro py_write_table(compiled_code, target_relation) %}\n{{ compiled_code }}\n# --- Autogenerated dbt materialization code. --- #\ndbt = dbtObj(spark.table)\ndf = model(dbt, spark)\n\n# make sure pyspark exists in the namepace, for 7.3.x-scala2.12 it does not exist\nimport pyspark\n# make sure pandas exists before using it\ntry:\n import pandas\n pandas_available = True\nexcept ImportError:\n pandas_available = False\n\n# make sure pyspark.pandas exists before using it\ntry:\n import pyspark.pandas\n pyspark_pandas_api_available = True\nexcept ImportError:\n pyspark_pandas_api_available = False\n\n# make sure databricks.koalas exists before using it\ntry:\n import databricks.koalas\n koalas_available = True\nexcept ImportError:\n koalas_available = False\n\n# preferentially convert pandas DataFrames to pandas-on-Spark or Koalas DataFrames first\n# since they know how to convert pandas DataFrames better than `spark.createDataFrame(df)`\n# and converting from pandas-on-Spark to Spark DataFrame has no overhead\nif pyspark_pandas_api_available and pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = pyspark.pandas.frame.DataFrame(df)\nelif koalas_available and pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = databricks.koalas.frame.DataFrame(df)\n\n# convert to pyspark.sql.dataframe.DataFrame\nif isinstance(df, pyspark.sql.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif pyspark_pandas_api_available and isinstance(df, pyspark.pandas.frame.DataFrame):\n df = df.to_spark()\nelif koalas_available and isinstance(df, databricks.koalas.frame.DataFrame):\n df = df.to_spark()\nelif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = spark.createDataFrame(df)\nelse:\n msg = f\"{type(df)} is not a supported type for dbt Python materialization\"\n raise Exception(msg)\n\ndf.write.mode(\"overwrite\").format(\"{{ config.get('file_format', 'delta') }}\").option(\"overwriteSchema\", \"true\").saveAsTable(\"{{ target_relation }}\")\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0646248, "supported_languages": null}, "macro.dbt_spark.py_script_comment": {"name": "py_script_comment", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n# how to execute python model in notebook\n# dbt = dbtObj(spark.table)\n# df = model(dbt, spark)\n{%endmacro%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.064667, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_hash_arguments": {"name": "spark__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_hash_arguments", "macro_sql": "{% macro spark__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as string ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.065334, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_string_as_time": {"name": "spark__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_string_as_time", "macro_sql": "{% macro spark__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"to_timestamp('\" ~ timestamp ~ \"')\" -%}\n {{ return(result) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.065428, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_merge_sql": {"name": "spark__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_merge_sql", "macro_sql": "{% macro spark__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n {% if target.is_iceberg %}\n {# create view only supports a name (no catalog, or schema) #}\n using {{ source.identifier }} as DBT_INTERNAL_SOURCE\n {% else %}\n using {{ source }} as DBT_INTERNAL_SOURCE\n {% endif %}\n on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}\n when matched\n {% if config.get(\"dbt_valid_to_current\") %}\n and ( DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or\n DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null )\n {% else %}\n and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null\n {% endif %}\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert *\n ;\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.065749, "supported_languages": null}, "macro.dbt_spark.spark_build_snapshot_staging_table": {"name": "spark_build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark_build_snapshot_staging_table", "macro_sql": "{% macro spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_identifier = target_relation.identifier ~ '__dbt_tmp' %}\n\n {% if target_relation.is_iceberg %}\n {# iceberg catalog does not support create view, but regular spark does. We removed the catalog and schema #}\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=none,\n database=none,\n type='view') -%}\n {% else %}\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=target_relation.schema,\n database=none,\n type='view') -%}\n {% endif %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {# needs to be a non-temp view so that its columns can be ascertained via `describe` #}\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_view_as(tmp_relation, select) }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.066113, "supported_languages": null}, "macro.dbt_spark.spark__post_snapshot": {"name": "spark__post_snapshot", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__post_snapshot", "macro_sql": "{% macro spark__post_snapshot(staging_relation) %}\n {% do adapter.drop_relation(staging_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.066187, "supported_languages": null}, "macro.dbt_spark.spark__create_columns": {"name": "spark__create_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__create_columns", "macro_sql": "{% macro spark__create_columns(relation, columns) %}\n {% if columns|length > 0 %}\n {% call statement() %}\n alter table {{ relation }} add columns (\n {% for column in columns %}\n `{{ column.name }}` {{ column.data_type }} {{- ',' if not loop.last -}}\n {% endfor %}\n );\n {% endcall %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0664508, "supported_languages": null}, "macro.dbt_spark.materialization_snapshot_spark": {"name": "materialization_snapshot_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.materialization_snapshot_spark", "macro_sql": "{% materialization snapshot, adapter='spark' %}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n {%- set file_format = config.get('file_format') or 'parquet' -%}\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=none,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if file_format not in ['delta', 'iceberg', 'hudi'] -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n Snapshot functionality requires file_format be set to 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n\n {%- if target_relation_exists -%}\n {%- if not target_relation.is_delta and not target_relation.is_iceberg and not target_relation.is_hudi -%}\n {% set invalid_format_msg -%}\n The existing table {{ model.schema }}.{{ target_table }} is in another format than 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n {% endif %}\n\n {% if not adapter.check_schema_exists(model.database, model.schema) %}\n {% do create_schema(model.schema) %}\n {% endif %}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", model['config'], target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {{ adapter.valid_snapshot_target(target_relation, columns) }}\n\n {% set staging_table = spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.create_schema", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.get_snapshot_table_column_names", "macro.dbt_spark.spark_build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.068236, "supported_languages": ["sql"]}, "macro.dbt_spark.spark__can_clone_table": {"name": "spark__can_clone_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.spark__can_clone_table", "macro_sql": "{% macro spark__can_clone_table() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.068533, "supported_languages": null}, "macro.dbt_spark.spark__create_or_replace_clone": {"name": "spark__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.spark__create_or_replace_clone", "macro_sql": "{% macro spark__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation }} shallow clone {{ defer_relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0686028, "supported_languages": null}, "macro.dbt_spark.materialization_clone_spark": {"name": "materialization_clone_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.materialization_clone_spark", "macro_sql": "\n\n{%- materialization clone, adapter='spark' -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if file_format != 'delta' -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n shallow clone requires file_format be set to 'delta'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {%- elif other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n {% endif %}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.06957, "supported_languages": ["sql"]}, "macro.dbt_spark.spark__get_merge_update_columns": {"name": "spark__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/column_helpers.sql", "original_file_path": "macros/materializations/incremental/column_helpers.sql", "unique_id": "macro.dbt_spark.spark__get_merge_update_columns", "macro_sql": "{% macro spark__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = None -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.070068, "supported_languages": null}, "macro.dbt_spark.dbt_spark_validate_get_file_format": {"name": "dbt_spark_validate_get_file_format", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_spark.dbt_spark_validate_get_file_format", "macro_sql": "{% macro dbt_spark_validate_get_file_format(raw_file_format) %}\n {#-- Validate the file format #}\n\n {% set accepted_formats = ['text', 'csv', 'json', 'jdbc', 'parquet', 'orc', 'hive', 'delta', 'iceberg', 'libsvm', 'hudi'] %}\n\n {% set invalid_file_format_msg -%}\n Invalid file format provided: {{ raw_file_format }}\n Expected one of: {{ accepted_formats | join(', ') }}\n {%- endset %}\n\n {% if raw_file_format not in accepted_formats %}\n {% do exceptions.raise_compiler_error(invalid_file_format_msg) %}\n {% endif %}\n\n {% do return(raw_file_format) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.07071, "supported_languages": null}, "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy": {"name": "dbt_spark_validate_get_incremental_strategy", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy", "macro_sql": "{% macro dbt_spark_validate_get_incremental_strategy(raw_strategy, file_format) %}\n {#-- Validate the incremental strategy #}\n\n {% set invalid_strategy_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n Expected one of: 'append', 'merge', 'insert_overwrite', 'microbatch'\n {%- endset %}\n\n {% set invalid_merge_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You can only choose this strategy when file_format is set to 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n\n {% set invalid_insert_overwrite_endpoint_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You cannot use this strategy when connecting via endpoint\n Use the 'append' or 'merge' strategy instead\n {%- endset %}\n\n {% if raw_strategy not in ['append', 'merge', 'insert_overwrite', 'microbatch'] %}\n {% do exceptions.raise_compiler_error(invalid_strategy_msg) %}\n {%-else %}\n {% if raw_strategy == 'merge' and file_format not in ['delta', 'iceberg', 'hudi'] %}\n {% do exceptions.raise_compiler_error(invalid_merge_msg) %}\n {% endif %}\n {% if raw_strategy in ['insert_overwrite', 'microbatch'] and target.endpoint %}\n {% do exceptions.raise_compiler_error(invalid_insert_overwrite_endpoint_msg) %}\n {% endif %}\n {% endif %}\n\n {% do return(raw_strategy) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.071131, "supported_languages": null}, "macro.dbt_spark.get_insert_overwrite_sql": {"name": "get_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.get_insert_overwrite_sql", "macro_sql": "{% macro get_insert_overwrite_sql(source_relation, target_relation, existing_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n {% if existing_relation.is_iceberg %}\n {# removed table from statement for iceberg #}\n insert overwrite {{ target_relation }}\n {# removed partition_cols for iceberg as well #}\n {% else %}\n insert overwrite table {{ target_relation }}\n {{ partition_cols(label=\"partition\") }}\n {% endif %}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0716958, "supported_languages": null}, "macro.dbt_spark.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(source_relation, target_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n insert into table {{ target_relation }}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.071851, "supported_languages": null}, "macro.dbt_spark.spark__get_merge_sql": {"name": "spark__get_merge_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.spark__get_merge_sql", "macro_sql": "{% macro spark__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %}\n {# need dest_columns for merge_exclude_columns, default to use \"*\" #}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n when matched then update set\n {% if update_columns -%}{%- for column_name in update_columns %}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {%- else %} * {% endif %}\n\n when not matched then insert *\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.07249, "supported_languages": null}, "macro.dbt_spark.dbt_spark_get_incremental_sql": {"name": "dbt_spark_get_incremental_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.dbt_spark_get_incremental_sql", "macro_sql": "{% macro dbt_spark_get_incremental_sql(strategy, source, target, existing, unique_key, incremental_predicates) %}\n {%- if strategy == 'append' -%}\n {#-- insert new records into existing table, without updating or overwriting #}\n {{ get_insert_into_sql(source, target) }}\n {%- elif strategy == 'insert_overwrite' -%}\n {#-- insert statements don't like CTEs, so support them via a temp view #}\n {{ get_insert_overwrite_sql(source, target, existing) }}\n {%- elif strategy == 'microbatch' -%}\n {#-- microbatch wraps insert_overwrite, and requires a partition_by config #}\n {% set missing_partition_key_microbatch_msg -%}\n dbt-spark 'microbatch' incremental strategy requires a `partition_by` config.\n Ensure you are using a `partition_by` column that is of grain {{ config.get('batch_size') }}.\n {%- endset %}\n\n {%- if not config.get('partition_by') -%}\n {{ exceptions.raise_compiler_error(missing_partition_key_microbatch_msg) }}\n {%- endif -%}\n {{ get_insert_overwrite_sql(source, target, existing) }}\n {%- elif strategy == 'merge' -%}\n {#-- merge all columns for datasources which implement MERGE INTO (e.g. databricks, iceberg) - schema changes are handled for us #}\n {{ get_merge_sql(target, source, unique_key, dest_columns=none, incremental_predicates=incremental_predicates) }}\n {%- else -%}\n {% set no_sql_for_strategy_msg -%}\n No known SQL for the incremental strategy provided: {{ strategy }}\n {%- endset %}\n {%- do exceptions.raise_compiler_error(no_sql_for_strategy_msg) -%}\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.get_insert_into_sql", "macro.dbt_spark.get_insert_overwrite_sql", "macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0729558, "supported_languages": null}, "macro.dbt_spark.materialization_incremental_spark": {"name": "materialization_incremental_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_spark.materialization_incremental_spark", "macro_sql": "{% materialization incremental, adapter='spark', supported_languages=['sql', 'python'] -%}\n {#-- Validate early so we don't run SQL if the file_format + strategy combo is invalid --#}\n {%- set raw_file_format = config.get('file_format', default='parquet') -%}\n {%- set raw_strategy = config.get('incremental_strategy') or 'append' -%}\n {%- set grant_config = config.get('grants') -%}\n\n {%- set file_format = dbt_spark_validate_get_file_format(raw_file_format) -%}\n {%- set strategy = dbt_spark_validate_get_incremental_strategy(raw_strategy, file_format) -%}\n\n {#-- Set vars --#}\n\n {%- set unique_key = config.get('unique_key', none) -%}\n {%- set partition_by = config.get('partition_by', none) -%}\n {%- set language = model['language'] -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) -%}\n {%- set target_relation = this -%}\n {%- set existing_relation = load_relation(this) -%}\n {% set tmp_relation = this.incorporate(path = {\"identifier\": this.identifier ~ '__dbt_tmp'}) -%}\n\n {#-- for SQL model we will create temp view that doesn't have database and schema --#}\n {%- if language == 'sql'-%}\n {%- set tmp_relation = tmp_relation.include(database=false, schema=false) -%}\n {%- endif -%}\n\n {#-- Set Overwrite Mode --#}\n {%- if strategy in ['insert_overwrite', 'microbatch'] and partition_by -%}\n {%- call statement() -%}\n set spark.sql.sources.partitionOverwriteMode = DYNAMIC\n {%- endcall -%}\n {%- endif -%}\n\n {#-- Run pre-hooks --#}\n {{ run_hooks(pre_hooks) }}\n\n {#-- Incremental run logic --#}\n {%- if existing_relation is none -%}\n {#-- Relation must be created --#}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {%- elif existing_relation.is_view or should_full_refresh() -%}\n {#-- Relation must be dropped & recreated --#}\n {% set is_delta = (file_format == 'delta' and existing_relation.is_delta) %}\n {% if not is_delta %} {#-- If Delta, we will `create or replace` below, so no need to drop --#}\n {% do adapter.drop_relation(existing_relation) %}\n {% endif %}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {%- else -%}\n {#-- Relation must be merged --#}\n {%- call statement('create_tmp_relation', language=language) -%}\n {{ create_table_as(True, tmp_relation, compiled_code, language) }}\n {%- endcall -%}\n {%- do process_schema_changes(on_schema_change, tmp_relation, existing_relation) -%}\n {%- call statement('main') -%}\n {{ dbt_spark_get_incremental_sql(strategy, tmp_relation, target_relation, existing_relation, unique_key, incremental_predicates) }}\n {%- endcall -%}\n {%- if language == 'python' -%}\n {#--\n This is yucky.\n See note in dbt-spark/dbt/include/spark/macros/adapters.sql\n re: python models and temporary views.\n\n Also, why do neither drop_relation or adapter.drop_relation work here?!\n --#}\n {% call statement('drop_relation') -%}\n drop table if exists {{ tmp_relation }}\n {%- endcall %}\n {%- endif -%}\n {%- endif -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt_spark.dbt_spark_validate_get_file_format", "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.load_relation", "macro.dbt.statement", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt_spark.persist_constraints", "macro.dbt.should_full_refresh", "macro.dbt.process_schema_changes", "macro.dbt_spark.dbt_spark_get_incremental_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0746171, "supported_languages": ["sql", "python"]}, "macro.dbt_spark.spark__date": {"name": "spark__date", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt_spark.spark__date", "macro_sql": "{% macro spark__date(year, month, day) -%}\n {%- set dt = modules.datetime.date(year, month, day) -%}\n {%- set iso_8601_formatted_date = dt.strftime('%Y-%m-%d') -%}\n to_date('{{ iso_8601_formatted_date }}', 'yyyy-MM-dd')\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.074813, "supported_languages": null}, "macro.dbt_spark.spark__concat": {"name": "spark__concat", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt_spark.spark__concat", "macro_sql": "{% macro spark__concat(fields) -%}\n concat({{ fields|join(', ') }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0749052, "supported_languages": null}, "macro.dbt_spark.assert_not_null": {"name": "assert_not_null", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/assert_not_null.sql", "original_file_path": "macros/utils/assert_not_null.sql", "unique_id": "macro.dbt_spark.assert_not_null", "macro_sql": "{% macro assert_not_null(function, arg) -%}\n {{ return(adapter.dispatch('assert_not_null', 'dbt')(function, arg)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.075047, "supported_languages": null}, "macro.dbt_spark.spark__assert_not_null": {"name": "spark__assert_not_null", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/assert_not_null.sql", "original_file_path": "macros/utils/assert_not_null.sql", "unique_id": "macro.dbt_spark.spark__assert_not_null", "macro_sql": "{% macro spark__assert_not_null(function, arg) %}\n\n coalesce({{function}}({{arg}}), nvl2({{function}}({{arg}}), assert_true({{function}}({{arg}}) is not null), null))\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.075142, "supported_languages": null}, "macro.dbt_spark.spark__dateadd": {"name": "spark__dateadd", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_spark.spark__dateadd", "macro_sql": "{% macro spark__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {%- set clock_component -%}\n {# make sure the dates + timestamps are real, otherwise raise an error asap #}\n to_unix_timestamp({{ assert_not_null('to_timestamp', from_date_or_timestamp) }})\n - to_unix_timestamp({{ assert_not_null('date', from_date_or_timestamp) }})\n {%- endset -%}\n\n {%- if datepart in ['day', 'week'] -%}\n\n {%- set multiplier = 7 if datepart == 'week' else 1 -%}\n\n to_timestamp(\n to_unix_timestamp(\n date_add(\n {{ assert_not_null('date', from_date_or_timestamp) }},\n cast({{interval}} * {{multiplier}} as int)\n )\n ) + {{clock_component}}\n )\n\n {%- elif datepart in ['month', 'quarter', 'year'] -%}\n\n {%- set multiplier -%}\n {%- if datepart == 'month' -%} 1\n {%- elif datepart == 'quarter' -%} 3\n {%- elif datepart == 'year' -%} 12\n {%- endif -%}\n {%- endset -%}\n\n to_timestamp(\n to_unix_timestamp(\n add_months(\n {{ assert_not_null('date', from_date_or_timestamp) }},\n cast({{interval}} * {{multiplier}} as int)\n )\n ) + {{clock_component}}\n )\n\n {%- elif datepart in ('hour', 'minute', 'second', 'millisecond', 'microsecond') -%}\n\n {%- set multiplier -%}\n {%- if datepart == 'hour' -%} 3600\n {%- elif datepart == 'minute' -%} 60\n {%- elif datepart == 'second' -%} 1\n {%- elif datepart == 'millisecond' -%} (1/1000000)\n {%- elif datepart == 'microsecond' -%} (1/1000000)\n {%- endif -%}\n {%- endset -%}\n\n to_timestamp(\n {{ assert_not_null('to_unix_timestamp', from_date_or_timestamp) }}\n + cast({{interval}} * {{multiplier}} as int)\n )\n\n {%- else -%}\n\n {{ exceptions.raise_compiler_error(\"macro dateadd not implemented for datepart ~ '\" ~ datepart ~ \"' ~ on Spark\") }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.07609, "supported_languages": null}, "macro.dbt_spark.spark__current_timestamp": {"name": "spark__current_timestamp", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/timestamps.sql", "original_file_path": "macros/utils/timestamps.sql", "unique_id": "macro.dbt_spark.spark__current_timestamp", "macro_sql": "{% macro spark__current_timestamp() -%}\n current_timestamp()\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.07618, "supported_languages": null}, "macro.dbt_spark.spark__escape_single_quotes": {"name": "spark__escape_single_quotes", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt_spark.spark__escape_single_quotes", "macro_sql": "{% macro spark__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"\\\\'\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.076289, "supported_languages": null}, "macro.dbt_spark.spark__listagg": {"name": "spark__listagg", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt_spark.spark__listagg", "macro_sql": "{% macro spark__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if order_by_clause %}\n {{ exceptions.warn(\"order_by_clause is not supported for listagg on Spark/Databricks\") }}\n {% endif %}\n\n {% set collect_list %} collect_list({{ measure }}) {% endset %}\n\n {% set limited %} slice({{ collect_list }}, 1, {{ limit_num }}) {% endset %}\n\n {% set collected = limited if limit_num else collect_list %}\n\n {% set final %} array_join({{ collected }}, {{ delimiter_text }}) {% endset %}\n\n {% do return(final) %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0765948, "supported_languages": null}, "macro.dbt_spark.spark__datediff": {"name": "spark__datediff", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_spark.spark__datediff", "macro_sql": "{% macro spark__datediff(first_date, second_date, datepart) %}\n\n {%- if datepart in ['day', 'week', 'month', 'quarter', 'year'] -%}\n\n {# make sure the dates are real, otherwise raise an error asap #}\n {% set first_date = assert_not_null('date', first_date) %}\n {% set second_date = assert_not_null('date', second_date) %}\n\n {%- endif -%}\n\n {%- if datepart == 'day' -%}\n\n datediff({{second_date}}, {{first_date}})\n\n {%- elif datepart == 'week' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(datediff({{second_date}}, {{first_date}})/7)\n else ceil(datediff({{second_date}}, {{first_date}})/7)\n end\n\n -- did we cross a week boundary (Sunday)?\n + case\n when {{first_date}} < {{second_date}} and dayofweek({{second_date}}) < dayofweek({{first_date}}) then 1\n when {{first_date}} > {{second_date}} and dayofweek({{second_date}}) > dayofweek({{first_date}}) then -1\n else 0 end\n\n {%- elif datepart == 'month' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(months_between(date({{second_date}}), date({{first_date}})))\n else ceil(months_between(date({{second_date}}), date({{first_date}})))\n end\n\n -- did we cross a month boundary?\n + case\n when {{first_date}} < {{second_date}} and dayofmonth({{second_date}}) < dayofmonth({{first_date}}) then 1\n when {{first_date}} > {{second_date}} and dayofmonth({{second_date}}) > dayofmonth({{first_date}}) then -1\n else 0 end\n\n {%- elif datepart == 'quarter' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(months_between(date({{second_date}}), date({{first_date}}))/3)\n else ceil(months_between(date({{second_date}}), date({{first_date}}))/3)\n end\n\n -- did we cross a quarter boundary?\n + case\n when {{first_date}} < {{second_date}} and (\n (dayofyear({{second_date}}) - (quarter({{second_date}}) * 365/4))\n < (dayofyear({{first_date}}) - (quarter({{first_date}}) * 365/4))\n ) then 1\n when {{first_date}} > {{second_date}} and (\n (dayofyear({{second_date}}) - (quarter({{second_date}}) * 365/4))\n > (dayofyear({{first_date}}) - (quarter({{first_date}}) * 365/4))\n ) then -1\n else 0 end\n\n {%- elif datepart == 'year' -%}\n\n year({{second_date}}) - year({{first_date}})\n\n {%- elif datepart in ('hour', 'minute', 'second', 'millisecond', 'microsecond') -%}\n\n {%- set divisor -%}\n {%- if datepart == 'hour' -%} 3600\n {%- elif datepart == 'minute' -%} 60\n {%- elif datepart == 'second' -%} 1\n {%- elif datepart == 'millisecond' -%} (1/1000)\n {%- elif datepart == 'microsecond' -%} (1/1000000)\n {%- endif -%}\n {%- endset -%}\n\n case when {{first_date}} < {{second_date}}\n then ceil((\n {# make sure the timestamps are real, otherwise raise an error asap #}\n {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', second_date)) }}\n - {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', first_date)) }}\n ) / {{divisor}})\n else floor((\n {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', second_date)) }}\n - {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', first_date)) }}\n ) / {{divisor}})\n end\n\n {% if datepart == 'millisecond' %}\n + cast(date_format({{second_date}}, 'SSS') as int)\n - cast(date_format({{first_date}}, 'SSS') as int)\n {% endif %}\n\n {% if datepart == 'microsecond' %}\n {% set capture_str = '[0-9]{4}-[0-9]{2}-[0-9]{2}.[0-9]{2}:[0-9]{2}:[0-9]{2}.([0-9]{6})' %}\n -- Spark doesn't really support microseconds, so this is a massive hack!\n -- It will only work if the timestamp-string is of the format\n -- 'yyyy-MM-dd-HH mm.ss.SSSSSS'\n + cast(regexp_extract({{second_date}}, '{{capture_str}}', 1) as int)\n - cast(regexp_extract({{first_date}}, '{{capture_str}}', 1) as int)\n {% endif %}\n\n {%- else -%}\n\n {{ exceptions.raise_compiler_error(\"macro datediff not implemented for datepart ~ '\" ~ datepart ~ \"' ~ on Spark\") }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.078332, "supported_languages": null}, "macro.dbt_spark.spark__safe_cast": {"name": "spark__safe_cast", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt_spark.spark__safe_cast", "macro_sql": "{% macro spark__safe_cast(field, type) %}\n{%- set field_clean = field.strip('\"').strip(\"'\") if (cast_from_string_unsupported_for(type) and field is string) else field -%}\ncast({{field_clean}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.cast_from_string_unsupported_for"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.078588, "supported_languages": null}, "macro.dbt_spark.cast_from_string_unsupported_for": {"name": "cast_from_string_unsupported_for", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt_spark.cast_from_string_unsupported_for", "macro_sql": "{% macro cast_from_string_unsupported_for(type) %}\n {{ return(type.lower().startswith('struct') or type.lower().startswith('array') or type.lower().startswith('map')) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0787148, "supported_languages": null}, "macro.dbt_spark.spark__any_value": {"name": "spark__any_value", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt_spark.spark__any_value", "macro_sql": "{% macro spark__any_value(expression) -%}\n {#-- return any value (non-deterministic) --#}\n first({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.078799, "supported_languages": null}, "macro.dbt_spark.spark__array_concat": {"name": "spark__array_concat", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt_spark.spark__array_concat", "macro_sql": "{% macro spark__array_concat(array_1, array_2) -%}\n concat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.078878, "supported_languages": null}, "macro.dbt_spark.spark__bool_or": {"name": "spark__bool_or", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt_spark.spark__bool_or", "macro_sql": "{% macro spark__bool_or(expression) -%}\n\n max({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.078954, "supported_languages": null}, "macro.dbt_spark.spark__split_part": {"name": "spark__split_part", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt_spark.spark__split_part", "macro_sql": "{% macro spark__split_part(string_text, delimiter_text, part_number) %}\n\n {% set delimiter_expr %}\n\n -- escape if starts with a special character\n case when regexp_extract({{ delimiter_text }}, '([^A-Za-z0-9])(.*)', 1) != '_'\n then concat('\\\\', {{ delimiter_text }})\n else {{ delimiter_text }} end\n\n {% endset %}\n\n {% if part_number >= 0 %}\n\n {% set split_part_expr %}\n\n split(\n {{ string_text }},\n {{ delimiter_expr }}\n )[({{ part_number - 1 if part_number > 0 else part_number }})]\n\n {% endset %}\n\n {% else %}\n\n {% set split_part_expr %}\n\n split(\n {{ string_text }},\n {{ delimiter_expr }}\n )[(\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 1 + {{ part_number }}\n )]\n\n {% endset %}\n\n {% endif %}\n\n {{ return(split_part_expr) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.079329, "supported_languages": null}, "macro.dbt_spark.spark__array_construct": {"name": "spark__array_construct", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt_spark.spark__array_construct", "macro_sql": "{% macro spark__array_construct(inputs, data_type) -%}\n array( {{ inputs|join(' , ') }} )\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.079419, "supported_languages": null}, "macro.dbt_spark.spark__array_append": {"name": "spark__array_append", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt_spark.spark__array_append", "macro_sql": "{% macro spark__array_append(array, new_element) -%}\n {{ array_concat(array, array_construct([new_element])) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.array_concat", "macro.dbt.array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.079514, "supported_languages": null}, "macro.dbt.run_hooks": {"name": "run_hooks", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0798872, "supported_languages": null}, "macro.dbt.make_hook_config": {"name": "make_hook_config", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.079977, "supported_languages": null}, "macro.dbt.before_begin": {"name": "before_begin", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.080045, "supported_languages": null}, "macro.dbt.in_transaction": {"name": "in_transaction", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0801091, "supported_languages": null}, "macro.dbt.after_commit": {"name": "after_commit", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.080172, "supported_languages": null}, "macro.dbt.set_sql_header": {"name": "set_sql_header", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0803041, "supported_languages": null}, "macro.dbt.should_full_refresh": {"name": "should_full_refresh", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.080466, "supported_languages": null}, "macro.dbt.should_store_failures": {"name": "should_store_failures", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.08062, "supported_languages": null}, "macro.dbt.snapshot_merge_sql": {"name": "snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__snapshot_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.080817, "supported_languages": null}, "macro.dbt.default__snapshot_merge_sql": {"name": "default__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n {%- set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() -%}\n\n merge into {{ target.render() }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}\n\n when matched\n {% if config.get(\"dbt_valid_to_current\") %}\n\t{% set source_unique_key = (\"DBT_INTERNAL_DEST.\" ~ columns.dbt_valid_to) | trim %}\n\t{% set target_unique_key = config.get('dbt_valid_to_current') | trim %}\n\tand ({{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null)\n\n {% else %}\n and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null\n {% endif %}\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.equals"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.081193, "supported_languages": null}, "macro.dbt.strategy_dispatch": {"name": "strategy_dispatch", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0821362, "supported_languages": null}, "macro.dbt.snapshot_hash_arguments": {"name": "snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.082276, "supported_languages": null}, "macro.dbt.default__snapshot_hash_arguments": {"name": "default__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0823748, "supported_languages": null}, "macro.dbt.snapshot_timestamp_strategy": {"name": "snapshot_timestamp_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set primary_key = config.get('unique_key') %}\n {% set updated_at = config.get('updated_at') %}\n {% set hard_deletes = adapter.get_hard_deletes_behavior(config) %}\n {% set invalidate_hard_deletes = hard_deletes == 'invalidate' %}\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.{{ columns.dbt_valid_from }} < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_args = api.Relation.scd_args(primary_key, updated_at) %}\n {% set scd_id_expr = snapshot_hash_arguments(scd_args) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes,\n \"hard_deletes\": hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.082809, "supported_languages": null}, "macro.dbt.snapshot_string_as_time": {"name": "snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__snapshot_string_as_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.08289, "supported_languages": null}, "macro.dbt.default__snapshot_string_as_time": {"name": "default__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.082976, "supported_languages": null}, "macro.dbt.snapshot_check_all_get_existing_columns": {"name": "snapshot_check_all_get_existing_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n {#-- N.B. The whitespace below is necessary to avoid edge case issue with comments --#}\n {#-- See: https://github.com/dbt-labs/dbt-core/issues/6781 --#}\n select {{ check_cols_config | join(', ') }} from (\n {{ node['compiled_code'] }}\n ) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0835998, "supported_languages": null}, "macro.dbt.snapshot_check_strategy": {"name": "snapshot_check_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set check_cols_config = config.get('check_cols') %}\n {% set primary_key = config.get('unique_key') %}\n {% set hard_deletes = adapter.get_hard_deletes_behavior(config) %}\n {% set invalidate_hard_deletes = hard_deletes == 'invalidate' %}\n {% set updated_at = config.get('updated_at') or snapshot_get_time() %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_args = api.Relation.scd_args(primary_key, updated_at) %}\n {% set scd_id_expr = snapshot_hash_arguments(scd_args) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes,\n \"hard_deletes\": hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.get_true_sql", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.084287, "supported_languages": null}, "macro.dbt.create_columns": {"name": "create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.085391, "supported_languages": null}, "macro.dbt.default__create_columns": {"name": "default__create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation.render() }} add column {{ adapter.quote(column.name) }} {{ column.expanded_data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.085542, "supported_languages": null}, "macro.dbt.post_snapshot": {"name": "post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.08562, "supported_languages": null}, "macro.dbt.default__post_snapshot": {"name": "default__post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.085664, "supported_languages": null}, "macro.dbt.get_true_sql": {"name": "get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_true_sql", "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_true_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.085738, "supported_languages": null}, "macro.dbt.default__get_true_sql": {"name": "default__get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__get_true_sql", "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.085789, "supported_languages": null}, "macro.dbt.snapshot_staging_table": {"name": "snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__snapshot_staging_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0858781, "supported_languages": null}, "macro.dbt.get_snapshot_table_column_names": {"name": "get_snapshot_table_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_snapshot_table_column_names", "macro_sql": "{% macro get_snapshot_table_column_names() %}\n {{ return({'dbt_valid_to': 'dbt_valid_to', 'dbt_valid_from': 'dbt_valid_from', 'dbt_scd_id': 'dbt_scd_id', 'dbt_updated_at': 'dbt_updated_at', 'dbt_is_deleted': 'dbt_is_deleted'}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.085985, "supported_languages": null}, "macro.dbt.default__snapshot_staging_table": {"name": "default__snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__snapshot_staging_table", "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n {% if strategy.hard_deletes == 'new_record' %}\n {% set new_scd_id = snapshot_hash_arguments([columns.dbt_scd_id, snapshot_get_time()]) %}\n {% endif %}\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }}\n from {{ target_relation }}\n where\n {% if config.get('dbt_valid_to_current') %}\n\t\t{% set source_unique_key = columns.dbt_valid_to | trim %}\n\t\t{% set target_unique_key = config.get('dbt_valid_to_current') | trim %}\n\n\t\t{# The exact equals semantics between NULL values depends on the current behavior flag set. Also, update records if the source field is null #}\n ( {{ equals(source_unique_key, target_unique_key) }} or {{ source_unique_key }} is null )\n {% else %}\n {{ columns.dbt_valid_to }} is null\n {% endif %}\n\n ),\n\n insertions_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ get_dbt_valid_to_current(strategy, columns) }},\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_to }}\n\n from snapshot_query\n ),\n\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n\n deletes_source_data as (\n\n select *, {{ unique_key_fields(strategy.unique_key) }}\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n {%- if strategy.hard_deletes == 'new_record' -%}\n ,'False' as {{ columns.dbt_is_deleted }}\n {%- endif %}\n\n from insertions_source_data as source_data\n left outer join snapshotted_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"snapshotted_data\") }}\n or ({{ unique_key_is_not_null(strategy.unique_key, \"snapshotted_data\") }} and (\n {{ strategy.row_changed }} {%- if strategy.hard_deletes == 'new_record' -%} or snapshotted_data.{{ columns.dbt_is_deleted }} = 'True' {% endif %}\n )\n\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.{{ columns.dbt_scd_id }}\n {%- if strategy.hard_deletes == 'new_record' -%}\n , snapshotted_data.{{ columns.dbt_is_deleted }}\n {%- endif %}\n\n from updates_source_data as source_data\n join snapshotted_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where (\n {{ strategy.row_changed }} {%- if strategy.hard_deletes == 'new_record' -%} or snapshotted_data.{{ columns.dbt_is_deleted }} = 'True' {% endif %}\n )\n )\n\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n ,\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},\n {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_to }},\n snapshotted_data.{{ columns.dbt_scd_id }}\n {%- if strategy.hard_deletes == 'new_record' -%}\n , snapshotted_data.{{ columns.dbt_is_deleted }}\n {%- endif %}\n from snapshotted_data\n left join deletes_source_data as source_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"source_data\") }}\n\n {%- if strategy.hard_deletes == 'new_record' %}\n and not (\n --avoid updating the record's valid_to if the latest entry is marked as deleted\n snapshotted_data.{{ columns.dbt_is_deleted }} = 'True'\n and\n {% if config.get('dbt_valid_to_current') -%}\n snapshotted_data.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }}\n {%- else -%}\n snapshotted_data.{{ columns.dbt_valid_to }} is null\n {%- endif %}\n )\n {%- endif %}\n )\n {%- endif %}\n\n {%- if strategy.hard_deletes == 'new_record' %}\n {% set snapshotted_cols = get_list_of_column_names(get_columns_in_relation(target_relation)) %}\n {% set source_sql_cols = get_column_schema_from_query(source_sql) %}\n ,\n deletion_records as (\n\n select\n 'insert' as dbt_change_type,\n {#/*\n If a column has been added to the source it won't yet exist in the\n snapshotted table so we insert a null value as a placeholder for the column.\n */#}\n {%- for col in source_sql_cols -%}\n {%- if col.name in snapshotted_cols -%}\n snapshotted_data.{{ adapter.quote(col.column) }},\n {%- else -%}\n NULL as {{ adapter.quote(col.column) }},\n {%- endif -%}\n {% endfor -%}\n {%- if strategy.unique_key | is_list -%}\n {%- for key in strategy.unique_key -%}\n snapshotted_data.{{ key }} as dbt_unique_key_{{ loop.index }},\n {% endfor -%}\n {%- else -%}\n snapshotted_data.dbt_unique_key as dbt_unique_key,\n {% endif -%}\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},\n {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},\n snapshotted_data.{{ columns.dbt_valid_to }} as {{ columns.dbt_valid_to }},\n {{ new_scd_id }} as {{ columns.dbt_scd_id }},\n 'True' as {{ columns.dbt_is_deleted }}\n from snapshotted_data\n left join deletes_source_data as source_data\n on {{ unique_key_join_on(strategy.unique_key, \"snapshotted_data\", \"source_data\") }}\n where {{ unique_key_is_null(strategy.unique_key, \"source_data\") }}\n and not (\n --avoid inserting a new record if the latest one is marked as deleted\n snapshotted_data.{{ columns.dbt_is_deleted }} = 'True'\n and\n {% if config.get('dbt_valid_to_current') -%}\n snapshotted_data.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }}\n {%- else -%}\n snapshotted_data.{{ columns.dbt_valid_to }} is null\n {%- endif %}\n )\n\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.hard_deletes == 'invalidate' or strategy.hard_deletes == 'new_record' %}\n union all\n select * from deletes\n {%- endif %}\n {%- if strategy.hard_deletes == 'new_record' %}\n union all\n select * from deletion_records\n {%- endif %}\n\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_hash_arguments", "macro.dbt.snapshot_get_time", "macro.dbt.unique_key_fields", "macro.dbt.equals", "macro.dbt.get_dbt_valid_to_current", "macro.dbt.unique_key_join_on", "macro.dbt.unique_key_is_null", "macro.dbt.unique_key_is_not_null", "macro.dbt.get_list_of_column_names", "macro.dbt.get_columns_in_relation", "macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.088119, "supported_languages": null}, "macro.dbt.build_snapshot_table": {"name": "build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__build_snapshot_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.088233, "supported_languages": null}, "macro.dbt.default__build_snapshot_table": {"name": "default__build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__build_snapshot_table", "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n\n select *,\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ get_dbt_valid_to_current(strategy, columns) }}\n {%- if strategy.hard_deletes == 'new_record' -%}\n , 'False' as {{ columns.dbt_is_deleted }}\n {% endif -%}\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.get_dbt_valid_to_current"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.088453, "supported_languages": null}, "macro.dbt.build_snapshot_staging_table": {"name": "build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set temp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, temp_relation, select) }}\n {% endcall %}\n\n {% do return(temp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.088635, "supported_languages": null}, "macro.dbt.get_updated_at_column_data_type": {"name": "get_updated_at_column_data_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_updated_at_column_data_type", "macro_sql": "{% macro get_updated_at_column_data_type(snapshot_sql) %}\n {% set snapshot_sql_column_schema = get_column_schema_from_query(snapshot_sql) %}\n {% set dbt_updated_at_data_type = null %}\n {% set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {% set ns.dbt_updated_at_data_type = null -%}\n {% for column in snapshot_sql_column_schema %}\n {% if ((column.column == 'dbt_updated_at') or (column.column == 'DBT_UPDATED_AT')) %}\n {% set ns.dbt_updated_at_data_type = column.dtype %}\n {% endif %}\n {% endfor %}\n {{ return(ns.dbt_updated_at_data_type or none) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.08889, "supported_languages": null}, "macro.dbt.check_time_data_types": {"name": "check_time_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.check_time_data_types", "macro_sql": "{% macro check_time_data_types(sql) %}\n {% set dbt_updated_at_data_type = get_updated_at_column_data_type(sql) %}\n {% set snapshot_get_time_data_type = get_snapshot_get_time_data_type() %}\n {% if snapshot_get_time_data_type is not none and dbt_updated_at_data_type is not none and snapshot_get_time_data_type != dbt_updated_at_data_type %}\n {% if exceptions.warn_snapshot_timestamp_data_types %}\n {{ exceptions.warn_snapshot_timestamp_data_types(snapshot_get_time_data_type, dbt_updated_at_data_type) }}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_updated_at_column_data_type", "macro.dbt.get_snapshot_get_time_data_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.089101, "supported_languages": null}, "macro.dbt.get_dbt_valid_to_current": {"name": "get_dbt_valid_to_current", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_dbt_valid_to_current", "macro_sql": "{% macro get_dbt_valid_to_current(strategy, columns) %}\n {% set dbt_valid_to_current = config.get('dbt_valid_to_current') or \"null\" %}\n coalesce(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}), {{dbt_valid_to_current}})\n as {{ columns.dbt_valid_to }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.08924, "supported_languages": null}, "macro.dbt.unique_key_fields": {"name": "unique_key_fields", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.unique_key_fields", "macro_sql": "{% macro unique_key_fields(unique_key) %}\n {% if unique_key | is_list %}\n {% for key in unique_key %}\n {{ key }} as dbt_unique_key_{{ loop.index }}\n {%- if not loop.last %} , {%- endif %}\n {% endfor %}\n {% else %}\n {{ unique_key }} as dbt_unique_key\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0893881, "supported_languages": null}, "macro.dbt.unique_key_join_on": {"name": "unique_key_join_on", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.unique_key_join_on", "macro_sql": "{% macro unique_key_join_on(unique_key, identifier, from_identifier) %}\n {% if unique_key | is_list %}\n {% for key in unique_key %}\n\t {% set source_unique_key = (identifier ~ \".dbt_unique_key_\" ~ loop.index) | trim %}\n\t {% set target_unique_key = (from_identifier ~ \".dbt_unique_key_\" ~ loop.index) | trim %}\n\t {{ equals(source_unique_key, target_unique_key) }}\n {%- if not loop.last %} and {%- endif %}\n {% endfor %}\n {% else %}\n {{ identifier }}.dbt_unique_key = {{ from_identifier }}.dbt_unique_key\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.equals"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.089632, "supported_languages": null}, "macro.dbt.unique_key_is_null": {"name": "unique_key_is_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.unique_key_is_null", "macro_sql": "{% macro unique_key_is_null(unique_key, identifier) %}\n {% if unique_key | is_list %}\n {{ identifier }}.dbt_unique_key_1 is null\n {% else %}\n {{ identifier }}.dbt_unique_key is null\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0897331, "supported_languages": null}, "macro.dbt.unique_key_is_not_null": {"name": "unique_key_is_not_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.unique_key_is_not_null", "macro_sql": "{% macro unique_key_is_not_null(unique_key, identifier) %}\n {% if unique_key | is_list %}\n {{ identifier }}.dbt_unique_key_1 is not null\n {% else %}\n {{ identifier }}.dbt_unique_key is not null\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.089823, "supported_languages": null}, "macro.dbt.materialization_snapshot_default": {"name": "materialization_snapshot_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot.sql", "original_file_path": "macros/materializations/snapshots/snapshot.sql", "unique_id": "macro.dbt.materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparision later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {# The model['config'] parameter below is no longer used, but passing anyway for compatibility #}\n {# It was a dictionary of config, instead of the config object from the context #}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", model['config'], target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set build_or_select_sql = build_sql %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {{ adapter.assert_valid_snapshot_target_given_strategy(target_relation, columns, strategy) }}\n\n {% set build_or_select_sql = snapshot_staging_table(strategy, sql, target_relation) %}\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set remove_columns = ['dbt_change_type', 'DBT_CHANGE_TYPE', 'dbt_unique_key', 'DBT_UNIQUE_KEY'] %}\n {% if unique_key | is_list %}\n {% for key in strategy.unique_key %}\n {{ remove_columns.append('dbt_unique_key_' + loop.index|string) }}\n {{ remove_columns.append('DBT_UNIQUE_KEY_' + loop.index|string) }}\n {% endfor %}\n {% endif %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'in', remove_columns)\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'in', remove_columns)\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n\n {{ check_time_data_types(build_or_select_sql) }}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_staging_table", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.check_time_data_types", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0919418, "supported_languages": ["sql"]}, "macro.dbt.materialization_test_default": {"name": "materialization_test_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/test.sql", "original_file_path": "macros/materializations/tests/test.sql", "unique_id": "macro.dbt.materialization_test_default", "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n {% set limit = config.get('limit') %}\n {% set sql_header = config.get('sql_header') if flags.REQUIRE_SQL_HEADER_IN_TEST_CONFIGS else none %}\n\n {% set sql_with_limit %}\n {{ get_limit_subquery_sql(sql, limit) }}\n {% endset %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% set store_failures_as = config.get('store_failures_as') %}\n -- if `--store-failures` is invoked via command line and `store_failures_as` is not set,\n -- config.get('store_failures_as', 'table') returns None, not 'table'\n {% if store_failures_as == none %}{% set store_failures_as = 'table' %}{% endif %}\n {% if store_failures_as not in ['table', 'view'] %}\n {{ exceptions.raise_compiler_error(\n \"'\" ~ store_failures_as ~ \"' is not a valid value for `store_failures_as`. \"\n \"Accepted values are: ['ephemeral', 'table', 'view']\"\n ) }}\n {% endif %}\n\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type=store_failures_as) -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {% if sql_header %}{{ sql_header }}{% endif %}\n {{ get_create_sql(target_relation, sql_with_limit) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {# Since the test failures have already been saved to the database, reuse that result rather than querying again #}\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql_with_limit %}\n\n {% endif %}\n\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {% if sql_header %}{{ sql_header }}{% endif %}\n {# The limit has already been included above, and we do not want to duplicate it again. We also want to be safe for macro overrides treating `limit` as a required parameter. #}\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit=none)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.get_limit_subquery_sql", "macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.get_create_sql", "macro.dbt.get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.093159, "supported_languages": ["sql"]}, "macro.dbt.get_test_sql": {"name": "get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0934138, "supported_languages": null}, "macro.dbt.default__get_test_sql": {"name": "default__get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.093545, "supported_languages": null}, "macro.dbt.get_unit_test_sql": {"name": "get_unit_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_unit_test_sql", "macro_sql": "{% macro get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n {{ adapter.dispatch('get_unit_test_sql', 'dbt')(main_sql, expected_fixture_sql, expected_column_names) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_unit_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.093638, "supported_languages": null}, "macro.dbt.default__get_unit_test_sql": {"name": "default__get_unit_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_unit_test_sql", "macro_sql": "{% macro default__get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n-- Build actual result given inputs\nwith dbt_internal_unit_test_actual as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%},{% endif %}{%- endfor -%}, {{ dbt.string_literal(\"actual\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ main_sql }}\n ) _dbt_internal_unit_test_actual\n),\n-- Build expected result\ndbt_internal_unit_test_expected as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%}, {% endif %}{%- endfor -%}, {{ dbt.string_literal(\"expected\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ expected_fixture_sql }}\n ) _dbt_internal_unit_test_expected\n)\n-- Union actual and expected results\nselect * from dbt_internal_unit_test_actual\nunion all\nselect * from dbt_internal_unit_test_expected\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.093896, "supported_languages": null}, "macro.dbt.get_where_subquery": {"name": "get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.get_where_subquery", "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_where_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.094045, "supported_languages": null}, "macro.dbt.default__get_where_subquery": {"name": "default__get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.default__get_where_subquery", "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.094226, "supported_languages": null}, "macro.dbt.materialization_unit_default": {"name": "materialization_unit_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/unit.sql", "original_file_path": "macros/materializations/tests/unit.sql", "unique_id": "macro.dbt.materialization_unit_default", "macro_sql": "{%- materialization unit, default -%}\n\n {% set relations = [] %}\n {% set sql_header = config.get('sql_header') if flags.REQUIRE_SQL_HEADER_IN_TEST_CONFIGS else none %}\n\n {% set expected_rows = config.get('expected_rows') %}\n {% set expected_sql = config.get('expected_sql') %}\n {% set tested_expected_column_names = expected_rows[0].keys() if (expected_rows | length ) > 0 else get_columns_in_query(sql) %}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {% do run_query(get_create_table_as_sql(True, temp_relation, get_empty_subquery_sql(sql))) %}\n {%- set columns_in_relation = adapter.get_columns_in_relation(temp_relation) -%}\n {%- set column_name_to_data_types = {} -%}\n {%- set column_name_to_quoted = {} -%}\n {%- for column in columns_in_relation -%}\n {%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n {%- do column_name_to_quoted.update({column.name|lower: column.quoted}) -%}\n {%- endfor -%}\n\n {%- set expected_column_names_quoted = [] -%}\n {%- for column_name in tested_expected_column_names -%}\n {%- do expected_column_names_quoted.append(column_name_to_quoted[column_name|lower]) -%}\n {%- endfor -%}\n\n {% if not expected_sql %}\n {% set expected_sql = get_expected_sql(expected_rows, column_name_to_data_types, column_name_to_quoted) %}\n {% endif %}\n {% set unit_test_sql = get_unit_test_sql(sql, expected_sql, expected_column_names_quoted) %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {% if sql_header %}{{ sql_header }}{% endif %}\n {{ unit_test_sql }}\n\n {%- endcall %}\n\n {% do adapter.drop_relation(temp_relation) %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt.make_temp_relation", "macro.dbt.run_query", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_empty_subquery_sql", "macro.dbt.get_expected_sql", "macro.dbt.get_unit_test_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.095192, "supported_languages": ["sql"]}, "macro.dbt.materialization_materialized_view_default": {"name": "materialization_materialized_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialization_materialized_view_default", "macro_sql": "{% materialization materialized_view, default %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n {% set intermediate_relation = make_intermediate_relation(target_relation) %}\n {% set backup_relation_type = target_relation.MaterializedView if existing_relation is none else existing_relation.type %}\n {% set backup_relation = make_backup_relation(target_relation, backup_relation_type) %}\n\n {{ materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% if build_sql == '' %}\n {{ materialized_view_execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.materialized_view_setup", "macro.dbt.materialized_view_get_build_sql", "macro.dbt.materialized_view_execute_no_op", "macro.dbt.materialized_view_execute_build_sql", "macro.dbt.materialized_view_teardown"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0959601, "supported_languages": ["sql"]}, "macro.dbt.materialized_view_setup": {"name": "materialized_view_setup", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_setup", "macro_sql": "{% macro materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) %}\n\n -- backup_relation and intermediate_relation should not already exist in the database\n -- it's possible these exist because of a previous run that exited unexpectedly\n {% set preexisting_backup_relation = load_cached_relation(backup_relation) %}\n {% set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.096139, "supported_languages": null}, "macro.dbt.materialized_view_teardown": {"name": "materialized_view_teardown", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_teardown", "macro_sql": "{% macro materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) %}\n\n -- drop the temp relations if they exist to leave the database clean for the next run\n {{ drop_relation_if_exists(backup_relation) }}\n {{ drop_relation_if_exists(intermediate_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.096252, "supported_languages": null}, "macro.dbt.materialized_view_get_build_sql": {"name": "materialized_view_get_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_get_build_sql", "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = '' %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation.render() ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation.render() ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt.get_replace_sql", "macro.dbt.get_materialized_view_configuration_changes", "macro.dbt.refresh_materialized_view", "macro.dbt.get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0968, "supported_languages": null}, "macro.dbt.materialized_view_execute_no_op": {"name": "materialized_view_execute_no_op", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_no_op", "macro_sql": "{% macro materialized_view_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.096912, "supported_languages": null}, "macro.dbt.materialized_view_execute_build_sql": {"name": "materialized_view_execute_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_build_sql", "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.097177, "supported_languages": null}, "macro.dbt.materialization_view_default": {"name": "materialization_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view.sql", "original_file_path": "macros/materializations/models/view.sql", "unique_id": "macro.dbt.materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.run_hooks", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0981472, "supported_languages": ["sql"]}, "macro.dbt.materialization_table_default": {"name": "materialization_table_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table.sql", "original_file_path": "macros/materializations/models/table.sql", "unique_id": "macro.dbt.materialization_table_default", "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n {% do create_indexes(intermediate_relation) %}\n\n -- cleanup\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.get_create_table_as_sql", "macro.dbt.create_indexes", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.099159, "supported_languages": ["sql"]}, "macro.dbt.get_quoted_csv": {"name": "get_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.0995798, "supported_languages": null}, "macro.dbt.diff_columns": {"name": "diff_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_columns", "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.099969, "supported_languages": null}, "macro.dbt.diff_column_data_types": {"name": "diff_column_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_column_data_types", "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n {{ return(adapter.dispatch('diff_column_data_types', 'dbt')(source_columns, target_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1000772, "supported_languages": null}, "macro.dbt.default__diff_column_data_types": {"name": "default__diff_column_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.default__diff_column_data_types", "macro_sql": "{% macro default__diff_column_data_types(source_columns, target_columns) %}\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.expanded_data_type != tc.expanded_data_type and not sc.can_expand_to(other_column=tc) %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.expanded_data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.100428, "supported_languages": null}, "macro.dbt.get_merge_update_columns": {"name": "get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_merge_update_columns", "macro_sql": "{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.100537, "supported_languages": null}, "macro.dbt.default__get_merge_update_columns": {"name": "default__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.default__get_merge_update_columns", "macro_sql": "{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = dest_columns | map(attribute=\"quoted\") | list -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.100951, "supported_languages": null}, "macro.dbt.get_merge_sql": {"name": "get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n -- back compat for old kwarg name\n {% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.101467, "supported_languages": null}, "macro.dbt.default__get_merge_sql": {"name": "default__get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set source_unique_key = (\"DBT_INTERNAL_SOURCE.\" ~ unique_key) | trim %}\n\t {% set target_unique_key = (\"DBT_INTERNAL_DEST.\" ~ unique_key) | trim %}\n\t {% set unique_key_match = equals(source_unique_key, target_unique_key) | trim %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{\"(\" ~ predicates | join(\") and (\") ~ \")\"}}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv", "macro.dbt.get_merge_update_columns", "macro.dbt.equals"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.102216, "supported_languages": null}, "macro.dbt.get_delete_insert_merge_sql": {"name": "get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1023428, "supported_languages": null}, "macro.dbt.default__get_delete_insert_merge_sql": {"name": "default__get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is string %}\n {% set unique_key = [unique_key] %}\n {% endif %}\n\n {%- set unique_key_str = unique_key|join(', ') -%}\n\n delete from {{ target }} as DBT_INTERNAL_DEST\n where ({{ unique_key_str }}) in (\n select distinct {{ unique_key_str }}\n from {{ source }} as DBT_INTERNAL_SOURCE\n )\n {%- if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {%- endif -%};\n\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.102666, "supported_languages": null}, "macro.dbt.get_insert_overwrite_merge_sql": {"name": "get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.102794, "supported_languages": null}, "macro.dbt.default__get_insert_overwrite_merge_sql": {"name": "default__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {#-- The only time include_sql_header is True: --#}\n {#-- BigQuery + insert_overwrite strategy + \"static\" partitions config --#}\n {#-- We should consider including the sql header at the materialization level instead --#}\n\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.103076, "supported_languages": null}, "macro.dbt.is_incremental": {"name": "is_incremental", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/is_incremental.sql", "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", "unique_id": "macro.dbt.is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.103338, "supported_languages": null}, "macro.dbt.get_incremental_append_sql": {"name": "get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_append_sql", "macro_sql": "{% macro get_incremental_append_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1037838, "supported_languages": null}, "macro.dbt.default__get_incremental_append_sql": {"name": "default__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_append_sql", "macro_sql": "{% macro default__get_incremental_append_sql(arg_dict) %}\n\n {% do return(get_insert_into_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1039, "supported_languages": null}, "macro.dbt.get_incremental_delete_insert_sql": {"name": "get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_delete_insert_sql", "macro_sql": "{% macro get_incremental_delete_insert_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_delete_insert_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.10399, "supported_languages": null}, "macro.dbt.default__get_incremental_delete_insert_sql": {"name": "default__get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_delete_insert_sql", "macro_sql": "{% macro default__get_incremental_delete_insert_sql(arg_dict) %}\n\n {% do return(get_delete_insert_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.10412, "supported_languages": null}, "macro.dbt.get_incremental_merge_sql": {"name": "get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_merge_sql", "macro_sql": "{% macro get_incremental_merge_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.104204, "supported_languages": null}, "macro.dbt.default__get_incremental_merge_sql": {"name": "default__get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_merge_sql", "macro_sql": "{% macro default__get_incremental_merge_sql(arg_dict) %}\n\n {% do return(get_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.104327, "supported_languages": null}, "macro.dbt.get_incremental_insert_overwrite_sql": {"name": "get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_insert_overwrite_sql", "macro_sql": "{% macro get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.104411, "supported_languages": null}, "macro.dbt.default__get_incremental_insert_overwrite_sql": {"name": "default__get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_insert_overwrite_sql", "macro_sql": "{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {% do return(get_insert_overwrite_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.104525, "supported_languages": null}, "macro.dbt.get_incremental_default_sql": {"name": "get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_default_sql", "macro_sql": "{% macro get_incremental_default_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_default_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1046069, "supported_languages": null}, "macro.dbt.default__get_incremental_default_sql": {"name": "default__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_default_sql", "macro_sql": "{% macro default__get_incremental_default_sql(arg_dict) %}\n\n {% do return(get_incremental_append_sql(arg_dict)) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.104674, "supported_languages": null}, "macro.dbt.get_incremental_microbatch_sql": {"name": "get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_microbatch_sql", "macro_sql": "{% macro get_incremental_microbatch_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_microbatch_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_microbatch_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.104752, "supported_languages": null}, "macro.dbt.default__get_incremental_microbatch_sql": {"name": "default__get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_microbatch_sql", "macro_sql": "{% macro default__get_incremental_microbatch_sql(arg_dict) %}\n\n {{ exceptions.raise_not_implemented('microbatch materialization strategy not implemented for adapter ' + adapter.type()) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1048262, "supported_languages": null}, "macro.dbt.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ temp_relation }}\n )\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.104952, "supported_languages": null}, "macro.dbt.materialization_incremental_default": {"name": "materialization_incremental_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/incremental.sql", "original_file_path": "macros/materializations/models/incremental/incremental.sql", "unique_id": "macro.dbt.materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% set relation_for_indexes = target_relation %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set relation_for_indexes = intermediate_relation %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% set relation_for_indexes = temp_relation %}\n {% set contract_config = config.get('contract') %}\n {% if not contract_config or not contract_config.enforced %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {% endif %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(relation_for_indexes) %}\n {% endif %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_temp_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.get_create_table_as_sql", "macro.dbt.run_query", "macro.dbt.process_schema_changes", "macro.dbt.statement", "macro.dbt.create_indexes", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1068742, "supported_languages": ["sql"]}, "macro.dbt.incremental_validate_on_schema_change": {"name": "incremental_validate_on_schema_change", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.incremental_validate_on_schema_change", "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1075, "supported_languages": null}, "macro.dbt.check_for_schema_changes": {"name": "check_for_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.check_for_schema_changes", "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n {{ return(adapter.dispatch('check_for_schema_changes', 'dbt')(source_relation, target_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__check_for_schema_changes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.107732, "supported_languages": null}, "macro.dbt.default__check_for_schema_changes": {"name": "default__check_for_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.default__check_for_schema_changes", "macro_sql": "{% macro default__check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.diff_columns", "macro.dbt.diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.10835, "supported_languages": null}, "macro.dbt.sync_column_schemas": {"name": "sync_column_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.sync_column_schemas", "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n {{ return(adapter.dispatch('sync_column_schemas', 'dbt')(on_schema_change, target_relation, schema_changes_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.108466, "supported_languages": null}, "macro.dbt.default__sync_column_schemas": {"name": "default__sync_column_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.default__sync_column_schemas", "macro_sql": "{% macro default__sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.alter_relation_add_remove_columns", "macro.dbt.alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.108983, "supported_languages": null}, "macro.dbt.process_schema_changes": {"name": "process_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.process_schema_changes", "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n {{ return(adapter.dispatch('process_schema_changes', 'dbt')(on_schema_change, source_relation, target_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__process_schema_changes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.109091, "supported_languages": null}, "macro.dbt.default__process_schema_changes": {"name": "default__process_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.default__process_schema_changes", "macro_sql": "{% macro default__process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n\n Additional troubleshooting context:\n Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}\n Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}\n New column types: {{ schema_changes_dict['new_target_types'] }}\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.check_for_schema_changes", "macro.dbt.sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.109462, "supported_languages": null}, "macro.dbt.can_clone_table": {"name": "can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.can_clone_table", "macro_sql": "{% macro can_clone_table() %}\n {{ return(adapter.dispatch('can_clone_table', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__can_clone_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.109591, "supported_languages": null}, "macro.dbt.default__can_clone_table": {"name": "default__can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.default__can_clone_table", "macro_sql": "{% macro default__can_clone_table() %}\n {{ return(False) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1096509, "supported_languages": null}, "macro.dbt.create_or_replace_clone": {"name": "create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.create_or_replace_clone", "macro_sql": "{% macro create_or_replace_clone(this_relation, defer_relation) %}\n {{ return(adapter.dispatch('create_or_replace_clone', 'dbt')(this_relation, defer_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_or_replace_clone"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.109792, "supported_languages": null}, "macro.dbt.default__create_or_replace_clone": {"name": "default__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.default__create_or_replace_clone", "macro_sql": "{% macro default__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation.render() }} clone {{ defer_relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.109867, "supported_languages": null}, "macro.dbt.materialization_clone_default": {"name": "materialization_clone_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/clone.sql", "original_file_path": "macros/materializations/models/clone/clone.sql", "unique_id": "macro.dbt.materialization_clone_default", "macro_sql": "{%- materialization clone, default -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n {%- set grant_config = config.get('grants') -%}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation.render() ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% if target_relation.database == defer_relation.database and\n target_relation.schema == defer_relation.schema and\n target_relation.identifier == defer_relation.identifier %}\n {{ log(\"Target relation and defer relation are the same, skipping clone for relation: \" ~ target_relation.render()) }}\n {% else %}\n {% call statement('main') %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endcall %}\n {% endif %}\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n\n {%- endif -%}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.111182, "supported_languages": ["sql"]}, "macro.dbt.materialization_seed_default": {"name": "materialization_seed_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/seed.sql", "original_file_path": "macros/materializations/seeds/seed.sql", "unique_id": "macro.dbt.materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparison later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation.render())) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.112628, "supported_languages": ["sql"]}, "macro.dbt.create_csv_table": {"name": "create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1130571, "supported_languages": null}, "macro.dbt.default__create_csv_table": {"name": "default__create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1134331, "supported_languages": null}, "macro.dbt.reset_csv_table": {"name": "reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__reset_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1135402, "supported_languages": null}, "macro.dbt.default__reset_csv_table": {"name": "default__reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation.render() %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.113757, "supported_languages": null}, "macro.dbt.get_csv_sql": {"name": "get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_csv_sql", "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_csv_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.113846, "supported_languages": null}, "macro.dbt.default__get_csv_sql": {"name": "default__get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_csv_sql", "macro_sql": "{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ create_or_truncate_sql }};\n -- dbt seed --\n {{ insert_sql }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.113909, "supported_languages": null}, "macro.dbt.get_binding_char": {"name": "get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_binding_char", "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.113974, "supported_languages": null}, "macro.dbt.default__get_binding_char": {"name": "default__get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_binding_char", "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.114034, "supported_languages": null}, "macro.dbt.get_batch_size": {"name": "get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_batch_size", "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_batch_size"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.114114, "supported_languages": null}, "macro.dbt.default__get_batch_size": {"name": "default__get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_batch_size", "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.11447, "supported_languages": null}, "macro.dbt.get_seed_column_quoted_csv": {"name": "get_seed_column_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1146848, "supported_languages": null}, "macro.dbt.load_csv_rows": {"name": "load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1148071, "supported_languages": null}, "macro.dbt.default__load_csv_rows": {"name": "default__load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_seed_column_quoted_csv", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.11533, "supported_languages": null}, "macro.dbt.scalar_function_sql": {"name": "scalar_function_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.scalar_function_sql", "macro_sql": "{% macro scalar_function_sql(target_relation) %}\n {{ return(adapter.dispatch('scalar_function_sql', 'dbt')(target_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__scalar_function_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.115777, "supported_languages": null}, "macro.dbt.default__scalar_function_sql": {"name": "default__scalar_function_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__scalar_function_sql", "macro_sql": "{% macro default__scalar_function_sql(target_relation) %}\n {{ scalar_function_create_replace_signature_sql(target_relation) }}\n {{ scalar_function_body_sql() }};\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.scalar_function_create_replace_signature_sql", "macro.dbt.scalar_function_body_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1158478, "supported_languages": null}, "macro.dbt.scalar_function_create_replace_signature_sql": {"name": "scalar_function_create_replace_signature_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.scalar_function_create_replace_signature_sql", "macro_sql": "{% macro scalar_function_create_replace_signature_sql(target_relation) %}\n {{ return(adapter.dispatch('scalar_function_create_replace_signature_sql', 'dbt')(target_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__scalar_function_create_replace_signature_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1159308, "supported_languages": null}, "macro.dbt.default__scalar_function_create_replace_signature_sql": {"name": "default__scalar_function_create_replace_signature_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__scalar_function_create_replace_signature_sql", "macro_sql": "{% macro default__scalar_function_create_replace_signature_sql(target_relation) %}\n CREATE OR REPLACE FUNCTION {{ target_relation.render() }} ({{ formatted_scalar_function_args_sql()}})\n RETURNS {{ model.returns.data_type }}\n {{ scalar_function_volatility_sql() }}\n AS\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.formatted_scalar_function_args_sql", "macro.dbt.scalar_function_volatility_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.11603, "supported_languages": null}, "macro.dbt.formatted_scalar_function_args_sql": {"name": "formatted_scalar_function_args_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.formatted_scalar_function_args_sql", "macro_sql": "{% macro formatted_scalar_function_args_sql() %}\n {{ return(adapter.dispatch('formatted_scalar_function_args_sql', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__formatted_scalar_function_args_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.116116, "supported_languages": null}, "macro.dbt.default__formatted_scalar_function_args_sql": {"name": "default__formatted_scalar_function_args_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__formatted_scalar_function_args_sql", "macro_sql": "{% macro default__formatted_scalar_function_args_sql() %}\n {% set args = [] %}\n {% for arg in model.arguments -%}\n {%- do args.append(arg.name ~ ' ' ~ arg.data_type) -%}\n {%- endfor %}\n {{ args | join(', ') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1162848, "supported_languages": null}, "macro.dbt.scalar_function_body_sql": {"name": "scalar_function_body_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.scalar_function_body_sql", "macro_sql": "{% macro scalar_function_body_sql() %}\n {{ return(adapter.dispatch('scalar_function_body_sql', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__scalar_function_body_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.116382, "supported_languages": null}, "macro.dbt.default__scalar_function_body_sql": {"name": "default__scalar_function_body_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__scalar_function_body_sql", "macro_sql": "{% macro default__scalar_function_body_sql() %}\n $$\n {{ model.compiled_code }}\n $$ LANGUAGE SQL\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1164339, "supported_languages": null}, "macro.dbt.scalar_function_volatility_sql": {"name": "scalar_function_volatility_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.scalar_function_volatility_sql", "macro_sql": "{% macro scalar_function_volatility_sql() %}\n {{ return(adapter.dispatch('scalar_function_volatility_sql', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__scalar_function_volatility_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.116509, "supported_languages": null}, "macro.dbt.default__scalar_function_volatility_sql": {"name": "default__scalar_function_volatility_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__scalar_function_volatility_sql", "macro_sql": "{% macro default__scalar_function_volatility_sql() %}\n {% set volatility = model.config.get('volatility') %}\n {% if volatility == 'deterministic' %}\n IMMUTABLE\n {% elif volatility == 'stable' %}\n STABLE\n {% elif volatility == 'non-deterministic' %}\n VOLATILE\n {% elif volatility != none %}\n {# This shouldn't happen unless a new volatility is invented #}\n {% do unsupported_volatility_warning(volatility) %}\n {% endif %}\n {# If no volatility is set, don't add anything and let the data warehouse default it #}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.unsupported_volatility_warning"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.116936, "supported_languages": null}, "macro.dbt.unsupported_volatility_warning": {"name": "unsupported_volatility_warning", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.unsupported_volatility_warning", "macro_sql": "{% macro unsupported_volatility_warning(volatility) %}\n {{ return(adapter.dispatch('unsupported_volatility_warning', 'dbt')(volatility)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__unsupported_volatility_warning"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1170602, "supported_languages": null}, "macro.dbt.default__unsupported_volatility_warning": {"name": "default__unsupported_volatility_warning", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/scalar.sql", "original_file_path": "macros/materializations/functions/scalar.sql", "unique_id": "macro.dbt.default__unsupported_volatility_warning", "macro_sql": "{% macro default__unsupported_volatility_warning(volatility) %}\n {% set msg = \"Found `\" ~ volatility ~ \"` volatility specified on function `\" ~ model.name ~ \"`. This volatility is not supported by \" ~ adapter.type() ~ \", and will be ignored\" %}\n {% do exceptions.warn(msg) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.117184, "supported_languages": null}, "macro.dbt.function_execute_build_sql": {"name": "function_execute_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/helpers.sql", "original_file_path": "macros/materializations/functions/helpers.sql", "unique_id": "macro.dbt.function_execute_build_sql", "macro_sql": "{% macro function_execute_build_sql(build_sql, existing_relation, target_relation) %}\n {{ return(adapter.dispatch('function_execute_build_sql', 'dbt')(build_sql, existing_relation, target_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__function_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.117384, "supported_languages": null}, "macro.dbt.default__function_execute_build_sql": {"name": "default__function_execute_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/helpers.sql", "original_file_path": "macros/materializations/functions/helpers.sql", "unique_id": "macro.dbt.default__function_execute_build_sql", "macro_sql": "{% macro default__function_execute_build_sql(build_sql, existing_relation, target_relation) %}\n\n {% set grant_config = config.get('grants') %}\n\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ adapter.commit() }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.117609, "supported_languages": null}, "macro.dbt.get_aggregate_function_create_replace_signature": {"name": "get_aggregate_function_create_replace_signature", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.get_aggregate_function_create_replace_signature", "macro_sql": "{% macro get_aggregate_function_create_replace_signature(target_relation) %}\n {{ return(adapter.dispatch('get_aggregate_function_create_replace_signature', 'dbt')(target_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_aggregate_function_create_replace_signature"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1179001, "supported_languages": null}, "macro.dbt.default__get_aggregate_function_create_replace_signature": {"name": "default__get_aggregate_function_create_replace_signature", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.default__get_aggregate_function_create_replace_signature", "macro_sql": "{% macro default__get_aggregate_function_create_replace_signature(target_relation) %}\n CREATE OR REPLACE AGGREGATE FUNCTION {{ target_relation.render() }} ({{ get_formatted_aggregate_function_args()}})\n RETURNS {{ model.returns.data_type }}\n {{ get_function_language_specifier() }}\n {{ get_aggregate_function_volatility_specifier() }}\n {% if model.get('language') == 'python' %}\n {{ get_function_python_options() }}\n {% endif %}\n AS\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_formatted_aggregate_function_args", "macro.dbt.get_function_language_specifier", "macro.dbt.get_aggregate_function_volatility_specifier", "macro.dbt.get_function_python_options"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.118081, "supported_languages": null}, "macro.dbt.get_formatted_aggregate_function_args": {"name": "get_formatted_aggregate_function_args", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.get_formatted_aggregate_function_args", "macro_sql": "{% macro get_formatted_aggregate_function_args() %}\n {{ return(adapter.dispatch('get_formatted_aggregate_function_args', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_formatted_aggregate_function_args"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.118166, "supported_languages": null}, "macro.dbt.default__get_formatted_aggregate_function_args": {"name": "default__get_formatted_aggregate_function_args", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.default__get_formatted_aggregate_function_args", "macro_sql": "{% macro default__get_formatted_aggregate_function_args() %}\n {# conveniently we can reuse the sql scalar function args #}\n {{ formatted_scalar_function_args_sql() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.formatted_scalar_function_args_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.118222, "supported_languages": null}, "macro.dbt.get_function_language_specifier": {"name": "get_function_language_specifier", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.get_function_language_specifier", "macro_sql": "{% macro get_function_language_specifier() %}\n {{ return(adapter.dispatch('get_function_language_specifier', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_function_language_specifier"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.118304, "supported_languages": null}, "macro.dbt.default__get_function_language_specifier": {"name": "default__get_function_language_specifier", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.default__get_function_language_specifier", "macro_sql": "{% macro default__get_function_language_specifier() %}\n {% set language = model.get('language') %}\n {% if language == 'sql' %}\n {# generally you dont need to specify the language for sql functions #}\n {% elif language == 'python' %}\n LANGUAGE PYTHON\n {% else %}\n {{ 'LANGUAGE ' ~ language.upper() }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.179839, "supported_languages": null}, "macro.dbt.get_aggregate_function_volatility_specifier": {"name": "get_aggregate_function_volatility_specifier", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.get_aggregate_function_volatility_specifier", "macro_sql": "{% macro get_aggregate_function_volatility_specifier() %}\n {{ return(adapter.dispatch('get_aggregate_function_volatility_specifier', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_aggregate_function_volatility_specifier"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.179981, "supported_languages": null}, "macro.dbt.default__get_aggregate_function_volatility_specifier": {"name": "default__get_aggregate_function_volatility_specifier", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.default__get_aggregate_function_volatility_specifier", "macro_sql": "{% macro default__get_aggregate_function_volatility_specifier() %}\n {{ scalar_function_volatility_sql() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.scalar_function_volatility_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.180064, "supported_languages": null}, "macro.dbt.get_function_python_options": {"name": "get_function_python_options", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.get_function_python_options", "macro_sql": "{% macro get_function_python_options() %}\n {{ return(adapter.dispatch('get_function_python_options', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_function_python_options"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.180147, "supported_languages": null}, "macro.dbt.default__get_function_python_options": {"name": "default__get_function_python_options", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/aggregate.sql", "original_file_path": "macros/materializations/functions/aggregate.sql", "unique_id": "macro.dbt.default__get_function_python_options", "macro_sql": "{% macro default__get_function_python_options() %}\n RUNTIME_VERSION = '{{ model.config.get('runtime_version') }}'\n HANDLER = '{{ model.config.get('entry_point') }}'\n {% set packages = model.config.get('packages', []) %}\n {% if packages %}\n PACKAGES = ('{{ packages | join(\"','\") }}')\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.18034, "supported_languages": null}, "macro.dbt.materialization_function_default": {"name": "materialization_function_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/functions/function.sql", "original_file_path": "macros/materializations/functions/function.sql", "unique_id": "macro.dbt.materialization_function_default", "macro_sql": "{% materialization function, default, supported_languages=['sql', 'python'] %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.Function) %}\n\n {{ run_hooks(pre_hooks) }}\n\n {% set function_config = this.get_function_config(model) %}\n {% set macro_name = this.get_function_macro_name(function_config) %}\n\n {# Doing this aliasing of adapter.dispatch is a hacky way to disable the static analysis of actually calling adapter.dispatch #}\n {# This is necessary because the static analysis breaks being able to dynamically pass a macro_name #}\n {% set _dispatch = adapter.dispatch %}\n\n {% set build_sql = _dispatch(macro_name, 'dbt')(target_relation) %}\n {{ function_execute_build_sql(build_sql, existing_relation, target_relation) }}\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.run_hooks", "macro.dbt.function_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.180772, "supported_languages": ["sql", "python"]}, "macro.dbt.generate_alias_name": {"name": "generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_alias_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.180958, "supported_languages": null}, "macro.dbt.default__generate_alias_name": {"name": "default__generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.default__generate_alias_name", "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name -%}\n\n {{ custom_alias_name | trim }}\n\n {%- elif node.version -%}\n\n {{ return(node.name ~ \"_v\" ~ (node.version | replace(\".\", \"_\"))) }}\n\n {%- else -%}\n\n {{ node.name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.181129, "supported_languages": null}, "macro.dbt.generate_schema_name": {"name": "generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.181337, "supported_languages": null}, "macro.dbt.default__generate_schema_name": {"name": "default__generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.default__generate_schema_name", "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1814568, "supported_languages": null}, "macro.dbt.generate_schema_name_for_env": {"name": "generate_schema_name_for_env", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.181586, "supported_languages": null}, "macro.dbt.generate_database_name": {"name": "generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__generate_database_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1817532, "supported_languages": null}, "macro.dbt.default__generate_database_name": {"name": "default__generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.181865, "supported_languages": null}, "macro.dbt.get_drop_sql": {"name": "get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.get_drop_sql", "macro_sql": "{%- macro get_drop_sql(relation) -%}\n {{- log('Applying DROP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.182065, "supported_languages": null}, "macro.dbt.default__get_drop_sql": {"name": "default__get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__get_drop_sql", "macro_sql": "{%- macro default__get_drop_sql(relation) -%}\n\n {%- if relation.is_view -%}\n {{ drop_view(relation) }}\n\n {%- elif relation.is_table -%}\n {{ drop_table(relation) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n\n {%- else -%}\n drop {{ relation.type }} if exists {{ relation.render() }} cascade\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.drop_view", "macro.dbt.drop_table", "macro.dbt.drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.182234, "supported_languages": null}, "macro.dbt.drop_relation": {"name": "drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.182316, "supported_languages": null}, "macro.dbt.default__drop_relation": {"name": "default__drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {{ get_drop_sql(relation) }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.182415, "supported_languages": null}, "macro.dbt.drop_relation_if_exists": {"name": "drop_relation_if_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1825101, "supported_languages": null}, "macro.dbt.get_replace_sql": {"name": "get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.get_replace_sql", "macro_sql": "{% macro get_replace_sql(existing_relation, target_relation, sql) %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {{- adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.182911, "supported_languages": null}, "macro.dbt.default__get_replace_sql": {"name": "default__get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.default__get_replace_sql", "macro_sql": "{% macro default__get_replace_sql(existing_relation, target_relation, sql) %}\n\n {# /* use a create or replace statement if possible */ #}\n\n {% set is_replaceable = existing_relation.type == target_relation.type and existing_relation.can_be_replaced %}\n\n {% if is_replaceable and existing_relation.is_view %}\n {{ get_replace_view_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_table %}\n {{ get_replace_table_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_materialized_view %}\n {{ get_replace_materialized_view_sql(target_relation, sql) }}\n\n {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}\n {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}\n {%- elif target_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_drop_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }}\n\n {# /* create target_relation in place by first backing up the existing relation */ #}\n {%- elif existing_relation.can_be_renamed -%}\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* no renaming is allowed, so just drop and create */ #}\n {%- else -%}\n {{ get_drop_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_replace_view_sql", "macro.dbt.get_replace_table_sql", "macro.dbt.get_replace_materialized_view_sql", "macro.dbt.get_create_intermediate_sql", "macro.dbt.get_create_backup_sql", "macro.dbt.get_rename_intermediate_sql", "macro.dbt.get_drop_backup_sql", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.183627, "supported_languages": null}, "macro.dbt.get_create_intermediate_sql": {"name": "get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.get_create_intermediate_sql", "macro_sql": "{%- macro get_create_intermediate_sql(relation, sql) -%}\n {{- log('Applying CREATE INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_intermediate_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_intermediate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.183841, "supported_languages": null}, "macro.dbt.default__get_create_intermediate_sql": {"name": "default__get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.default__get_create_intermediate_sql", "macro_sql": "{%- macro default__get_create_intermediate_sql(relation, sql) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n -- drop any pre-existing intermediate\n {{ get_drop_sql(intermediate_relation) }};\n\n {{ get_create_sql(intermediate_relation, sql) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.183991, "supported_languages": null}, "macro.dbt.drop_schema_named": {"name": "drop_schema_named", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/schema.sql", "original_file_path": "macros/relations/schema.sql", "unique_id": "macro.dbt.drop_schema_named", "macro_sql": "{% macro drop_schema_named(schema_name) %}\n {{ return(adapter.dispatch('drop_schema_named', 'dbt') (schema_name)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__drop_schema_named"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1841788, "supported_languages": null}, "macro.dbt.default__drop_schema_named": {"name": "default__drop_schema_named", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/schema.sql", "original_file_path": "macros/relations/schema.sql", "unique_id": "macro.dbt.default__drop_schema_named", "macro_sql": "{% macro default__drop_schema_named(schema_name) %}\n {% set schema_relation = api.Relation.create(schema=schema_name) %}\n {{ adapter.drop_schema(schema_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.184296, "supported_languages": null}, "macro.dbt.get_drop_backup_sql": {"name": "get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.get_drop_backup_sql", "macro_sql": "{%- macro get_drop_backup_sql(relation) -%}\n {{- log('Applying DROP BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_drop_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1844711, "supported_languages": null}, "macro.dbt.default__get_drop_backup_sql": {"name": "default__get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.default__get_drop_backup_sql", "macro_sql": "{%- macro default__get_drop_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n {{ get_drop_sql(backup_relation) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.184578, "supported_languages": null}, "macro.dbt.get_rename_sql": {"name": "get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.get_rename_sql", "macro_sql": "{%- macro get_rename_sql(relation, new_name) -%}\n {{- log('Applying RENAME to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_sql', 'dbt')(relation, new_name) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1848068, "supported_languages": null}, "macro.dbt.default__get_rename_sql": {"name": "default__get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__get_rename_sql", "macro_sql": "{%- macro default__get_rename_sql(relation, new_name) -%}\n\n {%- if relation.is_view -%}\n {{ get_rename_view_sql(relation, new_name) }}\n\n {%- elif relation.is_table -%}\n {{ get_rename_table_sql(relation, new_name) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_rename_materialized_view_sql(relation, new_name) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_rename_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.get_rename_view_sql", "macro.dbt.get_rename_table_sql", "macro.dbt.get_rename_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.184999, "supported_languages": null}, "macro.dbt.rename_relation": {"name": "rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__rename_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1850948, "supported_languages": null}, "macro.dbt.default__rename_relation": {"name": "default__rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation.render() }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.185228, "supported_languages": null}, "macro.dbt.get_create_backup_sql": {"name": "get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.get_create_backup_sql", "macro_sql": "{%- macro get_create_backup_sql(relation) -%}\n {{- log('Applying CREATE BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_create_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.185405, "supported_languages": null}, "macro.dbt.default__get_create_backup_sql": {"name": "default__get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.default__get_create_backup_sql", "macro_sql": "{%- macro default__get_create_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n -- drop any pre-existing backup\n {{ get_drop_sql(backup_relation) }};\n\n {{ get_rename_sql(relation, backup_relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1855412, "supported_languages": null}, "macro.dbt.get_create_sql": {"name": "get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.get_create_sql", "macro_sql": "{%- macro get_create_sql(relation, sql) -%}\n {{- log('Applying CREATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.185751, "supported_languages": null}, "macro.dbt.default__get_create_sql": {"name": "default__get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.default__get_create_sql", "macro_sql": "{%- macro default__get_create_sql(relation, sql) -%}\n\n {%- if relation.is_view -%}\n {{ get_create_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_table -%}\n {{ get_create_table_as_sql(False, relation, sql) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_create_materialized_view_as_sql(relation, sql) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_create_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.get_create_view_as_sql", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.185966, "supported_languages": null}, "macro.dbt.get_rename_intermediate_sql": {"name": "get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.get_rename_intermediate_sql", "macro_sql": "{%- macro get_rename_intermediate_sql(relation) -%}\n {{- log('Applying RENAME INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_intermediate_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_rename_intermediate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.186107, "supported_languages": null}, "macro.dbt.default__get_rename_intermediate_sql": {"name": "default__get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.default__get_rename_intermediate_sql", "macro_sql": "{%- macro default__get_rename_intermediate_sql(relation) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n {{ get_rename_sql(intermediate_relation, relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.186211, "supported_languages": null}, "macro.dbt.drop_materialized_view": {"name": "drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.drop_materialized_view", "macro_sql": "{% macro drop_materialized_view(relation) -%}\n {{- adapter.dispatch('drop_materialized_view', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.186338, "supported_languages": null}, "macro.dbt.default__drop_materialized_view": {"name": "default__drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.default__drop_materialized_view", "macro_sql": "{% macro default__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.186398, "supported_languages": null}, "macro.dbt.get_replace_materialized_view_sql": {"name": "get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.get_replace_materialized_view_sql", "macro_sql": "{% macro get_replace_materialized_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_materialized_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.186534, "supported_languages": null}, "macro.dbt.default__get_replace_materialized_view_sql": {"name": "default__get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.default__get_replace_materialized_view_sql", "macro_sql": "{% macro default__get_replace_materialized_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.186612, "supported_languages": null}, "macro.dbt.refresh_materialized_view": {"name": "refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.refresh_materialized_view", "macro_sql": "{% macro refresh_materialized_view(relation) %}\n {{- log('Applying REFRESH to: ' ~ relation) -}}\n {{- adapter.dispatch('refresh_materialized_view', 'dbt')(relation) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__refresh_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.186767, "supported_languages": null}, "macro.dbt.default__refresh_materialized_view": {"name": "default__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.default__refresh_materialized_view", "macro_sql": "{% macro default__refresh_materialized_view(relation) %}\n {{ exceptions.raise_compiler_error(\"`refresh_materialized_view` has not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.186836, "supported_languages": null}, "macro.dbt.get_rename_materialized_view_sql": {"name": "get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.get_rename_materialized_view_sql", "macro_sql": "{% macro get_rename_materialized_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_rename_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.18696, "supported_languages": null}, "macro.dbt.default__get_rename_materialized_view_sql": {"name": "default__get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.default__get_rename_materialized_view_sql", "macro_sql": "{% macro default__get_rename_materialized_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1871092, "supported_languages": null}, "macro.dbt.get_alter_materialized_view_as_sql": {"name": "get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_alter_materialized_view_as_sql", "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {{- adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n ) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.187374, "supported_languages": null}, "macro.dbt.default__get_alter_materialized_view_as_sql": {"name": "default__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_alter_materialized_view_as_sql", "macro_sql": "{% macro default__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1875808, "supported_languages": null}, "macro.dbt.get_materialized_view_configuration_changes": {"name": "get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_materialized_view_configuration_changes", "macro_sql": "{% macro get_materialized_view_configuration_changes(existing_relation, new_config) %}\n /* {#\n It's recommended that configuration changes be formatted as follows:\n {\"\": [{\"action\": \"\", \"context\": ...}]}\n\n For example:\n {\n \"indexes\": [\n {\"action\": \"drop\", \"context\": \"index_abc\"},\n {\"action\": \"create\", \"context\": {\"columns\": [\"column_1\", \"column_2\"], \"type\": \"hash\", \"unique\": True}},\n ],\n }\n\n Either way, `get_materialized_view_configuration_changes` needs to align with `get_alter_materialized_view_as_sql`.\n #} */\n {{- log('Determining configuration changes on: ' ~ existing_relation) -}}\n {%- do return(adapter.dispatch('get_materialized_view_configuration_changes', 'dbt')(existing_relation, new_config)) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_materialized_view_configuration_changes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1877382, "supported_languages": null}, "macro.dbt.default__get_materialized_view_configuration_changes": {"name": "default__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_materialized_view_configuration_changes", "macro_sql": "{% macro default__get_materialized_view_configuration_changes(existing_relation, new_config) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1878111, "supported_languages": null}, "macro.dbt.get_create_materialized_view_as_sql": {"name": "get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.get_create_materialized_view_as_sql", "macro_sql": "{% macro get_create_materialized_view_as_sql(relation, sql) -%}\n {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.187952, "supported_languages": null}, "macro.dbt.default__get_create_materialized_view_as_sql": {"name": "default__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.default__get_create_materialized_view_as_sql", "macro_sql": "{% macro default__get_create_materialized_view_as_sql(relation, sql) -%}\n {{ exceptions.raise_compiler_error(\n \"`get_create_materialized_view_as_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.188022, "supported_languages": null}, "macro.dbt.get_table_columns_and_constraints": {"name": "get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_table_columns_and_constraints", "macro_sql": "{%- macro get_table_columns_and_constraints() -%}\n {{ adapter.dispatch('get_table_columns_and_constraints', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.18837, "supported_languages": null}, "macro.dbt.default__get_table_columns_and_constraints": {"name": "default__get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_table_columns_and_constraints", "macro_sql": "{% macro default__get_table_columns_and_constraints() -%}\n {{ return(table_columns_and_constraints()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.188427, "supported_languages": null}, "macro.dbt.table_columns_and_constraints": {"name": "table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.table_columns_and_constraints", "macro_sql": "{% macro table_columns_and_constraints() %}\n {# loop through user_provided_columns to create DDL with data types and constraints #}\n {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%}\n {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%}\n (\n {% for c in raw_column_constraints -%}\n {{ c }}{{ \",\" if not loop.last or raw_model_constraints }}\n {% endfor %}\n {% for c in raw_model_constraints -%}\n {{ c }}{{ \",\" if not loop.last }}\n {% endfor -%}\n )\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.188666, "supported_languages": null}, "macro.dbt.get_assert_columns_equivalent": {"name": "get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_assert_columns_equivalent", "macro_sql": "\n\n{%- macro get_assert_columns_equivalent(sql) -%}\n {{ adapter.dispatch('get_assert_columns_equivalent', 'dbt')(sql) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1887438, "supported_languages": null}, "macro.dbt.default__get_assert_columns_equivalent": {"name": "default__get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_assert_columns_equivalent", "macro_sql": "{% macro default__get_assert_columns_equivalent(sql) -%}\n {{ return(assert_columns_equivalent(sql)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.188807, "supported_languages": null}, "macro.dbt.assert_columns_equivalent": {"name": "assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.assert_columns_equivalent", "macro_sql": "{% macro assert_columns_equivalent(sql) %}\n\n {#-- First ensure the user has defined 'columns' in yaml specification --#}\n {%- set user_defined_columns = model['columns'] -%}\n {%- if not user_defined_columns -%}\n {{ exceptions.raise_contract_error([], []) }}\n {%- endif -%}\n\n {#-- Obtain the column schema provided by sql file. #}\n {%- set sql_file_provided_columns = get_column_schema_from_query(sql, config.get('sql_header', none)) -%}\n {#--Obtain the column schema provided by the schema file by generating an 'empty schema' query from the model's columns. #}\n {%- set schema_file_provided_columns = get_column_schema_from_query(get_empty_schema_sql(user_defined_columns)) -%}\n\n {#-- create dictionaries with name and formatted data type and strings for exception #}\n {%- set sql_columns = format_columns(sql_file_provided_columns) -%}\n {%- set yaml_columns = format_columns(schema_file_provided_columns) -%}\n\n {%- if sql_columns|length != yaml_columns|length -%}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n\n {%- for sql_col in sql_columns -%}\n {%- set yaml_col = [] -%}\n {%- for this_col in yaml_columns -%}\n {%- if this_col['name'] == sql_col['name'] -%}\n {%- do yaml_col.append(this_col) -%}\n {%- break -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if not yaml_col -%}\n {#-- Column with name not found in yaml #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- if sql_col['formatted'] != yaml_col[0]['formatted'] -%}\n {#-- Column data types don't match #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- endfor -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_column_schema_from_query", "macro.dbt.get_empty_schema_sql", "macro.dbt.format_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.189343, "supported_languages": null}, "macro.dbt.format_columns": {"name": "format_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.format_columns", "macro_sql": "{% macro format_columns(columns) %}\n {% set formatted_columns = [] %}\n {% for column in columns %}\n {%- set formatted_column = adapter.dispatch('format_column', 'dbt')(column) -%}\n {%- do formatted_columns.append(formatted_column) -%}\n {% endfor %}\n {{ return(formatted_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__format_column"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1895099, "supported_languages": null}, "macro.dbt.default__format_column": {"name": "default__format_column", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__format_column", "macro_sql": "{% macro default__format_column(column) -%}\n {% set data_type = column.dtype %}\n {% set formatted = column.column.lower() ~ \" \" ~ data_type %}\n {{ return({'name': column.name, 'data_type': data_type, 'formatted': formatted}) }}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.189655, "supported_languages": null}, "macro.dbt.drop_table": {"name": "drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.drop_table", "macro_sql": "{% macro drop_table(relation) -%}\n {{- adapter.dispatch('drop_table', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.189782, "supported_languages": null}, "macro.dbt.default__drop_table": {"name": "default__drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.default__drop_table", "macro_sql": "{% macro default__drop_table(relation) -%}\n drop table if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.189846, "supported_languages": null}, "macro.dbt.get_replace_table_sql": {"name": "get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.get_replace_table_sql", "macro_sql": "{% macro get_replace_table_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_table_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_table_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.189981, "supported_languages": null}, "macro.dbt.default__get_replace_table_sql": {"name": "default__get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.default__get_replace_table_sql", "macro_sql": "{% macro default__get_replace_table_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.190054, "supported_languages": null}, "macro.dbt.get_rename_table_sql": {"name": "get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.get_rename_table_sql", "macro_sql": "{% macro get_rename_table_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_table_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_rename_table_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.190172, "supported_languages": null}, "macro.dbt.default__get_rename_table_sql": {"name": "default__get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.default__get_rename_table_sql", "macro_sql": "{% macro default__get_rename_table_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.190239, "supported_languages": null}, "macro.dbt.get_create_table_as_sql": {"name": "get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_create_table_as_sql", "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.190511, "supported_languages": null}, "macro.dbt.default__get_create_table_as_sql": {"name": "default__get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_create_table_as_sql", "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1907148, "supported_languages": null}, "macro.dbt.create_table_as": {"name": "create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {# backward compatibility for create_table_as that does not support language #}\n {% if language == \"sql\" %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code)}}\n {% else %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code, language) }}\n {% endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.190975, "supported_languages": null}, "macro.dbt.default__create_table_as": {"name": "default__create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced and (not temporary) %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_table_columns_and_constraints", "macro.dbt.get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1914492, "supported_languages": null}, "macro.dbt.default__get_column_names": {"name": "default__get_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_column_names", "macro_sql": "{% macro default__get_column_names() %}\n {#- loop through user_provided_columns to get column names -#}\n {%- set user_provided_columns = model['columns'] -%}\n {%- for i in user_provided_columns %}\n {%- set col = user_provided_columns[i] -%}\n {%- set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] -%}\n {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1917288, "supported_languages": null}, "macro.dbt.get_select_subquery": {"name": "get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_select_subquery", "macro_sql": "{% macro get_select_subquery(sql) %}\n {{ return(adapter.dispatch('get_select_subquery', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.191972, "supported_languages": null}, "macro.dbt.default__get_select_subquery": {"name": "default__get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_select_subquery", "macro_sql": "{% macro default__get_select_subquery(sql) %}\n select {{ adapter.dispatch('get_column_names', 'dbt')() }}\n from (\n {{ sql }}\n ) as model_subq\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1921458, "supported_languages": null}, "macro.dbt.drop_view": {"name": "drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.drop_view", "macro_sql": "{% macro drop_view(relation) -%}\n {{- adapter.dispatch('drop_view', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1923451, "supported_languages": null}, "macro.dbt.default__drop_view": {"name": "default__drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.default__drop_view", "macro_sql": "{% macro default__drop_view(relation) -%}\n drop view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.192408, "supported_languages": null}, "macro.dbt.get_replace_view_sql": {"name": "get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.get_replace_view_sql", "macro_sql": "{% macro get_replace_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.192696, "supported_languages": null}, "macro.dbt.default__get_replace_view_sql": {"name": "default__get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__get_replace_view_sql", "macro_sql": "{% macro default__get_replace_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.192768, "supported_languages": null}, "macro.dbt.create_or_replace_view": {"name": "create_or_replace_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.create_or_replace_view", "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.193269, "supported_languages": null}, "macro.dbt.handle_existing_table": {"name": "handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__handle_existing_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.193362, "supported_languages": null}, "macro.dbt.default__handle_existing_table": {"name": "default__handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation.render() ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1934772, "supported_languages": null}, "macro.dbt.get_rename_view_sql": {"name": "get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.get_rename_view_sql", "macro_sql": "{% macro get_rename_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_rename_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1936169, "supported_languages": null}, "macro.dbt.default__get_rename_view_sql": {"name": "default__get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.default__get_rename_view_sql", "macro_sql": "{% macro default__get_rename_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.193696, "supported_languages": null}, "macro.dbt.get_create_view_as_sql": {"name": "get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.get_create_view_as_sql", "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.193876, "supported_languages": null}, "macro.dbt.default__get_create_view_as_sql": {"name": "default__get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__get_create_view_as_sql", "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.193951, "supported_languages": null}, "macro.dbt.create_view_as": {"name": "create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.194039, "supported_languages": null}, "macro.dbt.default__create_view_as": {"name": "default__create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation.render() }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.194255, "supported_languages": null}, "macro.dbt.default__test_relationships": {"name": "default__test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/relationships.sql", "original_file_path": "macros/generic_test_sql/relationships.sql", "unique_id": "macro.dbt.default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.194412, "supported_languages": null}, "macro.dbt.default__test_not_null": {"name": "default__test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/not_null.sql", "original_file_path": "macros/generic_test_sql/not_null.sql", "unique_id": "macro.dbt.default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_store_failures"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.194565, "supported_languages": null}, "macro.dbt.default__test_unique": {"name": "default__test_unique", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/unique.sql", "original_file_path": "macros/generic_test_sql/unique.sql", "unique_id": "macro.dbt.default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.194686, "supported_languages": null}, "macro.dbt.default__test_accepted_values": {"name": "default__test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/accepted_values.sql", "original_file_path": "macros/generic_test_sql/accepted_values.sql", "unique_id": "macro.dbt.default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1949499, "supported_languages": null}, "macro.dbt.statement": {"name": "statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.statement", "macro_sql": "\n{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%}\n {%- if execute: -%}\n {%- set compiled_code = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime {} for node \"{}\"'.format(language, model['unique_id'])) }}\n {{ write(compiled_code) }}\n {%- endif -%}\n {%- if language == 'sql'-%}\n {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- elif language == 'python' -%}\n {%- set res = submit_python_job(model, compiled_code) -%}\n {#-- TODO: What should table be for python models? --#}\n {%- set table = None -%}\n {%- else -%}\n {% do exceptions.raise_compiler_error(\"statement macro didn't get supported language\") %}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1956532, "supported_languages": null}, "macro.dbt.noop_statement": {"name": "noop_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.195957, "supported_languages": null}, "macro.dbt.run_query": {"name": "run_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1961029, "supported_languages": null}, "macro.dbt.convert_datetime": {"name": "convert_datetime", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.196491, "supported_languages": null}, "macro.dbt.dates_in_range": {"name": "dates_in_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.convert_datetime"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.197057, "supported_languages": null}, "macro.dbt.partition_range": {"name": "partition_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.dates_in_range"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.197427, "supported_languages": null}, "macro.dbt.py_current_timestring": {"name": "py_current_timestring", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.197551, "supported_languages": null}, "macro.dbt.except": {"name": "except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.except", "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__except"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1976728, "supported_languages": null}, "macro.dbt.default__except": {"name": "default__except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.default__except", "macro_sql": "{% macro default__except() %}\n\n except\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.197716, "supported_languages": null}, "macro.dbt.get_intervals_between": {"name": "get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.get_intervals_between", "macro_sql": "{% macro get_intervals_between(start_date, end_date, datepart) -%}\n {{ return(adapter.dispatch('get_intervals_between', 'dbt')(start_date, end_date, datepart)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_intervals_between"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.197974, "supported_languages": null}, "macro.dbt.default__get_intervals_between": {"name": "default__get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__get_intervals_between", "macro_sql": "{% macro default__get_intervals_between(start_date, end_date, datepart) -%}\n {%- call statement('get_intervals_between', fetch_result=True) %}\n\n select {{ dbt.datediff(start_date, end_date, datepart) }}\n\n {%- endcall -%}\n\n {%- set value_list = load_result('get_intervals_between') -%}\n\n {%- if value_list and value_list['data'] -%}\n {%- set values = value_list['data'] | map(attribute=0) | list %}\n {{ return(values[0]) }}\n {%- else -%}\n {{ return(1) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.198269, "supported_languages": null}, "macro.dbt.date_spine": {"name": "date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.date_spine", "macro_sql": "{% macro date_spine(datepart, start_date, end_date) %}\n {{ return(adapter.dispatch('date_spine', 'dbt')(datepart, start_date, end_date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_spine"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.198399, "supported_languages": null}, "macro.dbt.default__date_spine": {"name": "default__date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__date_spine", "macro_sql": "{% macro default__date_spine(datepart, start_date, end_date) %}\n\n\n {# call as follows:\n\n date_spine(\n \"day\",\n \"to_date('01/01/2016', 'mm/dd/yyyy')\",\n \"dbt.dateadd(week, 1, current_date)\"\n ) #}\n\n\n with rawdata as (\n\n {{dbt.generate_series(\n dbt.get_intervals_between(start_date, end_date, datepart)\n )}}\n\n ),\n\n all_periods as (\n\n select (\n {{\n dbt.dateadd(\n datepart,\n \"row_number() over (order by 1) - 1\",\n start_date\n )\n }}\n ) as date_{{datepart}}\n from rawdata\n\n ),\n\n filtered as (\n\n select *\n from all_periods\n where date_{{datepart}} <= {{ end_date }}\n\n )\n\n select * from filtered\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.generate_series", "macro.dbt.get_intervals_between", "macro.dbt.dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.198598, "supported_languages": null}, "macro.dbt.date": {"name": "date", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt.date", "macro_sql": "{% macro date(year, month, day) %}\n {{ return(adapter.dispatch('date', 'dbt') (year, month, day)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__date"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.1988358, "supported_languages": null}, "macro.dbt.default__date": {"name": "default__date", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt.default__date", "macro_sql": "{% macro default__date(year, month, day) -%}\n {%- set dt = modules.datetime.date(year, month, day) -%}\n {%- set iso_8601_formatted_date = dt.strftime('%Y-%m-%d') -%}\n to_date('{{ iso_8601_formatted_date }}', 'YYYY-MM-DD')\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.198976, "supported_languages": null}, "macro.dbt.replace": {"name": "replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.replace", "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__replace"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.199136, "supported_languages": null}, "macro.dbt.default__replace": {"name": "default__replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.default__replace", "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.199224, "supported_languages": null}, "macro.dbt.concat": {"name": "concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.concat", "macro_sql": "{% macro concat(fields) -%}\n {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.19935, "supported_languages": null}, "macro.dbt.default__concat": {"name": "default__concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.default__concat", "macro_sql": "{% macro default__concat(fields) -%}\n {{ fields|join(' || ') }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.199418, "supported_languages": null}, "macro.dbt.get_powers_of_two": {"name": "get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.get_powers_of_two", "macro_sql": "{% macro get_powers_of_two(upper_bound) %}\n {{ return(adapter.dispatch('get_powers_of_two', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.199641, "supported_languages": null}, "macro.dbt.default__get_powers_of_two": {"name": "default__get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__get_powers_of_two", "macro_sql": "{% macro default__get_powers_of_two(upper_bound) %}\n\n {% if upper_bound <= 0 %}\n {{ exceptions.raise_compiler_error(\"upper bound must be positive\") }}\n {% endif %}\n\n {% for _ in range(1, 100) %}\n {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %}\n {% endfor %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.199845, "supported_languages": null}, "macro.dbt.generate_series": {"name": "generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.generate_series", "macro_sql": "{% macro generate_series(upper_bound) %}\n {{ return(adapter.dispatch('generate_series', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_series"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.199947, "supported_languages": null}, "macro.dbt.default__generate_series": {"name": "default__generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__generate_series", "macro_sql": "{% macro default__generate_series(upper_bound) %}\n\n {% set n = dbt.get_powers_of_two(upper_bound) %}\n\n with p as (\n select 0 as generated_number union all select 1\n ), unioned as (\n\n select\n\n {% for i in range(n) %}\n p{{i}}.generated_number * power(2, {{i}})\n {% if not loop.last %} + {% endif %}\n {% endfor %}\n + 1\n as generated_number\n\n from\n\n {% for i in range(n) %}\n p as p{{i}}\n {% if not loop.last %} cross join {% endif %}\n {% endfor %}\n\n )\n\n select *\n from unioned\n where generated_number <= {{upper_bound}}\n order by generated_number\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.200203, "supported_languages": null}, "macro.dbt.length": {"name": "length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.length", "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__length"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.200331, "supported_languages": null}, "macro.dbt.default__length": {"name": "default__length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.default__length", "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2003891, "supported_languages": null}, "macro.dbt.dateadd": {"name": "dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.dateadd", "macro_sql": "{% macro dateadd(datepart, interval, from_date_or_timestamp) %}\n {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.200618, "supported_languages": null}, "macro.dbt.default__dateadd": {"name": "default__dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.default__dateadd", "macro_sql": "{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n dateadd(\n {{ datepart }},\n {{ interval }},\n {{ from_date_or_timestamp }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.200736, "supported_languages": null}, "macro.dbt.intersect": {"name": "intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.intersect", "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__intersect"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2008681, "supported_languages": null}, "macro.dbt.default__intersect": {"name": "default__intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.default__intersect", "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.200911, "supported_languages": null}, "macro.dbt.escape_single_quotes": {"name": "escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.escape_single_quotes", "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__escape_single_quotes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2010481, "supported_languages": null}, "macro.dbt.default__escape_single_quotes": {"name": "default__escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.default__escape_single_quotes", "macro_sql": "{% macro default__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"''\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2011259, "supported_languages": null}, "macro.dbt.right": {"name": "right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.right", "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__right"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2012742, "supported_languages": null}, "macro.dbt.default__right": {"name": "default__right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.default__right", "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.201345, "supported_languages": null}, "macro.dbt.listagg": {"name": "listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.listagg", "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__listagg"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.201552, "supported_languages": null}, "macro.dbt.default__listagg": {"name": "default__listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.default__listagg", "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.201756, "supported_languages": null}, "macro.dbt.datediff": {"name": "datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.datediff", "macro_sql": "{% macro datediff(first_date, second_date, datepart) %}\n {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2019022, "supported_languages": null}, "macro.dbt.default__datediff": {"name": "default__datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.default__datediff", "macro_sql": "{% macro default__datediff(first_date, second_date, datepart) -%}\n\n datediff(\n {{ datepart }},\n {{ first_date }},\n {{ second_date }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2019808, "supported_languages": null}, "macro.dbt.safe_cast": {"name": "safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.safe_cast", "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.202115, "supported_languages": null}, "macro.dbt.default__safe_cast": {"name": "default__safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.default__safe_cast", "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.202183, "supported_languages": null}, "macro.dbt.equals": {"name": "equals", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/equals.sql", "original_file_path": "macros/utils/equals.sql", "unique_id": "macro.dbt.equals", "macro_sql": "{% macro equals(expr1, expr2) %}\n {{ return(adapter.dispatch('equals', 'dbt') (expr1, expr2)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__equals"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.20233, "supported_languages": null}, "macro.dbt.default__equals": {"name": "default__equals", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/equals.sql", "original_file_path": "macros/utils/equals.sql", "unique_id": "macro.dbt.default__equals", "macro_sql": "{% macro default__equals(expr1, expr2) -%}\n{%- if adapter.behavior.enable_truthy_nulls_equals_macro.no_warn %}\n case when (({{ expr1 }} = {{ expr2 }}) or ({{ expr1 }} is null and {{ expr2 }} is null))\n then 0\n else 1\n end = 0\n{%- else -%}\n ({{ expr1 }} = {{ expr2 }})\n{%- endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.202465, "supported_languages": null}, "macro.dbt.hash": {"name": "hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.hash", "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__hash"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.202585, "supported_languages": null}, "macro.dbt.default__hash": {"name": "default__hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.default__hash", "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.20266, "supported_languages": null}, "macro.dbt.cast_bool_to_text": {"name": "cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.cast_bool_to_text", "macro_sql": "{% macro cast_bool_to_text(field) %}\n {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast_bool_to_text"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.202775, "supported_languages": null}, "macro.dbt.default__cast_bool_to_text": {"name": "default__cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.default__cast_bool_to_text", "macro_sql": "{% macro default__cast_bool_to_text(field) %}\n cast({{ field }} as {{ api.Column.translate_type('string') }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.202849, "supported_languages": null}, "macro.dbt.cast": {"name": "cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast.sql", "original_file_path": "macros/utils/cast.sql", "unique_id": "macro.dbt.cast", "macro_sql": "{% macro cast(field, type) %}\n {{ return(adapter.dispatch('cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.202975, "supported_languages": null}, "macro.dbt.default__cast": {"name": "default__cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast.sql", "original_file_path": "macros/utils/cast.sql", "unique_id": "macro.dbt.default__cast", "macro_sql": "{% macro default__cast(field, type) %}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2030358, "supported_languages": null}, "macro.dbt.any_value": {"name": "any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.any_value", "macro_sql": "{% macro any_value(expression) -%}\n {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__any_value"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.203151, "supported_languages": null}, "macro.dbt.default__any_value": {"name": "default__any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.default__any_value", "macro_sql": "{% macro default__any_value(expression) -%}\n\n any_value({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.203209, "supported_languages": null}, "macro.dbt.position": {"name": "position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.position", "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__position"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2034051, "supported_languages": null}, "macro.dbt.default__position": {"name": "default__position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.default__position", "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2034712, "supported_languages": null}, "macro.dbt.string_literal": {"name": "string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.string_literal", "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2036622, "supported_languages": null}, "macro.dbt.default__string_literal": {"name": "default__string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.default__string_literal", "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2038422, "supported_languages": null}, "macro.dbt.type_string": {"name": "type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_string", "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_string"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.204191, "supported_languages": null}, "macro.dbt.default__type_string": {"name": "default__type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_string", "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.204264, "supported_languages": null}, "macro.dbt.type_timestamp": {"name": "type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_timestamp", "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2043588, "supported_languages": null}, "macro.dbt.default__type_timestamp": {"name": "default__type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_timestamp", "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.204451, "supported_languages": null}, "macro.dbt.type_float": {"name": "type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_float", "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_float"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.204534, "supported_languages": null}, "macro.dbt.default__type_float": {"name": "default__type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_float", "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.204611, "supported_languages": null}, "macro.dbt.type_numeric": {"name": "type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_numeric", "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_numeric"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.204698, "supported_languages": null}, "macro.dbt.default__type_numeric": {"name": "default__type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_numeric", "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.204826, "supported_languages": null}, "macro.dbt.type_bigint": {"name": "type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_bigint", "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_bigint"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2049239, "supported_languages": null}, "macro.dbt.default__type_bigint": {"name": "default__type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_bigint", "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.205009, "supported_languages": null}, "macro.dbt.type_int": {"name": "type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_int", "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_int"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2051, "supported_languages": null}, "macro.dbt.default__type_int": {"name": "default__type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_int", "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2051861, "supported_languages": null}, "macro.dbt.type_boolean": {"name": "type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_boolean", "macro_sql": "\n\n{%- macro type_boolean() -%}\n {{ return(adapter.dispatch('type_boolean', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_boolean"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2052732, "supported_languages": null}, "macro.dbt.default__type_boolean": {"name": "default__type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_boolean", "macro_sql": "{%- macro default__type_boolean() -%}\n {{ return(api.Column.translate_type(\"boolean\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.205344, "supported_languages": null}, "macro.dbt.array_concat": {"name": "array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.array_concat", "macro_sql": "{% macro array_concat(array_1, array_2) -%}\n {{ return(adapter.dispatch('array_concat', 'dbt')(array_1, array_2)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.205499, "supported_languages": null}, "macro.dbt.default__array_concat": {"name": "default__array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.default__array_concat", "macro_sql": "{% macro default__array_concat(array_1, array_2) -%}\n array_cat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.205569, "supported_languages": null}, "macro.dbt.bool_or": {"name": "bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.bool_or", "macro_sql": "{% macro bool_or(expression) -%}\n {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__bool_or"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2057018, "supported_languages": null}, "macro.dbt.default__bool_or": {"name": "default__bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.default__bool_or", "macro_sql": "{% macro default__bool_or(expression) -%}\n\n bool_or({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.205774, "supported_languages": null}, "macro.dbt.last_day": {"name": "last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.last_day", "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.205923, "supported_languages": null}, "macro.dbt.default_last_day": {"name": "default_last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default_last_day", "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.206113, "supported_languages": null}, "macro.dbt.default__last_day": {"name": "default__last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default__last_day", "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.206187, "supported_languages": null}, "macro.dbt.split_part": {"name": "split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.split_part", "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__split_part"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.206372, "supported_languages": null}, "macro.dbt.default__split_part": {"name": "default__split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.default__split_part", "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.206465, "supported_languages": null}, "macro.dbt._split_part_negative": {"name": "_split_part_negative", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt._split_part_negative", "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 + {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.206569, "supported_languages": null}, "macro.dbt.date_trunc": {"name": "date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.date_trunc", "macro_sql": "{% macro date_trunc(datepart, date) -%}\n {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2067, "supported_languages": null}, "macro.dbt.default__date_trunc": {"name": "default__date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.default__date_trunc", "macro_sql": "{% macro default__date_trunc(datepart, date) -%}\n date_trunc('{{datepart}}', {{date}})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.206762, "supported_languages": null}, "macro.dbt.array_construct": {"name": "array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.array_construct", "macro_sql": "{% macro array_construct(inputs=[], data_type=api.Column.translate_type('integer')) -%}\n {{ return(adapter.dispatch('array_construct', 'dbt')(inputs, data_type)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.20694, "supported_languages": null}, "macro.dbt.default__array_construct": {"name": "default__array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.default__array_construct", "macro_sql": "{% macro default__array_construct(inputs, data_type) -%}\n {% if inputs|length > 0 %}\n array[ {{ inputs|join(' , ') }} ]\n {% else %}\n array[]::{{data_type}}[]\n {% endif %}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2070599, "supported_languages": null}, "macro.dbt.array_append": {"name": "array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.array_append", "macro_sql": "{% macro array_append(array, new_element) -%}\n {{ return(adapter.dispatch('array_append', 'dbt')(array, new_element)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_append"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.207207, "supported_languages": null}, "macro.dbt.default__array_append": {"name": "default__array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.default__array_append", "macro_sql": "{% macro default__array_append(array, new_element) -%}\n array_append({{ array }}, {{ new_element }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2072709, "supported_languages": null}, "macro.dbt.create_schema": {"name": "create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__create_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2074199, "supported_languages": null}, "macro.dbt.default__create_schema": {"name": "default__create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.207508, "supported_languages": null}, "macro.dbt.drop_schema": {"name": "drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__drop_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.207584, "supported_languages": null}, "macro.dbt.default__drop_schema": {"name": "default__drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.207673, "supported_languages": null}, "macro.dbt.current_timestamp": {"name": "current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp", "macro_sql": "{%- macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2078972, "supported_languages": null}, "macro.dbt.default__current_timestamp": {"name": "default__current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter ' + adapter.type()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.207967, "supported_languages": null}, "macro.dbt.snapshot_get_time": {"name": "snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.snapshot_get_time", "macro_sql": "\n\n{%- macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.208035, "supported_languages": null}, "macro.dbt.default__snapshot_get_time": {"name": "default__snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() %}\n {{ current_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.208087, "supported_languages": null}, "macro.dbt.get_snapshot_get_time_data_type": {"name": "get_snapshot_get_time_data_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.get_snapshot_get_time_data_type", "macro_sql": "{% macro get_snapshot_get_time_data_type() %}\n {% set snapshot_time = adapter.dispatch('snapshot_get_time', 'dbt')() %}\n {% set time_data_type_sql = 'select ' ~ snapshot_time ~ ' as dbt_snapshot_time' %}\n {% set snapshot_time_column_schema = get_column_schema_from_query(time_data_type_sql) %}\n {% set time_data_type = snapshot_time_column_schema[0].dtype %}\n {{ return(time_data_type or none) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.default__snapshot_get_time", "macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.208278, "supported_languages": null}, "macro.dbt.current_timestamp_backcompat": {"name": "current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_backcompat", "macro_sql": "{% macro current_timestamp_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2083762, "supported_languages": null}, "macro.dbt.default__current_timestamp_backcompat": {"name": "default__current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_backcompat", "macro_sql": "{% macro default__current_timestamp_backcompat() %}\n current_timestamp::timestamp\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2084138, "supported_languages": null}, "macro.dbt.current_timestamp_in_utc_backcompat": {"name": "current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_in_utc_backcompat", "macro_sql": "{% macro current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_in_utc_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__current_timestamp_in_utc_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.208488, "supported_languages": null}, "macro.dbt.default__current_timestamp_in_utc_backcompat": {"name": "default__current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro default__current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp_backcompat", "macro.dbt.default__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.20863, "supported_languages": null}, "macro.dbt.get_create_index_sql": {"name": "get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.208879, "supported_languages": null}, "macro.dbt.default__get_create_index_sql": {"name": "default__get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.209077, "supported_languages": null}, "macro.dbt.create_indexes": {"name": "create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.209198, "supported_languages": null}, "macro.dbt.default__create_indexes": {"name": "default__create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_create_index_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.209381, "supported_languages": null}, "macro.dbt.get_drop_index_sql": {"name": "get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_drop_index_sql", "macro_sql": "{% macro get_drop_index_sql(relation, index_name) -%}\n {{ adapter.dispatch('get_drop_index_sql', 'dbt')(relation, index_name) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_drop_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.209468, "supported_languages": null}, "macro.dbt.default__get_drop_index_sql": {"name": "default__get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_drop_index_sql", "macro_sql": "{% macro default__get_drop_index_sql(relation, index_name) -%}\n {{ exceptions.raise_compiler_error(\"`get_drop_index_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.209532, "supported_languages": null}, "macro.dbt.get_show_indexes_sql": {"name": "get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_show_indexes_sql", "macro_sql": "{% macro get_show_indexes_sql(relation) -%}\n {{ adapter.dispatch('get_show_indexes_sql', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_show_indexes_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.209607, "supported_languages": null}, "macro.dbt.default__get_show_indexes_sql": {"name": "default__get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_show_indexes_sql", "macro_sql": "{% macro default__get_show_indexes_sql(relation) -%}\n {{ exceptions.raise_compiler_error(\"`get_show_indexes_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.209669, "supported_languages": null}, "macro.dbt.make_intermediate_relation": {"name": "make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_intermediate_relation", "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__make_intermediate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.209991, "supported_languages": null}, "macro.dbt.default__make_intermediate_relation": {"name": "default__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_intermediate_relation", "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.210067, "supported_languages": null}, "macro.dbt.make_temp_relation": {"name": "make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {#-- This ensures microbatch batches get unique temp relations to avoid clobbering --#}\n {% if suffix == '__dbt_tmp' and model.batch %}\n {% set suffix = suffix ~ '_' ~ model.batch.id %}\n {% endif %}\n\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2102349, "supported_languages": null}, "macro.dbt.default__make_temp_relation": {"name": "default__make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.21037, "supported_languages": null}, "macro.dbt.make_backup_relation": {"name": "make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_backup_relation", "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_backup_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.210477, "supported_languages": null}, "macro.dbt.default__make_backup_relation": {"name": "default__make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_backup_relation", "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.210685, "supported_languages": null}, "macro.dbt.truncate_relation": {"name": "truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__truncate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.210786, "supported_languages": null}, "macro.dbt.default__truncate_relation": {"name": "default__truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation.render() }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.210868, "supported_languages": null}, "macro.dbt.get_or_create_relation": {"name": "get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_or_create_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.210974, "supported_languages": null}, "macro.dbt.default__get_or_create_relation": {"name": "default__get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__get_or_create_relation", "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.211217, "supported_languages": null}, "macro.dbt.load_cached_relation": {"name": "load_cached_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_cached_relation", "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.211325, "supported_languages": null}, "macro.dbt.load_relation": {"name": "load_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.211393, "supported_languages": null}, "macro.dbt.collect_freshness": {"name": "collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__collect_freshness"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.211605, "supported_languages": null}, "macro.dbt.default__collect_freshness": {"name": "default__collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2118661, "supported_languages": null}, "macro.dbt.collect_freshness_custom_sql": {"name": "collect_freshness_custom_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.collect_freshness_custom_sql", "macro_sql": "{% macro collect_freshness_custom_sql(source, loaded_at_query) %}\n {{ return(adapter.dispatch('collect_freshness_custom_sql', 'dbt')(source, loaded_at_query))}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__collect_freshness_custom_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.211964, "supported_languages": null}, "macro.dbt.default__collect_freshness_custom_sql": {"name": "default__collect_freshness_custom_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.default__collect_freshness_custom_sql", "macro_sql": "{% macro default__collect_freshness_custom_sql(source, loaded_at_query) %}\n {% call statement('collect_freshness_custom_sql', fetch_result=True, auto_begin=False) -%}\n with source_query as (\n {{ loaded_at_query }}\n )\n select\n (select * from source_query) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n {% endcall %}\n {{ return(load_result('collect_freshness_custom_sql')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.212112, "supported_languages": null}, "macro.dbt.validate_sql": {"name": "validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.validate_sql", "macro_sql": "{% macro validate_sql(sql) -%}\n {{ return(adapter.dispatch('validate_sql', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__validate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.212257, "supported_languages": null}, "macro.dbt.default__validate_sql": {"name": "default__validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.default__validate_sql", "macro_sql": "{% macro default__validate_sql(sql) -%}\n {% call statement('validate_sql') -%}\n explain {{ sql }}\n {% endcall %}\n {{ return(load_result('validate_sql')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.212362, "supported_languages": null}, "macro.dbt.copy_grants": {"name": "copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.copy_grants", "macro_sql": "{% macro copy_grants() %}\n {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.213102, "supported_languages": null}, "macro.dbt.default__copy_grants": {"name": "default__copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__copy_grants", "macro_sql": "{% macro default__copy_grants() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.213182, "supported_languages": null}, "macro.dbt.support_multiple_grantees_per_dcl_statement": {"name": "support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement", "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2132711, "supported_languages": null}, "macro.dbt.default__support_multiple_grantees_per_dcl_statement": {"name": "default__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2133272, "supported_languages": null}, "macro.dbt.should_revoke": {"name": "should_revoke", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.should_revoke", "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.213482, "supported_languages": null}, "macro.dbt.get_show_grant_sql": {"name": "get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_show_grant_sql", "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_show_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.213568, "supported_languages": null}, "macro.dbt.default__get_show_grant_sql": {"name": "default__get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_show_grant_sql", "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.213629, "supported_languages": null}, "macro.dbt.get_grant_sql": {"name": "get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_grant_sql", "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2137332, "supported_languages": null}, "macro.dbt.default__get_grant_sql": {"name": "default__get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_grant_sql", "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation.render() }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.21383, "supported_languages": null}, "macro.dbt.get_revoke_sql": {"name": "get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_revoke_sql", "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_revoke_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.213931, "supported_languages": null}, "macro.dbt.default__get_revoke_sql": {"name": "default__get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_revoke_sql", "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation.render() }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.214027, "supported_languages": null}, "macro.dbt.get_dcl_statement_list": {"name": "get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_dcl_statement_list", "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_dcl_statement_list"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.214132, "supported_languages": null}, "macro.dbt.default__get_dcl_statement_list": {"name": "default__get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_dcl_statement_list", "macro_sql": "\n\n{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}\n {#\n -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.\n -- Depending on whether this database supports multiple grantees per statement, pass in the list of\n -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.\n -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`\n #}\n {%- set dcl_statements = [] -%}\n {%- for privilege, grantees in grant_config.items() %}\n {%- if support_multiple_grantees_per_dcl_statement() and grantees -%}\n {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}\n {%- do dcl_statements.append(dcl) -%}\n {%- else -%}\n {%- for grantee in grantees -%}\n {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}\n {%- do dcl_statements.append(dcl) -%}\n {% endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {{ return(dcl_statements) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2144508, "supported_languages": null}, "macro.dbt.call_dcl_statements": {"name": "call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.call_dcl_statements", "macro_sql": "{% macro call_dcl_statements(dcl_statement_list) %}\n {{ return(adapter.dispatch(\"call_dcl_statements\", \"dbt\")(dcl_statement_list)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.214543, "supported_languages": null}, "macro.dbt.default__call_dcl_statements": {"name": "default__call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__call_dcl_statements", "macro_sql": "{% macro default__call_dcl_statements(dcl_statement_list) %}\n {#\n -- By default, supply all grant + revoke statements in a single semicolon-separated block,\n -- so that they're all processed together.\n\n -- Some databases do not support this. Those adapters will need to override this macro\n -- to run each statement individually.\n #}\n {% call statement('grants') %}\n {% for dcl_statement in dcl_statement_list %}\n {{ dcl_statement }};\n {% endfor %}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2146711, "supported_languages": null}, "macro.dbt.apply_grants": {"name": "apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.apply_grants", "macro_sql": "{% macro apply_grants(relation, grant_config, should_revoke) %}\n {{ return(adapter.dispatch(\"apply_grants\", \"dbt\")(relation, grant_config, should_revoke)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.214787, "supported_languages": null}, "macro.dbt.default__apply_grants": {"name": "default__apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__apply_grants", "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation.render() ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.get_show_grant_sql", "macro.dbt.get_dcl_statement_list", "macro.dbt.call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.21541, "supported_languages": null}, "macro.dbt.get_show_sql": {"name": "get_show_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_show_sql", "macro_sql": "{% macro get_show_sql(compiled_code, sql_header, limit) -%}\n {%- if sql_header is not none -%}\n {{ sql_header }}\n {%- endif %}\n {{ get_limit_subquery_sql(compiled_code, limit) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_limit_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2156138, "supported_languages": null}, "macro.dbt.get_limit_subquery_sql": {"name": "get_limit_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_limit_subquery_sql", "macro_sql": "\n{%- macro get_limit_subquery_sql(sql, limit) -%}\n {{ adapter.dispatch('get_limit_sql', 'dbt')(sql, limit) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_limit_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2157152, "supported_languages": null}, "macro.dbt.default__get_limit_sql": {"name": "default__get_limit_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.default__get_limit_sql", "macro_sql": "{% macro default__get_limit_sql(sql, limit) %}\n {{ sql }}\n {% if limit is not none %}\n limit {{ limit }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.215811, "supported_languages": null}, "macro.dbt.alter_column_comment": {"name": "alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2161121, "supported_languages": null}, "macro.dbt.default__alter_column_comment": {"name": "default__alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.216202, "supported_languages": null}, "macro.dbt.alter_relation_comment": {"name": "alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__alter_relation_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.216311, "supported_languages": null}, "macro.dbt.default__alter_relation_comment": {"name": "default__alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.216398, "supported_languages": null}, "macro.dbt.persist_docs": {"name": "persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.216532, "supported_languages": null}, "macro.dbt.validate_doc_columns": {"name": "validate_doc_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.validate_doc_columns", "macro_sql": "{% macro validate_doc_columns(relation, column_dict, existing_column_names) %}\n {% set existing_lower = existing_column_names | map(\"lower\") | list %}\n {% set missing = [] %}\n {% for col_name in column_dict %}\n {% if col_name | lower not in existing_lower %}\n {% do missing.append(col_name) %}\n {% endif %}\n {% endfor %}\n {% if missing | length > 0 %}\n {{ exceptions.warn(\"In relation \" ~ relation.render() ~ \": The following columns are specified in the schema but are not present in the database: \" ~ missing | join(\", \")) }}\n {% endif %}\n {% set filtered = {} %}\n {% for col_name in column_dict if col_name | lower in existing_lower %}\n {% do filtered.update({col_name: column_dict[col_name]}) %}\n {% endfor %}\n {{ return(filtered) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.216981, "supported_languages": null}, "macro.dbt.default__persist_docs": {"name": "default__persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% set existing_columns = adapter.get_columns_in_relation(relation) | map(attribute=\"name\") | list %}\n {% set filtered_columns = validate_doc_columns(relation, model.columns, existing_columns) %}\n {% set alter_comment_sql = alter_column_comment(relation, filtered_columns) %}\n {% if alter_comment_sql and alter_comment_sql | trim | length > 0 %}\n {% do run_query(alter_comment_sql) %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.validate_doc_columns", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2174249, "supported_languages": null}, "macro.dbt.get_catalog_relations": {"name": "get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog_relations", "macro_sql": "{% macro get_catalog_relations(information_schema, relations) -%}\n {{ return(adapter.dispatch('get_catalog_relations', 'dbt')(information_schema, relations)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2178628, "supported_languages": null}, "macro.dbt.default__get_catalog_relations": {"name": "default__get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog_relations", "macro_sql": "{% macro default__get_catalog_relations(information_schema, relations) -%}\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog_relations not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.217987, "supported_languages": null}, "macro.dbt.get_catalog": {"name": "get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.218086, "supported_languages": null}, "macro.dbt.default__get_catalog": {"name": "default__get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.218208, "supported_languages": null}, "macro.dbt.information_schema_name": {"name": "information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__information_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.218299, "supported_languages": null}, "macro.dbt.default__information_schema_name": {"name": "default__information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2183769, "supported_languages": null}, "macro.dbt.list_schemas": {"name": "list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__list_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.218462, "supported_languages": null}, "macro.dbt.default__list_schemas": {"name": "default__list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.information_schema_name", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2185779, "supported_languages": null}, "macro.dbt.check_schema_exists": {"name": "check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__check_schema_exists"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.218684, "supported_languages": null}, "macro.dbt.default__check_schema_exists": {"name": "default__check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.replace", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.218834, "supported_languages": null}, "macro.dbt.list_relations_without_caching": {"name": "list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.218924, "supported_languages": null}, "macro.dbt.default__list_relations_without_caching": {"name": "default__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.219004, "supported_languages": null}, "macro.dbt.get_catalog_for_single_relation": {"name": "get_catalog_for_single_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog_for_single_relation", "macro_sql": "{% macro get_catalog_for_single_relation(relation) %}\n {{ return(adapter.dispatch('get_catalog_for_single_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_catalog_for_single_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2191591, "supported_languages": null}, "macro.dbt.default__get_catalog_for_single_relation": {"name": "default__get_catalog_for_single_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog_for_single_relation", "macro_sql": "{% macro default__get_catalog_for_single_relation(relation) %}\n {{ exceptions.raise_not_implemented(\n 'get_catalog_for_single_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2192378, "supported_languages": null}, "macro.dbt.get_relations": {"name": "get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relations", "macro_sql": "{% macro get_relations() %}\n {{ return(adapter.dispatch('get_relations', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.219313, "supported_languages": null}, "macro.dbt.default__get_relations": {"name": "default__get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relations", "macro_sql": "{% macro default__get_relations() %}\n {{ exceptions.raise_not_implemented(\n 'get_relations macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.219388, "supported_languages": null}, "macro.dbt.get_relation_last_modified": {"name": "get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relation_last_modified", "macro_sql": "{% macro get_relation_last_modified(information_schema, relations) %}\n {{ return(adapter.dispatch('get_relation_last_modified', 'dbt')(information_schema, relations)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_relation_last_modified"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.21949, "supported_languages": null}, "macro.dbt.default__get_relation_last_modified": {"name": "default__get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relation_last_modified", "macro_sql": "{% macro default__get_relation_last_modified(information_schema, relations) %}\n {{ exceptions.raise_not_implemented(\n 'get_relation_last_modified macro not implemented for adapter ' + adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2195961, "supported_languages": null}, "macro.dbt.get_columns_in_relation": {"name": "get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.220182, "supported_languages": null}, "macro.dbt.default__get_columns_in_relation": {"name": "default__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.220267, "supported_languages": null}, "macro.dbt.sql_convert_columns_in_relation": {"name": "sql_convert_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.220414, "supported_languages": null}, "macro.dbt.get_list_of_column_names": {"name": "get_list_of_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_list_of_column_names", "macro_sql": "\n\n{%- macro get_list_of_column_names(columns) -%}\n {% set col_names = [] %}\n {% for col in columns %}\n {% do col_names.append(col.name) %}\n {% endfor %}\n {{ return(col_names) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2205608, "supported_languages": null}, "macro.dbt.get_empty_subquery_sql": {"name": "get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_subquery_sql", "macro_sql": "{% macro get_empty_subquery_sql(select_sql, select_sql_header=none) -%}\n {{ return(adapter.dispatch('get_empty_subquery_sql', 'dbt')(select_sql, select_sql_header)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.220671, "supported_languages": null}, "macro.dbt.default__get_empty_subquery_sql": {"name": "default__get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_subquery_sql", "macro_sql": "{% macro default__get_empty_subquery_sql(select_sql, select_sql_header=none) %}\n {%- if select_sql_header is not none -%}\n {{ select_sql_header }}\n {%- endif -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.220783, "supported_languages": null}, "macro.dbt.get_empty_schema_sql": {"name": "get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_schema_sql", "macro_sql": "{% macro get_empty_schema_sql(columns) -%}\n {{ return(adapter.dispatch('get_empty_schema_sql', 'dbt')(columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_schema_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.220872, "supported_languages": null}, "macro.dbt.default__get_empty_schema_sql": {"name": "default__get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_schema_sql", "macro_sql": "{% macro default__get_empty_schema_sql(columns) %}\n {%- set col_err = [] -%}\n {%- set col_naked_numeric = [] -%}\n select\n {% for i in columns %}\n {%- set col = columns[i] -%}\n {%- if col['data_type'] is not defined -%}\n {%- do col_err.append(col['name']) -%}\n {#-- If this column's type is just 'numeric' then it is missing precision/scale, raise a warning --#}\n {%- elif col['data_type'].strip().lower() in ('numeric', 'decimal', 'number') -%}\n {%- do col_naked_numeric.append(col['name']) -%}\n {%- endif -%}\n {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %}\n {{ cast('null', col['data_type']) }} as {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n {%- if (col_err | length) > 0 -%}\n {{ exceptions.column_type_missing(column_names=col_err) }}\n {%- elif (col_naked_numeric | length) > 0 -%}\n {{ exceptions.warn(\"Detected columns with numeric type and unspecified precision/scale, this can lead to unintended rounding: \" ~ col_naked_numeric ~ \"`\") }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2214458, "supported_languages": null}, "macro.dbt.get_column_schema_from_query": {"name": "get_column_schema_from_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_column_schema_from_query", "macro_sql": "{% macro get_column_schema_from_query(select_sql, select_sql_header=none) -%}\n {% set columns = [] %}\n {# -- Using an 'empty subquery' here to get the same schema as the given select_sql statement, without necessitating a data scan.#}\n {% set sql = get_empty_subquery_sql(select_sql, select_sql_header) %}\n {% set column_schema = adapter.get_column_schema_from_query(sql) %}\n {{ return(column_schema) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2216241, "supported_languages": null}, "macro.dbt.get_columns_in_query": {"name": "get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.22172, "supported_languages": null}, "macro.dbt.default__get_columns_in_query": {"name": "default__get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n {{ get_empty_subquery_sql(select_sql) }}\n {% endcall %}\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.221895, "supported_languages": null}, "macro.dbt.alter_column_type": {"name": "alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.222016, "supported_languages": null}, "macro.dbt.default__alter_column_type": {"name": "default__alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation.render() }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation.render() }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation.render() }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation.render() }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.222333, "supported_languages": null}, "macro.dbt.alter_relation_add_remove_columns": {"name": "alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_relation_add_remove_columns", "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_relation_add_remove_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2224672, "supported_languages": null}, "macro.dbt.default__alter_relation_add_remove_columns": {"name": "default__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_relation_add_remove_columns", "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation.render() }}\n\n {% for column in add_columns %}\n add column {{ column.quoted }} {{ column.expanded_data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.quoted }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.222859, "supported_languages": null}, "macro.dbt.get_fixture_sql": {"name": "get_fixture_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.get_fixture_sql", "macro_sql": "{% macro get_fixture_sql(rows, column_name_to_data_types) %}\n-- Fixture for {{ model.name }}\n{% set default_row = {} %}\n\n{%- if not column_name_to_data_types -%}\n{#-- Use defer_relation IFF it is available in the manifest and 'this' is missing from the database --#}\n{%- set this_or_defer_relation = defer_relation if (defer_relation and not load_relation(this)) else this -%}\n{%- set columns_in_relation = adapter.get_columns_in_relation(this_or_defer_relation) -%}\n\n{%- set column_name_to_data_types = {} -%}\n{%- set column_name_to_quoted = {} -%}\n{%- for column in columns_in_relation -%}\n\n{#-- This needs to be a case-insensitive comparison --#}\n{%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n{%- do column_name_to_quoted.update({column.name|lower: column.quoted}) -%}\n{%- endfor -%}\n{%- endif -%}\n\n{%- if not column_name_to_data_types -%}\n {{ exceptions.raise_compiler_error(\"Not able to get columns for unit test '\" ~ model.name ~ \"' from relation \" ~ this ~ \" because the relation doesn't exist\") }}\n{%- endif -%}\n\n{%- for column_name, column_type in column_name_to_data_types.items() -%}\n {%- do default_row.update({column_name: (safe_cast(\"null\", column_type) | trim )}) -%}\n{%- endfor -%}\n\n{{ validate_fixture_rows(rows, row_number) }}\n\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\n{%- set default_row_copy = default_row.copy() -%}\n{%- do default_row_copy.update(formatted_row) -%}\nselect\n{%- for column_name, column_value in default_row_copy.items() %} {{ column_value }} as {{ column_name_to_quoted[column_name] }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n\n{%- if (rows | length) == 0 -%}\n select\n {%- for column_name, column_value in default_row.items() %} {{ column_value }} as {{ column_name_to_quoted[column_name] }}{% if not loop.last -%},{%- endif %}\n {%- endfor %}\n limit 0\n{%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_relation", "macro.dbt.safe_cast", "macro.dbt.validate_fixture_rows", "macro.dbt.format_row"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.224936, "supported_languages": null}, "macro.dbt.get_expected_sql": {"name": "get_expected_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.get_expected_sql", "macro_sql": "{% macro get_expected_sql(rows, column_name_to_data_types, column_name_to_quoted) %}\n\n{%- if (rows | length) == 0 -%}\n select * from dbt_internal_unit_test_actual\n limit 0\n{%- else -%}\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\nselect\n{%- for column_name, column_value in formatted_row.items() %} {{ column_value }} as {{ column_name_to_quoted[column_name] }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n{%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.format_row"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.225257, "supported_languages": null}, "macro.dbt.format_row": {"name": "format_row", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.format_row", "macro_sql": "\n\n{%- macro format_row(row, column_name_to_data_types) -%}\n {#-- generate case-insensitive formatted row --#}\n {% set formatted_row = {} %}\n {%- for column_name, column_value in row.items() -%}\n {% set column_name = column_name|lower %}\n\n {%- if column_name not in column_name_to_data_types %}\n {#-- if user-provided row contains column name that relation does not contain, raise an error --#}\n {% set fixture_name = \"expected output\" if model.resource_type == 'unit_test' else (\"'\" ~ model.name ~ \"'\") %}\n {{ exceptions.raise_compiler_error(\n \"Invalid column name: '\" ~ column_name ~ \"' in unit test fixture for \" ~ fixture_name ~ \".\"\n \"\\nAccepted columns for \" ~ fixture_name ~ \" are: \" ~ (column_name_to_data_types.keys()|list)\n ) }}\n {%- endif -%}\n\n {%- set column_type = column_name_to_data_types[column_name] %}\n\n {#-- For string fixture values, strip varchar length to prevent silent truncation (GH-11974) --#}\n {%- if column_value is string and 'varying' in column_type -%}\n {%- set column_type = column_type.split('(')[0] -%}\n {%- endif -%}\n\n {#-- sanitize column_value: wrap yaml strings in quotes, apply cast --#}\n {%- set column_value_clean = column_value -%}\n {%- if column_value is string -%}\n {%- set column_value_clean = dbt.string_literal(dbt.escape_single_quotes(column_value)) -%}\n {%- elif column_value is none -%}\n {%- set column_value_clean = 'null' -%}\n {%- endif -%}\n\n {%- set row_update = {column_name: safe_cast(column_value_clean, column_type) } -%}\n {%- do formatted_row.update(row_update) -%}\n {%- endfor -%}\n {{ return(formatted_row) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.string_literal", "macro.dbt.escape_single_quotes", "macro.dbt.safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.226022, "supported_languages": null}, "macro.dbt.validate_fixture_rows": {"name": "validate_fixture_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.validate_fixture_rows", "macro_sql": "{%- macro validate_fixture_rows(rows, row_number) -%}\n {{ return(adapter.dispatch('validate_fixture_rows', 'dbt')(rows, row_number)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__validate_fixture_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.226166, "supported_languages": null}, "macro.dbt.default__validate_fixture_rows": {"name": "default__validate_fixture_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.default__validate_fixture_rows", "macro_sql": "{%- macro default__validate_fixture_rows(rows, row_number) -%}\n {# This is an abstract method for adapter overrides as needed #}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.226222, "supported_languages": null}, "macro.dbt.resolve_model_name": {"name": "resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.resolve_model_name", "macro_sql": "{% macro resolve_model_name(input_model_name) %}\n {{ return(adapter.dispatch('resolve_model_name', 'dbt')(input_model_name)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.226644, "supported_languages": null}, "macro.dbt.default__resolve_model_name": {"name": "default__resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.default__resolve_model_name", "macro_sql": "\n\n{%- macro default__resolve_model_name(input_model_name) -%}\n {{ input_model_name | string | replace('\"', '\\\"') }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2267241, "supported_languages": null}, "macro.dbt.build_ref_function": {"name": "build_ref_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_ref_function", "macro_sql": "{% macro build_ref_function(model) %}\n\n {%- set ref_dict = {} -%}\n {%- for _ref in model.refs -%}\n {% set _ref_args = [_ref.get('package'), _ref['name']] if _ref.get('package') else [_ref['name'],] %}\n {%- set resolved = ref(*_ref_args, v=_ref.get('version')) -%}\n\n {#\n We want to get the string of the returned relation by calling .render() in order to skip sample/empty\n mode rendering logic. However, people override the default ref macro, and often return a string instead\n of a relation (like the ref macro does by default). Thus, to make sure we dont blow things up, we have\n to ensure the resolved relation has a .render() method.\n #}\n {%- if resolved.render is defined and resolved.render is callable -%}\n {%- set resolved = resolved.render() -%}\n {%- endif -%}\n\n {%- if _ref.get('version') -%}\n {% do _ref_args.extend([\"v\" ~ _ref['version']]) %}\n {%- endif -%}\n {%- do ref_dict.update({_ref_args | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef ref(*args, **kwargs):\n refs = {{ ref_dict | tojson }}\n key = '.'.join(args)\n version = kwargs.get(\"v\") or kwargs.get(\"version\")\n if version:\n key += f\".v{version}\"\n dbt_load_df_function = kwargs.get(\"dbt_load_df_function\")\n return dbt_load_df_function(refs[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.22717, "supported_languages": null}, "macro.dbt.build_source_function": {"name": "build_source_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_source_function", "macro_sql": "{% macro build_source_function(model) %}\n\n {%- set source_dict = {} -%}\n {%- for _source in model.sources -%}\n {%- set resolved = source(*_source) -%}\n {%- do source_dict.update({_source | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef source(*args, dbt_load_df_function):\n sources = {{ source_dict | tojson }}\n key = '.'.join(args)\n return dbt_load_df_function(sources[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.227363, "supported_languages": null}, "macro.dbt.build_config_dict": {"name": "build_config_dict", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_config_dict", "macro_sql": "{% macro build_config_dict(model) %}\n {%- set config_dict = {} -%}\n {% set config_dbt_used = zip(model.config.config_keys_used, model.config.config_keys_defaults) | list %}\n {%- for key, default in config_dbt_used -%}\n {# weird type testing with enum, would be much easier to write this logic in Python! #}\n {%- if key == \"language\" -%}\n {%- set value = \"python\" -%}\n {%- endif -%}\n {%- set value = model.config.get(key, default) -%}\n {%- do config_dict.update({key: value}) -%}\n {%- endfor -%}\n {# Handle dbt.config.meta_get() calls - use separate dict to avoid overwriting native configs #}\n {%- set meta_dict = {} -%}\n {%- if model.config.meta_keys_used -%}\n {% set meta_dbt_used = zip(model.config.meta_keys_used, model.config.meta_keys_defaults) | list %}\n {%- for key, default in meta_dbt_used -%}\n {%- if model.config.meta and key in model.config.meta -%}\n {%- set value = model.config.meta[key] -%}\n {%- else -%}\n {%- set value = default -%}\n {%- endif -%}\n {%- do meta_dict.update({key: value}) -%}\n {%- endfor -%}\n {%- endif -%}\nconfig_dict = {{ config_dict }}\nmeta_dict = {{ meta_dict }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.227892, "supported_languages": null}, "macro.dbt.py_script_postfix": {"name": "py_script_postfix", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_postfix", "macro_sql": "{% macro py_script_postfix(model) %}\n# This part is user provided model code\n# you will need to copy the next section to run the code\n# COMMAND ----------\n# this part is dbt logic for get ref work, do not modify\n\n{{ build_ref_function(model ) }}\n{{ build_source_function(model ) }}\n{{ build_config_dict(model) }}\n\nclass config:\n def __init__(self, *args, **kwargs):\n pass\n\n @staticmethod\n def get(key, default=None):\n return config_dict.get(key, default)\n\n @staticmethod\n def meta_get(key, default=None):\n return meta_dict.get(key, default)\n\nclass this:\n \"\"\"dbt.this() or dbt.this.identifier\"\"\"\n database = \"{{ this.database }}\"\n schema = \"{{ this.schema }}\"\n identifier = \"{{ this.identifier }}\"\n {% set this_relation_name = resolve_model_name(this) %}\n def __repr__(self):\n return '{{ this_relation_name }}'\n\n\nclass dbtObj:\n def __init__(self, load_df_function) -> None:\n self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function)\n self.ref = lambda *args, **kwargs: ref(*args, **kwargs, dbt_load_df_function=load_df_function)\n self.config = config\n self.this = this()\n self.is_incremental = {{ is_incremental() }}\n\n# COMMAND ----------\n{{py_script_comment()}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.build_ref_function", "macro.dbt.build_source_function", "macro.dbt.build_config_dict", "macro.dbt.resolve_model_name", "macro.dbt.is_incremental", "macro.dbt.py_script_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.22812, "supported_languages": null}, "macro.dbt.py_script_comment": {"name": "py_script_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n{%endmacro%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2281642, "supported_languages": null}, "macro.dbt.test_unique": {"name": "test_unique", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_unique"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.2283971, "supported_languages": null}, "macro.dbt.test_not_null": {"name": "test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.228558, "supported_languages": null}, "macro.dbt.test_accepted_values": {"name": "test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.228699, "supported_languages": null}, "macro.dbt.test_relationships": {"name": "test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_relationships"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "config": {"meta": {}, "docs": {"show": true, "node_color": null}}, "patch_path": null, "arguments": [], "created_at": 1783011794.228839, "supported_languages": null}}, "docs": {"doc.dbt.__overview__": {"name": "__overview__", "resource_type": "doc", "package_name": "dbt", "path": "overview.md", "original_file_path": "docs/overview.md", "unique_id": "doc.dbt.__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion"}}, "exposures": {}, "metrics": {}, "groups": {}, "selectors": {}, "disabled": {}, "parent_map": {"model.dbt_factory.orders_raw": [], "model.dbt_factory.orders_daily": ["model.dbt_factory.orders_raw"], "test.dbt_factory.not_null_orders_raw_customer_name.f1166f8d91": ["model.dbt_factory.orders_raw"], "test.dbt_factory.unique_orders_daily_order_date.8b58585cb3": ["model.dbt_factory.orders_daily"], "test.dbt_factory.not_null_orders_daily_order_date.e8076bc258": ["model.dbt_factory.orders_daily"]}, "child_map": {"model.dbt_factory.orders_raw": ["model.dbt_factory.orders_daily", "test.dbt_factory.not_null_orders_raw_customer_name.f1166f8d91"], "model.dbt_factory.orders_daily": ["test.dbt_factory.not_null_orders_daily_order_date.e8076bc258", "test.dbt_factory.unique_orders_daily_order_date.8b58585cb3"], "test.dbt_factory.not_null_orders_raw_customer_name.f1166f8d91": [], "test.dbt_factory.unique_orders_daily_order_date.8b58585cb3": [], "test.dbt_factory.not_null_orders_daily_order_date.e8076bc258": []}, "group_map": {}, "saved_queries": {}, "semantic_models": {}, "unit_tests": {}, "functions": {}} \ No newline at end of file diff --git a/contrib/dbt_factory/tests/__init__.py b/contrib/dbt_factory/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/contrib/dbt_factory/tests/conftest.py b/contrib/dbt_factory/tests/conftest.py new file mode 100644 index 00000000..5517527b --- /dev/null +++ b/contrib/dbt_factory/tests/conftest.py @@ -0,0 +1,74 @@ +import pytest +from databricks_dbt_factory.SpecsHandler import SpecsHandler +from databricks_dbt_factory.DbtFactory import DbtFactory +from databricks_dbt_factory.DbtTask import DbtTaskOptions +from databricks_dbt_factory.TaskFactory import ( + ModelTaskFactory, + SnapshotTaskFactory, + SeedTaskFactory, + TestTaskFactory, + DbtDependencyResolver, +) + + +@pytest.fixture +def file_handler() -> SpecsHandler: + return SpecsHandler() + + +@pytest.fixture +def dbt_factory(file_handler: SpecsHandler): + return create_dbt_factory(file_handler) + + +@pytest.fixture +def dbt_factory_with_deps(file_handler: SpecsHandler): + return create_dbt_factory(file_handler, dbt_deps_enabled=True) + + +@pytest.fixture +def dbt_factory_with_deps_selected(file_handler: SpecsHandler): + return create_dbt_factory( + file_handler, + dbt_deps_enabled=True, + dbt_tasks_deps=["diamonds_prices", "second_dbt_model"], + ) + + +@pytest.fixture +def notebook_factory(file_handler: SpecsHandler): + return create_dbt_factory(file_handler, task_type="notebook", notebook_path="./notebooks/dbt_runner.py") + + +@pytest.fixture +def dbt_factory_bundled(file_handler: SpecsHandler): + return create_dbt_factory(file_handler, bundle_tests=True) + + +def create_dbt_factory( + handler: SpecsHandler, + dbt_deps_enabled: bool = False, + dbt_tasks_deps: list[str] | None = None, + task_type: str = "dbt", + notebook_path: str | None = None, + bundle_tests: bool = False, +) -> DbtFactory: + resolver = DbtDependencyResolver() + task_options = DbtTaskOptions( + source="GIT", + environment_key="Default", + dbt_deps_enabled=dbt_deps_enabled, + dbt_tasks_deps=dbt_tasks_deps, + task_type=task_type, + notebook_path=notebook_path, + ) + dbt_options = "--target dev" + + task_factories = { + "model": ModelTaskFactory(resolver, task_options, dbt_options), + "snapshot": SnapshotTaskFactory(resolver, task_options, dbt_options), + "seed": SeedTaskFactory(resolver, task_options, dbt_options), + "test": TestTaskFactory(resolver, task_options, dbt_options), + } + + return DbtFactory(handler, task_factories, bundle_tests=bundle_tests) diff --git a/contrib/dbt_factory/tests/test_app.py b/contrib/dbt_factory/tests/test_app.py new file mode 100644 index 00000000..dff91c0d --- /dev/null +++ b/contrib/dbt_factory/tests/test_app.py @@ -0,0 +1,370 @@ +import os +from tempfile import NamedTemporaryFile +from pathlib import Path +import pytest +import yaml + +from databricks_dbt_factory.main import main, parse_args + +BASE_PATH = str(Path(__file__).resolve().parent) + + +def test_main_given_default_args(monkeypatch): + """Test the main function for job spec generation.""" + dbt_manifest_path = BASE_PATH + "/test_data/manifest.json" + input_job_spec_path = BASE_PATH + "/test_data/job_definition_template.yaml" + expected_job_definition_path = BASE_PATH + "/test_data/job_definition_default.yaml" + + with NamedTemporaryFile(suffix=".yaml", delete=False) as temp_file: + target_job_spec_path = temp_file.name + + monkeypatch.setattr( + "sys.argv", + [ + "main.py", + "--dbt-manifest-path", + dbt_manifest_path, + "--input-job-spec-path", + input_job_spec_path, + "--target-job-spec-path", + target_job_spec_path, + ], + ) + + try: + main() + + with open(expected_job_definition_path, "r", encoding="utf-8") as file: + expected_job_definition = yaml.safe_load(file) + + with open(target_job_spec_path, "r", encoding="utf-8") as file: + job_definition = yaml.safe_load(file) + + assert job_definition == expected_job_definition + finally: + if os.path.exists(target_job_spec_path): + os.remove(target_job_spec_path) + + +def test_main_notebook_mode_auto_copies_runner_notebook_next_to_spec(monkeypatch, tmp_path): + """Without --project-directory, the factory copies the runner notebook next to the + generated job spec and emits `notebook_path: ./run_dbt_command.py`.""" + target_job_spec_path = tmp_path / "job_definition.yaml" + + monkeypatch.setattr( + "sys.argv", + [ + "main.py", + "--dbt-manifest-path", + BASE_PATH + "/test_data/manifest.json", + "--input-job-spec-path", + BASE_PATH + "/test_data/job_definition_template.yaml", + "--target-job-spec-path", + str(target_job_spec_path), + "--task-type", + "notebook", + ], + ) + + main() + + copied_notebook = tmp_path / "run_dbt_command.py" + assert copied_notebook.exists(), "runner notebook should have been copied next to the job spec" + assert "dbtRunner" in copied_notebook.read_text(), "copied file should be the packaged runner" + + with open(target_job_spec_path, "r", encoding="utf-8") as file: + job_definition = yaml.safe_load(file) + + tasks = job_definition["resources"]["jobs"]["dbt_sql_job"]["tasks"] + for task in tasks: + assert task["notebook_task"]["notebook_path"] == "./run_dbt_command.py" + + +def test_main_notebook_mode_auto_copies_runner_notebook_to_project_root(monkeypatch, tmp_path): + """With a relative --project-directory (e.g. `../`), the factory copies the runner to + the computed project root and emits a matching relative notebook_path from the spec.""" + spec_dir = tmp_path / "resources" + spec_dir.mkdir() + target_job_spec_path = spec_dir / "job_definition.yaml" + + monkeypatch.setattr( + "sys.argv", + [ + "main.py", + "--dbt-manifest-path", + BASE_PATH + "/test_data/manifest.json", + "--input-job-spec-path", + BASE_PATH + "/test_data/job_definition_template.yaml", + "--target-job-spec-path", + str(target_job_spec_path), + "--task-type", + "notebook", + "--project-directory", + "../", + ], + ) + + main() + + copied_notebook = tmp_path / "run_dbt_command.py" + assert copied_notebook.exists(), "runner should have been copied to the project root (one level up from the spec)" + assert not (spec_dir / "run_dbt_command.py").exists(), "runner should NOT be copied next to the spec in this case" + + with open(target_job_spec_path, "r", encoding="utf-8") as file: + job_definition = yaml.safe_load(file) + + tasks = job_definition["resources"]["jobs"]["dbt_sql_job"]["tasks"] + for task in tasks: + assert task["notebook_task"]["notebook_path"] == "../run_dbt_command.py" + # With the runner at project root, CWD at runtime = project root. We explicitly + # pin project_directory to "." so the spec is self-documenting (the user's original + # "../" would resolve one level too high and has been rewritten). + assert task["notebook_task"]["base_parameters"]["project_directory"] == "." + + +def test_main_notebook_mode(monkeypatch): + """Test the main function for notebook task type generation.""" + dbt_manifest_path = BASE_PATH + "/test_data/manifest.json" + input_job_spec_path = BASE_PATH + "/test_data/job_definition_template.yaml" + expected_job_definition_path = BASE_PATH + "/test_data/job_definition_notebook_default.yaml" + + with NamedTemporaryFile(suffix=".yaml", delete=False) as temp_file: + target_job_spec_path = temp_file.name + + monkeypatch.setattr( + "sys.argv", + [ + "main.py", + "--dbt-manifest-path", + dbt_manifest_path, + "--input-job-spec-path", + input_job_spec_path, + "--target-job-spec-path", + target_job_spec_path, + "--task-type", + "notebook", + "--notebook-path", + "./notebooks/dbt_runner.py", + ], + ) + + try: + main() + + with open(expected_job_definition_path, "r", encoding="utf-8") as file: + expected_job_definition = yaml.safe_load(file) + + with open(target_job_spec_path, "r", encoding="utf-8") as file: + job_definition = yaml.safe_load(file) + + assert job_definition == expected_job_definition + finally: + if os.path.exists(target_job_spec_path): + os.remove(target_job_spec_path) + + +def test_main_all_args(monkeypatch): + """Test the main function for job spec generation.""" + dbt_manifest_path = BASE_PATH + "/test_data/manifest.json" + input_job_spec_path = BASE_PATH + "/test_data/job_definition_template.yaml" + expected_job_definition_path = BASE_PATH + "/test_data/job_definition_deps_selected.yaml" + + with NamedTemporaryFile(suffix=".yaml", delete=False) as temp_file: + target_job_spec_path = temp_file.name + + new_job_name = "test_job" + warehouse_id = "1234567890abcdef" + schema = "dqx_test" + catalog = "main" + profiles_dir = "profiles_dir" + project_dir = "/project_dir" + extra_dbt_command_options = '"--upgrade"' + + # Mock command-line arguments + monkeypatch.setattr( + "sys.argv", + [ + "main.py", + "--new-job-name", + new_job_name, + "--dbt-manifest-path", + dbt_manifest_path, + "--input-job-spec-path", + input_job_spec_path, + "--target-job-spec-path", + target_job_spec_path, + "--target", + "dev", + "--environment-key", + "Default", + "--source", + "GIT", + "--enable-dbt-deps", + "--dbt-tasks-deps", + "diamonds_prices,second_dbt_model", + "--warehouse_id", + warehouse_id, + "--schema", + schema, + "--catalog", + catalog, + "--profiles-directory", + profiles_dir, + "--project-directory", + project_dir, + "--extra-dbt-command-options", + extra_dbt_command_options, + ], + ) + + try: + main() + + with open(expected_job_definition_path, "r", encoding="utf-8") as file: + expected_job_definition = yaml.safe_load(file) + + with open(target_job_spec_path, "r", encoding="utf-8") as file: + job_definition = yaml.safe_load(file) + + expected_job_definition = update_spec( + expected_job_definition, + new_job_name, + warehouse_id, + schema, + catalog, + profiles_dir, + project_dir, + extra_dbt_command_options, + ) + + assert job_definition == expected_job_definition + finally: + if os.path.exists(target_job_spec_path): + os.remove(target_job_spec_path) + + +REQUIRED_ARGS = [ + "--dbt-manifest-path", + "manifest.json", + "--input-job-spec-path", + "in.yaml", + "--target-job-spec-path", + "out.yaml", +] + + +def test_explicit_environment_key_with_job_cluster_key_is_rejected(monkeypatch): + monkeypatch.setattr( + "sys.argv", + ["main.py", *REQUIRED_ARGS, "--job-cluster-key", "foo", "--environment-key", "Default"], + ) + with pytest.raises(SystemExit): + parse_args() + + +def test_job_cluster_key_alone_parses(monkeypatch): + monkeypatch.setattr("sys.argv", ["main.py", *REQUIRED_ARGS, "--job-cluster-key", "foo"]) + args = parse_args() + assert args.job_cluster_key == "foo" + assert args.environment_key is None + + +def test_environment_key_alone_parses(monkeypatch): + monkeypatch.setattr("sys.argv", ["main.py", *REQUIRED_ARGS, "--environment-key", "Default"]) + args = parse_args() + assert args.environment_key == "Default" + assert args.job_cluster_key is None + + +def test_boolean_flags_default(monkeypatch): + monkeypatch.setattr("sys.argv", ["main.py", *REQUIRED_ARGS]) + args = parse_args() + assert args.run_tests is True # tests enabled by default + assert args.bundle_tests is False + assert args.enable_dbt_deps is False + assert args.dry_run is False + + +def test_boolean_flags_toggled(monkeypatch): + monkeypatch.setattr( + "sys.argv", + [ + "main.py", + *REQUIRED_ARGS, + "--no-run-tests", + "--bundle-tests", + "--enable-dbt-deps", + "--dry-run", + ], + ) + args = parse_args() + assert args.run_tests is False + assert args.bundle_tests is True + assert args.enable_dbt_deps is True + assert args.dry_run is True + + +def test_notebook_task_type_with_warehouse_id_is_rejected(monkeypatch): + monkeypatch.setattr( + "sys.argv", + [ + "main.py", + *REQUIRED_ARGS, + "--task-type", + "notebook", + "--notebook-path", + "/n", + "--warehouse_id", + "wh123", + ], + ) + with pytest.raises(SystemExit): + parse_args() + + +def remove_target_from_spec(expected_job_definition): + """Remove 'target' key from dbt_task in the expected job definition.""" + spec = dict(expected_job_definition) + + for task in expected_job_definition["resources"]["jobs"]["dbt_sql_job"]["tasks"]: + if "dbt_task" in task: + task["dbt_task"].pop("source", None) + + return spec + + +def update_spec( + expected_spec: dict, + new_job_name: str, + warehouse_id: str, + schema: str, + catalog: str, + profiles_dir: str, + project_dir: str, + extra_dbt_command_options: str, +) -> dict: + """Update the job specification with new parameters.""" + spec = dict(expected_spec) + + # Update job name + spec["resources"]["jobs"][new_job_name] = spec["resources"]["jobs"].pop("dbt_sql_job") + spec["resources"]["jobs"][new_job_name]["name"] = new_job_name + + # Add warehouse_id under dbt_task + for task in spec["resources"]["jobs"][new_job_name]["tasks"]: + if "dbt_task" in task: + task["dbt_task"]["schema"] = schema + task["dbt_task"]["catalog"] = catalog + task["dbt_task"]["warehouse_id"] = warehouse_id + task["dbt_task"]["project_directory"] = project_dir + task["dbt_task"]["profiles_directory"] = profiles_dir + + # Update commands with extra dbt command options + updated_commands = [] + for command in task["dbt_task"]["commands"]: + if "--target dev" in command: + command = command.replace("--target dev", f"--target dev {extra_dbt_command_options}") + updated_commands.append(command) + task["dbt_task"]["commands"] = updated_commands + + return spec diff --git a/contrib/dbt_factory/tests/test_data/job_definition.yaml b/contrib/dbt_factory/tests/test_data/job_definition.yaml new file mode 100644 index 00000000..88fe0908 --- /dev/null +++ b/contrib/dbt_factory/tests/test_data/job_definition.yaml @@ -0,0 +1,342 @@ +resources: + jobs: + dbt_sql_job: + name: dbt_job + git_source: + git_url: https://github.com/mwojtyczka/dbt-demo.git + git_provider: gitHub + git_branch: main + queue: + enabled: true + environments: + - environment_key: Default + spec: + client: '1' + dependencies: + - dbt-databricks + tasks: + - task_key: model_dbt_demo_diamonds_list_colors + depends_on: + - task_key: model_dbt_demo_diamonds_four_cs + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select diamonds_list_colors --target dev + source: GIT + - task_key: model_dbt_demo_diamonds_prices + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select diamonds_prices --target dev + source: GIT + - task_key: model_dbt_demo_diamonds_four_cs + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select diamonds_four_cs --target dev + source: GIT + - task_key: model_dbt_demo_zzz_game_details + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select zzz_game_details --target dev + source: GIT + - task_key: model_dbt_demo_second_dbt_model + depends_on: + - task_key: model_dbt_demo_first_dbt_model + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select second_dbt_model --target dev + source: GIT + - task_key: model_dbt_demo_first_dbt_model + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select first_dbt_model --target dev + source: GIT + - task_key: test_dbt_demo_zzz_game_details_check_scores + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select zzz_game_details_check_scores --target dev + source: GIT + - task_key: test_dbt_demo_zzz_game_details_check_dates + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select zzz_game_details_check_dates --target dev + source: GIT + - task_key: test_dbt_demo_zzz_win_loss_records_check_records + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select zzz_win_loss_records_check_records --target dev + source: GIT + - task_key: test_dbt_demo_diamonds_list_colors_test + depends_on: + - task_key: model_dbt_demo_diamonds_list_colors + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select diamonds_list_colors_test --target dev + source: GIT + - task_key: seed_dbt_demo_country_codes + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt seed --select country_codes --target dev + source: GIT + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select unique_zzz_game_details_game_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_game_details_game_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_game_details_home --target dev + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_game_details_visitor --target dev + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_game_details_home_score --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_game_details_visitor_score --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_game_details_winner --target dev + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_game_details_date --target dev + source: GIT + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select unique_first_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_first_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_unique_second_dbt_model_id_3d7f6966a4 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select unique_second_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_second_dbt_model_id_c791c712a2 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_second_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_second_dbt_model_amount_usd_28ae55e16f + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_second_dbt_model_amount_usd --target dev + source: GIT + - task_key: model_dbt_demo_changed_country + depends_on: + - task_key: snapshot_dbt_demo_country_snapshot + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select changed_country --target dev + source: GIT + - task_key: model_dbt_demo_country + depends_on: + - task_key: seed_dbt_demo_country_codes + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select country --target dev + source: GIT + - task_key: snapshot_dbt_demo_country_snapshot + depends_on: + - task_key: model_dbt_demo_country + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt snapshot --select country_snapshot --target dev + source: GIT + - task_key: model_dbt_demo_zzz_win_loss_records + depends_on: + - task_key: model_dbt_demo_zzz_game_details + - task_key: test_dbt_demo_zzz_game_details_check_scores + - task_key: test_dbt_demo_zzz_game_details_check_dates + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select zzz_win_loss_records --target dev + source: GIT + - task_key: test_dbt_demo_unique_zzz_win_loss_records_team_b989062105 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select unique_zzz_win_loss_records_team --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_team_9a18f33f86 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_win_loss_records_team --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_wins_f6815357fe + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_win_loss_records_wins --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_losses_82bf4ea601 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_win_loss_records_losses --target dev + source: GIT diff --git a/contrib/dbt_factory/tests/test_data/job_definition_default.yaml b/contrib/dbt_factory/tests/test_data/job_definition_default.yaml new file mode 100644 index 00000000..8f836f24 --- /dev/null +++ b/contrib/dbt_factory/tests/test_data/job_definition_default.yaml @@ -0,0 +1,272 @@ +resources: + jobs: + dbt_sql_job: + name: dbt_job + git_source: + git_url: https://github.com/mwojtyczka/dbt-demo.git + git_provider: gitHub + git_branch: main + queue: + enabled: true + environments: + - environment_key: Default + spec: + client: '1' + dependencies: + - dbt-databricks + tasks: + - task_key: model_dbt_demo_diamonds_list_colors + depends_on: + - task_key: model_dbt_demo_diamonds_four_cs + environment_key: Default + dbt_task: + commands: + - dbt run --select diamonds_list_colors + - task_key: model_dbt_demo_diamonds_prices + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select diamonds_prices + - task_key: model_dbt_demo_diamonds_four_cs + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select diamonds_four_cs + - task_key: model_dbt_demo_zzz_game_details + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select zzz_game_details + - task_key: model_dbt_demo_second_dbt_model + depends_on: + - task_key: model_dbt_demo_first_dbt_model + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + environment_key: Default + dbt_task: + commands: + - dbt run --select second_dbt_model + - task_key: model_dbt_demo_first_dbt_model + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select first_dbt_model + - task_key: test_dbt_demo_zzz_game_details_check_scores + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select zzz_game_details_check_scores + - task_key: test_dbt_demo_zzz_game_details_check_dates + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select zzz_game_details_check_dates + - task_key: test_dbt_demo_zzz_win_loss_records_check_records + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select zzz_win_loss_records_check_records + - task_key: test_dbt_demo_diamonds_list_colors_test + depends_on: + - task_key: model_dbt_demo_diamonds_list_colors + environment_key: Default + dbt_task: + commands: + - dbt test --select diamonds_list_colors_test + - task_key: seed_dbt_demo_country_codes + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt seed --select country_codes + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_zzz_game_details_game_id + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_game_id + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_home + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_visitor + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_home_score + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_visitor_score + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_winner + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_date + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_first_dbt_model_id + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_first_dbt_model_id + - task_key: test_dbt_demo_unique_second_dbt_model_id_3d7f6966a4 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_second_dbt_model_id + - task_key: test_dbt_demo_not_null_second_dbt_model_id_c791c712a2 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_second_dbt_model_id + - task_key: test_dbt_demo_not_null_second_dbt_model_amount_usd_28ae55e16f + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_second_dbt_model_amount_usd + - task_key: model_dbt_demo_changed_country + depends_on: + - task_key: snapshot_dbt_demo_country_snapshot + environment_key: Default + dbt_task: + commands: + - dbt run --select changed_country + - task_key: model_dbt_demo_country + depends_on: + - task_key: seed_dbt_demo_country_codes + environment_key: Default + dbt_task: + commands: + - dbt run --select country + - task_key: snapshot_dbt_demo_country_snapshot + depends_on: + - task_key: model_dbt_demo_country + environment_key: Default + dbt_task: + commands: + - dbt snapshot --select country_snapshot + - task_key: model_dbt_demo_zzz_win_loss_records + depends_on: + - task_key: model_dbt_demo_zzz_game_details + - task_key: test_dbt_demo_zzz_game_details_check_scores + - task_key: test_dbt_demo_zzz_game_details_check_dates + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + environment_key: Default + dbt_task: + commands: + - dbt run --select zzz_win_loss_records + - task_key: test_dbt_demo_unique_zzz_win_loss_records_team_b989062105 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_zzz_win_loss_records_team + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_team_9a18f33f86 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_win_loss_records_team + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_wins_f6815357fe + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_win_loss_records_wins + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_losses_82bf4ea601 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_win_loss_records_losses diff --git a/contrib/dbt_factory/tests/test_data/job_definition_deps_selected.yaml b/contrib/dbt_factory/tests/test_data/job_definition_deps_selected.yaml new file mode 100644 index 00000000..3ab7d4ea --- /dev/null +++ b/contrib/dbt_factory/tests/test_data/job_definition_deps_selected.yaml @@ -0,0 +1,309 @@ +resources: + jobs: + dbt_sql_job: + name: dbt_job + git_source: + git_url: https://github.com/mwojtyczka/dbt-demo.git + git_provider: gitHub + git_branch: main + queue: + enabled: true + environments: + - environment_key: Default + spec: + client: '1' + dependencies: + - dbt-databricks + tasks: + - task_key: model_dbt_demo_diamonds_list_colors + depends_on: + - task_key: model_dbt_demo_diamonds_four_cs + environment_key: Default + dbt_task: + commands: + - dbt run --select diamonds_list_colors --target dev + source: GIT + - task_key: model_dbt_demo_diamonds_prices + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select diamonds_prices --target dev + source: GIT + - task_key: model_dbt_demo_diamonds_four_cs + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select diamonds_four_cs --target dev + source: GIT + - task_key: model_dbt_demo_zzz_game_details + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select zzz_game_details --target dev + source: GIT + - task_key: model_dbt_demo_second_dbt_model + depends_on: + - task_key: model_dbt_demo_first_dbt_model + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select second_dbt_model --target dev + source: GIT + - task_key: model_dbt_demo_first_dbt_model + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select first_dbt_model --target dev + source: GIT + - task_key: test_dbt_demo_zzz_game_details_check_scores + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select zzz_game_details_check_scores --target dev + source: GIT + - task_key: test_dbt_demo_zzz_game_details_check_dates + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select zzz_game_details_check_dates --target dev + source: GIT + - task_key: test_dbt_demo_zzz_win_loss_records_check_records + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select zzz_win_loss_records_check_records --target dev + source: GIT + - task_key: test_dbt_demo_diamonds_list_colors_test + depends_on: + - task_key: model_dbt_demo_diamonds_list_colors + environment_key: Default + dbt_task: + commands: + - dbt test --select diamonds_list_colors_test --target dev + source: GIT + - task_key: seed_dbt_demo_country_codes + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt seed --select country_codes --target dev + source: GIT + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_zzz_game_details_game_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_game_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_home --target dev + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_visitor --target dev + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_home_score --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_visitor_score --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_winner --target dev + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_date --target dev + source: GIT + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_first_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_first_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_unique_second_dbt_model_id_3d7f6966a4 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_second_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_second_dbt_model_id_c791c712a2 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_second_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_second_dbt_model_amount_usd_28ae55e16f + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_second_dbt_model_amount_usd --target dev + source: GIT + - task_key: model_dbt_demo_changed_country + depends_on: + - task_key: snapshot_dbt_demo_country_snapshot + environment_key: Default + dbt_task: + commands: + - dbt run --select changed_country --target dev + source: GIT + - task_key: model_dbt_demo_country + depends_on: + - task_key: seed_dbt_demo_country_codes + environment_key: Default + dbt_task: + commands: + - dbt run --select country --target dev + source: GIT + - task_key: snapshot_dbt_demo_country_snapshot + depends_on: + - task_key: model_dbt_demo_country + environment_key: Default + dbt_task: + commands: + - dbt snapshot --select country_snapshot --target dev + source: GIT + - task_key: model_dbt_demo_zzz_win_loss_records + depends_on: + - task_key: model_dbt_demo_zzz_game_details + - task_key: test_dbt_demo_zzz_game_details_check_scores + - task_key: test_dbt_demo_zzz_game_details_check_dates + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + environment_key: Default + dbt_task: + commands: + - dbt run --select zzz_win_loss_records --target dev + source: GIT + - task_key: test_dbt_demo_unique_zzz_win_loss_records_team_b989062105 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_zzz_win_loss_records_team --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_team_9a18f33f86 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_win_loss_records_team --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_wins_f6815357fe + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_win_loss_records_wins --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_losses_82bf4ea601 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_win_loss_records_losses --target dev + source: GIT diff --git a/contrib/dbt_factory/tests/test_data/job_definition_no_deps.yaml b/contrib/dbt_factory/tests/test_data/job_definition_no_deps.yaml new file mode 100644 index 00000000..0876d986 --- /dev/null +++ b/contrib/dbt_factory/tests/test_data/job_definition_no_deps.yaml @@ -0,0 +1,307 @@ +resources: + jobs: + dbt_sql_job: + name: dbt_job + git_source: + git_url: https://github.com/mwojtyczka/dbt-demo.git + git_provider: gitHub + git_branch: main + queue: + enabled: true + environments: + - environment_key: Default + spec: + client: '1' + dependencies: + - dbt-databricks + tasks: + - task_key: model_dbt_demo_diamonds_list_colors + depends_on: + - task_key: model_dbt_demo_diamonds_four_cs + environment_key: Default + dbt_task: + commands: + - dbt run --select diamonds_list_colors --target dev + source: GIT + - task_key: model_dbt_demo_diamonds_prices + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select diamonds_prices --target dev + source: GIT + - task_key: model_dbt_demo_diamonds_four_cs + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select diamonds_four_cs --target dev + source: GIT + - task_key: model_dbt_demo_zzz_game_details + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select zzz_game_details --target dev + source: GIT + - task_key: model_dbt_demo_second_dbt_model + depends_on: + - task_key: model_dbt_demo_first_dbt_model + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + environment_key: Default + dbt_task: + commands: + - dbt run --select second_dbt_model --target dev + source: GIT + - task_key: model_dbt_demo_first_dbt_model + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select first_dbt_model --target dev + source: GIT + - task_key: test_dbt_demo_zzz_game_details_check_scores + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select zzz_game_details_check_scores --target dev + source: GIT + - task_key: test_dbt_demo_zzz_game_details_check_dates + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select zzz_game_details_check_dates --target dev + source: GIT + - task_key: test_dbt_demo_zzz_win_loss_records_check_records + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select zzz_win_loss_records_check_records --target dev + source: GIT + - task_key: test_dbt_demo_diamonds_list_colors_test + depends_on: + - task_key: model_dbt_demo_diamonds_list_colors + environment_key: Default + dbt_task: + commands: + - dbt test --select diamonds_list_colors_test --target dev + source: GIT + - task_key: seed_dbt_demo_country_codes + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt seed --select country_codes --target dev + source: GIT + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_zzz_game_details_game_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_game_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_home --target dev + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_visitor --target dev + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_home_score --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_visitor_score --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_winner --target dev + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_date --target dev + source: GIT + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_first_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_first_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_unique_second_dbt_model_id_3d7f6966a4 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_second_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_second_dbt_model_id_c791c712a2 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_second_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_second_dbt_model_amount_usd_28ae55e16f + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_second_dbt_model_amount_usd --target dev + source: GIT + - task_key: model_dbt_demo_changed_country + depends_on: + - task_key: snapshot_dbt_demo_country_snapshot + environment_key: Default + dbt_task: + commands: + - dbt run --select changed_country --target dev + source: GIT + - task_key: model_dbt_demo_country + depends_on: + - task_key: seed_dbt_demo_country_codes + environment_key: Default + dbt_task: + commands: + - dbt run --select country --target dev + source: GIT + - task_key: snapshot_dbt_demo_country_snapshot + depends_on: + - task_key: model_dbt_demo_country + environment_key: Default + dbt_task: + commands: + - dbt snapshot --select country_snapshot --target dev + source: GIT + - task_key: model_dbt_demo_zzz_win_loss_records + depends_on: + - task_key: model_dbt_demo_zzz_game_details + - task_key: test_dbt_demo_zzz_game_details_check_scores + - task_key: test_dbt_demo_zzz_game_details_check_dates + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + environment_key: Default + dbt_task: + commands: + - dbt run --select zzz_win_loss_records --target dev + source: GIT + - task_key: test_dbt_demo_unique_zzz_win_loss_records_team_b989062105 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_zzz_win_loss_records_team --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_team_9a18f33f86 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_win_loss_records_team --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_wins_f6815357fe + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_win_loss_records_wins --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_losses_82bf4ea601 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_win_loss_records_losses --target dev + source: GIT diff --git a/contrib/dbt_factory/tests/test_data/job_definition_notebook_default.yaml b/contrib/dbt_factory/tests/test_data/job_definition_notebook_default.yaml new file mode 100644 index 00000000..d3dcedb2 --- /dev/null +++ b/contrib/dbt_factory/tests/test_data/job_definition_notebook_default.yaml @@ -0,0 +1,307 @@ +resources: + jobs: + dbt_sql_job: + name: dbt_job + git_source: + git_url: https://github.com/mwojtyczka/dbt-demo.git + git_provider: gitHub + git_branch: main + queue: + enabled: true + environments: + - environment_key: Default + spec: + client: '1' + dependencies: + - dbt-databricks + tasks: + - task_key: model_dbt_demo_diamonds_list_colors + depends_on: + - task_key: model_dbt_demo_diamonds_four_cs + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select diamonds_list_colors"]' + - task_key: model_dbt_demo_diamonds_prices + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select diamonds_prices"]' + - task_key: model_dbt_demo_diamonds_four_cs + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select diamonds_four_cs"]' + - task_key: model_dbt_demo_zzz_game_details + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select zzz_game_details"]' + - task_key: model_dbt_demo_second_dbt_model + depends_on: + - task_key: model_dbt_demo_first_dbt_model + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select second_dbt_model"]' + - task_key: model_dbt_demo_first_dbt_model + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select first_dbt_model"]' + - task_key: test_dbt_demo_zzz_game_details_check_scores + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select zzz_game_details_check_scores"]' + - task_key: test_dbt_demo_zzz_game_details_check_dates + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select zzz_game_details_check_dates"]' + - task_key: test_dbt_demo_zzz_win_loss_records_check_records + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select zzz_win_loss_records_check_records"]' + - task_key: test_dbt_demo_diamonds_list_colors_test + depends_on: + - task_key: model_dbt_demo_diamonds_list_colors + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select diamonds_list_colors_test"]' + - task_key: seed_dbt_demo_country_codes + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt seed --select country_codes"]' + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select unique_zzz_game_details_game_id"]' + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_game_id"]' + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_home"]' + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle"]' + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_visitor"]' + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle"]' + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_home_score"]' + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_visitor_score"]' + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_winner"]' + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle"]' + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_date"]' + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select unique_first_dbt_model_id"]' + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_first_dbt_model_id"]' + - task_key: test_dbt_demo_unique_second_dbt_model_id_3d7f6966a4 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select unique_second_dbt_model_id"]' + - task_key: test_dbt_demo_not_null_second_dbt_model_id_c791c712a2 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_second_dbt_model_id"]' + - task_key: test_dbt_demo_not_null_second_dbt_model_amount_usd_28ae55e16f + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_second_dbt_model_amount_usd"]' + - task_key: model_dbt_demo_changed_country + depends_on: + - task_key: snapshot_dbt_demo_country_snapshot + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select changed_country"]' + - task_key: model_dbt_demo_country + depends_on: + - task_key: seed_dbt_demo_country_codes + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select country"]' + - task_key: snapshot_dbt_demo_country_snapshot + depends_on: + - task_key: model_dbt_demo_country + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt snapshot --select country_snapshot"]' + - task_key: model_dbt_demo_zzz_win_loss_records + depends_on: + - task_key: model_dbt_demo_zzz_game_details + - task_key: test_dbt_demo_zzz_game_details_check_scores + - task_key: test_dbt_demo_zzz_game_details_check_dates + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select zzz_win_loss_records"]' + - task_key: test_dbt_demo_unique_zzz_win_loss_records_team_b989062105 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select unique_zzz_win_loss_records_team"]' + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_team_9a18f33f86 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_win_loss_records_team"]' + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_wins_f6815357fe + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_win_loss_records_wins"]' + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_losses_82bf4ea601 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_win_loss_records_losses"]' diff --git a/contrib/dbt_factory/tests/test_data/job_definition_notebook_no_deps.yaml b/contrib/dbt_factory/tests/test_data/job_definition_notebook_no_deps.yaml new file mode 100644 index 00000000..0c9b453e --- /dev/null +++ b/contrib/dbt_factory/tests/test_data/job_definition_notebook_no_deps.yaml @@ -0,0 +1,342 @@ +resources: + jobs: + dbt_sql_job: + name: dbt_job + git_source: + git_url: https://github.com/mwojtyczka/dbt-demo.git + git_provider: gitHub + git_branch: main + queue: + enabled: true + environments: + - environment_key: Default + spec: + client: '1' + dependencies: + - dbt-databricks + tasks: + - task_key: model_dbt_demo_diamonds_list_colors + depends_on: + - task_key: model_dbt_demo_diamonds_four_cs + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select diamonds_list_colors --target dev"]' + source: GIT + - task_key: model_dbt_demo_diamonds_prices + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select diamonds_prices --target dev"]' + source: GIT + - task_key: model_dbt_demo_diamonds_four_cs + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select diamonds_four_cs --target dev"]' + source: GIT + - task_key: model_dbt_demo_zzz_game_details + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select zzz_game_details --target dev"]' + source: GIT + - task_key: model_dbt_demo_second_dbt_model + depends_on: + - task_key: model_dbt_demo_first_dbt_model + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select second_dbt_model --target dev"]' + source: GIT + - task_key: model_dbt_demo_first_dbt_model + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select first_dbt_model --target dev"]' + source: GIT + - task_key: test_dbt_demo_zzz_game_details_check_scores + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select zzz_game_details_check_scores --target dev"]' + source: GIT + - task_key: test_dbt_demo_zzz_game_details_check_dates + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select zzz_game_details_check_dates --target dev"]' + source: GIT + - task_key: test_dbt_demo_zzz_win_loss_records_check_records + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select zzz_win_loss_records_check_records --target dev"]' + source: GIT + - task_key: test_dbt_demo_diamonds_list_colors_test + depends_on: + - task_key: model_dbt_demo_diamonds_list_colors + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select diamonds_list_colors_test --target dev"]' + source: GIT + - task_key: seed_dbt_demo_country_codes + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt seed --select country_codes --target dev"]' + source: GIT + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select unique_zzz_game_details_game_id --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_game_id --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_home --target dev"]' + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_visitor --target dev"]' + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_home_score --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_visitor_score --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_winner --target dev"]' + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_date --target dev"]' + source: GIT + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select unique_first_dbt_model_id --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_first_dbt_model_id --target dev"]' + source: GIT + - task_key: test_dbt_demo_unique_second_dbt_model_id_3d7f6966a4 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select unique_second_dbt_model_id --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_second_dbt_model_id_c791c712a2 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_second_dbt_model_id --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_second_dbt_model_amount_usd_28ae55e16f + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_second_dbt_model_amount_usd --target dev"]' + source: GIT + - task_key: model_dbt_demo_changed_country + depends_on: + - task_key: snapshot_dbt_demo_country_snapshot + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select changed_country --target dev"]' + source: GIT + - task_key: model_dbt_demo_country + depends_on: + - task_key: seed_dbt_demo_country_codes + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select country --target dev"]' + source: GIT + - task_key: snapshot_dbt_demo_country_snapshot + depends_on: + - task_key: model_dbt_demo_country + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt snapshot --select country_snapshot --target dev"]' + source: GIT + - task_key: model_dbt_demo_zzz_win_loss_records + depends_on: + - task_key: model_dbt_demo_zzz_game_details + - task_key: test_dbt_demo_zzz_game_details_check_scores + - task_key: test_dbt_demo_zzz_game_details_check_dates + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select zzz_win_loss_records --target dev"]' + source: GIT + - task_key: test_dbt_demo_unique_zzz_win_loss_records_team_b989062105 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select unique_zzz_win_loss_records_team --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_team_9a18f33f86 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_win_loss_records_team --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_wins_f6815357fe + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_win_loss_records_wins --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_losses_82bf4ea601 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_win_loss_records_losses --target dev"]' + source: GIT diff --git a/contrib/dbt_factory/tests/test_data/job_definition_template.yaml b/contrib/dbt_factory/tests/test_data/job_definition_template.yaml new file mode 100644 index 00000000..6570068c --- /dev/null +++ b/contrib/dbt_factory/tests/test_data/job_definition_template.yaml @@ -0,0 +1,16 @@ +resources: + jobs: + dbt_sql_job: + name: dbt_job + git_source: + git_url: https://github.com/mwojtyczka/dbt-demo.git + git_provider: gitHub + git_branch: main + queue: + enabled: true + environments: + - environment_key: Default + spec: + client: '1' + dependencies: + - dbt-databricks diff --git a/contrib/dbt_factory/tests/test_data/manifest.json b/contrib/dbt_factory/tests/test_data/manifest.json new file mode 100644 index 00000000..44282d07 --- /dev/null +++ b/contrib/dbt_factory/tests/test_data/manifest.json @@ -0,0 +1 @@ +{"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v12.json", "dbt_version": "1.8.7", "generated_at": "2024-12-06T13:55:35.349350Z", "invocation_id": "a2d9eb4c-6787-4ede-b5ce-44e61f3769b2", "env": {}, "project_name": "dbt_demo", "project_id": "f005e70093f1ddd3daa6316f5ec6f69e", "user_id": "b802ff80-7d9c-4428-8a39-5f56cf164bf8", "send_anonymous_usage_stats": true, "adapter_type": "databricks"}, "nodes": {"model.dbt_demo.diamonds_list_colors": {"database": "dbt_demo", "schema": "dbt_dev", "name": "diamonds_list_colors", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/diamonds_list_colors.sql", "original_file_path": "src/models/sql_model1/diamonds_list_colors.sql", "unique_id": "model.dbt_demo.diamonds_list_colors", "fqn": ["dbt_demo", "sql_model1", "diamonds_list_colors"], "alias": "diamonds_list_colors", "checksum": {"name": "sha256", "checksum": "fb8e076fbc40ae30c2441fbc660922d1d3428cf1557cc8f400dbd69caa78c266"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/diamonds_list_colors.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "created_at": 1733486794.194898, "relation_name": "`dbt_demo`.`dbt_dev`.`diamonds_list_colors`", "raw_code": "-- create a view\nselect distinct color\nfrom {{ ref('diamonds_four_cs') }}\nsort by color asc", "language": "sql", "refs": [{"name": "diamonds_four_cs", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.diamonds_four_cs"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/diamonds_list_colors.sql", "compiled": true, "compiled_code": "-- create a view\nselect distinct color\nfrom `dbt_demo`.`dbt_dev`.`diamonds_four_cs`\nsort by color asc", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.diamonds_prices": {"database": "dbt_demo", "schema": "dbt_dev", "name": "diamonds_prices", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/diamonds_prices.sql", "original_file_path": "src/models/sql_model1/diamonds_prices.sql", "unique_id": "model.dbt_demo.diamonds_prices", "fqn": ["dbt_demo", "sql_model1", "diamonds_prices"], "alias": "diamonds_prices", "checksum": {"name": "sha256", "checksum": "809b1318095c2871ab3e9d98af3cc11233e1fbb791e19682fbb38070074e53d2"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/diamonds_prices.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "created_at": 1733486794.207654, "relation_name": "`dbt_demo`.`dbt_dev`.`diamonds_prices`", "raw_code": "-- create a view\nselect\n account_id,\n workspace_id,\n usage_date,\n sum(usage_quantity) as quantity,\n sum({{ cents_to_dollars('usage_quantity') }}) as amount_usd\nfrom system.billing.usage\ngroup by account_id, workspace_id, usage_date", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt_demo.cents_to_dollars"], "nodes": []}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/diamonds_prices.sql", "compiled": true, "compiled_code": "-- create a view\nselect\n account_id,\n workspace_id,\n usage_date,\n sum(usage_quantity) as quantity,\n sum(\n (usage_quantity / 100)::numeric(16, 2)\n) as amount_usd\nfrom system.billing.usage\ngroup by account_id, workspace_id, usage_date", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.diamonds_four_cs": {"database": "dbt_demo", "schema": "dbt_dev", "name": "diamonds_four_cs", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/diamonds_four_cs.sql", "original_file_path": "src/models/sql_model1/diamonds_four_cs.sql", "unique_id": "model.dbt_demo.diamonds_four_cs", "fqn": ["dbt_demo", "sql_model1", "diamonds_four_cs"], "alias": "diamonds_four_cs", "checksum": {"name": "sha256", "checksum": "85a0d2493625371fdfbd2d8261dcc22335301c88e08791638ad760e6cb7dd614"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/diamonds_four_cs.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733486794.264364, "relation_name": "`dbt_demo`.`dbt_dev`.`diamonds_four_cs`", "raw_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n{{ config(\n materialized='table',\n file_format='delta'\n) }}\nselect\n carat,\n cut,\n color,\n clarity\n-- get table specification from schema.yml\nfrom {{ source('raw', 'diamonds') }} -- equivalent of: from hive_metastore.default.diamonds", "language": "sql", "refs": [], "sources": [["raw", "diamonds"]], "metrics": [], "depends_on": {"macros": [], "nodes": ["source.dbt_demo.raw.diamonds"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/diamonds_four_cs.sql", "compiled": true, "compiled_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n\nselect\n carat,\n cut,\n color,\n clarity\n-- get table specification from schema.yml\nfrom `hive_metastore`.`default`.`diamonds` -- equivalent of: from hive_metastore.default.diamonds", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.zzz_game_details": {"database": "dbt_demo", "schema": "dbt_dev", "name": "zzz_game_details", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/zzz_game_details.sql", "original_file_path": "src/models/sql_model1/zzz_game_details.sql", "unique_id": "model.dbt_demo.zzz_game_details", "fqn": ["dbt_demo", "sql_model1", "zzz_game_details"], "alias": "zzz_game_details", "checksum": {"name": "sha256", "checksum": "07a0ddce4e171f5838f27124b3574a77cbab23fac66bba7a1b8f87f6bfbd653f"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {"game_id": {"name": "game_id", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "home": {"name": "home", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "visitor": {"name": "visitor", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "home_score": {"name": "home_score", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "visitor_score": {"name": "visitor_score", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "winner": {"name": "winner", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "date": {"name": "date", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model1/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model1/zzz_game_details.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733486794.43343, "relation_name": "`dbt_demo`.`dbt_dev`.`zzz_game_details`", "raw_code": "-- Create a table that provides full details for each game, including\n-- the game ID, the home and visiting teams' city names and scores,\n-- the game winner's city name, and the game date.\n{{ config(\n materialized='table',\n file_format='delta'\n) }}\n-- Step 4 of 4: Replace the visitor team IDs with their city names.\nselect\n game_id,\n home,\n t.team_city as visitor,\n home_score,\n visitor_score,\n -- Step 3 of 4: Display the city name for each game's winner.\n game_date as date,\n case\n when\n home_score > visitor_score\n then\n home\n when\n visitor_score > home_score\n then\n t.team_city\n end as winner\nfrom (\n -- Step 2 of 4: Replace the home team IDs with their actual city names.\n select\n game_id,\n t.team_city as home,\n home_score,\n visitor_team_id,\n visitor_score,\n game_date\n from (\n -- Step 1 of 4: Combine data from various tables (for example, game and team IDs, scores, dates).\n select\n g.game_id,\n go.home_team_id,\n gs.home_team_score as home_score,\n go.visitor_team_id,\n gs.visitor_team_score as visitor_score,\n g.game_date\n from\n marcin_demo.dbt.{{ target.name }}_zzz_games as g,\n marcin_demo.dbt.zzz_game_opponents as go,\n marcin_demo.dbt.zzz_game_scores as gs\n where\n g.game_id = go.game_id\n and g.game_id = gs.game_id\n ) as all_ids,\n marcin_demo.dbt.zzz_teams as t\n where\n all_ids.home_team_id = t.team_id\n) as visitor_ids,\n marcin_demo.dbt.zzz_teams as t\nwhere\n visitor_ids.visitor_team_id = t.team_id\norder by game_date desc", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/zzz_game_details.sql", "compiled": true, "compiled_code": "-- Create a table that provides full details for each game, including\n-- the game ID, the home and visiting teams' city names and scores,\n-- the game winner's city name, and the game date.\n\n-- Step 4 of 4: Replace the visitor team IDs with their city names.\nselect\n game_id,\n home,\n t.team_city as visitor,\n home_score,\n visitor_score,\n -- Step 3 of 4: Display the city name for each game's winner.\n game_date as date,\n case\n when\n home_score > visitor_score\n then\n home\n when\n visitor_score > home_score\n then\n t.team_city\n end as winner\nfrom (\n -- Step 2 of 4: Replace the home team IDs with their actual city names.\n select\n game_id,\n t.team_city as home,\n home_score,\n visitor_team_id,\n visitor_score,\n game_date\n from (\n -- Step 1 of 4: Combine data from various tables (for example, game and team IDs, scores, dates).\n select\n g.game_id,\n go.home_team_id,\n gs.home_team_score as home_score,\n go.visitor_team_id,\n gs.visitor_team_score as visitor_score,\n g.game_date\n from\n marcin_demo.dbt.dev_zzz_games as g,\n marcin_demo.dbt.zzz_game_opponents as go,\n marcin_demo.dbt.zzz_game_scores as gs\n where\n g.game_id = go.game_id\n and g.game_id = gs.game_id\n ) as all_ids,\n marcin_demo.dbt.zzz_teams as t\n where\n all_ids.home_team_id = t.team_id\n) as visitor_ids,\n marcin_demo.dbt.zzz_teams as t\nwhere\n visitor_ids.visitor_team_id = t.team_id\norder by game_date desc", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.second_dbt_model": {"database": "dbt_demo", "schema": "dbt_dev", "name": "second_dbt_model", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model2/second_dbt_model.sql", "original_file_path": "src/models/sql_model2/second_dbt_model.sql", "unique_id": "model.dbt_demo.second_dbt_model", "fqn": ["dbt_demo", "sql_model2", "second_dbt_model"], "alias": "second_dbt_model", "checksum": {"name": "sha256", "checksum": "226c0e1d47e6f9bee7430c7acffadc0d239fa185558a2de703c5f0ab0b01d218"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": true, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "A starter dbt model", "columns": {"id": {"name": "id", "description": "The primary key for this table", "meta": {}, "data_type": "int", "constraints": [], "quote": null, "tags": []}, "amount_usd": {"name": "amount_usd", "description": "The primary key for this table", "meta": {}, "data_type": "decimal", "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model2/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model2/second_dbt_model.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "contract": {"enforced": true}}, "created_at": 1733486794.536133, "relation_name": "`dbt_demo`.`dbt_dev`.`second_dbt_model`", "raw_code": "-- Use the `ref` function to select from other models\n\nselect\n *, {{ cents_to_dollars('id') }} as amount_usd\nfrom {{ ref('first_dbt_model') }}\nwhere id = 1", "language": "sql", "refs": [{"name": "first_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt_demo.cents_to_dollars"], "nodes": ["model.dbt_demo.first_dbt_model"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model2/second_dbt_model.sql", "compiled": true, "compiled_code": "-- Use the `ref` function to select from other models\n\nselect\n *, \n (id / 100)::numeric(16, 2)\n as amount_usd\nfrom `dbt_demo`.`dbt_dev`.`first_dbt_model`\nwhere id = 1", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": true, "alias_types": true, "checksum": "d4b533efdb37067787094e25d1860ddcc473cb5cf92ebe2c358011ff2866658f"}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.first_dbt_model": {"database": "dbt_demo", "schema": "dbt_dev", "name": "first_dbt_model", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model2/first_dbt_model.sql", "original_file_path": "src/models/sql_model2/first_dbt_model.sql", "unique_id": "model.dbt_demo.first_dbt_model", "fqn": ["dbt_demo", "sql_model2", "first_dbt_model"], "alias": "first_dbt_model", "checksum": {"name": "sha256", "checksum": "91d9a8b1e5c899463e52c50492b7b3c7df99fa6635c868f9165ca83e5f3b13f7"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": true, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "A starter dbt model", "columns": {"id": {"name": "id", "description": "The primary key for this table", "meta": {}, "data_type": "int", "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model2/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model2/first_dbt_model.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "contract": {"enforced": true}, "materialized": "table"}, "created_at": 1733486794.534828, "relation_name": "`dbt_demo`.`dbt_dev`.`first_dbt_model`", "raw_code": "/*\n Welcome to your first dbt model!\n Did you know that you can also configure models directly within SQL files?\n This will override configurations stated in dbt_project.yml\n\n Try changing \"table\" to \"view\" below\n*/\n\n{{ config(materialized='table') }}\n\nwith source_data as (\n\n select 1 as id\n union all\n select 2 as id\n\n)\n\nselect *\nfrom source_data\n\n/*\n Uncomment the line below to remove records with null `id` values\n*/\n\n-- where id is not null", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model2/first_dbt_model.sql", "compiled": true, "compiled_code": "/*\n Welcome to your first dbt model!\n Did you know that you can also configure models directly within SQL files?\n This will override configurations stated in dbt_project.yml\n\n Try changing \"table\" to \"view\" below\n*/\n\n\n\nwith source_data as (\n\n select 1 as id\n union all\n select 2 as id\n\n)\n\nselect *\nfrom source_data\n\n/*\n Uncomment the line below to remove records with null `id` values\n*/\n\n-- where id is not null", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": true, "alias_types": true, "checksum": "a31b0e38eb3a9f0c0ed2f45bd615732c39e48b7f698f50a36748a687fd6d7289"}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "test.dbt_demo.zzz_game_details_check_scores": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "zzz_game_details_check_scores", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/zzz_game_details_check_scores.sql", "original_file_path": "tests/data_tests/zzz_game_details_check_scores.sql", "unique_id": "test.dbt_demo.zzz_game_details_check_scores", "fqn": ["dbt_demo", "data_tests", "zzz_game_details_check_scores"], "alias": "zzz_game_details_check_scores", "checksum": {"name": "sha256", "checksum": "49cca9739f2738a4609833d267ade1a137756523f0659ec02f0bd31adb3c5789"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.31489, "relation_name": null, "raw_code": "-- This sport allows no negative scores or tie games.\n-- For this test to pass, this query must return no results.\n\nselect home_score, visitor_score\nfrom {{ ref('zzz_game_details') }}\nwhere home_score < 0\nor visitor_score < 0\nor home_score = visitor_score", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "test.dbt_demo.zzz_game_details_check_dates": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "zzz_game_details_check_dates", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/zzz_game_details_check_dates.sql", "original_file_path": "tests/data_tests/zzz_game_details_check_dates.sql", "unique_id": "test.dbt_demo.zzz_game_details_check_dates", "fqn": ["dbt_demo", "data_tests", "zzz_game_details_check_dates"], "alias": "zzz_game_details_check_dates", "checksum": {"name": "sha256", "checksum": "ac2ae3e40e5b734327e8a60a8c944e535c472314feee78d9320da1f7ed515918"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.3213592, "relation_name": null, "raw_code": "-- This season's games happened between 2020-12-12 and 2021-02-06.\n-- For this test to pass, this query must return no results.\n\nselect date\nfrom {{ ref('zzz_game_details') }}\nwhere date < '2020-12-12'\nor date > '2021-02-06'", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "test.dbt_demo.zzz_win_loss_records_check_records": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "zzz_win_loss_records_check_records", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/zzz_win_loss_records_check_records.sql", "original_file_path": "tests/data_tests/zzz_win_loss_records_check_records.sql", "unique_id": "test.dbt_demo.zzz_win_loss_records_check_records", "fqn": ["dbt_demo", "data_tests", "zzz_win_loss_records_check_records"], "alias": "zzz_win_loss_records_check_records", "checksum": {"name": "sha256", "checksum": "8840acce153171ad015abfe10626d1220cc9533e98c859a2b483f0d25d2e4903"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.323633, "relation_name": null, "raw_code": "-- Each team participated in 4 games this season.\n-- For this test to pass, this query must return no results.\n\nselect wins, losses\nfrom {{ ref('zzz_win_loss_records') }}\nwhere wins < 0 or wins > 4\nor losses < 0 or losses > 4\nor (wins + losses) > 4", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "test.dbt_demo.diamonds_list_colors_test": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "diamonds_list_colors_test", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/diamonds_list_colors_test.sql", "original_file_path": "tests/data_tests/diamonds_list_colors_test.sql", "unique_id": "test.dbt_demo.diamonds_list_colors_test", "fqn": ["dbt_demo", "data_tests", "diamonds_list_colors_test"], "alias": "diamonds_list_colors_test", "checksum": {"name": "sha256", "checksum": "4a4d3ee2fa6aad9a7fc4b248111aa6432b67374e217793fbf29d92c2504f13be"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.3264809, "relation_name": null, "raw_code": "SELECT color, COUNT(*)\nFROM {{ ref('diamonds_list_colors') }}\nGROUP BY color\nHAVING COUNT(*) > 1", "language": "sql", "refs": [{"name": "diamonds_list_colors", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.diamonds_list_colors"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "seed.dbt_demo.country_codes": {"database": "dbt_demo", "schema": "dbt_dev", "name": "country_codes", "resource_type": "seed", "package_name": "dbt_demo", "path": "country_codes.csv", "original_file_path": "src/seeds/country_codes.csv", "unique_id": "seed.dbt_demo.country_codes", "fqn": ["dbt_demo", "country_codes"], "alias": "country_codes", "checksum": {"name": "sha256", "checksum": "b54c38e4337b053b07cdc37fa5364e8f159b4e37b77d36281a8d2d11a2c09b8c"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "delimiter": ",", "quote_columns": null}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.388313, "relation_name": "`dbt_demo`.`dbt_dev`.`country_codes`", "raw_code": "", "root_path": "/Users/marcin.wojtyczka/Documents/repos/dbt_demo", "depends_on": {"macros": []}}, "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_zzz_game_details_game_id", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_zzz_game_details_game_id.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218", "fqn": ["dbt_demo", "models", "sql_model1", "unique_zzz_game_details_game_id"], "alias": "unique_zzz_game_details_game_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.462498, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "game_id", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "unique", "kwargs": {"column_name": "game_id", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_game_id", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_game_id.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_game_id"], "alias": "not_null_zzz_game_details_game_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4634, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "game_id", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "game_id", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_home", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_home.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_home"], "alias": "not_null_zzz_game_details_home", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.464213, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "home", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "home", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle", "resource_type": "test", "package_name": "dbt_demo", "path": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40", "fqn": ["dbt_demo", "models", "sql_model1", "accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle"], "alias": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"alias": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3"}, "created_at": 1733486794.4649322, "relation_name": null, "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3\") }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_accepted_values", "macro.dbt.get_where_subquery"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "home", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "accepted_values", "kwargs": {"values": ["Amsterdam", "San Francisco", "Seattle"], "column_name": "home", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_visitor", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_visitor.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_visitor"], "alias": "not_null_zzz_game_details_visitor", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.474986, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "visitor", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "visitor", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle", "resource_type": "test", "package_name": "dbt_demo", "path": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d", "fqn": ["dbt_demo", "models", "sql_model1", "accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle"], "alias": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"alias": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272"}, "created_at": 1733486794.4757178, "relation_name": null, "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272\") }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_accepted_values", "macro.dbt.get_where_subquery"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "visitor", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "accepted_values", "kwargs": {"values": ["Amsterdam", "San Francisco", "Seattle"], "column_name": "visitor", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_home_score", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_home_score.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_home_score"], "alias": "not_null_zzz_game_details_home_score", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4776351, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "home_score", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "home_score", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_visitor_score", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_visitor_score.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_visitor_score"], "alias": "not_null_zzz_game_details_visitor_score", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4783318, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "visitor_score", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "visitor_score", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_winner", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_winner.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_winner"], "alias": "not_null_zzz_game_details_winner", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.479018, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "winner", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "winner", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle", "resource_type": "test", "package_name": "dbt_demo", "path": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c", "fqn": ["dbt_demo", "models", "sql_model1", "accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle"], "alias": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"alias": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca"}, "created_at": 1733486794.4797802, "relation_name": null, "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca\") }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_accepted_values", "macro.dbt.get_where_subquery"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "winner", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "accepted_values", "kwargs": {"values": ["Amsterdam", "San Francisco", "Seattle"], "column_name": "winner", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_date", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_date.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_date"], "alias": "not_null_zzz_game_details_date", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4815848, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "date", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "date", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.unique_first_dbt_model_id.9dda05743e": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_first_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_first_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.unique_first_dbt_model_id.9dda05743e", "fqn": ["dbt_demo", "models", "sql_model2", "unique_first_dbt_model_id"], "alias": "unique_first_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.536581, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "first_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.first_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.first_dbt_model", "attached_node": "model.dbt_demo.first_dbt_model", "test_metadata": {"name": "unique", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('first_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_first_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_first_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061", "fqn": ["dbt_demo", "models", "sql_model2", "not_null_first_dbt_model_id"], "alias": "not_null_first_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.537291, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "first_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.first_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.first_dbt_model", "attached_node": "model.dbt_demo.first_dbt_model", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('first_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_second_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_second_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4", "fqn": ["dbt_demo", "models", "sql_model2", "unique_second_dbt_model_id"], "alias": "unique_second_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.537977, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "second_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.second_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.second_dbt_model", "attached_node": "model.dbt_demo.second_dbt_model", "test_metadata": {"name": "unique", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('second_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_second_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_second_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2", "fqn": ["dbt_demo", "models", "sql_model2", "not_null_second_dbt_model_id"], "alias": "not_null_second_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.538719, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "second_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.second_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.second_dbt_model", "attached_node": "model.dbt_demo.second_dbt_model", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('second_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_second_dbt_model_amount_usd", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_second_dbt_model_amount_usd.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f", "fqn": ["dbt_demo", "models", "sql_model2", "not_null_second_dbt_model_amount_usd"], "alias": "not_null_second_dbt_model_amount_usd", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.539395, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "second_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.second_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "amount_usd", "file_key_name": "models.second_dbt_model", "attached_node": "model.dbt_demo.second_dbt_model", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "amount_usd", "model": "{{ get_where_subquery(ref('second_dbt_model')) }}"}, "namespace": null}}, "model.dbt_demo.changed_country": {"database": "dbt_demo", "schema": "dbt_dev", "name": "changed_country", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/changed_country.sql", "original_file_path": "src/models/sql_model1/changed_country.sql", "unique_id": "model.dbt_demo.changed_country", "fqn": ["dbt_demo", "sql_model1", "changed_country"], "alias": "changed_country", "checksum": {"name": "sha256", "checksum": "e1f9599d0e633856772d321bfc75b89ead550df1ba8fe3dc40a8102c9cc2eb9d"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/changed_country.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733490496.062186, "relation_name": "`dbt_demo`.`dbt_dev`.`changed_country`", "raw_code": "{{ config(\n materialized='table',\n file_format='delta'\n) }}\nselect * from {{ ref('country_snapshot') }}", "language": "sql", "refs": [{"name": "country_snapshot", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["snapshot.dbt_demo.country_snapshot"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/changed_country.sql", "compiled": true, "compiled_code": "\nselect * from `dbt_demo`.`dbt_dev`.`country_snapshot`", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.country": {"database": "dbt_demo", "schema": "dbt_dev", "name": "country", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/country.sql", "original_file_path": "src/models/sql_model1/country.sql", "unique_id": "model.dbt_demo.country", "fqn": ["dbt_demo", "sql_model1", "country"], "alias": "country", "checksum": {"name": "sha256", "checksum": "be347ee69b8c27dec884b8bd692e1aa0ffdac4fb2578c410d048dc9ed16b0b08"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/country.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733492304.094096, "relation_name": "`dbt_demo`.`dbt_dev`.`country`", "raw_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n{{ config(\n materialized='table',\n file_format='delta'\n) }}\nselect\n country_code,\n country_name\nfrom {{ ref('country_codes') }} -- use seed data to populate the table", "language": "sql", "refs": [{"name": "country_codes", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["seed.dbt_demo.country_codes"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/country.sql", "compiled": true, "compiled_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n\nselect\n country_code,\n country_name\nfrom `dbt_demo`.`dbt_dev`.`country_codes` -- use seed data to populate the table", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "snapshot.dbt_demo.country_snapshot": {"database": "dbt_demo", "schema": "dbt_dev", "name": "country_snapshot", "resource_type": "snapshot", "package_name": "dbt_demo", "path": "country_snapshot.sql", "original_file_path": "src/snapshots/country_snapshot.sql", "unique_id": "snapshot.dbt_demo.country_snapshot", "fqn": ["dbt_demo", "country_snapshot", "country_snapshot"], "alias": "country_snapshot", "checksum": {"name": "sha256", "checksum": "100b8c3c320732e98704fe520db0c0bcbf9f1fccf565fccd5807b87cd0d01984"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "snapshot", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "country_code", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "strategy": "check", "target_schema": "dbt_dev", "target_database": "dbt_demo", "updated_at": null, "check_cols": ["country_code"]}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"target_database": "dbt_demo", "target_schema": "dbt_dev", "unique_key": "country_code", "strategy": "check", "check_cols": ["country_code"]}, "created_at": 1733493205.419168, "relation_name": "`dbt_demo`.`dbt_dev`.`country_snapshot`", "raw_code": "\n\n{% set unique_key = 'country_code' %}\n{% set strategy = 'check' %}\n{% set check_cols = ['country_code'] %}\n{% set target_database = 'dbt_demo' %}\n{% set target_schema = 'dbt_dev' %}\n\n{{\n config(\n target_database=target_database,\n target_schema=target_schema,\n unique_key=unique_key,\n strategy=strategy,\n check_cols=check_cols\n )\n}}\n\n-- SQL query for the snapshot\nSELECT *\nFROM {{ ref('country') }}\n\n", "language": "sql", "refs": [{"name": "country", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.country"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "model.dbt_demo.zzz_win_loss_records": {"database": "dbt_demo", "schema": "dbt_dev", "name": "zzz_win_loss_records", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/zzz_win_loss_records.sql", "original_file_path": "src/models/sql_model1/zzz_win_loss_records.sql", "unique_id": "model.dbt_demo.zzz_win_loss_records", "fqn": ["dbt_demo", "sql_model1", "zzz_win_loss_records"], "alias": "zzz_win_loss_records", "checksum": {"name": "sha256", "checksum": "2af4334f4afcfc71dbb39dd2bb88e64cf6e49fb20ec401ef0abf345aa827eb69"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "", "columns": {"team": {"name": "team", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "wins": {"name": "wins", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "losses": {"name": "losses", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model1/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model1/zzz_win_loss_records.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "created_at": 1733493328.3391528, "relation_name": "`dbt_demo`.`dbt_dev`.`zzz_win_loss_records`", "raw_code": "-- Step 2 of 2: Calculate the number of wins and losses for each team.\nselect\n winner as team,\n count(winner) as wins,\n -- Each team played in 4 games.\n (4 - count(winner)) as losses\nfrom (\n -- Step 1 of 2: Determine the winner and loser for each game.\n select\n game_id,\n winner,\n case\n when\n home = winner\n then\n visitor\n else\n home\n end as loser\n from {{ ref('zzz_game_details') }}\n)\ngroup by winner\norder by wins desc", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/zzz_win_loss_records.sql", "compiled": true, "compiled_code": "-- Step 2 of 2: Calculate the number of wins and losses for each team.\nselect\n winner as team,\n count(winner) as wins,\n -- Each team played in 4 games.\n (4 - count(winner)) as losses\nfrom (\n -- Step 1 of 2: Determine the winner and loser for each game.\n select\n game_id,\n winner,\n case\n when\n home = winner\n then\n visitor\n else\n home\n end as loser\n from `dbt_demo`.`dbt_dev`.`zzz_game_details`\n)\ngroup by winner\norder by wins desc", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_zzz_win_loss_records_team", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_zzz_win_loss_records_team.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105", "fqn": ["dbt_demo", "models", "sql_model1", "unique_zzz_win_loss_records_team"], "alias": "unique_zzz_win_loss_records_team", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.3774579, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "team", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "unique", "kwargs": {"column_name": "team", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_win_loss_records_team", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_win_loss_records_team.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_win_loss_records_team"], "alias": "not_null_zzz_win_loss_records_team", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.378356, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "team", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "team", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_win_loss_records_wins", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_win_loss_records_wins.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_win_loss_records_wins"], "alias": "not_null_zzz_win_loss_records_wins", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.3790798, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "wins", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "wins", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_win_loss_records_losses", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_win_loss_records_losses.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_win_loss_records_losses"], "alias": "not_null_zzz_win_loss_records_losses", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.379801, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "losses", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "losses", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}}, "sources": {"source.dbt_demo.raw.diamonds": {"database": "hive_metastore", "schema": "default", "name": "diamonds", "resource_type": "source", "package_name": "dbt_demo", "path": "src/models/sql_model1/sources.yml", "original_file_path": "src/models/sql_model1/sources.yml", "unique_id": "source.dbt_demo.raw.diamonds", "fqn": ["dbt_demo", "sql_model1", "raw", "diamonds"], "source_name": "raw", "source_description": "", "loader": "", "identifier": "diamonds", "quoting": {"database": null, "schema": null, "identifier": null, "column": null}, "loaded_at_field": null, "freshness": {"warn_after": {"count": null, "period": null}, "error_after": {"count": null, "period": null}, "filter": null}, "external": null, "description": "", "columns": {}, "meta": {}, "source_meta": {}, "tags": [], "config": {"enabled": true}, "patch_path": null, "unrendered_config": {}, "relation_name": "`hive_metastore`.`default`.`diamonds`", "created_at": 1733486794.541405}}, "macros": {"macro.dbt_demo.cents_to_dollars": {"name": "cents_to_dollars", "resource_type": "macro", "package_name": "dbt_demo", "path": "src/macros/centrs_to_dollars.sql", "original_file_path": "src/macros/centrs_to_dollars.sql", "unique_id": "macro.dbt_demo.cents_to_dollars", "macro_sql": "{% macro cents_to_dollars(column_name, scale=2) %}\n ({{ column_name }} / 100)::numeric(16, {{ scale }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.526016, "supported_languages": null}, "macro.dbt_databricks.materialization_materialized_view_databricks": {"name": "materialization_materialized_view_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialization_materialized_view_databricks", "macro_sql": "{% materialization materialized_view, adapter = 'databricks' %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation) %}\n\n {% if build_sql == '' %}\n {{ materialized_view_execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.run_hooks", "macro.dbt_databricks.materialized_view_get_build_sql", "macro.dbt_databricks.materialized_view_execute_no_op", "macro.dbt_databricks.materialized_view_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5290601, "supported_languages": ["sql"]}, "macro.dbt_databricks.materialized_view_get_build_sql": {"name": "materialized_view_get_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialized_view_get_build_sql", "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, None, None) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = \"\" %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt_databricks.get_replace_sql", "macro.dbt.get_materialized_view_configuration_changes", "macro.dbt.refresh_materialized_view", "macro.dbt_databricks.get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.530187, "supported_languages": null}, "macro.dbt_databricks.materialized_view_execute_no_op": {"name": "materialized_view_execute_no_op", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialized_view_execute_no_op", "macro_sql": "{% macro materialized_view_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5303931, "supported_languages": null}, "macro.dbt_databricks.materialized_view_execute_build_sql": {"name": "materialized_view_execute_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialized_view_execute_build_sql", "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {%- if build_sql is string %}\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n {%- else %}\n {%- for sql in build_sql %}\n {% call statement(name=\"main\") %}\n {{ sql }}\n {% endcall %}\n {% endfor %}\n {% endif %}\n\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model, for_relation=False) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.531114, "supported_languages": null}, "macro.dbt_databricks.materialization_view_databricks": {"name": "materialization_view_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "unique_id": "macro.dbt_databricks.materialization_view_databricks", "macro_sql": "{% materialization view, adapter='databricks' -%}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n {% set tags = config.get('databricks_tags') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and not old_relation.is_view -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=True) %}\n\n {%- do apply_tags(target_relation, tags) -%}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.apply_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.532544, "supported_languages": ["sql"]}, "macro.dbt_databricks.materialization_table_databricks": {"name": "materialization_table_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_databricks.materialization_table_databricks", "macro_sql": "{% materialization table, adapter = 'databricks', supported_languages=['sql', 'python'] %}\n {{ log(\"MATERIALIZING TABLE\") }}\n {%- set language = model['language'] -%}\n {%- set identifier = model['alias'] -%}\n {%- set grant_config = config.get('grants') -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- set tags = config.get('databricks_tags') -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier, needs_information=True) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n\n {{ run_hooks(pre_hooks) }}\n\n -- setup: if the target relation already exists, drop it\n -- in case if the existing and future table is delta, we want to do a\n -- create or replace table instead of dropping, so we don't have the table unavailable\n {% if old_relation and (not (old_relation.is_delta and config.get('file_format', default='delta') == 'delta')) or (old_relation.is_materialized_view or old_relation.is_streaming_table) -%}\n {{ adapter.drop_relation(old_relation) }}\n {%- endif %}\n\n -- build model\n\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n {% if language==\"python\" %}\n {% do apply_tblproperties(target_relation, tblproperties) %}\n {% endif %}\n {%- do apply_tags(target_relation, tags) -%}\n\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {% do optimize(target_relation) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]})}}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.apply_tblproperties", "macro.dbt_databricks.apply_tags", "macro.dbt.persist_docs", "macro.dbt_databricks.persist_constraints", "macro.dbt_databricks.optimize"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5349078, "supported_languages": ["sql", "python"]}, "macro.dbt_databricks.databricks_build_snapshot_staging_table": {"name": "databricks_build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_databricks.databricks_build_snapshot_staging_table", "macro_sql": "{% macro databricks_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_identifier = target_relation.identifier ~ '__dbt_tmp' %}\n\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=target_relation.schema,\n database=target_relation.database,\n type='view') -%}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {# needs to be a non-temp view so that its columns can be ascertained via `describe` #}\n {% call statement('build_snapshot_staging_relation') %}\n create or replace view {{ tmp_relation }}\n as\n {{ select }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_staging_table", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5406778, "supported_languages": null}, "macro.dbt_databricks.materialization_snapshot_databricks": {"name": "materialization_snapshot_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_databricks.materialization_snapshot_databricks", "macro_sql": "{% materialization snapshot, adapter='databricks' %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n {%- set file_format = config.get('file_format', 'delta') -%}\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = databricks__get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table',\n needs_information=True) -%}\n\n {%- if file_format not in ['delta', 'hudi'] -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n Snapshot functionality requires file_format be set to 'delta' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n\n {%- if target_relation_exists -%}\n {%- if not target_relation.is_delta and not target_relation.is_hudi -%}\n {% set invalid_format_msg -%}\n The existing table {{ model.schema }}.{{ target_table }} is in another format than 'delta' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n {% endif %}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model, for_relation=False) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% if target_relation.database is none %}\n {% set staging_table = spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% else %}\n {% set staging_table = databricks_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% endif %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% call statement_with_staging_table('main', staging_table) %}\n {{ final_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model, for_relation=True) %}\n\n {% endif %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.statement", "macro.dbt.persist_docs", "macro.dbt_spark.spark_build_snapshot_staging_table", "macro.dbt_databricks.databricks_build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt_databricks.statement_with_staging_table", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.544019, "supported_languages": ["sql"]}, "macro.dbt_databricks.materialization_streaming_table_databricks": {"name": "materialization_streaming_table_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.materialization_streaming_table_databricks", "macro_sql": "{% materialization streaming_table, adapter='databricks' %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.StreamingTable) %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% set build_sql = streaming_table_get_build_sql(existing_relation, target_relation) %}\n\n {% if build_sql == '' %}\n {{ streaming_table_execute_no_op(target_relation) }}\n {% else %}\n {{ streaming_table_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.run_hooks", "macro.dbt_databricks.streaming_table_get_build_sql", "macro.dbt_databricks.streaming_table_execute_no_op", "macro.dbt_databricks.streaming_table_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.546883, "supported_languages": ["sql"]}, "macro.dbt_databricks.streaming_table_get_build_sql": {"name": "streaming_table_get_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.streaming_table_get_build_sql", "macro_sql": "{% macro streaming_table_get_build_sql(existing_relation, target_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_streaming_table_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_streaming_table %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_streaming_table_configuration_changes(existing_relation, config) %}\n {% if configuration_changes is none %}\n {% set build_sql = refresh_streaming_table(target_relation, sql) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_streaming_table_as_sql(target_relation, configuration_changes, sql, existing_relation, None, None) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = \"\" %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt_databricks.get_create_streaming_table_as_sql", "macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_streaming_table_configuration_changes", "macro.dbt_databricks.refresh_streaming_table", "macro.dbt_databricks.get_alter_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5479271, "supported_languages": null}, "macro.dbt_databricks.streaming_table_execute_no_op": {"name": "streaming_table_execute_no_op", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.streaming_table_execute_no_op", "macro_sql": "{% macro streaming_table_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.548121, "supported_languages": null}, "macro.dbt_databricks.streaming_table_execute_build_sql": {"name": "streaming_table_execute_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.streaming_table_execute_build_sql", "macro_sql": "{% macro streaming_table_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {%- if build_sql is string %}\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n {%- else %}\n {%- for sql in build_sql %}\n {% call statement(name=\"main\") %}\n {{ sql }}\n {% endcall %}\n {% endfor %}\n {% endif %}\n\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model, for_relation=False) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.548807, "supported_languages": null}, "macro.dbt_databricks.databricks__can_clone_table": {"name": "databricks__can_clone_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_databricks.databricks__can_clone_table", "macro_sql": "{% macro databricks__can_clone_table() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.550354, "supported_languages": null}, "macro.dbt_databricks.databricks__create_or_replace_clone": {"name": "databricks__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_databricks.databricks__create_or_replace_clone", "macro_sql": "{% macro databricks__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace\n table {{ this_relation }}\n shallow clone {{ defer_relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.550466, "supported_languages": null}, "macro.dbt_databricks.materialization_clone_databricks": {"name": "materialization_clone_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_databricks.materialization_clone_databricks", "macro_sql": "\n\n{%- materialization clone, adapter='databricks' -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n {% endif %}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.552143, "supported_languages": ["sql"]}, "macro.dbt_databricks.dbt_databricks_validate_get_file_format": {"name": "dbt_databricks_validate_get_file_format", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_databricks.dbt_databricks_validate_get_file_format", "macro_sql": "{% macro dbt_databricks_validate_get_file_format(raw_file_format) %}\n {#-- Validate the file format #}\n\n {% set accepted_formats = ['text', 'csv', 'json', 'jdbc', 'parquet', 'orc', 'hive', 'delta', 'libsvm', 'hudi'] %}\n\n {% set invalid_file_format_msg -%}\n Invalid file format provided: {{ raw_file_format }}\n Expected one of: {{ accepted_formats | join(', ') }}\n {%- endset %}\n\n {% if raw_file_format not in accepted_formats %}\n {% do exceptions.raise_compiler_error(invalid_file_format_msg) %}\n {% endif %}\n\n {% do return(raw_file_format) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.553088, "supported_languages": null}, "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy": {"name": "dbt_databricks_validate_get_incremental_strategy", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy", "macro_sql": "{% macro dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format) %}\n {#-- Validate the incremental strategy #}\n\n {% set invalid_strategy_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n Expected one of: 'merge', 'replace_where', 'append', 'insert_overwrite'\n {%- endset %}\n\n {% set invalid_delta_only_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You can only choose this strategy when file_format is set to 'delta'\n {%- endset %}\n\n {% set invalid_insert_overwrite_endpoint_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You cannot use this strategy when connecting via warehouse\n Use the 'merge' or 'replace_where' strategy instead\n {%- endset %}\n\n {% if raw_strategy not in ['append', 'merge', 'insert_overwrite', 'replace_where'] %}\n {% do exceptions.raise_compiler_error(invalid_strategy_msg) %}\n {%-else %}\n {% if raw_strategy == 'merge' and file_format not in ['delta', 'hudi'] %}\n {% do exceptions.raise_compiler_error(invalid_delta_only_msg) %}\n {% endif %}\n {% if raw_strategy == 'replace_where' and file_format not in ['delta'] %}\n {% do exceptions.raise_compiler_error(invalid_delta_only_msg) %}\n {% endif %}\n {% endif %}\n\n {% do return(raw_strategy) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.55381, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_default_sql": {"name": "databricks__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_default_sql", "macro_sql": "{% macro databricks__get_incremental_default_sql(arg_dict) %}\n {{ return(get_incremental_merge_sql(arg_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.55716, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_append_sql": {"name": "databricks__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_append_sql", "macro_sql": "{% macro databricks__get_incremental_append_sql(arg_dict) %}\n {% do return(get_insert_into_sql(arg_dict[\"temp_relation\"], arg_dict[\"target_relation\"])) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5573308, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_replace_where_sql": {"name": "databricks__get_incremental_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_replace_where_sql", "macro_sql": "{% macro databricks__get_incremental_replace_where_sql(arg_dict) %}\n {% do return(get_replace_where_sql(arg_dict)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_where_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.557461, "supported_languages": null}, "macro.dbt_databricks.get_incremental_replace_where_sql": {"name": "get_incremental_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_incremental_replace_where_sql", "macro_sql": "{% macro get_incremental_replace_where_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_replace_where_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_replace_where_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.557615, "supported_languages": null}, "macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql": {"name": "databricks__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql", "macro_sql": "{% macro databricks__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) %}\n {{ return(get_insert_overwrite_sql(source, target)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5577788, "supported_languages": null}, "macro.dbt_databricks.get_insert_overwrite_sql": {"name": "get_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_insert_overwrite_sql", "macro_sql": "{% macro get_insert_overwrite_sql(source_relation, target_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n insert overwrite table {{ target_relation }}\n {{ partition_cols(label=\"partition\") }}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.558079, "supported_languages": null}, "macro.dbt_databricks.get_replace_where_sql": {"name": "get_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_replace_where_sql", "macro_sql": "{% macro get_replace_where_sql(args_dict) -%}\n {%- set predicates = args_dict['incremental_predicates'] -%}\n {%- set target_relation = args_dict['target_relation'] -%}\n {%- set temp_relation = args_dict['temp_relation'] -%}\n\n insert into {{ target_relation }}\n {% if predicates %}\n {% if predicates is sequence and predicates is not string %}\n replace where {{ predicates | join(' and ') }}\n {% else %}\n replace where {{ predicates }}\n {% endif %}\n {% endif %}\n table {{ temp_relation }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5584981, "supported_languages": null}, "macro.dbt_databricks.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(source_relation, target_relation) %}\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) | map(attribute=\"quoted\") | list -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) | map(attribute=\"quoted\") | list -%}\n {{ insert_into_sql_impl(target_relation, dest_columns, source_relation, source_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.insert_into_sql_impl"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.558819, "supported_languages": null}, "macro.dbt_databricks.insert_into_sql_impl": {"name": "insert_into_sql_impl", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.insert_into_sql_impl", "macro_sql": "{% macro insert_into_sql_impl(target_relation, dest_columns, source_relation, source_columns) %}\n {%- set common_columns = [] -%}\n {%- for dest_col in dest_columns -%}\n {%- if dest_col in source_columns -%}\n {%- do common_columns.append(dest_col) -%}\n {%- else -%}\n {%- do common_columns.append('DEFAULT') -%}\n {%- endif -%}\n {%- endfor -%}\n {%- set dest_cols_csv = dest_columns | join(', ') -%}\n {%- set source_cols_csv = common_columns | join(', ') -%}\ninsert into table {{ target_relation }} ({{ dest_cols_csv }})\nselect {{source_cols_csv}} from {{ source_relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5592902, "supported_languages": null}, "macro.dbt_databricks.databricks__get_merge_sql": {"name": "databricks__get_merge_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_merge_sql", "macro_sql": "{% macro databricks__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %}\n {# need dest_columns for merge_exclude_columns, default to use \"*\" #}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target) -%}\n {%- set source_columns = (adapter.get_columns_in_relation(source) | map(attribute='quoted') | list)-%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} <=> DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} <=> DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n when matched then update set {{ get_merge_update_set(update_columns, on_schema_change, source_columns) }}\n when not matched then insert {{ get_merge_insert(on_schema_change, source_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.incremental_validate_on_schema_change", "macro.dbt.get_merge_update_columns", "macro.dbt_databricks.get_merge_update_set", "macro.dbt_databricks.get_merge_insert"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5605829, "supported_languages": null}, "macro.dbt_databricks.get_merge_update_set": {"name": "get_merge_update_set", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_merge_update_set", "macro_sql": "{% macro get_merge_update_set(update_columns, on_schema_change, source_columns) %}\n {%- if update_columns -%}\n {%- for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n {%- elif on_schema_change == 'ignore' -%}\n *\n {%- else -%}\n {%- for column in source_columns -%}\n {{ column }} = DBT_INTERNAL_SOURCE.{{ column }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.560978, "supported_languages": null}, "macro.dbt_databricks.get_merge_insert": {"name": "get_merge_insert", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_merge_insert", "macro_sql": "{% macro get_merge_insert(on_schema_change, source_columns) %}\n {%- if on_schema_change == 'ignore' -%}\n *\n {%- else -%}\n ({{ source_columns | join(\", \") }}) VALUES (\n {%- for column in source_columns -%}\n DBT_INTERNAL_SOURCE.{{ column }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %})\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.561265, "supported_languages": null}, "macro.dbt_databricks.materialization_incremental_databricks": {"name": "materialization_incremental_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_databricks.materialization_incremental_databricks", "macro_sql": "{% materialization incremental, adapter='databricks', supported_languages=['sql', 'python'] -%}\n {#-- Validate early so we don't run SQL if the file_format + strategy combo is invalid --#}\n {%- set raw_file_format = config.get('file_format', default='delta') -%}\n {%- set raw_strategy = config.get('incremental_strategy') or 'merge' -%}\n {%- set grant_config = config.get('grants') -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- set tags = config.get('databricks_tags') -%}\n {%- set unique_tmp_table_suffix = config.get('unique_tmp_table_suffix', False) | as_bool -%}\n\n {%- set file_format = dbt_databricks_validate_get_file_format(raw_file_format) -%}\n {%- set incremental_strategy = dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format) -%}\n\n {#-- Set vars --#}\n\n {%- set incremental_predicates = config.get('predicates', default=none) or config.get('incremental_predicates', default=none) -%}\n {%- set unique_key = config.get('unique_key', none) -%}\n {%- set partition_by = config.get('partition_by', none) -%}\n {%- set language = model['language'] -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set existing_relation = adapter.get_relation(database=this.database, schema=this.schema, identifier=this.identifier, needs_information=True) -%}\n {%- if unique_tmp_table_suffix == True and raw_strategy == 'replace_where' and raw_file_format == 'delta' -%}\n {%- set temp_relation_suffix = adapter.generate_unique_temporary_table_suffix() -%}\n {%- else -%}\n {%- set temp_relation_suffix = '__dbt_tmp' -%}\n {%- endif -%}\n\n {#-- Set Overwrite Mode to STATIC for initial replace --#}\n {%- if incremental_strategy == 'insert_overwrite' and should_full_refresh() -%}\n {%- call statement() -%}\n set spark.sql.sources.partitionOverwriteMode = STATIC\n {%- endcall -%}\n {%- endif -%}\n\n {#-- Run pre-hooks --#}\n {{ run_hooks(pre_hooks) }}\n\n {#-- Incremental run logic --#}\n {%- if existing_relation is none -%}\n {#-- Relation must be created --#}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {% do apply_tags(target_relation, tags) %}\n {%- if language == 'python' -%}\n {%- do apply_tblproperties(target_relation, tblproperties) %}\n {%- endif -%}\n\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n {%- elif existing_relation.is_view or existing_relation.is_materialized_view or existing_relation.is_streaming_table or should_full_refresh() -%}\n {#-- Relation must be dropped & recreated --#}\n {% set is_delta = (file_format == 'delta' and existing_relation.is_delta) %}\n {% if not is_delta %} {#-- If Delta, we will `create or replace` below, so no need to drop --#}\n {% do adapter.drop_relation(existing_relation) %}\n {% endif %}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% if not existing_relation.is_view %}\n {% do persist_constraints(target_relation, model) %}\n {% endif %}\n {% do apply_tags(target_relation, tags) %}\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n {%- else -%}\n {#-- Set Overwrite Mode to DYNAMIC for subsequent incremental operations --#}\n {%- if incremental_strategy == 'insert_overwrite' and partition_by -%}\n {%- call statement() -%}\n set spark.sql.sources.partitionOverwriteMode = DYNAMIC\n {%- endcall -%}\n {%- endif -%}\n {#-- Relation must be merged --#}\n {%- set _existing_config = adapter.get_relation_config(existing_relation) -%}\n {%- set model_config = adapter.get_config_from_model(config.model) -%}\n {%- set _configuration_changes = model_config.get_changeset(_existing_config) -%}\n {%- set temp_relation = databricks__make_temp_relation(target_relation, suffix=temp_relation_suffix, as_table=language != 'sql') -%}\n {%- call statement('create_temp_relation', language=language) -%}\n {{ create_table_as(True, temp_relation, compiled_code, language) }}\n {%- endcall -%}\n {%- do process_schema_changes(on_schema_change, temp_relation, existing_relation) -%}\n {%- set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) -%}\n {%- set strategy_arg_dict = ({\n 'target_relation': target_relation,\n 'temp_relation': temp_relation,\n 'unique_key': unique_key,\n 'dest_columns': none,\n 'incremental_predicates': incremental_predicates}) -%}\n {%- set build_sql = strategy_sql_macro_func(strategy_arg_dict) -%}\n {%- if language == 'sql' -%}\n {%- call statement('main') -%}\n {{ build_sql }}\n {%- endcall -%}\n {%- elif language == 'python' -%}\n {%- call statement_with_staging_table('main', temp_relation) -%}\n {{ build_sql }}\n {%- endcall -%}\n {#--\n This is yucky.\n See note in dbt-spark/dbt/include/spark/macros/adapters.sql\n re: python models and temporary views.\n\n Also, why does not either drop_relation or adapter.drop_relation work here?!\n --#}\n {%- endif -%}\n {% do apply_liquid_clustered_cols(target_relation) %}\n {% if _configuration_changes is not none %}\n {% set tags = _configuration_changes.changes.get(\"tags\", None) %}\n {% set tblproperties = _configuration_changes.changes.get(\"tblproperties\", None) %}\n {% if tags is not none %}\n {% do apply_tags(target_relation, tags.set_tags, tags.unset_tags) %}\n {%- endif -%}\n {% if tblproperties is not none %}\n {% do apply_tblproperties(target_relation, tblproperties.tblproperties) %}\n {%- endif -%}\n {%- endif -%}\n {% do persist_docs(target_relation, model, for_relation=True) %}\n {%- endif -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n {% do optimize(target_relation) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.dbt_databricks_validate_get_file_format", "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt_databricks.persist_constraints", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_tblproperties", "macro.dbt.persist_docs", "macro.dbt_databricks.databricks__make_temp_relation", "macro.dbt.process_schema_changes", "macro.dbt_databricks.statement_with_staging_table", "macro.dbt_databricks.apply_liquid_clustered_cols", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.optimize"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.566802, "supported_languages": ["sql", "python"]}, "macro.dbt_databricks.databricks__get_binding_char": {"name": "databricks__get_binding_char", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__get_binding_char", "macro_sql": "{% macro databricks__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5697222, "supported_languages": null}, "macro.dbt_databricks.databricks__load_csv_rows": {"name": "databricks__load_csv_rows", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__load_csv_rows", "macro_sql": "{% macro databricks__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n {% set column_override = model['config'].get('column_types', {}) %}\n {% set must_cast = model['config'].get(\"file_format\", \"delta\") == \"parquet\" %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert {% if loop.index0 == 0 -%} overwrite {% else -%} into {% endif -%} {{ this.render() }} values\n {% for row in chunk -%}\n ({%- for col_name in agate_table.column_names -%}\n {%- if must_cast -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n cast({{ get_binding_char() }} as {{type}})\n {%- else -%}\n {{ get_binding_char() }}\n {%- endif -%}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True, close_cursor=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.571152, "supported_languages": null}, "macro.dbt_databricks.databricks__reset_csv_table": {"name": "databricks__reset_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__reset_csv_table", "macro_sql": "{% macro databricks__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% if old_relation %}\n {% if old_relation.is_delta and config.get('file_format', default='delta') == 'delta' %}\n {% set sql = create_or_replace_csv_table(model, agate_table, True) %}\n {% else %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% endif %}\n {% else %}\n {% set sql = create_csv_table(model, agate_table) %}\n {% endif %}\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.create_or_replace_csv_table", "macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.57173, "supported_languages": null}, "macro.dbt_databricks.create_or_replace_csv_table": {"name": "create_or_replace_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.create_or_replace_csv_table", "macro_sql": "{% macro create_or_replace_csv_table(model, agate_table, replace=False) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {%- set column_comment = config.persist_column_docs() and model.columns %}\n {%- set identifier = model['alias'] -%}\n {%- set relation = api.Relation.create(database=database, schema=schema, identifier=identifier, type='table') -%}\n {%- set replace_clause = \"\" -%}\n {%- if replace -%}\n {%- set replace_clause = \"or replace\" -%}\n {%- endif -%}\n\n {% set sql %}\n create {{replace_clause}} table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {%- set column_comment_clause = \"\" -%}\n {%- if column_comment and col_name in model.columns.keys() -%} \n {%- set comment = model.columns[col_name]['description'] | replace(\"'\", \"\\\\'\") -%}\n {%- if comment and comment != \"\" -%}\n {%- set column_comment_clause = \"comment '\" ~ comment ~ \"'\" -%}\n {%- endif -%}\n {%- endif -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {{ column_comment_clause }}{%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {{ file_format_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause(relation) }}\n {{ comment_clause() }}\n {{ tblproperties_clause() }}\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.file_format_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_databricks.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_databricks.tblproperties_clause", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.57331, "supported_languages": null}, "macro.dbt_databricks.databricks__create_csv_table": {"name": "databricks__create_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__create_csv_table", "macro_sql": "{% macro databricks__create_csv_table(model, agate_table) %}\n {{ return(create_or_replace_csv_table(model, agate_table)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.create_or_replace_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.573471, "supported_languages": null}, "macro.dbt_databricks.materialization_seed_databricks": {"name": "materialization_seed_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/seeds.sql", "original_file_path": "macros/materializations/seeds/seeds.sql", "unique_id": "macro.dbt_databricks.materialization_seed_databricks", "macro_sql": "{% materialization seed, adapter='databricks' %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier, needs_information=True) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and (old_relation.is_view or old_relation.is_materialized_view)) -%}\n {%- set exists_as_streaming_table = (old_relation is not none and old_relation.is_streaming_table) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparision later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view or a materialized view\".format(old_relation)) }}\n {% elif exists_as_streaming_table %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a streaming table\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n -- No need to persist docs, already handled in seed create\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.576501, "supported_languages": ["sql"]}, "macro.dbt_databricks.databricks__generate_database_name": {"name": "databricks__generate_database_name", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt_databricks.databricks__generate_database_name", "macro_sql": "{% macro databricks__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n {{ return(default_database) }}\n {%- else -%}\n {{ return(custom_database_name) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.576849, "supported_languages": null}, "macro.dbt_databricks.databricks__get_drop_sql": {"name": "databricks__get_drop_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt_databricks.databricks__get_drop_sql", "macro_sql": "{% macro databricks__get_drop_sql(relation) -%}\n {%- if relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n {%- elif relation.is_streaming_table-%}\n {{ drop_streaming_table(relation) }}\n {%- elif relation.is_view -%}\n {{ drop_view(relation) }}\n {%- else -%}\n {{ drop_table(relation) }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.drop_materialized_view", "macro.dbt_databricks.drop_streaming_table", "macro.dbt.drop_view", "macro.dbt.drop_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.577309, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_relation": {"name": "databricks__drop_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_relation", "macro_sql": "{% macro databricks__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {{ get_drop_sql(relation) }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.577478, "supported_languages": null}, "macro.dbt_databricks.get_replace_sql": {"name": "get_replace_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.get_replace_sql", "macro_sql": "{% macro get_replace_sql(existing_relation, target_relation, sql) %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {% do return(adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.57809, "supported_languages": null}, "macro.dbt_databricks.databricks__get_replace_sql": {"name": "databricks__get_replace_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.databricks__get_replace_sql", "macro_sql": "{% macro databricks__get_replace_sql(existing_relation, target_relation, sql) %}\n {# /* use a create or replace statement if possible */ #}\n\n {% set is_replaceable = existing_relation.type == target_relation_type and existing_relation.can_be_replaced %}\n\n {% if is_replaceable and existing_relation.is_view %}\n {{ get_replace_view_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_table %}\n {{ get_replace_table_sql(target_relation, sql) }}\n\n {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}\n {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}\n {{ return([\n get_create_intermediate_sql(target_relation, sql),\n get_create_backup_sql(existing_relation),\n get_rename_intermediate_sql(target_relation),\n get_drop_backup_sql(existing_relation)\n ]) }}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}\n {%- elif target_relation.can_be_renamed -%}\n {{ return([\n get_create_intermediate_sql(target_relation, sql),\n get_drop_sql(existing_relation),\n get_rename_intermediate_sql(target_relation),\n ]) }}\n\n {# /* create target_relation in place by first backing up the existing relation */ #}\n {%- elif existing_relation.can_be_renamed -%}\n {{ return([\n get_create_backup_sql(existing_relation),\n get_create_sql(target_relation, sql),\n get_drop_backup_sql(existing_relation)\n ]) }}\n\n {# /* no renaming is allowed, so just drop and create */ #}\n {%- else -%}\n {{ return([\n get_drop_sql(existing_relation),\n get_create_sql(target_relation, sql)\n ]) }}\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_replace_view_sql", "macro.dbt.get_replace_table_sql", "macro.dbt.get_create_intermediate_sql", "macro.dbt.get_create_backup_sql", "macro.dbt.get_rename_intermediate_sql", "macro.dbt.get_drop_backup_sql", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.579058, "supported_languages": null}, "macro.dbt_databricks.fetch_tags": {"name": "fetch_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.fetch_tags", "macro_sql": "{% macro fetch_tags(relation) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Tags are only supported for Unity Catalog\") }}\n {%- endif %}\n {% call statement('list_tags', fetch_result=True) -%}\n {{ fetch_tags_sql(relation) }}\n {% endcall %}\n {% do return(load_result('list_tags').table) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.fetch_tags_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5802681, "supported_languages": null}, "macro.dbt_databricks.fetch_tags_sql": {"name": "fetch_tags_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.fetch_tags_sql", "macro_sql": "{% macro fetch_tags_sql(relation) -%}\n SELECT tag_name, tag_value\n FROM `system`.`information_schema`.`table_tags`\n WHERE catalog_name = '{{ relation.database|lower }}' \n AND schema_name = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}'\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5804422, "supported_languages": null}, "macro.dbt_databricks.apply_tags": {"name": "apply_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.apply_tags", "macro_sql": "{% macro apply_tags(relation, set_tags, unset_tags=[]) -%}\n {%- if (set_tags or unset_tags) and relation.is_hive_metastore() -%}\n {{ exceptions.raise_compiler_error(\"Tags are only supported for Unity Catalog\") }}\n {%- endif -%}\n {%- if set_tags %}\n {%- call statement('set_tags') -%}\n {{ alter_set_tags(relation, set_tags) }}\n {%- endcall -%}\n {%- endif %}\n {%- if unset_tags %}\n {%- call statement('unset_tags') -%}\n {{ alter_unset_tags(relation, unset_tags) }}\n {%- endcall -%}\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.alter_set_tags", "macro.dbt_databricks.alter_unset_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.580887, "supported_languages": null}, "macro.dbt_databricks.alter_set_tags": {"name": "alter_set_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.alter_set_tags", "macro_sql": "{% macro alter_set_tags(relation, tags) -%}\n ALTER {{ relation.type }} {{ relation }} SET TAGS (\n {% for tag in tags -%}\n '{{ tag }}' = '{{ tags[tag] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n )\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.581147, "supported_languages": null}, "macro.dbt_databricks.alter_unset_tags": {"name": "alter_unset_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.alter_unset_tags", "macro_sql": "{% macro alter_unset_tags(relation, tags) -%}\n ALTER {{ relation.type }} {{ relation }} UNSET TAGS (\n {% for tag in tags -%}\n '{{ tag }}' {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n )\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.58137, "supported_languages": null}, "macro.dbt_databricks.location_clause": {"name": "location_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/location.sql", "original_file_path": "macros/relations/location.sql", "unique_id": "macro.dbt_databricks.location_clause", "macro_sql": "{% macro location_clause(relation) %}\n {%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n {%- set file_format = config.get('file_format', default='delta') -%}\n {%- set identifier = model['alias'] -%}\n {%- if location_root is not none %}\n location '{{ location_root }}/{{ identifier }}'\n {%- elif (not relation.is_hive_metastore()) and file_format != 'delta' -%}\n {{ exceptions.raise_compiler_error(\n 'Incompatible configuration: `location_root` must be set when using a non-delta file format with Unity Catalog'\n ) }}\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5819929, "supported_languages": null}, "macro.dbt_databricks.optimize": {"name": "optimize", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.optimize", "macro_sql": "{% macro optimize(relation) %}\n {{ return(adapter.dispatch('optimize', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__optimize"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.582578, "supported_languages": null}, "macro.dbt_databricks.databricks__optimize": {"name": "databricks__optimize", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.databricks__optimize", "macro_sql": "\n\n{%- macro databricks__optimize(relation) -%}\n {%- if var('DATABRICKS_SKIP_OPTIMIZE', 'false')|lower != 'true' and\n var('databricks_skip_optimize', 'false')|lower != 'true' and\n config.get('file_format', 'delta') == 'delta' -%}\n {%- if (config.get('zorder', False) or config.get('liquid_clustered_by', False)) -%}\n {%- call statement('run_optimize_stmt') -%}\n {{ get_optimize_sql(relation) }}\n {%- endcall -%}\n {%- endif -%}\n {%- endif -%}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.get_optimize_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.583037, "supported_languages": null}, "macro.dbt_databricks.get_optimize_sql": {"name": "get_optimize_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.get_optimize_sql", "macro_sql": "{%- macro get_optimize_sql(relation) %}\n optimize {{ relation }}\n {%- if config.get('zorder', False) and config.get('file_format', 'delta') == 'delta' %}\n {%- if config.get('liquid_clustered_by', False) %}\n {{ exceptions.warn(\"Both zorder and liquid_clustered_by are set but they are incompatible. zorder will be ignored.\") }}\n {%- else %}\n {%- set zorder = config.get('zorder', none) %}\n {# TODO: predicates here? WHERE ... #}\n {%- if zorder is sequence and zorder is not string %}\n zorder by (\n {%- for col in zorder %}\n {{ col }}{% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- else %}\n zorder by ({{zorder}})\n {%- endif %}\n {%- endif %}\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.583699, "supported_languages": null}, "macro.dbt_databricks.file_format_clause": {"name": "file_format_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/file_format.sql", "original_file_path": "macros/relations/file_format.sql", "unique_id": "macro.dbt_databricks.file_format_clause", "macro_sql": "{% macro file_format_clause() %}\n {%- set file_format = config.get('file_format', default='delta') -%}\n using {{ file_format }}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.583901, "supported_languages": null}, "macro.dbt_databricks.liquid_clustered_cols": {"name": "liquid_clustered_cols", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/liquid_clustering.sql", "original_file_path": "macros/relations/liquid_clustering.sql", "unique_id": "macro.dbt_databricks.liquid_clustered_cols", "macro_sql": "{% macro liquid_clustered_cols() -%}\n {%- set cols = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n CLUSTER BY ({{ cols | join(', ') }})\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5845, "supported_languages": null}, "macro.dbt_databricks.apply_liquid_clustered_cols": {"name": "apply_liquid_clustered_cols", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/liquid_clustering.sql", "original_file_path": "macros/relations/liquid_clustering.sql", "unique_id": "macro.dbt_databricks.apply_liquid_clustered_cols", "macro_sql": "{% macro apply_liquid_clustered_cols(target_relation) -%}\n {%- set cols = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {%- call statement('set_cluster_by_columns') -%}\n ALTER {{ target_relation.type }} {{ target_relation }} CLUSTER BY ({{ cols | join(', ') }})\n {%- endcall -%}\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.584919, "supported_languages": null}, "macro.dbt_databricks.tblproperties_clause": {"name": "tblproperties_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.tblproperties_clause", "macro_sql": "{% macro tblproperties_clause() -%}\n {{ return(adapter.dispatch('tblproperties_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.58541, "supported_languages": null}, "macro.dbt_databricks.databricks__tblproperties_clause": {"name": "databricks__tblproperties_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.databricks__tblproperties_clause", "macro_sql": "{% macro databricks__tblproperties_clause() -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- if tblproperties is not none %}\n tblproperties (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}' {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5857081, "supported_languages": null}, "macro.dbt_databricks.apply_tblproperties": {"name": "apply_tblproperties", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.apply_tblproperties", "macro_sql": "{% macro apply_tblproperties(relation, tblproperties) -%}\n {% if tblproperties %}\n {%- call statement('apply_tblproperties') -%}\n ALTER {{ relation.type }} {{ relation }} SET TBLPROPERTIES (\n {% for tblproperty in tblproperties -%}\n '{{ tblproperty }}' = '{{ tblproperties[tblproperty] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n )\n {%- endcall -%}\n {% endif %}\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.58604, "supported_languages": null}, "macro.dbt_databricks.persist_constraints": {"name": "persist_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.persist_constraints", "macro_sql": "{% macro persist_constraints(relation, model) %}\n {{ return(adapter.dispatch('persist_constraints', 'dbt')(relation, model)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.599794, "supported_languages": null}, "macro.dbt_databricks.databricks__persist_constraints": {"name": "databricks__persist_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__persist_constraints", "macro_sql": "{% macro databricks__persist_constraints(relation, model) %}\n {%- set contract_config = config.get('contract') -%}\n {% set has_model_contract = contract_config and contract_config.enforced %}\n {% set has_databricks_constraints = config.get('persist_constraints', False) %}\n\n {% if (has_model_contract or has_databricks_constraints) %}\n {% if config.get('file_format', 'delta') != 'delta' %}\n {# Constraints are only supported for delta tables #}\n {{ exceptions.warn(\"Constraints not supported for file format: \" ~ config.get('file_format')) }}\n {% elif relation.is_view %}\n {# Constraints are not supported for views. This point in the code should not have been reached. #}\n {{ exceptions.raise_compiler_error(\"Constraints not supported for views.\") }}\n {% elif is_incremental() %}\n {# Constraints are not applied for incremental updates. This point in the code should not have been reached #}\n {{ exceptions.raise_compiler_error(\"Constraints are not applied for incremental updates. Full refresh is required to update constraints.\") }}\n {% else %}\n {% do alter_column_set_constraints(relation, model) %}\n {% do alter_table_add_constraints(relation, model) %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.is_incremental", "macro.dbt_databricks.alter_column_set_constraints", "macro.dbt_databricks.alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.600513, "supported_languages": null}, "macro.dbt_databricks.alter_table_add_constraints": {"name": "alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.alter_table_add_constraints", "macro_sql": "{% macro alter_table_add_constraints(relation, constraints) %}\n {{ return(adapter.dispatch('alter_table_add_constraints', 'dbt')(relation, constraints)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.600693, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_table_add_constraints": {"name": "databricks__alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__alter_table_add_constraints", "macro_sql": "{% macro databricks__alter_table_add_constraints(relation, model) %}\n {% set constraints = get_model_constraints(model) %}\n {% set statements = get_constraints_sql(relation, constraints, model) %}\n {% for stmt in statements %}\n {% call statement() %}\n {{ stmt }}\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_model_constraints", "macro.dbt_databricks.get_constraints_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6010072, "supported_languages": null}, "macro.dbt_databricks.get_model_constraints": {"name": "get_model_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_model_constraints", "macro_sql": "{% macro get_model_constraints(model) %}\n {% set constraints = model.get('constraints', []) %}\n {% if config.get('persist_constraints', False) and model.get('meta', {}).get('constraints') is sequence %}\n {# Databricks constraints implementation. Constraints are in the meta property. #}\n {% set db_constraints = model.get('meta', {}).get('constraints', []) %}\n {% set constraints = databricks_constraints_to_dbt(db_constraints) %}\n {% endif %}\n {{ return(constraints) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks_constraints_to_dbt"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.601491, "supported_languages": null}, "macro.dbt_databricks.get_column_constraints": {"name": "get_column_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_column_constraints", "macro_sql": "{% macro get_column_constraints(column) %}\n {% set constraints = column.get('constraints', []) %}\n {% if config.get('persist_constraints', False) and column.get('meta', {}).get('constraint') %}\n {# Databricks constraints implementation. Constraint is in the meta property. #}\n {% set db_constraints = [column.get('meta', {}).get('constraint')] %}\n {% set constraints = databricks_constraints_to_dbt(db_constraints, column) %}\n {% endif %}\n {{ return(constraints) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks_constraints_to_dbt"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6019762, "supported_languages": null}, "macro.dbt_databricks.alter_column_set_constraints": {"name": "alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.alter_column_set_constraints", "macro_sql": "{% macro alter_column_set_constraints(relation, column_dict) %}\n {{ return(adapter.dispatch('alter_column_set_constraints', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.602155, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_column_set_constraints": {"name": "databricks__alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__alter_column_set_constraints", "macro_sql": "{% macro databricks__alter_column_set_constraints(relation, model) %}\n {% set column_dict = model.columns %}\n {% for column_name in column_dict %}\n {% set column = column_dict[column_name] %}\n {% set constraints = get_column_constraints(column) %}\n {% set statements = get_constraints_sql(relation, constraints, model, column) %}\n {% for stmt in statements %}\n {% call statement() %}\n {{ stmt }}\n {% endcall %}\n {% endfor %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_column_constraints", "macro.dbt_databricks.get_constraints_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.602596, "supported_languages": null}, "macro.dbt_databricks.get_constraints_sql": {"name": "get_constraints_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_constraints_sql", "macro_sql": "{% macro get_constraints_sql(relation, constraints, model, column={}) %}\n {% set statements = [] %}\n -- Hack so that not null constraints will be applied before primary key constraints\n {% for constraint in constraints|sort(attribute='type') %}\n {% if constraint %}\n {% set constraint_statements = get_constraint_sql(relation, constraint, model, column) %}\n {% for statement in constraint_statements %}\n {% if statement %}\n {% do statements.append(statement) %}\n {% endif %}\n {% endfor %}\n {% endif %}\n {% endfor %}\n\n {{ return(statements) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_constraint_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.603168, "supported_languages": null}, "macro.dbt_databricks.get_constraint_sql": {"name": "get_constraint_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_constraint_sql", "macro_sql": "{% macro get_constraint_sql(relation, constraint, model, column={}) %}\n {% set statements = [] %}\n {% set type = constraint.get(\"type\", \"\") %}\n\n {% if type == 'check' %}\n {% set expression = constraint.get(\"expression\", \"\") %}\n {% if not expression %}\n {{ exceptions.raise_compiler_error('Invalid check constraint expression') }}\n {% endif %}\n\n {% set name = constraint.get(\"name\") %}\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5 (relation.identifier ~ \";\" ~ column.get(\"name\", \"\") ~ \";\" ~ expression ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %}\n {% endif %}\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" check (\" ~ expression ~ \");\" %}\n {% do statements.append(stmt) %}\n {% elif type == 'not_null' %}\n {% set column_names = constraint.get(\"columns\", []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if column %}\n {% set quoted_name = adapter.quote(column['name']) if column['quote'] else column['name'] %}\n {% set stmt = \"alter table \" ~ relation ~ \" change column \" ~ quoted_name ~ \" set not null \" ~ (constraint.expression or \"\") ~ \";\" %}\n {% do statements.append(stmt) %}\n {% else %}\n {{ exceptions.warn('not_null constraint on invalid column: ' ~ column_name) }}\n {% endif %}\n {% endfor %}\n {% elif type == 'primary_key' %}\n {% if constraint.get('warn_unenforced') %}\n {{ exceptions.warn(\"unenforced constraint type: \" ~ type)}}\n {% endif %}\n {% set column_names = constraint.get(\"columns\", []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% set quoted_names = [] %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if not column %}\n {{ exceptions.warn('Invalid primary key column: ' ~ column_name) }}\n {% else %}\n {% set quoted_name = adapter.quote(column['name']) if column['quote'] else column['name'] %}\n {% do quoted_names.append(quoted_name) %}\n {% endif %}\n {% endfor %}\n\n {% set joined_names = quoted_names|join(\", \") %}\n\n {% set name = constraint.get(\"name\") %}\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"primary_key;\" ~ relation.identifier ~ \";\" ~ column_names ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %}\n {% endif %}\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" primary key(\" ~ joined_names ~ \");\" %}\n {% do statements.append(stmt) %}\n {% elif type == 'foreign_key' %}\n\n {% if constraint.get('warn_unenforced') %}\n {{ exceptions.warn(\"unenforced constraint type: \" ~ constraint.type)}}\n {% endif %}\n\n {% set name = constraint.get(\"name\") %}\n \n {% if constraint.get('expression') %}\n\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"foreign_key;\" ~ relation.identifier ~ \";\" ~ constraint.get('expression') ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %} \n {% endif %}\n\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" foreign key\" ~ constraint.get('expression') %}\n {% else %}\n {% set column_names = constraint.get(\"columns\", []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% set quoted_names = [] %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if not column %}\n {{ exceptions.warn('Invalid foreign key column: ' ~ column_name) }}\n {% else %}\n {% set quoted_name = adapter.quote(column['name']) if column['quote'] else column['name'] %}\n {% do quoted_names.append(quoted_name) %}\n {% endif %}\n {% endfor %}\n\n {% set joined_names = quoted_names|join(\", \") %}\n\n {% set parent = constraint.get(\"to\") %}\n {% if not parent %}\n {{ exceptions.raise_compiler_error('No parent table defined for foreign key: ' ~ expression) }}\n {% endif %}\n {% if not \".\" in parent %}\n {% set parent = relation.schema ~ \".\" ~ parent%}\n {% endif %}\n\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"foreign_key;\" ~ relation.identifier ~ \";\" ~ column_names ~ \";\" ~ parent ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %} \n {% endif %}\n\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" foreign key(\" ~ joined_names ~ \") references \" ~ parent %}\n {% set parent_columns = constraint.get(\"to_columns\") %}\n {% if parent_columns %}\n {% set stmt = stmt ~ \"(\" ~ parent_columns|join(\", \") ~ \")\"%}\n {% endif %}\n {% endif %}\n {% set stmt = stmt ~ \";\" %}\n {% do statements.append(stmt) %}\n {% elif constraint.get('warn_unsupported') %}\n {{ exceptions.warn(\"unsupported constraint type: \" ~ constraint.type)}}\n {% endif %}\n\n {{ return(statements) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6086242, "supported_languages": null}, "macro.dbt_databricks.databricks_constraints_to_dbt": {"name": "databricks_constraints_to_dbt", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks_constraints_to_dbt", "macro_sql": "{% macro databricks_constraints_to_dbt(constraints, column) %}\n {# convert constraints defined using the original databricks format #}\n {% set dbt_constraints = [] %}\n {% for constraint in constraints %}\n {% if constraint.get and constraint.get(\"type\") %}\n {# already in model contract format #}\n {% do dbt_constraints.append(constraint) %}\n {% else %}\n {% if column %}\n {% if constraint == \"not_null\" %}\n {% do dbt_constraints.append({\"type\": \"not_null\", \"columns\": [column.get(\"name\")]}) %}\n {% else %}\n {{ exceptions.raise_compiler_error('Invalid constraint for column ' ~ column.get(\"name\", \"\") ~ '. Only `not_null` is supported.') }}\n {% endif %}\n {% else %}\n {% set name = constraint['name'] %}\n {% if not name %}\n {{ exceptions.raise_compiler_error('Invalid check constraint name') }}\n {% endif %}\n {% set condition = constraint['condition'] %}\n {% if not condition %}\n {{ exceptions.raise_compiler_error('Invalid check constraint condition') }}\n {% endif %}\n {% do dbt_constraints.append({\"name\": name, \"type\": \"check\", \"expression\": condition}) %}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(dbt_constraints) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.609767, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_sql": {"name": "databricks__get_create_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_sql", "macro_sql": "{%- macro databricks__get_create_sql(relation, sql) -%}\n {%- if relation.is_view -%}\n {{ get_create_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_table -%}\n {{ get_create_table_as_sql(False, relation, sql) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_create_materialized_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_streaming_table -%}\n {{ get_create_streaming_table_as_sql(relation, sql) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_create_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.get_create_view_as_sql", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt_databricks.get_create_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.610386, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_materialized_view": {"name": "databricks__drop_materialized_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_materialized_view", "macro_sql": "{% macro databricks__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.610508, "supported_languages": null}, "macro.dbt_databricks.databricks__refresh_materialized_view": {"name": "databricks__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt_databricks.databricks__refresh_materialized_view", "macro_sql": "{% macro databricks__refresh_materialized_view(relation) -%}\n refresh materialized view {{ relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.61062, "supported_languages": null}, "macro.dbt_databricks.get_alter_materialized_view_as_sql": {"name": "get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_materialized_view_as_sql", "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {%- do return(adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n )) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.611409, "supported_languages": null}, "macro.dbt_databricks.databricks__get_materialized_view_configuration_changes": {"name": "databricks__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_materialized_view_configuration_changes", "macro_sql": "\n\n\n{%- macro databricks__get_materialized_view_configuration_changes(existing_relation, new_config) -%}\n {%- set _existing_materialized_view = adapter.get_relation_config(existing_relation) -%}\n {%- set materialized_view = adapter.get_config_from_model(config.model) -%}\n {%- set _configuration_changes = materialized_view.get_changeset(_existing_materialized_view) -%}\n {% do return(_configuration_changes) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6116831, "supported_languages": null}, "macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql": {"name": "databricks__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql", "macro_sql": "{% macro databricks__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n {% do return(get_replace_sql(existing_relation, relation, sql)) %}\n\n -- otherwise apply individual changes as needed\n {% else %}\n {% do return(get_alter_mv_internal(relation, configuration_changes)) %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_alter_mv_internal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.611993, "supported_languages": null}, "macro.dbt_databricks.get_alter_mv_internal": {"name": "get_alter_mv_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_mv_internal", "macro_sql": "{% macro get_alter_mv_internal(relation, configuration_changes) %}\n {%- set refresh = configuration_changes.changes[\"refresh\"] -%}\n -- Currently only schedule can be altered\n ALTER MATERIALIZED VIEW {{ relation }}\n {{ get_alter_sql_refresh_schedule(refresh.cron, refresh.time_zone_value, refresh.is_altered) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_alter_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6122189, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_materialized_view_as_sql": {"name": "databricks__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_materialized_view_as_sql", "macro_sql": "{% macro databricks__get_create_materialized_view_as_sql(relation, sql) -%}\n {%- set materialized_view = adapter.get_config_from_model(config.model) -%}\n {%- set partition_by = materialized_view.config[\"partition_by\"].partition_by -%}\n {%- set tblproperties = materialized_view.config[\"tblproperties\"].tblproperties -%}\n {%- set comment = materialized_view.config[\"comment\"].comment -%}\n {%- set refresh = materialized_view.config[\"refresh\"] -%}\n create materialized view {{ relation }}\n {{ get_create_sql_partition_by(partition_by) }}\n {{ get_create_sql_comment(comment) }}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {{ get_create_sql_refresh_schedule(refresh.cron, refresh.time_zone_value) }}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties", "macro.dbt_databricks.get_create_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.612938, "supported_languages": null}, "macro.dbt_databricks.drop_streaming_table": {"name": "drop_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/drop.sql", "original_file_path": "macros/relations/streaming_table/drop.sql", "unique_id": "macro.dbt_databricks.drop_streaming_table", "macro_sql": "{% macro drop_streaming_table(relation) -%}\n {{ return(adapter.dispatch('drop_streaming_table', 'dbt')(relation)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.default__drop_streaming_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.613157, "supported_languages": null}, "macro.dbt_databricks.default__drop_streaming_table": {"name": "default__drop_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/drop.sql", "original_file_path": "macros/relations/streaming_table/drop.sql", "unique_id": "macro.dbt_databricks.default__drop_streaming_table", "macro_sql": "{% macro default__drop_streaming_table(relation) -%}\n drop table if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6132412, "supported_languages": null}, "macro.dbt_databricks.refresh_streaming_table": {"name": "refresh_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/refresh.sql", "original_file_path": "macros/relations/streaming_table/refresh.sql", "unique_id": "macro.dbt_databricks.refresh_streaming_table", "macro_sql": "{% macro refresh_streaming_table(relation, sql) -%}\n {{ adapter.dispatch('refresh_streaming_table', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__refresh_streaming_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.61346, "supported_languages": null}, "macro.dbt_databricks.databricks__refresh_streaming_table": {"name": "databricks__refresh_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/refresh.sql", "original_file_path": "macros/relations/streaming_table/refresh.sql", "unique_id": "macro.dbt_databricks.databricks__refresh_streaming_table", "macro_sql": "{% macro databricks__refresh_streaming_table(relation, sql) -%}\n create or refresh streaming table {{ relation }}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6135712, "supported_languages": null}, "macro.dbt_databricks.get_alter_streaming_table_as_sql": {"name": "get_alter_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_streaming_table_as_sql", "macro_sql": "{% macro get_alter_streaming_table_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {%- do return(adapter.dispatch('get_alter_streaming_table_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n )) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.615469, "supported_languages": null}, "macro.dbt_databricks.get_streaming_table_configuration_changes": {"name": "get_streaming_table_configuration_changes", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_streaming_table_configuration_changes", "macro_sql": "{% macro get_streaming_table_configuration_changes(existing_relation, new_config) -%}\n {{- log('Determining configuration changes on: ' ~ existing_relation) -}}\n {%- do return(adapter.dispatch('get_streaming_table_configuration_changes', 'dbt')(existing_relation, new_config)) -%}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_streaming_table_configuration_changes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6158211, "supported_languages": null}, "macro.dbt_databricks.databricks__get_streaming_table_configuration_changes": {"name": "databricks__get_streaming_table_configuration_changes", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_streaming_table_configuration_changes", "macro_sql": "\n\n{%- macro databricks__get_streaming_table_configuration_changes(existing_relation, new_config) -%}\n {%- set _existing_streaming_table = adapter.get_relation_config(existing_relation) -%}\n {%- set streaming_table = adapter.get_config_from_model(config.model) -%}\n {%- set _configuration_changes = streaming_table.get_changeset(_existing_streaming_table) -%}\n {% do return(_configuration_changes) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6161752, "supported_languages": null}, "macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql": {"name": "databricks__get_alter_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql", "macro_sql": "{% macro databricks__get_alter_streaming_table_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n {% do return(get_replace_sql(existing_relation, relation, sql)) %}\n\n -- otherwise apply individual changes as needed\n {% else %}\n {%- set alter_statement = get_alter_st_internal(relation, configuration_changes) -%}\n {%- set create_statement = get_create_st_internal(relation, configuration_changes, sql) -%}\n {%- set return_statements = [] -%}\n {%- if create_statement -%}\n {{ return_statements.append(create_statement) }}\n {%- endif -%}\n {%- if alter_statement -%}\n {{ return_statements.append(alter_statement) }}\n {%- endif -%}\n {% do return(return_statements) %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_alter_st_internal", "macro.dbt_databricks.get_create_st_internal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.616995, "supported_languages": null}, "macro.dbt_databricks.get_create_st_internal": {"name": "get_create_st_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_create_st_internal", "macro_sql": "{% macro get_create_st_internal(relation, configuration_changes, sql) %}\n {%- set partition_by = configuration_changes.changes[\"partition_by\"].partition_by -%}\n {%- set tblproperties = configuration_changes.changes[\"tblproperties\"].tblproperties -%}\n {%- set comment = configuration_changes.changes[\"comment\"].comment -%}\n CREATE OR REFRESH STREAMING TABLE {{ relation }}\n {% if partition_by -%}\n {{ get_create_sql_partition_by(partition_by) }}\n {%- endif %}\n {% if comment -%}\n {{ get_create_sql_comment(comment) }}\n {%- endif %}\n {% if tblproperties -%}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {%- endif %}\n AS {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6175182, "supported_languages": null}, "macro.dbt_databricks.get_alter_st_internal": {"name": "get_alter_st_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_st_internal", "macro_sql": "{% macro get_alter_st_internal(relation, configuration_changes) %}\n {%- set refresh = configuration_changes.changes[\"refresh\"] -%}\n {%- if refresh and refresh.cron -%}\n ALTER STREAMING TABLE {{ relation }}\n {{ get_alter_sql_refresh_schedule(refresh.cron, refresh.time_zone_value, False) -}}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_alter_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6177878, "supported_languages": null}, "macro.dbt_databricks.get_create_streaming_table_as_sql": {"name": "get_create_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/create.sql", "original_file_path": "macros/relations/streaming_table/create.sql", "unique_id": "macro.dbt_databricks.get_create_streaming_table_as_sql", "macro_sql": "{% macro get_create_streaming_table_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_streaming_table_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.618218, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_streaming_table_as_sql": {"name": "databricks__get_create_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/create.sql", "original_file_path": "macros/relations/streaming_table/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_streaming_table_as_sql", "macro_sql": "{% macro databricks__get_create_streaming_table_as_sql(relation, sql) -%}\n {%- set streaming_table = adapter.get_config_from_model(config.model) -%}\n {%- set partition_by = streaming_table.config[\"partition_by\"].partition_by -%}\n {%- set tblproperties = streaming_table.config[\"tblproperties\"].tblproperties -%}\n {%- set comment = streaming_table.config[\"comment\"].comment -%}\n {%- set refresh = streaming_table.config[\"refresh\"] -%}\n\n CREATE STREAMING TABLE {{ relation }}\n {{ get_create_sql_partition_by(partition_by) }}\n {{ get_create_sql_comment(comment) }}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {{ get_create_sql_refresh_schedule(refresh.cron, refresh.time_zone_value) }}\n AS {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties", "macro.dbt_databricks.get_create_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.618762, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_comment": {"name": "get_create_sql_comment", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/comment.sql", "original_file_path": "macros/relations/components/comment.sql", "unique_id": "macro.dbt_databricks.get_create_sql_comment", "macro_sql": "{%- macro get_create_sql_comment(comment) -%}\n{% if comment is string -%}\n COMMENT '{{ comment }}'\n{%- endif -%}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6189408, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_refresh_schedule": {"name": "get_create_sql_refresh_schedule", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/refresh_schedule.sql", "original_file_path": "macros/relations/components/refresh_schedule.sql", "unique_id": "macro.dbt_databricks.get_create_sql_refresh_schedule", "macro_sql": "{% macro get_create_sql_refresh_schedule(cron, time_zone_value) %}\n {%- if cron -%}\n SCHEDULE CRON '{{ cron }}'{%- if time_zone_value %} AT TIME ZONE '{{ time_zone_value }}'{%- endif -%}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.619374, "supported_languages": null}, "macro.dbt_databricks.get_alter_sql_refresh_schedule": {"name": "get_alter_sql_refresh_schedule", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/refresh_schedule.sql", "original_file_path": "macros/relations/components/refresh_schedule.sql", "unique_id": "macro.dbt_databricks.get_alter_sql_refresh_schedule", "macro_sql": "{% macro get_alter_sql_refresh_schedule(cron, time_zone_value, is_altered) %}\n {%- if cron -%}\n {%- if is_altered -%}\n ALTER SCHEDULE CRON '{{ cron }}'{%- if time_zone_value %} AT TIME ZONE '{{ time_zone_value }}'{%- endif -%}\n {%- else -%}\n ADD SCHEDULE CRON '{{ cron }}'{%- if time_zone_value %} AT TIME ZONE '{{ time_zone_value }}'{%- endif -%}\n {%- endif -%}\n {%- else -%}\n DROP SCHEDULE\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.619678, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_partition_by": {"name": "get_create_sql_partition_by", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/partitioning.sql", "original_file_path": "macros/relations/components/partitioning.sql", "unique_id": "macro.dbt_databricks.get_create_sql_partition_by", "macro_sql": "{% macro get_create_sql_partition_by(partition_by) -%}\n{%- if partition_by -%}\n PARTITIONED BY ({%- for col in partition_by -%}{{ col }}{% if not loop.last %}, {% endif %}{%- endfor %})\n{%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.619945, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_tblproperties": {"name": "get_create_sql_tblproperties", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/tblproperties.sql", "original_file_path": "macros/relations/components/tblproperties.sql", "unique_id": "macro.dbt_databricks.get_create_sql_tblproperties", "macro_sql": "{% macro get_create_sql_tblproperties(tblproperties) %}\n {%- if tblproperties and tblproperties|length>0 -%}\n TBLPROPERTIES (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}'{%- if not loop.last -%}, {% endif -%}\n {% endfor -%}\n )\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.62031, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_table": {"name": "databricks__drop_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_table", "macro_sql": "{% macro databricks__drop_table(relation) -%}\n drop table if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.620424, "supported_languages": null}, "macro.dbt_databricks.databricks__create_table_as": {"name": "databricks__create_table_as", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.databricks__create_table_as", "macro_sql": "{% macro databricks__create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {%- if language == 'sql' -%}\n {%- if temporary -%}\n {{ create_temporary_view(relation, compiled_code) }}\n {%- else -%}\n {%- set file_format = config.get('file_format', default='delta') -%}\n {% if file_format == 'delta' %}\n create or replace table {{ relation }}\n {% else %}\n create table {{ relation }}\n {% endif %}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config and contract_config.enforced %}\n {{ get_assert_columns_equivalent(compiled_code) }}\n {%- set compiled_code = get_select_subquery(compiled_code) %}\n {% endif %}\n {{ file_format_clause() }}\n {{ options_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ liquid_clustered_cols() }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause(relation) }}\n {{ comment_clause() }}\n {{ tblproperties_clause() }}\n as\n {{ compiled_code }}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {#--\n N.B. Python models _can_ write to temp views HOWEVER they use a different session\n and have already expired by the time they need to be used (I.E. in merges for incremental models)\n\n TODO: Deep dive into spark sessions to see if we can reuse a single session for an entire\n dbt invocation.\n --#}\n {{ databricks__py_write_table(compiled_code=compiled_code, target_relation=relation) }}\n {%- endif -%}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.create_temporary_view", "macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_select_subquery", "macro.dbt_databricks.file_format_clause", "macro.dbt_spark.options_clause", "macro.dbt_spark.partition_cols", "macro.dbt_databricks.liquid_clustered_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_databricks.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_databricks.tblproperties_clause", "macro.dbt_databricks.databricks__py_write_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6221719, "supported_languages": null}, "macro.dbt_databricks.databricks__options_clause": {"name": "databricks__options_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.databricks__options_clause", "macro_sql": "{% macro databricks__options_clause() -%}\n {%- set options = config.get('options') -%}\n {%- if config.get('file_format', default='delta') == 'hudi' -%}\n {%- set unique_key = config.get('unique_key') -%}\n {%- if unique_key is not none and options is none -%}\n {%- set options = {'primaryKey': config.get('unique_key')} -%}\n {%- elif unique_key is not none and options is not none and 'primaryKey' not in options -%}\n {%- set _ = options.update({'primaryKey': config.get('unique_key')}) -%}\n {%- elif options is not none and 'primaryKey' in options and options['primaryKey'] != unique_key -%}\n {{ exceptions.raise_compiler_error(\"unique_key and options('primaryKey') should be the same column(s).\") }}\n {%- endif %}\n {%- endif %}\n\n {%- if options is not none %}\n options (\n {%- for option in options -%}\n {{ option }} \"{{ options[option] }}\" {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.623204, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_view": {"name": "databricks__drop_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_view", "macro_sql": "{% macro databricks__drop_view(relation) -%}\n drop view if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6233542, "supported_languages": null}, "macro.dbt_databricks.databricks__create_view_as": {"name": "databricks__create_view_as", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt_databricks.databricks__create_view_as", "macro_sql": "{% macro databricks__create_view_as(relation, sql) -%}\n create or replace view {{ relation }}\n {% if config.persist_column_docs() -%}\n {% set model_columns = model.columns %}\n {% set query_columns = get_columns_in_query(sql) %}\n {% if query_columns %}\n (\n {{ get_persist_docs_column_list(model_columns, query_columns) }}\n )\n {% endif %}\n {% endif %}\n {{ comment_clause() }}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config and contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n {{ tblproperties_clause() }}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt_databricks.get_persist_docs_column_list", "macro.dbt_spark.comment_clause", "macro.dbt.get_assert_columns_equivalent", "macro.dbt_databricks.tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6240902, "supported_languages": null}, "macro.dbt_databricks.statement_with_staging_table": {"name": "statement_with_staging_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt_databricks.statement_with_staging_table", "macro_sql": "{% macro statement_with_staging_table(name=None, staging_table=None, fetch_result=False, auto_begin=True) -%}\n {%- if execute: -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- set res, table = adapter.execute(sql, auto_begin=auto_begin, fetch=fetch_result, staging_table=staging_table) -%}\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6248162, "supported_languages": null}, "macro.dbt_databricks.databricks__dateadd": {"name": "databricks__dateadd", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_databricks.databricks__dateadd", "macro_sql": "{% macro databricks__dateadd(datepart, interval, from_date_or_timestamp) %}\n {%- if adapter.compare_dbr_version(10, 4) >= 0 -%}\n timestampadd({{datepart}}, {{interval}}, {{from_date_or_timestamp}})\n {%- else -%}\n {{ spark__dateadd(datepart, interval, from_date_or_timestamp) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6251829, "supported_languages": null}, "macro.dbt_databricks.databricks__datediff": {"name": "databricks__datediff", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_databricks.databricks__datediff", "macro_sql": "{% macro databricks__datediff(first_date, second_date, datepart) %}\n {%- if adapter.compare_dbr_version(10, 4) >= 0 -%}\n timestampdiff({{datepart}}, {{date_trunc(datepart, first_date)}}, {{date_trunc(datepart, second_date)}})\n {%- else -%}\n {{ spark__datediff(first_date, second_date, datepart) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.date_trunc", "macro.dbt_spark.spark__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6255841, "supported_languages": null}, "macro.dbt_databricks.databricks_copy_into": {"name": "databricks_copy_into", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/copy_into.sql", "original_file_path": "macros/adapters/copy_into.sql", "unique_id": "macro.dbt_databricks.databricks_copy_into", "macro_sql": "{% macro databricks_copy_into(\n target_table,\n source,\n file_format,\n expression_list=none,\n source_credential=none,\n source_encryption=none,\n validate=none,\n files=none,\n pattern=none,\n format_options=none,\n copy_options=none) -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=target.database,\n schema=target.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- set source_clause -%}\n {%- if expression_list -%}\n ( select {{ expression_list }} from '{{ source }}' )\n {%- else -%}\n '{{ source }}'\n {%- endif -%}\n {%- if source_credential or source_encryption %}\n WITH (\n {%- if source_credential %}\n credential (\n {%- for name in source_credential -%}\n '{{ name }}' = '{{ source_credential[name] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {%- if source_encryption %}\n encryption (\n {%- for name in source_encryption -%}\n '{{ name }}' = '{{ source_encryption[name] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n )\n {%- endif -%}\n {%- endset -%}\n\n {% set query %}\n copy into {{ target_relation }}\n from {{ source_clause }}\n fileformat = {{ file_format }}\n {% if validate -%} validate {{ validate }} {%- endif %}\n {% if files and pattern %}\n {{ exceptions.raise_compiler_error(\"You can only specify one of 'files' or 'pattern'\") }}\n {% endif %}\n {% if files -%}\n files = (\n {%- for file in files -%}\n '{{ file }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% if pattern -%}\n pattern = '{{ pattern }}'\n {%- endif %}\n {% if format_options -%}\n format_options (\n {%- for key in format_options -%}\n '{{ key }}' = '{{ format_options[key] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% if copy_options -%}\n copy_options (\n {%- for key in copy_options -%}\n '{{ key }}' = '{{ copy_options[key] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% endset %}\n\n {% do log(\"Running COPY INTO\" ~ adapter.redact_credentials(query), info=True) %}\n {% do run_query(query) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.62939, "supported_languages": null}, "macro.dbt_databricks.databricks__make_temp_relation": {"name": "databricks__make_temp_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.databricks__make_temp_relation", "macro_sql": "{% macro databricks__make_temp_relation(base_relation, suffix='__dbt_tmp', as_table=False) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {%- if as_table -%}\n {% set tmp_relation = api.Relation.create(\n identifier=tmp_identifier,\n schema=base_relation.schema,\n database=base_relation.database,\n type='table') %}\n {%- else -%}\n {% set tmp_relation = api.Relation.create(identifier=tmp_identifier, type='view') %}\n {%- endif -%}\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.630194, "supported_languages": null}, "macro.dbt_databricks.databricks__get_or_create_relation": {"name": "databricks__get_or_create_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.databricks__get_or_create_relation", "macro_sql": "{% macro databricks__get_or_create_relation(database, schema, identifier, type, needs_information=False) %}\n {%- set target_relation = adapter.get_relation(\n database=database,\n schema=schema,\n identifier=identifier,\n needs_information=needs_information) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6306949, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog": {"name": "databricks__get_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog", "macro_sql": "{% macro databricks__get_catalog(information_schema, schemas) -%}\n\n {% set query %}\n with tables as (\n {{ databricks__get_catalog_tables_sql(information_schema) }}\n {{ databricks__get_catalog_schemas_where_clause_sql(information_schema.database, schemas) }}\n ),\n columns as (\n {{ databricks__get_catalog_columns_sql(information_schema) }}\n {{ databricks__get_catalog_schemas_where_clause_sql(information_schema.database, schemas) }}\n )\n {{ databricks__get_catalog_results_sql() }}\n {%- endset -%}\n\n {{ return(run_query(query)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql", "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.633368, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_relations": {"name": "databricks__get_catalog_relations", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_relations", "macro_sql": "{% macro databricks__get_catalog_relations(information_schema, relations) -%}\n\n {% set query %}\n with tables as (\n {{ databricks__get_catalog_tables_sql(information_schema) }}\n {{ databricks__get_catalog_relations_where_clause_sql(relations) }}\n ),\n columns as (\n {{ databricks__get_catalog_columns_sql(information_schema) }}\n {{ databricks__get_catalog_relations_where_clause_sql(relations) }}\n )\n {{ databricks__get_catalog_results_sql() }}\n {%- endset -%}\n\n {{ return(run_query(query)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql", "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.63369, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_tables_sql": {"name": "databricks__get_catalog_tables_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro_sql": "{% macro databricks__get_catalog_tables_sql(information_schema) -%}\n select\n table_catalog as table_database,\n table_schema,\n table_name,\n lower(table_type) as table_type,\n comment as table_comment,\n table_owner,\n 'Last Modified' as `stats:last_modified:label`,\n last_altered as `stats:last_modified:value`,\n 'The timestamp for last update/change' as `stats:last_modified:description`,\n (last_altered is not null and table_type not ilike '%VIEW%') as `stats:last_modified:include`\n from `system`.`information_schema`.`tables`\n where table_catalog = '{{ information_schema.database|lower }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6338282, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_columns_sql": {"name": "databricks__get_catalog_columns_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro_sql": "{% macro databricks__get_catalog_columns_sql(information_schema) -%}\n select\n table_catalog as table_database,\n table_schema,\n table_name,\n column_name,\n ordinal_position as column_index,\n lower(data_type) as column_type,\n comment as column_comment\n from `system`.`information_schema`.`columns`\n where table_catalog = '{{ information_schema.database|lower }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.633953, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_results_sql": {"name": "databricks__get_catalog_results_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro_sql": "{% macro databricks__get_catalog_results_sql() -%}\n select *\n from tables\n join columns using (table_database, table_schema, table_name)\n order by column_index\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.634028, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql": {"name": "databricks__get_catalog_schemas_where_clause_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql", "macro_sql": "{% macro databricks__get_catalog_schemas_where_clause_sql(catalog, schemas) -%}\n where table_catalog = '{{ catalog|lower }}' and ({%- for relation in schemas -%}\n table_schema = '{{ relation[1]|lower }}'{%- if not loop.last %} or {% endif -%}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.63427, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql": {"name": "databricks__get_catalog_relations_where_clause_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql", "macro_sql": "{% macro databricks__get_catalog_relations_where_clause_sql(catalog, relations) -%}\n where table_catalog = '{{ catalog|lower }}' and (\n {%- for relation in relations -%}\n {% if relation.schema and relation.identifier %}\n (\n table_schema = '{{ relation.schema|lower }}'\n and table_name = '{{ relation.identifier|lower }}'\n )\n {% elif relation.schema %}\n (\n table_schema = '{{ relation.schema|lower }}'\n )\n {% else %}\n {% do exceptions.raise_compiler_error(\n '`get_catalog_relations` requires a list of relations, each with a schema'\n ) %}\n {% endif %}\n\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6348171, "supported_languages": null}, "macro.dbt_databricks.current_catalog": {"name": "current_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.current_catalog", "macro_sql": "{% macro current_catalog() -%}\n {{ return(adapter.dispatch('current_catalog', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__current_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6351402, "supported_languages": null}, "macro.dbt_databricks.databricks__current_catalog": {"name": "databricks__current_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.databricks__current_catalog", "macro_sql": "{% macro databricks__current_catalog() -%}\n {% call statement('current_catalog', fetch_result=True) %}\n select current_catalog()\n {% endcall %}\n {% do return(load_result('current_catalog').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.635343, "supported_languages": null}, "macro.dbt_databricks.use_catalog": {"name": "use_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.use_catalog", "macro_sql": "{% macro use_catalog(catalog) -%}\n {{ return(adapter.dispatch('use_catalog', 'dbt')(catalog)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__use_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6355011, "supported_languages": null}, "macro.dbt_databricks.databricks__use_catalog": {"name": "databricks__use_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.databricks__use_catalog", "macro_sql": "{% macro databricks__use_catalog(catalog) -%}\n {% call statement() %}\n use catalog {{ adapter.quote(catalog)|lower }}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.635657, "supported_languages": null}, "macro.dbt_databricks.databricks__py_write_table": {"name": "databricks__py_write_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.databricks__py_write_table", "macro_sql": "{% macro databricks__py_write_table(compiled_code, target_relation) %}\n{{ compiled_code }}\n# --- Autogenerated dbt materialization code. --- #\ndbt = dbtObj(spark.table)\ndf = model(dbt, spark)\n\nimport pyspark\n\n{{ py_try_import('pyspark.sql.connect.dataframe', 'newer_pyspark_available') }}\n{{ py_try_import('pandas', 'pandas_available') }}\n{{ py_try_import('pyspark.pandas', 'pyspark_pandas_api_available') }}\n{{ py_try_import('databricks.koalas', 'koalas_available') }}\n\n# preferentially convert pandas DataFrames to pandas-on-Spark or Koalas DataFrames first\n# since they know how to convert pandas DataFrames better than `spark.createDataFrame(df)`\n# and converting from pandas-on-Spark to Spark DataFrame has no overhead\n\nif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n if pyspark_pandas_api_available:\n df = pyspark.pandas.frame.DataFrame(df)\n elif koalas_available:\n df = databricks.koalas.frame.DataFrame(df)\n\n# convert to pyspark.sql.dataframe.DataFrame\nif isinstance(df, pyspark.sql.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif newer_pyspark_available and isinstance(df, pyspark.sql.connect.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif pyspark_pandas_api_available and isinstance(df, pyspark.pandas.frame.DataFrame):\n df = df.to_spark()\nelif koalas_available and isinstance(df, databricks.koalas.frame.DataFrame):\n df = df.to_spark()\nelif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = spark.createDataFrame(df)\nelse:\n msg = f\"{type(df)} is not a supported type for dbt Python materialization\"\n raise Exception(msg)\n\nwriter = (\n df.write\n .mode(\"overwrite\")\n .option(\"overwriteSchema\", \"true\")\n{{ py_get_writer_options()|indent(8, True) }}\n)\n\nwriter.saveAsTable(\"{{ target_relation }}\")\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.py_try_import", "macro.dbt_databricks.py_get_writer_options"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6376588, "supported_languages": null}, "macro.dbt_databricks.py_get_writer_options": {"name": "py_get_writer_options", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.py_get_writer_options", "macro_sql": "\n\n{%- macro py_get_writer_options() -%}\n{%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n{%- set file_format = config.get('file_format', validator=validation.any[basestring])|default('delta', true) -%}\n{%- set partition_by = config.get('partition_by', validator=validation.any[list, basestring]) -%}\n{%- set liquid_clustered_by = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}\n{%- set clustered_by = config.get('clustered_by', validator=validation.any[list, basestring]) -%}\n{%- set buckets = config.get('buckets', validator=validation.any[int]) -%}\n.format(\"{{ file_format }}\")\n{%- if location_root is not none %}\n{%- set identifier = model['alias'] %}\n{%- if is_incremental() %}\n{%- set identifier = identifier + '__dbt_tmp' %}\n{%- endif %}\n.option(\"path\", \"{{ location_root }}/{{ identifier }}\")\n{%- endif -%}\n{%- if partition_by is not none -%}\n {%- if partition_by is string -%}\n {%- set partition_by = [partition_by] -%}\n {%- endif %}\n.partitionBy({{ partition_by }})\n{%- endif -%}\n{%- if liquid_clustered_by and not is_incremental() -%}\n {%- if liquid_clustered_by is string -%}\n {%- set liquid_clustered_by = [liquid_clustered_by] -%}\n {%- endif %}\n.clusterBy({{ liquid_clustered_by }})\n{%- endif -%}\n{%- if (clustered_by is not none) and (buckets is not none) -%}\n {%- if clustered_by is string -%}\n {%- set clustered_by = [clustered_by] -%}\n {%- endif %}\n.bucketBy({{ buckets }}, {{ clustered_by }})\n{%- endif -%}\n{% endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.is_incremental"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6390772, "supported_languages": null}, "macro.dbt_databricks.py_try_import": {"name": "py_try_import", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.py_try_import", "macro_sql": "{% macro py_try_import(library, var_name) -%}\n# make sure {{ library }} exists before using it\ntry:\n import {{ library }}\n {{ var_name }} = True\nexcept ImportError:\n {{ var_name }} = False\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.639234, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_column_comment": {"name": "databricks__alter_column_comment", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.databricks__alter_column_comment", "macro_sql": "{% macro databricks__alter_column_comment(relation, column_dict) %}\n {% if config.get('file_format', default='delta') in ['delta', 'hudi'] %}\n {% for column_name in column_dict %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = comment | replace('\\'', '\\\\\\'') %}\n {% set comment_query %}\n alter table {{ relation }} change column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% endset %}\n {% do run_query(comment_query) %}\n {% endfor %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.641151, "supported_languages": null}, "macro.dbt_databricks.alter_table_comment": {"name": "alter_table_comment", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.alter_table_comment", "macro_sql": "{% macro alter_table_comment(relation, model) %}\n {% set comment_query %}\n comment on table {{ relation|lower }} is '{{ model.description | replace(\"'\", \"\\\\'\") }}'\n {% endset %}\n {% do run_query(comment_query) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.641404, "supported_languages": null}, "macro.dbt_databricks.get_columns_comments": {"name": "get_columns_comments", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.get_columns_comments", "macro_sql": "{% macro get_columns_comments(relation) -%}\n {% call statement('get_columns_comments', fetch_result=True) -%}\n describe table {{ relation|lower }}\n {% endcall %}\n\n {% do return(load_result('get_columns_comments').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6416378, "supported_languages": null}, "macro.dbt_databricks.databricks__persist_docs": {"name": "databricks__persist_docs", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.databricks__persist_docs", "macro_sql": "{% macro databricks__persist_docs(relation, model, for_relation, for_columns) -%}\n {%- if for_relation and config.persist_relation_docs() and model.description %}\n {% do alter_table_comment(relation, model) %}\n {% endif %}\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {%- set existing_columns = adapter.get_columns_in_relation(relation) -%}\n {%- set columns_to_persist_docs = adapter.get_persist_doc_columns(existing_columns, model.columns) -%}\n {% do alter_column_comment(relation, columns_to_persist_docs) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.alter_table_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.642109, "supported_languages": null}, "macro.dbt_databricks.get_column_comment_sql": {"name": "get_column_comment_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.get_column_comment_sql", "macro_sql": "{% macro get_column_comment_sql(column_name, column_dict) -%}\n {% if column_name in column_dict and column_dict[column_name][\"description\"] -%}\n {% set escaped_description = column_dict[column_name][\"description\"] | replace(\"'\", \"\\\\'\") %}\n {% set column_comment_clause = \"comment '\" ~ escaped_description ~ \"'\" %}\n {%- endif -%}\n {{ adapter.quote(column_name) }} {{ column_comment_clause }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.642476, "supported_languages": null}, "macro.dbt_databricks.get_persist_docs_column_list": {"name": "get_persist_docs_column_list", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.get_persist_docs_column_list", "macro_sql": "{% macro get_persist_docs_column_list(model_columns, query_columns) %}\n {% for column_name in query_columns %}\n {{ get_column_comment_sql(column_name, model_columns) }}\n {{- \", \" if not loop.last else \"\" }}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_column_comment_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.642708, "supported_languages": null}, "macro.dbt_databricks.databricks__list_relations_without_caching": {"name": "databricks__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__list_relations_without_caching", "macro_sql": "{% macro databricks__list_relations_without_caching(schema_relation) %}\n {{ return(adapter.get_relations_without_caching(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6461172, "supported_languages": null}, "macro.dbt_databricks.show_table_extended": {"name": "show_table_extended", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_table_extended", "macro_sql": "{% macro show_table_extended(schema_relation) %}\n {{ return(adapter.dispatch('show_table_extended', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_table_extended"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646267, "supported_languages": null}, "macro.dbt_databricks.databricks__show_table_extended": {"name": "databricks__show_table_extended", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_table_extended", "macro_sql": "{% macro databricks__show_table_extended(schema_relation) %}\n {% call statement('show_table_extended', fetch_result=True) -%}\n show table extended in {{ schema_relation.without_identifier()|lower }} like '{{ schema_relation.identifier|lower }}'\n {% endcall %}\n\n {% do return(load_result('show_table_extended').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646537, "supported_languages": null}, "macro.dbt_databricks.show_tables": {"name": "show_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_tables", "macro_sql": "{% macro show_tables(relation) %}\n {{ return(adapter.dispatch('show_tables', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_tables"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646763, "supported_languages": null}, "macro.dbt_databricks.databricks__show_tables": {"name": "databricks__show_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_tables", "macro_sql": "{% macro databricks__show_tables(relation) %}\n {% call statement('show_tables', fetch_result=True) -%}\n show tables in {{ relation|lower }}\n {% endcall %}\n\n {% do return(load_result('show_tables').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646989, "supported_languages": null}, "macro.dbt_databricks.show_views": {"name": "show_views", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_views", "macro_sql": "{% macro show_views(relation) %}\n {{ return(adapter.dispatch('show_views', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_views"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.647151, "supported_languages": null}, "macro.dbt_databricks.databricks__show_views": {"name": "databricks__show_views", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_views", "macro_sql": "{% macro databricks__show_views(relation) %}\n {% call statement('show_views', fetch_result=True) -%}\n show views in {{ relation|lower }}\n {% endcall %}\n\n {% do return(load_result('show_views').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.647396, "supported_languages": null}, "macro.dbt_databricks.databricks__get_relation_last_modified": {"name": "databricks__get_relation_last_modified", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__get_relation_last_modified", "macro_sql": "{% macro databricks__get_relation_last_modified(information_schema, relations) -%}\n\n {%- call statement('last_modified', fetch_result=True) -%}\n {% if information_schema.is_hive_metastore() %}\n {%- for relation in relations -%}\n select '{{ relation.schema }}' as schema,\n '{{ relation.identifier }}' as identifier,\n max(timestamp) as last_modified,\n {{ current_timestamp() }} as snapshotted_at\n from (describe history {{ relation.schema|lower }}.{{ relation.identifier|lower }})\n {% if not loop.last %}\n union all\n {% endif %}\n {%- endfor -%}\n {% else %}\n select table_schema as schema,\n table_name as identifier,\n last_altered as last_modified,\n {{ current_timestamp() }} as snapshotted_at\n from `system`.`information_schema`.`tables`\n where table_catalog = '{{ information_schema.database|lower }}' and\n (\n {%- for relation in relations -%}\n (table_schema = '{{ relation.schema|lower }}' and\n table_name = '{{ relation.identifier|lower }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n {% endif %}\n {%- endcall -%}\n\n {{ return(load_result('last_modified')) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6481018, "supported_languages": null}, "macro.dbt_databricks.get_view_description": {"name": "get_view_description", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_view_description", "macro_sql": "{% macro get_view_description(relation) %}\n {% call statement('get_view_description', fetch_result=True) -%}\n select *\n from `system`.`information_schema`.`views`\n where table_catalog = '{{ relation.database|lower }}'\n and table_schema = '{{ relation.schema|lower }}'\n and table_name = '{{ relation.identifier|lower }}'\n {%- endcall -%}\n\n {% do return(load_result('get_view_description').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.648402, "supported_languages": null}, "macro.dbt_databricks.get_uc_tables": {"name": "get_uc_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_uc_tables", "macro_sql": "{% macro get_uc_tables(relation) %}\n {% call statement('get_uc_tables', fetch_result=True) -%}\n select\n table_name,\n if(table_type in ('EXTERNAL', 'MANAGED', 'MANAGED_SHALLOW_CLONE', 'EXTERNAL_SHALLOW_CLONE'), 'table', lower(table_type)) as table_type,\n lower(data_source_format) as file_format,\n table_owner\n from `system`.`information_schema`.`tables`\n where table_catalog = '{{ relation.database|lower }}'\n and table_schema = '{{ relation.schema|lower }}'\n {% if relation.identifier %}\n and table_name = '{{ relation.identifier|lower }}'\n {% endif %}\n {% endcall %}\n\n {% do return(load_result('get_uc_tables').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.648753, "supported_languages": null}, "macro.dbt_spark.tblproperties_clause": {"name": "tblproperties_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.tblproperties_clause", "macro_sql": "{% macro tblproperties_clause() %}\n {{ return(adapter.dispatch('tblproperties_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.663456, "supported_languages": null}, "macro.dbt_spark.spark__tblproperties_clause": {"name": "spark__tblproperties_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__tblproperties_clause", "macro_sql": "{% macro spark__tblproperties_clause() -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- if tblproperties is not none %}\n tblproperties (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}' {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.663767, "supported_languages": null}, "macro.dbt_spark.file_format_clause": {"name": "file_format_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.file_format_clause", "macro_sql": "{% macro file_format_clause() %}\n {{ return(adapter.dispatch('file_format_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__file_format_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.663905, "supported_languages": null}, "macro.dbt_spark.spark__file_format_clause": {"name": "spark__file_format_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__file_format_clause", "macro_sql": "{% macro spark__file_format_clause() %}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n {%- if file_format is not none %}\n using {{ file_format }}\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6641312, "supported_languages": null}, "macro.dbt_spark.location_clause": {"name": "location_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.location_clause", "macro_sql": "{% macro location_clause() %}\n {{ return(adapter.dispatch('location_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__location_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.664276, "supported_languages": null}, "macro.dbt_spark.spark__location_clause": {"name": "spark__location_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__location_clause", "macro_sql": "{% macro spark__location_clause() %}\n {%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n {%- set identifier = model['alias'] -%}\n {%- if location_root is not none %}\n location '{{ location_root }}/{{ identifier }}'\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.664563, "supported_languages": null}, "macro.dbt_spark.options_clause": {"name": "options_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.options_clause", "macro_sql": "{% macro options_clause() -%}\n {{ return(adapter.dispatch('options_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__options_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6646988, "supported_languages": null}, "macro.dbt_spark.spark__options_clause": {"name": "spark__options_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__options_clause", "macro_sql": "{% macro spark__options_clause() -%}\n {%- set options = config.get('options') -%}\n {%- if config.get('file_format') == 'hudi' -%}\n {%- set unique_key = config.get('unique_key') -%}\n {%- if unique_key is not none and options is none -%}\n {%- set options = {'primaryKey': config.get('unique_key')} -%}\n {%- elif unique_key is not none and options is not none and 'primaryKey' not in options -%}\n {%- set _ = options.update({'primaryKey': config.get('unique_key')}) -%}\n {%- elif options is not none and 'primaryKey' in options and options['primaryKey'] != unique_key -%}\n {{ exceptions.raise_compiler_error(\"unique_key and options('primaryKey') should be the same column(s).\") }}\n {%- endif %}\n {%- endif %}\n\n {%- if options is not none %}\n options (\n {%- for option in options -%}\n {{ option }} \"{{ options[option] }}\" {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6656451, "supported_languages": null}, "macro.dbt_spark.comment_clause": {"name": "comment_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.comment_clause", "macro_sql": "{% macro comment_clause() %}\n {{ return(adapter.dispatch('comment_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__comment_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.665798, "supported_languages": null}, "macro.dbt_spark.spark__comment_clause": {"name": "spark__comment_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__comment_clause", "macro_sql": "{% macro spark__comment_clause() %}\n {%- set raw_persist_docs = config.get('persist_docs', {}) -%}\n\n {%- if raw_persist_docs is mapping -%}\n {%- set raw_relation = raw_persist_docs.get('relation', false) -%}\n {%- if raw_relation -%}\n comment '{{ model.description | replace(\"'\", \"\\\\'\") }}'\n {% endif %}\n {%- elif raw_persist_docs -%}\n {{ exceptions.raise_compiler_error(\"Invalid value provided for 'persist_docs'. Expected dict but got value: \" ~ raw_persist_docs) }}\n {% endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.666194, "supported_languages": null}, "macro.dbt_spark.partition_cols": {"name": "partition_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.partition_cols", "macro_sql": "{% macro partition_cols(label, required=false) %}\n {{ return(adapter.dispatch('partition_cols', 'dbt')(label, required)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.666384, "supported_languages": null}, "macro.dbt_spark.spark__partition_cols": {"name": "spark__partition_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__partition_cols", "macro_sql": "{% macro spark__partition_cols(label, required=false) %}\n {%- set cols = config.get('partition_by', validator=validation.any[list, basestring]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {{ label }} (\n {%- for item in cols -%}\n {{ item }}\n {%- if not loop.last -%},{%- endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.666821, "supported_languages": null}, "macro.dbt_spark.clustered_cols": {"name": "clustered_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.clustered_cols", "macro_sql": "{% macro clustered_cols(label, required=false) %}\n {{ return(adapter.dispatch('clustered_cols', 'dbt')(label, required)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__clustered_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.667007, "supported_languages": null}, "macro.dbt_spark.spark__clustered_cols": {"name": "spark__clustered_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__clustered_cols", "macro_sql": "{% macro spark__clustered_cols(label, required=false) %}\n {%- set cols = config.get('clustered_by', validator=validation.any[list, basestring]) -%}\n {%- set buckets = config.get('buckets', validator=validation.any[int]) -%}\n {%- if (cols is not none) and (buckets is not none) %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {{ label }} (\n {%- for item in cols -%}\n {{ item }}\n {%- if not loop.last -%},{%- endif -%}\n {%- endfor -%}\n ) into {{ buckets }} buckets\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6675909, "supported_languages": null}, "macro.dbt_spark.fetch_tbl_properties": {"name": "fetch_tbl_properties", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.fetch_tbl_properties", "macro_sql": "{% macro fetch_tbl_properties(relation) -%}\n {% call statement('list_properties', fetch_result=True) -%}\n SHOW TBLPROPERTIES {{ relation }}\n {% endcall %}\n {% do return(load_result('list_properties').table) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.667808, "supported_languages": null}, "macro.dbt_spark.create_temporary_view": {"name": "create_temporary_view", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.create_temporary_view", "macro_sql": "{% macro create_temporary_view(relation, compiled_code) -%}\n {{ return(adapter.dispatch('create_temporary_view', 'dbt')(relation, compiled_code)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__create_temporary_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.667983, "supported_languages": null}, "macro.dbt_spark.spark__create_temporary_view": {"name": "spark__create_temporary_view", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_temporary_view", "macro_sql": "{% macro spark__create_temporary_view(relation, compiled_code) -%}\n create or replace temporary view {{ relation }} as\n {{ compiled_code }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.668092, "supported_languages": null}, "macro.dbt_spark.spark__create_table_as": {"name": "spark__create_table_as", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_table_as", "macro_sql": "{%- macro spark__create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {%- if language == 'sql' -%}\n {%- if temporary -%}\n {{ create_temporary_view(relation, compiled_code) }}\n {%- else -%}\n {% if config.get('file_format', validator=validation.any[basestring]) in ['delta', 'iceberg'] %}\n create or replace table {{ relation }}\n {% else %}\n create table {{ relation }}\n {% endif %}\n {%- set contract_config = config.get('contract') -%}\n {%- if contract_config.enforced -%}\n {{ get_assert_columns_equivalent(compiled_code) }}\n {%- set compiled_code = get_select_subquery(compiled_code) %}\n {% endif %}\n {{ file_format_clause() }}\n {{ options_clause() }}\n {{ tblproperties_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause() }}\n {{ comment_clause() }}\n\n as\n {{ compiled_code }}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {#--\n N.B. Python models _can_ write to temp views HOWEVER they use a different session\n and have already expired by the time they need to be used (I.E. in merges for incremental models)\n\n TODO: Deep dive into spark sessions to see if we can reuse a single session for an entire\n dbt invocation.\n --#}\n {{ py_write_table(compiled_code=compiled_code, target_relation=relation) }}\n {%- endif -%}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_spark.create_temporary_view", "macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_select_subquery", "macro.dbt_spark.file_format_clause", "macro.dbt_spark.options_clause", "macro.dbt_spark.tblproperties_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_spark.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_spark.py_write_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.668963, "supported_languages": null}, "macro.dbt_spark.persist_constraints": {"name": "persist_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.persist_constraints", "macro_sql": "{% macro persist_constraints(relation, model) %}\n {{ return(adapter.dispatch('persist_constraints', 'dbt')(relation, model)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.669142, "supported_languages": null}, "macro.dbt_spark.spark__persist_constraints": {"name": "spark__persist_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__persist_constraints", "macro_sql": "{% macro spark__persist_constraints(relation, model) %}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config.enforced and config.get('file_format', 'delta') == 'delta' %}\n {% do alter_table_add_constraints(relation, model.constraints) %}\n {% do alter_column_set_constraints(relation, model.columns) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.alter_table_add_constraints", "macro.dbt_spark.alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.669481, "supported_languages": null}, "macro.dbt_spark.alter_table_add_constraints": {"name": "alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.alter_table_add_constraints", "macro_sql": "{% macro alter_table_add_constraints(relation, constraints) %}\n {{ return(adapter.dispatch('alter_table_add_constraints', 'dbt')(relation, constraints)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.669651, "supported_languages": null}, "macro.dbt_spark.spark__alter_table_add_constraints": {"name": "spark__alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_table_add_constraints", "macro_sql": "{% macro spark__alter_table_add_constraints(relation, constraints) %}\n {% for constraint in constraints %}\n {% if constraint.type == 'check' and not is_incremental() %}\n {%- set constraint_hash = local_md5(column_name ~ \";\" ~ constraint.expression ~ \";\" ~ loop.index) -%}\n {% call statement() %}\n alter table {{ relation }} add constraint {{ constraint.name if constraint.name else constraint_hash }} check ({{ constraint.expression }});\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.is_incremental", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.670092, "supported_languages": null}, "macro.dbt_spark.alter_column_set_constraints": {"name": "alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.alter_column_set_constraints", "macro_sql": "{% macro alter_column_set_constraints(relation, column_dict) %}\n {{ return(adapter.dispatch('alter_column_set_constraints', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.67027, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_set_constraints": {"name": "spark__alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_set_constraints", "macro_sql": "{% macro spark__alter_column_set_constraints(relation, column_dict) %}\n {% for column_name in column_dict %}\n {% set constraints = column_dict[column_name]['constraints'] %}\n {% for constraint in constraints %}\n {% if constraint.type != 'not_null' %}\n {{ exceptions.warn('Invalid constraint for column ' ~ column_name ~ '. Only `not_null` is supported.') }}\n {% else %}\n {% set quoted_name = adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name %}\n {% call statement() %}\n alter table {{ relation }} change column {{ quoted_name }} set not null {{ constraint.expression or \"\" }};\n {% endcall %}\n {% endif %}\n {% endfor %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.670861, "supported_languages": null}, "macro.dbt_spark.get_column_comment_sql": {"name": "get_column_comment_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_column_comment_sql", "macro_sql": "{% macro get_column_comment_sql(column_name, column_dict) -%}\n {% if column_name in column_dict and column_dict[column_name][\"description\"] -%}\n {% set escaped_description = column_dict[column_name][\"description\"] | replace(\"'\", \"\\\\'\") %}\n {% set column_comment_clause = \"comment '\" ~ escaped_description ~ \"'\" %}\n {%- endif -%}\n {{ adapter.quote(column_name) }} {{ column_comment_clause }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.671293, "supported_languages": null}, "macro.dbt_spark.get_persist_docs_column_list": {"name": "get_persist_docs_column_list", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_persist_docs_column_list", "macro_sql": "{% macro get_persist_docs_column_list(model_columns, query_columns) %}\n {% for column_name in query_columns %}\n {{ get_column_comment_sql(column_name, model_columns) }}\n {{- \", \" if not loop.last else \"\" }}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.get_column_comment_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.671527, "supported_languages": null}, "macro.dbt_spark.spark__create_view_as": {"name": "spark__create_view_as", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_view_as", "macro_sql": "{% macro spark__create_view_as(relation, sql) -%}\n create or replace view {{ relation }}\n {% if config.persist_column_docs() -%}\n {% set model_columns = model.columns %}\n {% set query_columns = get_columns_in_query(sql) %}\n (\n {{ get_persist_docs_column_list(model_columns, query_columns) }}\n )\n {% endif %}\n {{ comment_clause() }}\n {%- set contract_config = config.get('contract') -%}\n {%- if contract_config.enforced -%}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt_spark.get_persist_docs_column_list", "macro.dbt_spark.comment_clause", "macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6719759, "supported_languages": null}, "macro.dbt_spark.spark__create_schema": {"name": "spark__create_schema", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_schema", "macro_sql": "{% macro spark__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{relation}}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6721191, "supported_languages": null}, "macro.dbt_spark.spark__drop_schema": {"name": "spark__drop_schema", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__drop_schema", "macro_sql": "{% macro spark__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation }} cascade\n {%- endcall -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672259, "supported_languages": null}, "macro.dbt_spark.get_columns_in_relation_raw": {"name": "get_columns_in_relation_raw", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_columns_in_relation_raw", "macro_sql": "{% macro get_columns_in_relation_raw(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation_raw', 'dbt')(relation)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__get_columns_in_relation_raw"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672405, "supported_languages": null}, "macro.dbt_spark.spark__get_columns_in_relation_raw": {"name": "spark__get_columns_in_relation_raw", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__get_columns_in_relation_raw", "macro_sql": "{% macro spark__get_columns_in_relation_raw(relation) -%}\n {% call statement('get_columns_in_relation_raw', fetch_result=True) %}\n describe extended {{ relation }}\n {% endcall %}\n {% do return(load_result('get_columns_in_relation_raw').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672622, "supported_languages": null}, "macro.dbt_spark.spark__get_columns_in_relation": {"name": "spark__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__get_columns_in_relation", "macro_sql": "{% macro spark__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n describe extended {{ relation.include(schema=(schema is not none)) }}\n {% endcall %}\n {% do return(load_result('get_columns_in_relation').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672888, "supported_languages": null}, "macro.dbt_spark.spark__list_relations_without_caching": {"name": "spark__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__list_relations_without_caching", "macro_sql": "{% macro spark__list_relations_without_caching(relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n show table extended in {{ relation.schema }} like '*'\n {% endcall %}\n\n {% do return(load_result('list_relations_without_caching').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.673117, "supported_languages": null}, "macro.dbt_spark.list_relations_show_tables_without_caching": {"name": "list_relations_show_tables_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.list_relations_show_tables_without_caching", "macro_sql": "{% macro list_relations_show_tables_without_caching(schema_relation) %}\n {#-- Spark with iceberg tables don't work with show table extended for #}\n {#-- V2 iceberg tables #}\n {#-- https://issues.apache.org/jira/browse/SPARK-33393 #}\n {% call statement('list_relations_without_caching_show_tables', fetch_result=True) -%}\n show tables in {{ schema_relation.schema }} like '*'\n {% endcall %}\n\n {% do return(load_result('list_relations_without_caching_show_tables').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6733599, "supported_languages": null}, "macro.dbt_spark.describe_table_extended_without_caching": {"name": "describe_table_extended_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.describe_table_extended_without_caching", "macro_sql": "{% macro describe_table_extended_without_caching(table_name) %}\n {#-- Spark with iceberg tables don't work with show table extended for #}\n {#-- V2 iceberg tables #}\n {#-- https://issues.apache.org/jira/browse/SPARK-33393 #}\n {% call statement('describe_table_extended_without_caching', fetch_result=True) -%}\n describe extended {{ table_name }}\n {% endcall %}\n {% do return(load_result('describe_table_extended_without_caching').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6736, "supported_languages": null}, "macro.dbt_spark.spark__list_schemas": {"name": "spark__list_schemas", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__list_schemas", "macro_sql": "{% macro spark__list_schemas(database) -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n show databases\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.673813, "supported_languages": null}, "macro.dbt_spark.spark__rename_relation": {"name": "spark__rename_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__rename_relation", "macro_sql": "{% macro spark__rename_relation(from_relation, to_relation) -%}\n {% call statement('rename_relation') -%}\n {% if not from_relation.type %}\n {% do exceptions.raise_database_error(\"Cannot rename a relation with a blank type: \" ~ from_relation.identifier) %}\n {% elif from_relation.type in ('table') %}\n alter table {{ from_relation }} rename to {{ to_relation }}\n {% elif from_relation.type == 'view' %}\n alter view {{ from_relation }} rename to {{ to_relation }}\n {% else %}\n {% do exceptions.raise_database_error(\"Unknown type '\" ~ from_relation.type ~ \"' for relation: \" ~ from_relation.identifier) %}\n {% endif %}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674293, "supported_languages": null}, "macro.dbt_spark.spark__drop_relation": {"name": "spark__drop_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__drop_relation", "macro_sql": "{% macro spark__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674477, "supported_languages": null}, "macro.dbt_spark.spark__generate_database_name": {"name": "spark__generate_database_name", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__generate_database_name", "macro_sql": "{% macro spark__generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(None) %}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674613, "supported_languages": null}, "macro.dbt_spark.spark__persist_docs": {"name": "spark__persist_docs", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__persist_docs", "macro_sql": "{% macro spark__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do alter_column_comment(relation, model.columns) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674838, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_comment": {"name": "spark__alter_column_comment", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_comment", "macro_sql": "{% macro spark__alter_column_comment(relation, column_dict) %}\n {% if config.get('file_format', validator=validation.any[basestring]) in ['delta', 'hudi', 'iceberg'] %}\n {% for column_name in column_dict %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = comment | replace('\\'', '\\\\\\'') %}\n {% set comment_query %}\n {% if relation.is_iceberg %}\n alter table {{ relation }} alter column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% else %}\n alter table {{ relation }} change column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% endif %}\n {% endset %}\n {% do run_query(comment_query) %}\n {% endfor %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.675686, "supported_languages": null}, "macro.dbt_spark.spark__make_temp_relation": {"name": "spark__make_temp_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__make_temp_relation", "macro_sql": "{% macro spark__make_temp_relation(base_relation, suffix) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {% set tmp_relation = base_relation.incorporate(path = {\n \"identifier\": tmp_identifier\n }) -%}\n\n {%- set tmp_relation = tmp_relation.include(database=false, schema=false) -%}\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6760252, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_type": {"name": "spark__alter_column_type", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_type", "macro_sql": "{% macro spark__alter_column_type(relation, column_name, new_column_type) -%}\n {% call statement('alter_column_type') %}\n alter table {{ relation }} alter column {{ column_name }} type {{ new_column_type }};\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.67621, "supported_languages": null}, "macro.dbt_spark.spark__alter_relation_add_remove_columns": {"name": "spark__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_relation_add_remove_columns", "macro_sql": "{% macro spark__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if remove_columns %}\n {% if relation.is_delta %}\n {% set platform_name = 'Delta Lake' %}\n {% elif relation.is_iceberg %}\n {% set platform_name = 'Iceberg' %}\n {% else %}\n {% set platform_name = 'Apache Spark' %}\n {% endif %}\n {{ exceptions.raise_compiler_error(platform_name + ' does not support dropping columns from tables') }}\n {% endif %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% if add_columns %} add columns {% endif %}\n {% for column in add_columns %}\n {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.676926, "supported_languages": null}, "macro.dbt_spark.spark__copy_grants": {"name": "spark__copy_grants", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__copy_grants", "macro_sql": "{% macro spark__copy_grants() %}\n\n {% if config.materialized == 'view' %}\n {#-- Spark views don't copy grants when they're replaced --#}\n {{ return(False) }}\n\n {% else %}\n {#-- This depends on how we're replacing the table, which depends on its file format\n -- Just play it safe by assuming that grants have been copied over, and need to be checked / possibly revoked\n -- We can make this more efficient in the future\n #}\n {{ return(True) }}\n\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.677521, "supported_languages": null}, "macro.dbt_spark.spark__get_grant_sql": {"name": "spark__get_grant_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__get_grant_sql", "macro_sql": "\n\n\n{%- macro spark__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ adapter.quote(grantees[0]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6777, "supported_languages": null}, "macro.dbt_spark.spark__get_revoke_sql": {"name": "spark__get_revoke_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__get_revoke_sql", "macro_sql": "\n\n\n{%- macro spark__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ adapter.quote(grantees[0]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.677873, "supported_languages": null}, "macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement": {"name": "spark__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n\n{%- macro spark__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(False) }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.677966, "supported_languages": null}, "macro.dbt_spark.spark__call_dcl_statements": {"name": "spark__call_dcl_statements", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__call_dcl_statements", "macro_sql": "{% macro spark__call_dcl_statements(dcl_statement_list) %}\n {% for dcl_statement in dcl_statement_list %}\n {% call statement('grant_or_revoke') %}\n {{ dcl_statement }}\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.678161, "supported_languages": null}, "macro.dbt_spark.spark__get_binding_char": {"name": "spark__get_binding_char", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__get_binding_char", "macro_sql": "{% macro spark__get_binding_char() %}\n {{ return('?' if target.method == 'odbc' else '%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.679692, "supported_languages": null}, "macro.dbt_spark.spark__reset_csv_table": {"name": "spark__reset_csv_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__reset_csv_table", "macro_sql": "{% macro spark__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% if old_relation %}\n {{ adapter.drop_relation(old_relation) }}\n {% endif %}\n {% set sql = create_csv_table(model, agate_table) %}\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6799822, "supported_languages": null}, "macro.dbt_spark.spark__load_csv_rows": {"name": "spark__load_csv_rows", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__load_csv_rows", "macro_sql": "{% macro spark__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n {% set column_override = model['config'].get('column_types', {}) %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} values\n {% for row in chunk -%}\n ({%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n cast({{ get_binding_char() }} as {{type}})\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.681087, "supported_languages": null}, "macro.dbt_spark.spark__create_csv_table": {"name": "spark__create_csv_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__create_csv_table", "macro_sql": "{% macro spark__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {{ file_format_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause() }}\n {{ comment_clause() }}\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.file_format_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_spark.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6820521, "supported_languages": null}, "macro.dbt_spark.materialization_view_spark": {"name": "materialization_view_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "unique_id": "macro.dbt_spark.materialization_view_spark", "macro_sql": "{% materialization view, adapter='spark' -%}\n {{ return(create_or_replace_view()) }}\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.create_or_replace_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.682219, "supported_languages": ["sql"]}, "macro.dbt_spark.materialization_table_spark": {"name": "materialization_table_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.materialization_table_spark", "macro_sql": "{% materialization table, adapter = 'spark', supported_languages=['sql', 'python'] %}\n {%- set language = model['language'] -%}\n {%- set identifier = model['alias'] -%}\n {%- set grant_config = config.get('grants') -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n\n {{ run_hooks(pre_hooks) }}\n\n -- setup: if the target relation already exists, drop it\n -- in case if the existing and future table is delta or iceberg, we want to do a\n -- create or replace table instead of dropping, so we don't have the table unavailable\n {% if old_relation is not none %}\n {% set is_delta = (old_relation.is_delta and config.get('file_format', validator=validation.any[basestring]) == 'delta') %}\n {% set is_iceberg = (old_relation.is_iceberg and config.get('file_format', validator=validation.any[basestring]) == 'iceberg') %}\n {% set old_relation_type = old_relation.type %}\n {% else %}\n {% set is_delta = false %}\n {% set is_iceberg = false %}\n {% set old_relation_type = target_relation.type %}\n {% endif %}\n\n {% if not is_delta and not is_iceberg %}\n {% set existing_relation = target_relation %}\n {{ adapter.drop_relation(existing_relation.incorporate(type=old_relation_type)) }}\n {% endif %}\n\n -- build model\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]})}}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt_spark.persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6857522, "supported_languages": ["sql", "python"]}, "macro.dbt_spark.py_write_table": {"name": "py_write_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.py_write_table", "macro_sql": "{% macro py_write_table(compiled_code, target_relation) %}\n{{ compiled_code }}\n# --- Autogenerated dbt materialization code. --- #\ndbt = dbtObj(spark.table)\ndf = model(dbt, spark)\n\n# make sure pyspark exists in the namepace, for 7.3.x-scala2.12 it does not exist\nimport pyspark\n# make sure pandas exists before using it\ntry:\n import pandas\n pandas_available = True\nexcept ImportError:\n pandas_available = False\n\n# make sure pyspark.pandas exists before using it\ntry:\n import pyspark.pandas\n pyspark_pandas_api_available = True\nexcept ImportError:\n pyspark_pandas_api_available = False\n\n# make sure databricks.koalas exists before using it\ntry:\n import databricks.koalas\n koalas_available = True\nexcept ImportError:\n koalas_available = False\n\n# preferentially convert pandas DataFrames to pandas-on-Spark or Koalas DataFrames first\n# since they know how to convert pandas DataFrames better than `spark.createDataFrame(df)`\n# and converting from pandas-on-Spark to Spark DataFrame has no overhead\nif pyspark_pandas_api_available and pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = pyspark.pandas.frame.DataFrame(df)\nelif koalas_available and pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = databricks.koalas.frame.DataFrame(df)\n\n# convert to pyspark.sql.dataframe.DataFrame\nif isinstance(df, pyspark.sql.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif pyspark_pandas_api_available and isinstance(df, pyspark.pandas.frame.DataFrame):\n df = df.to_spark()\nelif koalas_available and isinstance(df, databricks.koalas.frame.DataFrame):\n df = df.to_spark()\nelif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = spark.createDataFrame(df)\nelse:\n msg = f\"{type(df)} is not a supported type for dbt Python materialization\"\n raise Exception(msg)\n\ndf.write.mode(\"overwrite\").format(\"{{ config.get('file_format', 'delta') }}\").option(\"overwriteSchema\", \"true\").saveAsTable(\"{{ target_relation }}\")\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.686034, "supported_languages": null}, "macro.dbt_spark.py_script_comment": {"name": "py_script_comment", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n# how to execute python model in notebook\n# dbt = dbtObj(spark.table)\n# df = model(dbt, spark)\n{%endmacro%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.686104, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_hash_arguments": {"name": "spark__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_hash_arguments", "macro_sql": "{% macro spark__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as string ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.692856, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_string_as_time": {"name": "spark__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_string_as_time", "macro_sql": "{% macro spark__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"to_timestamp('\" ~ timestamp ~ \"')\" -%}\n {{ return(result) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.693005, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_merge_sql": {"name": "spark__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_merge_sql", "macro_sql": "{% macro spark__snapshot_merge_sql(target, source, insert_cols) -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n {% if target.is_iceberg %}\n {# create view only supports a name (no catalog, or schema) #}\n using {{ source.identifier }} as DBT_INTERNAL_SOURCE\n {% else %}\n using {{ source }} as DBT_INTERNAL_SOURCE\n {% endif %}\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert *\n ;\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.693233, "supported_languages": null}, "macro.dbt_spark.spark_build_snapshot_staging_table": {"name": "spark_build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark_build_snapshot_staging_table", "macro_sql": "{% macro spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_identifier = target_relation.identifier ~ '__dbt_tmp' %}\n\n {% if target_relation.is_iceberg %}\n {# iceberg catalog does not support create view, but regular spark does. We removed the catalog and schema #}\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=none,\n database=none,\n type='view') -%}\n {% else %}\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=target_relation.schema,\n database=none,\n type='view') -%}\n {% endif %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {# needs to be a non-temp view so that its columns can be ascertained via `describe` #}\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_view_as(tmp_relation, select) }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.693888, "supported_languages": null}, "macro.dbt_spark.spark__post_snapshot": {"name": "spark__post_snapshot", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__post_snapshot", "macro_sql": "{% macro spark__post_snapshot(staging_relation) %}\n {% do adapter.drop_relation(staging_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.694009, "supported_languages": null}, "macro.dbt_spark.spark__create_columns": {"name": "spark__create_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__create_columns", "macro_sql": "{% macro spark__create_columns(relation, columns) %}\n {% if columns|length > 0 %}\n {% call statement() %}\n alter table {{ relation }} add columns (\n {% for column in columns %}\n `{{ column.name }}` {{ column.data_type }} {{- ',' if not loop.last -}}\n {% endfor %}\n );\n {% endcall %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6943328, "supported_languages": null}, "macro.dbt_spark.materialization_snapshot_spark": {"name": "materialization_snapshot_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.materialization_snapshot_spark", "macro_sql": "{% materialization snapshot, adapter='spark' %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n {%- set file_format = config.get('file_format', 'parquet') -%}\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=none,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if file_format not in ['delta', 'iceberg', 'hudi'] -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n Snapshot functionality requires file_format be set to 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n\n {%- if target_relation_exists -%}\n {%- if not target_relation.is_delta and not target_relation.is_iceberg and not target_relation.is_hudi -%}\n {% set invalid_format_msg -%}\n The existing table {{ model.schema }}.{{ target_table }} is in another format than 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n {% endif %}\n\n {% if not adapter.check_schema_exists(model.database, model.schema) %}\n {% do create_schema(model.schema) %}\n {% endif %}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.create_schema", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt_spark.spark_build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6975749, "supported_languages": ["sql"]}, "macro.dbt_spark.spark__can_clone_table": {"name": "spark__can_clone_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.spark__can_clone_table", "macro_sql": "{% macro spark__can_clone_table() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6994078, "supported_languages": null}, "macro.dbt_spark.spark__create_or_replace_clone": {"name": "spark__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.spark__create_or_replace_clone", "macro_sql": "{% macro spark__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation }} shallow clone {{ defer_relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.699519, "supported_languages": null}, "macro.dbt_spark.materialization_clone_spark": {"name": "materialization_clone_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.materialization_clone_spark", "macro_sql": "\n\n{%- materialization clone, adapter='spark' -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if file_format != 'delta' -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n shallow clone requires file_format be set to 'delta'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {%- elif other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n {% endif %}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.70128, "supported_languages": ["sql"]}, "macro.dbt_spark.spark__get_merge_update_columns": {"name": "spark__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/column_helpers.sql", "original_file_path": "macros/materializations/incremental/column_helpers.sql", "unique_id": "macro.dbt_spark.spark__get_merge_update_columns", "macro_sql": "{% macro spark__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = None -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.702069, "supported_languages": null}, "macro.dbt_spark.dbt_spark_validate_get_file_format": {"name": "dbt_spark_validate_get_file_format", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_spark.dbt_spark_validate_get_file_format", "macro_sql": "{% macro dbt_spark_validate_get_file_format(raw_file_format) %}\n {#-- Validate the file format #}\n\n {% set accepted_formats = ['text', 'csv', 'json', 'jdbc', 'parquet', 'orc', 'hive', 'delta', 'iceberg', 'libsvm', 'hudi'] %}\n\n {% set invalid_file_format_msg -%}\n Invalid file format provided: {{ raw_file_format }}\n Expected one of: {{ accepted_formats | join(', ') }}\n {%- endset %}\n\n {% if raw_file_format not in accepted_formats %}\n {% do exceptions.raise_compiler_error(invalid_file_format_msg) %}\n {% endif %}\n\n {% do return(raw_file_format) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.703007, "supported_languages": null}, "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy": {"name": "dbt_spark_validate_get_incremental_strategy", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy", "macro_sql": "{% macro dbt_spark_validate_get_incremental_strategy(raw_strategy, file_format) %}\n {#-- Validate the incremental strategy #}\n\n {% set invalid_strategy_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n Expected one of: 'append', 'merge', 'insert_overwrite'\n {%- endset %}\n\n {% set invalid_merge_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You can only choose this strategy when file_format is set to 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n\n {% set invalid_insert_overwrite_endpoint_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You cannot use this strategy when connecting via endpoint\n Use the 'append' or 'merge' strategy instead\n {%- endset %}\n\n {% if raw_strategy not in ['append', 'merge', 'insert_overwrite'] %}\n {% do exceptions.raise_compiler_error(invalid_strategy_msg) %}\n {%-else %}\n {% if raw_strategy == 'merge' and file_format not in ['delta', 'iceberg', 'hudi'] %}\n {% do exceptions.raise_compiler_error(invalid_merge_msg) %}\n {% endif %}\n {% if raw_strategy == 'insert_overwrite' and target.endpoint %}\n {% do exceptions.raise_compiler_error(invalid_insert_overwrite_endpoint_msg) %}\n {% endif %}\n {% endif %}\n\n {% do return(raw_strategy) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.703701, "supported_languages": null}, "macro.dbt_spark.get_insert_overwrite_sql": {"name": "get_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.get_insert_overwrite_sql", "macro_sql": "{% macro get_insert_overwrite_sql(source_relation, target_relation, existing_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n {% if existing_relation.is_iceberg %}\n {# removed table from statement for iceberg #}\n insert overwrite {{ target_relation }}\n {# removed partition_cols for iceberg as well #}\n {% else %}\n insert overwrite table {{ target_relation }}\n {{ partition_cols(label=\"partition\") }}\n {% endif %}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.705375, "supported_languages": null}, "macro.dbt_spark.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(source_relation, target_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n insert into table {{ target_relation }}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.705639, "supported_languages": null}, "macro.dbt_spark.spark__get_merge_sql": {"name": "spark__get_merge_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.spark__get_merge_sql", "macro_sql": "{% macro spark__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %}\n {# need dest_columns for merge_exclude_columns, default to use \"*\" #}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n when matched then update set\n {% if update_columns -%}{%- for column_name in update_columns %}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {%- else %} * {% endif %}\n\n when not matched then insert *\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.706816, "supported_languages": null}, "macro.dbt_spark.dbt_spark_get_incremental_sql": {"name": "dbt_spark_get_incremental_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.dbt_spark_get_incremental_sql", "macro_sql": "{% macro dbt_spark_get_incremental_sql(strategy, source, target, existing, unique_key, incremental_predicates) %}\n {%- if strategy == 'append' -%}\n {#-- insert new records into existing table, without updating or overwriting #}\n {{ get_insert_into_sql(source, target) }}\n {%- elif strategy == 'insert_overwrite' -%}\n {#-- insert statements don't like CTEs, so support them via a temp view #}\n {{ get_insert_overwrite_sql(source, target, existing) }}\n {%- elif strategy == 'merge' -%}\n {#-- merge all columns for datasources which implement MERGE INTO (e.g. databricks, iceberg) - schema changes are handled for us #}\n {{ get_merge_sql(target, source, unique_key, dest_columns=none, incremental_predicates=incremental_predicates) }}\n {%- else -%}\n {% set no_sql_for_strategy_msg -%}\n No known SQL for the incremental strategy provided: {{ strategy }}\n {%- endset %}\n {%- do exceptions.raise_compiler_error(no_sql_for_strategy_msg) -%}\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.get_insert_into_sql", "macro.dbt_spark.get_insert_overwrite_sql", "macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.707412, "supported_languages": null}, "macro.dbt_spark.materialization_incremental_spark": {"name": "materialization_incremental_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_spark.materialization_incremental_spark", "macro_sql": "{% materialization incremental, adapter='spark', supported_languages=['sql', 'python'] -%}\n {#-- Validate early so we don't run SQL if the file_format + strategy combo is invalid --#}\n {%- set raw_file_format = config.get('file_format', default='parquet') -%}\n {%- set raw_strategy = config.get('incremental_strategy') or 'append' -%}\n {%- set grant_config = config.get('grants') -%}\n\n {%- set file_format = dbt_spark_validate_get_file_format(raw_file_format) -%}\n {%- set strategy = dbt_spark_validate_get_incremental_strategy(raw_strategy, file_format) -%}\n\n {#-- Set vars --#}\n\n {%- set unique_key = config.get('unique_key', none) -%}\n {%- set partition_by = config.get('partition_by', none) -%}\n {%- set language = model['language'] -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) -%}\n {%- set target_relation = this -%}\n {%- set existing_relation = load_relation(this) -%}\n {%- set tmp_relation = make_temp_relation(this) -%}\n\n {#-- for SQL model we will create temp view that doesn't have database and schema --#}\n {%- if language == 'sql'-%}\n {%- set tmp_relation = tmp_relation.include(database=false, schema=false) -%}\n {%- endif -%}\n\n {#-- Set Overwrite Mode --#}\n {%- if strategy == 'insert_overwrite' and partition_by -%}\n {%- call statement() -%}\n set spark.sql.sources.partitionOverwriteMode = DYNAMIC\n {%- endcall -%}\n {%- endif -%}\n\n {#-- Run pre-hooks --#}\n {{ run_hooks(pre_hooks) }}\n\n {#-- Incremental run logic --#}\n {%- if existing_relation is none -%}\n {#-- Relation must be created --#}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {%- elif existing_relation.is_view or should_full_refresh() -%}\n {#-- Relation must be dropped & recreated --#}\n {% set is_delta = (file_format == 'delta' and existing_relation.is_delta) %}\n {% if not is_delta %} {#-- If Delta, we will `create or replace` below, so no need to drop --#}\n {% do adapter.drop_relation(existing_relation) %}\n {% endif %}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {%- else -%}\n {#-- Relation must be merged --#}\n {%- call statement('create_tmp_relation', language=language) -%}\n {{ create_table_as(True, tmp_relation, compiled_code, language) }}\n {%- endcall -%}\n {%- do process_schema_changes(on_schema_change, tmp_relation, existing_relation) -%}\n {%- call statement('main') -%}\n {{ dbt_spark_get_incremental_sql(strategy, tmp_relation, target_relation, existing_relation, unique_key, incremental_predicates) }}\n {%- endcall -%}\n {%- if language == 'python' -%}\n {#--\n This is yucky.\n See note in dbt-spark/dbt/include/spark/macros/adapters.sql\n re: python models and temporary views.\n\n Also, why do neither drop_relation or adapter.drop_relation work here?!\n --#}\n {% call statement('drop_relation') -%}\n drop table if exists {{ tmp_relation }}\n {%- endcall %}\n {%- endif -%}\n {%- endif -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt_spark.dbt_spark_validate_get_file_format", "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.load_relation", "macro.dbt.make_temp_relation", "macro.dbt.statement", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt_spark.persist_constraints", "macro.dbt.should_full_refresh", "macro.dbt.process_schema_changes", "macro.dbt_spark.dbt_spark_get_incremental_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.710429, "supported_languages": ["sql", "python"]}, "macro.dbt_spark.spark__concat": {"name": "spark__concat", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt_spark.spark__concat", "macro_sql": "{% macro spark__concat(fields) -%}\n concat({{ fields|join(', ') }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.710591, "supported_languages": null}, "macro.dbt_spark.assert_not_null": {"name": "assert_not_null", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/assert_not_null.sql", "original_file_path": "macros/utils/assert_not_null.sql", "unique_id": "macro.dbt_spark.assert_not_null", "macro_sql": "{% macro assert_not_null(function, arg) -%}\n {{ return(adapter.dispatch('assert_not_null', 'dbt')(function, arg)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.710856, "supported_languages": null}, "macro.dbt_spark.spark__assert_not_null": {"name": "spark__assert_not_null", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/assert_not_null.sql", "original_file_path": "macros/utils/assert_not_null.sql", "unique_id": "macro.dbt_spark.spark__assert_not_null", "macro_sql": "{% macro spark__assert_not_null(function, arg) %}\n\n coalesce({{function}}({{arg}}), nvl2({{function}}({{arg}}), assert_true({{function}}({{arg}}) is not null), null))\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.711019, "supported_languages": null}, "macro.dbt_spark.spark__dateadd": {"name": "spark__dateadd", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_spark.spark__dateadd", "macro_sql": "{% macro spark__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {%- set clock_component -%}\n {# make sure the dates + timestamps are real, otherwise raise an error asap #}\n to_unix_timestamp({{ assert_not_null('to_timestamp', from_date_or_timestamp) }})\n - to_unix_timestamp({{ assert_not_null('date', from_date_or_timestamp) }})\n {%- endset -%}\n\n {%- if datepart in ['day', 'week'] -%}\n\n {%- set multiplier = 7 if datepart == 'week' else 1 -%}\n\n to_timestamp(\n to_unix_timestamp(\n date_add(\n {{ assert_not_null('date', from_date_or_timestamp) }},\n cast({{interval}} * {{multiplier}} as int)\n )\n ) + {{clock_component}}\n )\n\n {%- elif datepart in ['month', 'quarter', 'year'] -%}\n\n {%- set multiplier -%}\n {%- if datepart == 'month' -%} 1\n {%- elif datepart == 'quarter' -%} 3\n {%- elif datepart == 'year' -%} 12\n {%- endif -%}\n {%- endset -%}\n\n to_timestamp(\n to_unix_timestamp(\n add_months(\n {{ assert_not_null('date', from_date_or_timestamp) }},\n cast({{interval}} * {{multiplier}} as int)\n )\n ) + {{clock_component}}\n )\n\n {%- elif datepart in ('hour', 'minute', 'second', 'millisecond', 'microsecond') -%}\n\n {%- set multiplier -%}\n {%- if datepart == 'hour' -%} 3600\n {%- elif datepart == 'minute' -%} 60\n {%- elif datepart == 'second' -%} 1\n {%- elif datepart == 'millisecond' -%} (1/1000000)\n {%- elif datepart == 'microsecond' -%} (1/1000000)\n {%- endif -%}\n {%- endset -%}\n\n to_timestamp(\n {{ assert_not_null('to_unix_timestamp', from_date_or_timestamp) }}\n + cast({{interval}} * {{multiplier}} as int)\n )\n\n {%- else -%}\n\n {{ exceptions.raise_compiler_error(\"macro dateadd not implemented for datepart ~ '\" ~ datepart ~ \"' ~ on Spark\") }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.713269, "supported_languages": null}, "macro.dbt_spark.spark__current_timestamp": {"name": "spark__current_timestamp", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/timestamps.sql", "original_file_path": "macros/utils/timestamps.sql", "unique_id": "macro.dbt_spark.spark__current_timestamp", "macro_sql": "{% macro spark__current_timestamp() -%}\n current_timestamp()\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.713365, "supported_languages": null}, "macro.dbt_spark.spark__escape_single_quotes": {"name": "spark__escape_single_quotes", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt_spark.spark__escape_single_quotes", "macro_sql": "{% macro spark__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"\\\\'\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.713526, "supported_languages": null}, "macro.dbt_spark.spark__listagg": {"name": "spark__listagg", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt_spark.spark__listagg", "macro_sql": "{% macro spark__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if order_by_clause %}\n {{ exceptions.warn(\"order_by_clause is not supported for listagg on Spark/Databricks\") }}\n {% endif %}\n\n {% set collect_list %} collect_list({{ measure }}) {% endset %}\n\n {% set limited %} slice({{ collect_list }}, 1, {{ limit_num }}) {% endset %}\n\n {% set collected = limited if limit_num else collect_list %}\n\n {% set final %} array_join({{ collected }}, {{ delimiter_text }}) {% endset %}\n\n {% do return(final) %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.714139, "supported_languages": null}, "macro.dbt_spark.spark__datediff": {"name": "spark__datediff", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_spark.spark__datediff", "macro_sql": "{% macro spark__datediff(first_date, second_date, datepart) %}\n\n {%- if datepart in ['day', 'week', 'month', 'quarter', 'year'] -%}\n\n {# make sure the dates are real, otherwise raise an error asap #}\n {% set first_date = assert_not_null('date', first_date) %}\n {% set second_date = assert_not_null('date', second_date) %}\n\n {%- endif -%}\n\n {%- if datepart == 'day' -%}\n\n datediff({{second_date}}, {{first_date}})\n\n {%- elif datepart == 'week' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(datediff({{second_date}}, {{first_date}})/7)\n else ceil(datediff({{second_date}}, {{first_date}})/7)\n end\n\n -- did we cross a week boundary (Sunday)?\n + case\n when {{first_date}} < {{second_date}} and dayofweek({{second_date}}) < dayofweek({{first_date}}) then 1\n when {{first_date}} > {{second_date}} and dayofweek({{second_date}}) > dayofweek({{first_date}}) then -1\n else 0 end\n\n {%- elif datepart == 'month' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(months_between(date({{second_date}}), date({{first_date}})))\n else ceil(months_between(date({{second_date}}), date({{first_date}})))\n end\n\n -- did we cross a month boundary?\n + case\n when {{first_date}} < {{second_date}} and dayofmonth({{second_date}}) < dayofmonth({{first_date}}) then 1\n when {{first_date}} > {{second_date}} and dayofmonth({{second_date}}) > dayofmonth({{first_date}}) then -1\n else 0 end\n\n {%- elif datepart == 'quarter' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(months_between(date({{second_date}}), date({{first_date}}))/3)\n else ceil(months_between(date({{second_date}}), date({{first_date}}))/3)\n end\n\n -- did we cross a quarter boundary?\n + case\n when {{first_date}} < {{second_date}} and (\n (dayofyear({{second_date}}) - (quarter({{second_date}}) * 365/4))\n < (dayofyear({{first_date}}) - (quarter({{first_date}}) * 365/4))\n ) then 1\n when {{first_date}} > {{second_date}} and (\n (dayofyear({{second_date}}) - (quarter({{second_date}}) * 365/4))\n > (dayofyear({{first_date}}) - (quarter({{first_date}}) * 365/4))\n ) then -1\n else 0 end\n\n {%- elif datepart == 'year' -%}\n\n year({{second_date}}) - year({{first_date}})\n\n {%- elif datepart in ('hour', 'minute', 'second', 'millisecond', 'microsecond') -%}\n\n {%- set divisor -%}\n {%- if datepart == 'hour' -%} 3600\n {%- elif datepart == 'minute' -%} 60\n {%- elif datepart == 'second' -%} 1\n {%- elif datepart == 'millisecond' -%} (1/1000)\n {%- elif datepart == 'microsecond' -%} (1/1000000)\n {%- endif -%}\n {%- endset -%}\n\n case when {{first_date}} < {{second_date}}\n then ceil((\n {# make sure the timestamps are real, otherwise raise an error asap #}\n {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', second_date)) }}\n - {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', first_date)) }}\n ) / {{divisor}})\n else floor((\n {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', second_date)) }}\n - {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', first_date)) }}\n ) / {{divisor}})\n end\n\n {% if datepart == 'millisecond' %}\n + cast(date_format({{second_date}}, 'SSS') as int)\n - cast(date_format({{first_date}}, 'SSS') as int)\n {% endif %}\n\n {% if datepart == 'microsecond' %}\n {% set capture_str = '[0-9]{4}-[0-9]{2}-[0-9]{2}.[0-9]{2}:[0-9]{2}:[0-9]{2}.([0-9]{6})' %}\n -- Spark doesn't really support microseconds, so this is a massive hack!\n -- It will only work if the timestamp-string is of the format\n -- 'yyyy-MM-dd-HH mm.ss.SSSSSS'\n + cast(regexp_extract({{second_date}}, '{{capture_str}}', 1) as int)\n - cast(regexp_extract({{first_date}}, '{{capture_str}}', 1) as int)\n {% endif %}\n\n {%- else -%}\n\n {{ exceptions.raise_compiler_error(\"macro datediff not implemented for datepart ~ '\" ~ datepart ~ \"' ~ on Spark\") }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7200809, "supported_languages": null}, "macro.dbt_spark.spark__safe_cast": {"name": "spark__safe_cast", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt_spark.spark__safe_cast", "macro_sql": "{% macro spark__safe_cast(field, type) %}\n{%- set field_clean = field.strip('\"').strip(\"'\") if (cast_from_string_unsupported_for(type) and field is string) else field -%}\ncast({{field_clean}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.cast_from_string_unsupported_for"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7204401, "supported_languages": null}, "macro.dbt_spark.cast_from_string_unsupported_for": {"name": "cast_from_string_unsupported_for", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt_spark.cast_from_string_unsupported_for", "macro_sql": "{% macro cast_from_string_unsupported_for(type) %}\n {{ return(type.lower().startswith('struct') or type.lower().startswith('array') or type.lower().startswith('map')) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.720673, "supported_languages": null}, "macro.dbt_spark.spark__any_value": {"name": "spark__any_value", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt_spark.spark__any_value", "macro_sql": "{% macro spark__any_value(expression) -%}\n {#-- return any value (non-deterministic) --#}\n first({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.720806, "supported_languages": null}, "macro.dbt_spark.spark__array_concat": {"name": "spark__array_concat", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt_spark.spark__array_concat", "macro_sql": "{% macro spark__array_concat(array_1, array_2) -%}\n concat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.720957, "supported_languages": null}, "macro.dbt_spark.spark__bool_or": {"name": "spark__bool_or", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt_spark.spark__bool_or", "macro_sql": "{% macro spark__bool_or(expression) -%}\n\n max({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.721088, "supported_languages": null}, "macro.dbt_spark.spark__split_part": {"name": "spark__split_part", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt_spark.spark__split_part", "macro_sql": "{% macro spark__split_part(string_text, delimiter_text, part_number) %}\n\n {% set delimiter_expr %}\n\n -- escape if starts with a special character\n case when regexp_extract({{ delimiter_text }}, '([^A-Za-z0-9])(.*)', 1) != '_'\n then concat('\\\\', {{ delimiter_text }})\n else {{ delimiter_text }} end\n\n {% endset %}\n\n {% if part_number >= 0 %}\n\n {% set split_part_expr %}\n\n split(\n {{ string_text }},\n {{ delimiter_expr }}\n )[({{ part_number - 1 if part_number > 0 else part_number }})]\n\n {% endset %}\n\n {% else %}\n\n {% set split_part_expr %}\n\n split(\n {{ string_text }},\n {{ delimiter_expr }}\n )[(\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 1 + {{ part_number }}\n )]\n\n {% endset %}\n\n {% endif %}\n\n {{ return(split_part_expr) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.722142, "supported_languages": null}, "macro.dbt_spark.spark__array_construct": {"name": "spark__array_construct", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt_spark.spark__array_construct", "macro_sql": "{% macro spark__array_construct(inputs, data_type) -%}\n array( {{ inputs|join(' , ') }} )\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.722311, "supported_languages": null}, "macro.dbt_spark.spark__array_append": {"name": "spark__array_append", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt_spark.spark__array_append", "macro_sql": "{% macro spark__array_append(array, new_element) -%}\n {{ array_concat(array, array_construct([new_element])) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.array_concat", "macro.dbt.array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.722492, "supported_languages": null}, "macro.dbt.run_hooks": {"name": "run_hooks", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.72342, "supported_languages": null}, "macro.dbt.make_hook_config": {"name": "make_hook_config", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723584, "supported_languages": null}, "macro.dbt.before_begin": {"name": "before_begin", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723713, "supported_languages": null}, "macro.dbt.in_transaction": {"name": "in_transaction", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723829, "supported_languages": null}, "macro.dbt.after_commit": {"name": "after_commit", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723947, "supported_languages": null}, "macro.dbt.set_sql_header": {"name": "set_sql_header", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.724263, "supported_languages": null}, "macro.dbt.should_full_refresh": {"name": "should_full_refresh", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.72451, "supported_languages": null}, "macro.dbt.should_store_failures": {"name": "should_store_failures", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.724753, "supported_languages": null}, "macro.dbt.snapshot_merge_sql": {"name": "snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__snapshot_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7252579, "supported_languages": null}, "macro.dbt.default__snapshot_merge_sql": {"name": "default__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n {%- set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() -%}\n\n merge into {{ target.render() }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}\n\n when matched\n and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.725742, "supported_languages": null}, "macro.dbt.strategy_dispatch": {"name": "strategy_dispatch", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7284422, "supported_languages": null}, "macro.dbt.snapshot_hash_arguments": {"name": "snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7285912, "supported_languages": null}, "macro.dbt.default__snapshot_hash_arguments": {"name": "default__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.728775, "supported_languages": null}, "macro.dbt.snapshot_timestamp_strategy": {"name": "snapshot_timestamp_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set primary_key = config.get('unique_key') %}\n {% set updated_at = config.get('updated_at') %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes') or false %}\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.{{ columns.dbt_valid_from }} < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7294762, "supported_languages": null}, "macro.dbt.snapshot_string_as_time": {"name": "snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__snapshot_string_as_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7296178, "supported_languages": null}, "macro.dbt.default__snapshot_string_as_time": {"name": "default__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.729759, "supported_languages": null}, "macro.dbt.snapshot_check_all_get_existing_columns": {"name": "snapshot_check_all_get_existing_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n {#-- N.B. The whitespace below is necessary to avoid edge case issue with comments --#}\n {#-- See: https://github.com/dbt-labs/dbt-core/issues/6781 --#}\n select {{ check_cols_config | join(', ') }} from (\n {{ node['compiled_code'] }}\n ) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7308888, "supported_languages": null}, "macro.dbt.snapshot_check_strategy": {"name": "snapshot_check_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set check_cols_config = config.get('check_cols') %}\n {% set primary_key = config.get('unique_key') %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes') or false %}\n {% set updated_at = config.get('updated_at') or snapshot_get_time() %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.get_true_sql", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7321742, "supported_languages": null}, "macro.dbt.create_columns": {"name": "create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__create_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.73771, "supported_languages": null}, "macro.dbt.default__create_columns": {"name": "default__create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation.render() }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.737952, "supported_languages": null}, "macro.dbt.post_snapshot": {"name": "post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.738092, "supported_languages": null}, "macro.dbt.default__post_snapshot": {"name": "default__post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.738164, "supported_languages": null}, "macro.dbt.get_true_sql": {"name": "get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_true_sql", "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_true_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.738283, "supported_languages": null}, "macro.dbt.default__get_true_sql": {"name": "default__get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__get_true_sql", "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7383811, "supported_languages": null}, "macro.dbt.snapshot_staging_table": {"name": "snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__snapshot_staging_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7385468, "supported_languages": null}, "macro.dbt.get_snapshot_table_column_names": {"name": "get_snapshot_table_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_snapshot_table_column_names", "macro_sql": "{% macro get_snapshot_table_column_names() %}\n {{ return({'dbt_valid_to': 'dbt_valid_to', 'dbt_valid_from': 'dbt_valid_from', 'dbt_scd_id': 'dbt_scd_id', 'dbt_updated_at': 'dbt_updated_at'}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7387378, "supported_languages": null}, "macro.dbt.default__snapshot_staging_table": {"name": "default__snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__snapshot_staging_table", "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where {{ columns.dbt_valid_to }} is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }},\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_to }}\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.{{ columns.dbt_scd_id }}\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},\n {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_to }},\n snapshotted_data.{{ columns.dbt_scd_id }}\n\n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.739788, "supported_languages": null}, "macro.dbt.build_snapshot_table": {"name": "build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__build_snapshot_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7399502, "supported_languages": null}, "macro.dbt.default__build_snapshot_table": {"name": "default__build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__build_snapshot_table", "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n\n select *,\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }}\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7403069, "supported_languages": null}, "macro.dbt.build_snapshot_staging_table": {"name": "build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set temp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, temp_relation, select) }}\n {% endcall %}\n\n {% do return(temp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.740653, "supported_languages": null}, "macro.dbt.get_updated_at_column_data_type": {"name": "get_updated_at_column_data_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_updated_at_column_data_type", "macro_sql": "{% macro get_updated_at_column_data_type(snapshot_sql) %}\n {% set snapshot_sql_column_schema = get_column_schema_from_query(snapshot_sql) %}\n {% set dbt_updated_at_data_type = null %}\n {% set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {% set ns.dbt_updated_at_data_type = null -%}\n {% for column in snapshot_sql_column_schema %}\n {% if ((column.column == 'dbt_updated_at') or (column.column == 'DBT_UPDATED_AT')) %}\n {% set ns.dbt_updated_at_data_type = column.dtype %}\n {% endif %}\n {% endfor %}\n {{ return(ns.dbt_updated_at_data_type or none) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.741133, "supported_languages": null}, "macro.dbt.check_time_data_types": {"name": "check_time_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.check_time_data_types", "macro_sql": "{% macro check_time_data_types(sql) %}\n {% set dbt_updated_at_data_type = get_updated_at_column_data_type(sql) %}\n {% set snapshot_get_time_data_type = get_snapshot_get_time_data_type() %}\n {% if snapshot_get_time_data_type is not none and dbt_updated_at_data_type is not none and snapshot_get_time_data_type != dbt_updated_at_data_type %}\n {% if exceptions.warn_snapshot_timestamp_data_types %}\n {{ exceptions.warn_snapshot_timestamp_data_types(snapshot_get_time_data_type, dbt_updated_at_data_type) }}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_updated_at_column_data_type", "macro.dbt.get_snapshot_get_time_data_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.741546, "supported_languages": null}, "macro.dbt.materialization_snapshot_default": {"name": "materialization_snapshot_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot.sql", "original_file_path": "macros/materializations/snapshots/snapshot.sql", "unique_id": "macro.dbt.materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparision later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {# The model['config'] parameter below is no longer used, but passing anyway for compatibility #}\n {# It was a dictionary of config, instead of the config object from the context #}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", model['config'], target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set build_or_select_sql = build_sql %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {{ adapter.valid_snapshot_target(target_relation, columns) }}\n\n {% set build_or_select_sql = snapshot_staging_table(strategy, sql, target_relation) %}\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n\n {{ check_time_data_types(build_or_select_sql) }}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_staging_table", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.check_time_data_types", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.746002, "supported_languages": ["sql"]}, "macro.dbt.materialization_test_default": {"name": "materialization_test_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/test.sql", "original_file_path": "macros/materializations/tests/test.sql", "unique_id": "macro.dbt.materialization_test_default", "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% set store_failures_as = config.get('store_failures_as') %}\n -- if `--store-failures` is invoked via command line and `store_failures_as` is not set,\n -- config.get('store_failures_as', 'table') returns None, not 'table'\n {% if store_failures_as == none %}{% set store_failures_as = 'table' %}{% endif %}\n {% if store_failures_as not in ['table', 'view'] %}\n {{ exceptions.raise_compiler_error(\n \"'\" ~ store_failures_as ~ \"' is not a valid value for `store_failures_as`. \"\n \"Accepted values are: ['ephemeral', 'table', 'view']\"\n ) }}\n {% endif %}\n\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type=store_failures_as) -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ get_create_sql(target_relation, sql) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql %}\n\n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.get_create_sql", "macro.dbt.get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7482162, "supported_languages": ["sql"]}, "macro.dbt.get_test_sql": {"name": "get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.749175, "supported_languages": null}, "macro.dbt.default__get_test_sql": {"name": "default__get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7494242, "supported_languages": null}, "macro.dbt.get_unit_test_sql": {"name": "get_unit_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_unit_test_sql", "macro_sql": "{% macro get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n {{ adapter.dispatch('get_unit_test_sql', 'dbt')(main_sql, expected_fixture_sql, expected_column_names) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_unit_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.749598, "supported_languages": null}, "macro.dbt.default__get_unit_test_sql": {"name": "default__get_unit_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_unit_test_sql", "macro_sql": "{% macro default__get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n-- Build actual result given inputs\nwith dbt_internal_unit_test_actual as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%},{% endif %}{%- endfor -%}, {{ dbt.string_literal(\"actual\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ main_sql }}\n ) _dbt_internal_unit_test_actual\n),\n-- Build expected result\ndbt_internal_unit_test_expected as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%}, {% endif %}{%- endfor -%}, {{ dbt.string_literal(\"expected\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ expected_fixture_sql }}\n ) _dbt_internal_unit_test_expected\n)\n-- Union actual and expected results\nselect * from dbt_internal_unit_test_actual\nunion all\nselect * from dbt_internal_unit_test_expected\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7500942, "supported_languages": null}, "macro.dbt.get_where_subquery": {"name": "get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.get_where_subquery", "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_where_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7504158, "supported_languages": null}, "macro.dbt.default__get_where_subquery": {"name": "default__get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.default__get_where_subquery", "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7507288, "supported_languages": null}, "macro.dbt.materialization_unit_default": {"name": "materialization_unit_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/unit.sql", "original_file_path": "macros/materializations/tests/unit.sql", "unique_id": "macro.dbt.materialization_unit_default", "macro_sql": "{%- materialization unit, default -%}\n\n {% set relations = [] %}\n\n {% set expected_rows = config.get('expected_rows') %}\n {% set expected_sql = config.get('expected_sql') %}\n {% set tested_expected_column_names = expected_rows[0].keys() if (expected_rows | length ) > 0 else get_columns_in_query(sql) %} %}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {% do run_query(get_create_table_as_sql(True, temp_relation, get_empty_subquery_sql(sql))) %}\n {%- set columns_in_relation = adapter.get_columns_in_relation(temp_relation) -%}\n {%- set column_name_to_data_types = {} -%}\n {%- for column in columns_in_relation -%}\n {%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n {%- endfor -%}\n\n {% if not expected_sql %}\n {% set expected_sql = get_expected_sql(expected_rows, column_name_to_data_types) %}\n {% endif %}\n {% set unit_test_sql = get_unit_test_sql(sql, expected_sql, tested_expected_column_names) %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ unit_test_sql }}\n\n {%- endcall %}\n\n {% do adapter.drop_relation(temp_relation) %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt.make_temp_relation", "macro.dbt.run_query", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_empty_subquery_sql", "macro.dbt.get_expected_sql", "macro.dbt.get_unit_test_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7521741, "supported_languages": ["sql"]}, "macro.dbt.materialization_materialized_view_default": {"name": "materialization_materialized_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialization_materialized_view_default", "macro_sql": "{% materialization materialized_view, default %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n {% set intermediate_relation = make_intermediate_relation(target_relation) %}\n {% set backup_relation_type = target_relation.MaterializedView if existing_relation is none else existing_relation.type %}\n {% set backup_relation = make_backup_relation(target_relation, backup_relation_type) %}\n\n {{ materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% if build_sql == '' %}\n {{ materialized_view_execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.materialized_view_setup", "macro.dbt.materialized_view_get_build_sql", "macro.dbt.materialized_view_execute_no_op", "macro.dbt.materialized_view_execute_build_sql", "macro.dbt.materialized_view_teardown"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.756201, "supported_languages": ["sql"]}, "macro.dbt.materialized_view_setup": {"name": "materialized_view_setup", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_setup", "macro_sql": "{% macro materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) %}\n\n -- backup_relation and intermediate_relation should not already exist in the database\n -- it's possible these exist because of a previous run that exited unexpectedly\n {% set preexisting_backup_relation = load_cached_relation(backup_relation) %}\n {% set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.756525, "supported_languages": null}, "macro.dbt.materialized_view_teardown": {"name": "materialized_view_teardown", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_teardown", "macro_sql": "{% macro materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) %}\n\n -- drop the temp relations if they exist to leave the database clean for the next run\n {{ drop_relation_if_exists(backup_relation) }}\n {{ drop_relation_if_exists(intermediate_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.756736, "supported_languages": null}, "macro.dbt.materialized_view_get_build_sql": {"name": "materialized_view_get_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_get_build_sql", "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = '' %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation.render() ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation.render() ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt.get_replace_sql", "macro.dbt.get_materialized_view_configuration_changes", "macro.dbt.refresh_materialized_view", "macro.dbt.get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.757851, "supported_languages": null}, "macro.dbt.materialized_view_execute_no_op": {"name": "materialized_view_execute_no_op", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_no_op", "macro_sql": "{% macro materialized_view_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.75805, "supported_languages": null}, "macro.dbt.materialized_view_execute_build_sql": {"name": "materialized_view_execute_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_build_sql", "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7585862, "supported_languages": null}, "macro.dbt.materialization_view_default": {"name": "materialization_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view.sql", "original_file_path": "macros/materializations/models/view.sql", "unique_id": "macro.dbt.materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.run_hooks", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.760961, "supported_languages": ["sql"]}, "macro.dbt.materialization_table_default": {"name": "materialization_table_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table.sql", "original_file_path": "macros/materializations/models/table.sql", "unique_id": "macro.dbt.materialization_table_default", "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.get_create_table_as_sql", "macro.dbt.create_indexes", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.763325, "supported_languages": ["sql"]}, "macro.dbt.get_quoted_csv": {"name": "get_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7646668, "supported_languages": null}, "macro.dbt.diff_columns": {"name": "diff_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_columns", "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.765106, "supported_languages": null}, "macro.dbt.diff_column_data_types": {"name": "diff_column_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_column_data_types", "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type and not sc.can_expand_to(other_column=tc) %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.76564, "supported_languages": null}, "macro.dbt.get_merge_update_columns": {"name": "get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_merge_update_columns", "macro_sql": "{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.765835, "supported_languages": null}, "macro.dbt.default__get_merge_update_columns": {"name": "default__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.default__get_merge_update_columns", "macro_sql": "{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = dest_columns | map(attribute=\"quoted\") | list -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7664142, "supported_languages": null}, "macro.dbt.get_merge_sql": {"name": "get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n -- back compat for old kwarg name\n {% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.771425, "supported_languages": null}, "macro.dbt.default__get_merge_sql": {"name": "default__get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{\"(\" ~ predicates | join(\") and (\") ~ \")\"}}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv", "macro.dbt.get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.772757, "supported_languages": null}, "macro.dbt.get_delete_insert_merge_sql": {"name": "get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.77298, "supported_languages": null}, "macro.dbt.default__get_delete_insert_merge_sql": {"name": "default__get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not string %}\n delete from {{target }}\n using {{ source }}\n where (\n {% for key in unique_key %}\n {{ source }}.{{ key }} = {{ target }}.{{ key }}\n {{ \"and \" if not loop.last}}\n {% endfor %}\n {% if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {% endif %}\n );\n {% else %}\n delete from {{ target }}\n where (\n {{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n )\n {%- if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {%- endif -%};\n\n {% endif %}\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.773788, "supported_languages": null}, "macro.dbt.get_insert_overwrite_merge_sql": {"name": "get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.774013, "supported_languages": null}, "macro.dbt.default__get_insert_overwrite_merge_sql": {"name": "default__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {#-- The only time include_sql_header is True: --#}\n {#-- BigQuery + insert_overwrite strategy + \"static\" partitions config --#}\n {#-- We should consider including the sql header at the materialization level instead --#}\n\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.774538, "supported_languages": null}, "macro.dbt.is_incremental": {"name": "is_incremental", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/is_incremental.sql", "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", "unique_id": "macro.dbt.is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.775049, "supported_languages": null}, "macro.dbt.get_incremental_append_sql": {"name": "get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_append_sql", "macro_sql": "{% macro get_incremental_append_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.775977, "supported_languages": null}, "macro.dbt.default__get_incremental_append_sql": {"name": "default__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_append_sql", "macro_sql": "{% macro default__get_incremental_append_sql(arg_dict) %}\n\n {% do return(get_insert_into_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.776169, "supported_languages": null}, "macro.dbt.get_incremental_delete_insert_sql": {"name": "get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_delete_insert_sql", "macro_sql": "{% macro get_incremental_delete_insert_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_delete_insert_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.776327, "supported_languages": null}, "macro.dbt.default__get_incremental_delete_insert_sql": {"name": "default__get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_delete_insert_sql", "macro_sql": "{% macro default__get_incremental_delete_insert_sql(arg_dict) %}\n\n {% do return(get_delete_insert_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7765691, "supported_languages": null}, "macro.dbt.get_incremental_merge_sql": {"name": "get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_merge_sql", "macro_sql": "{% macro get_incremental_merge_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.776725, "supported_languages": null}, "macro.dbt.default__get_incremental_merge_sql": {"name": "default__get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_merge_sql", "macro_sql": "{% macro default__get_incremental_merge_sql(arg_dict) %}\n\n {% do return(get_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7769668, "supported_languages": null}, "macro.dbt.get_incremental_insert_overwrite_sql": {"name": "get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_insert_overwrite_sql", "macro_sql": "{% macro get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7771251, "supported_languages": null}, "macro.dbt.default__get_incremental_insert_overwrite_sql": {"name": "default__get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_insert_overwrite_sql", "macro_sql": "{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {% do return(get_insert_overwrite_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777345, "supported_languages": null}, "macro.dbt.get_incremental_default_sql": {"name": "get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_default_sql", "macro_sql": "{% macro get_incremental_default_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_default_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777499, "supported_languages": null}, "macro.dbt.default__get_incremental_default_sql": {"name": "default__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_default_sql", "macro_sql": "{% macro default__get_incremental_default_sql(arg_dict) %}\n\n {% do return(get_incremental_append_sql(arg_dict)) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777623, "supported_languages": null}, "macro.dbt.get_incremental_microbatch_sql": {"name": "get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_microbatch_sql", "macro_sql": "{% macro get_incremental_microbatch_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_microbatch_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_microbatch_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777774, "supported_languages": null}, "macro.dbt.default__get_incremental_microbatch_sql": {"name": "default__get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_microbatch_sql", "macro_sql": "{% macro default__get_incremental_microbatch_sql(arg_dict) %}\n\n {{ exceptions.raise_not_implemented('microbatch materialization strategy not implemented for adapter ' + adapter.type()) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7779121, "supported_languages": null}, "macro.dbt.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ temp_relation }}\n )\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7781398, "supported_languages": null}, "macro.dbt.materialization_incremental_default": {"name": "materialization_incremental_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/incremental.sql", "original_file_path": "macros/materializations/models/incremental/incremental.sql", "unique_id": "macro.dbt.materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% set contract_config = config.get('contract') %}\n {% if not contract_config or not contract_config.enforced %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {% endif %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_temp_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.get_create_table_as_sql", "macro.dbt.run_query", "macro.dbt.process_schema_changes", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.782465, "supported_languages": ["sql"]}, "macro.dbt.incremental_validate_on_schema_change": {"name": "incremental_validate_on_schema_change", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.incremental_validate_on_schema_change", "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7869802, "supported_languages": null}, "macro.dbt.check_for_schema_changes": {"name": "check_for_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.check_for_schema_changes", "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.diff_columns", "macro.dbt.diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.787961, "supported_languages": null}, "macro.dbt.sync_column_schemas": {"name": "sync_column_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.sync_column_schemas", "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.alter_relation_add_remove_columns", "macro.dbt.alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.788921, "supported_languages": null}, "macro.dbt.process_schema_changes": {"name": "process_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.process_schema_changes", "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n\n Additional troubleshooting context:\n Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}\n Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}\n New column types: {{ schema_changes_dict['new_target_types'] }}\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.check_for_schema_changes", "macro.dbt.sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.789614, "supported_languages": null}, "macro.dbt.can_clone_table": {"name": "can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.can_clone_table", "macro_sql": "{% macro can_clone_table() %}\n {{ return(adapter.dispatch('can_clone_table', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__can_clone_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.789815, "supported_languages": null}, "macro.dbt.default__can_clone_table": {"name": "default__can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.default__can_clone_table", "macro_sql": "{% macro default__can_clone_table() %}\n {{ return(False) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7899132, "supported_languages": null}, "macro.dbt.create_or_replace_clone": {"name": "create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.create_or_replace_clone", "macro_sql": "{% macro create_or_replace_clone(this_relation, defer_relation) %}\n {{ return(adapter.dispatch('create_or_replace_clone', 'dbt')(this_relation, defer_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_or_replace_clone"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.790163, "supported_languages": null}, "macro.dbt.default__create_or_replace_clone": {"name": "default__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.default__create_or_replace_clone", "macro_sql": "{% macro default__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation.render() }} clone {{ defer_relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.790299, "supported_languages": null}, "macro.dbt.materialization_clone_default": {"name": "materialization_clone_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/clone.sql", "original_file_path": "macros/materializations/models/clone/clone.sql", "unique_id": "macro.dbt.materialization_clone_default", "macro_sql": "{%- materialization clone, default -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation.render() ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {% if target_relation and defer_relation and target_relation == defer_relation %}\n {{ log(\"Target relation and defer relation are the same, skipping clone for relation: \" ~ target_relation.render()) }}\n {% else %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endif %}\n\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n\n {%- endif -%}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.793341, "supported_languages": ["sql"]}, "macro.dbt.materialization_seed_default": {"name": "materialization_seed_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/seed.sql", "original_file_path": "macros/materializations/seeds/seed.sql", "unique_id": "macro.dbt.materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparison later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation.render())) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.796114, "supported_languages": ["sql"]}, "macro.dbt.create_csv_table": {"name": "create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80035, "supported_languages": null}, "macro.dbt.default__create_csv_table": {"name": "default__create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80112, "supported_languages": null}, "macro.dbt.reset_csv_table": {"name": "reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__reset_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8013299, "supported_languages": null}, "macro.dbt.default__reset_csv_table": {"name": "default__reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation.render() %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.801737, "supported_languages": null}, "macro.dbt.get_csv_sql": {"name": "get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_csv_sql", "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_csv_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.801903, "supported_languages": null}, "macro.dbt.default__get_csv_sql": {"name": "default__get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_csv_sql", "macro_sql": "{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ create_or_truncate_sql }};\n -- dbt seed --\n {{ insert_sql }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80202, "supported_languages": null}, "macro.dbt.get_binding_char": {"name": "get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_binding_char", "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8021429, "supported_languages": null}, "macro.dbt.default__get_binding_char": {"name": "default__get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_binding_char", "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.802243, "supported_languages": null}, "macro.dbt.get_batch_size": {"name": "get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_batch_size", "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_batch_size"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8023849, "supported_languages": null}, "macro.dbt.default__get_batch_size": {"name": "default__get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_batch_size", "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.802489, "supported_languages": null}, "macro.dbt.get_seed_column_quoted_csv": {"name": "get_seed_column_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8028958, "supported_languages": null}, "macro.dbt.load_csv_rows": {"name": "load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.803055, "supported_languages": null}, "macro.dbt.default__load_csv_rows": {"name": "default__load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_seed_column_quoted_csv", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8041549, "supported_languages": null}, "macro.dbt.generate_alias_name": {"name": "generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_alias_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.804568, "supported_languages": null}, "macro.dbt.default__generate_alias_name": {"name": "default__generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.default__generate_alias_name", "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name -%}\n\n {{ custom_alias_name | trim }}\n\n {%- elif node.version -%}\n\n {{ return(node.name ~ \"_v\" ~ (node.version | replace(\".\", \"_\"))) }}\n\n {%- else -%}\n\n {{ node.name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.804885, "supported_languages": null}, "macro.dbt.generate_schema_name": {"name": "generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8053389, "supported_languages": null}, "macro.dbt.default__generate_schema_name": {"name": "default__generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.default__generate_schema_name", "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8055522, "supported_languages": null}, "macro.dbt.generate_schema_name_for_env": {"name": "generate_schema_name_for_env", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8057861, "supported_languages": null}, "macro.dbt.generate_database_name": {"name": "generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__generate_database_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.806116, "supported_languages": null}, "macro.dbt.default__generate_database_name": {"name": "default__generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.806323, "supported_languages": null}, "macro.dbt.get_drop_sql": {"name": "get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.get_drop_sql", "macro_sql": "{%- macro get_drop_sql(relation) -%}\n {{- log('Applying DROP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.806918, "supported_languages": null}, "macro.dbt.default__get_drop_sql": {"name": "default__get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__get_drop_sql", "macro_sql": "{%- macro default__get_drop_sql(relation) -%}\n\n {%- if relation.is_view -%}\n {{ drop_view(relation) }}\n\n {%- elif relation.is_table -%}\n {{ drop_table(relation) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n\n {%- else -%}\n drop {{ relation.type }} if exists {{ relation.render() }} cascade\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.drop_view", "macro.dbt.drop_table", "macro.dbt.drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.807232, "supported_languages": null}, "macro.dbt.drop_relation": {"name": "drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8073862, "supported_languages": null}, "macro.dbt.default__drop_relation": {"name": "default__drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {{ get_drop_sql(relation) }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.807553, "supported_languages": null}, "macro.dbt.drop_relation_if_exists": {"name": "drop_relation_if_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.807711, "supported_languages": null}, "macro.dbt.get_replace_sql": {"name": "get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.get_replace_sql", "macro_sql": "{% macro get_replace_sql(existing_relation, target_relation, sql) %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {{- adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.808418, "supported_languages": null}, "macro.dbt.default__get_replace_sql": {"name": "default__get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.default__get_replace_sql", "macro_sql": "{% macro default__get_replace_sql(existing_relation, target_relation, sql) %}\n\n {# /* use a create or replace statement if possible */ #}\n\n {% set is_replaceable = existing_relation.type == target_relation_type and existing_relation.can_be_replaced %}\n\n {% if is_replaceable and existing_relation.is_view %}\n {{ get_replace_view_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_table %}\n {{ get_replace_table_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_materialized_view %}\n {{ get_replace_materialized_view_sql(target_relation, sql) }}\n\n {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}\n {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}\n {%- elif target_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_drop_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }}\n\n {# /* create target_relation in place by first backing up the existing relation */ #}\n {%- elif existing_relation.can_be_renamed -%}\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* no renaming is allowed, so just drop and create */ #}\n {%- else -%}\n {{ get_drop_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_replace_view_sql", "macro.dbt.get_replace_table_sql", "macro.dbt.get_replace_materialized_view_sql", "macro.dbt.get_create_intermediate_sql", "macro.dbt.get_create_backup_sql", "macro.dbt.get_rename_intermediate_sql", "macro.dbt.get_drop_backup_sql", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80947, "supported_languages": null}, "macro.dbt.get_create_intermediate_sql": {"name": "get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.get_create_intermediate_sql", "macro_sql": "{%- macro get_create_intermediate_sql(relation, sql) -%}\n {{- log('Applying CREATE INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_intermediate_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_create_intermediate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.809802, "supported_languages": null}, "macro.dbt.default__get_create_intermediate_sql": {"name": "default__get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.default__get_create_intermediate_sql", "macro_sql": "{%- macro default__get_create_intermediate_sql(relation, sql) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n -- drop any pre-existing intermediate\n {{ get_drop_sql(intermediate_relation) }};\n\n {{ get_create_sql(intermediate_relation, sql) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.810016, "supported_languages": null}, "macro.dbt.drop_schema_named": {"name": "drop_schema_named", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/schema.sql", "original_file_path": "macros/relations/schema.sql", "unique_id": "macro.dbt.drop_schema_named", "macro_sql": "{% macro drop_schema_named(schema_name) %}\n {{ return(adapter.dispatch('drop_schema_named', 'dbt') (schema_name)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__drop_schema_named"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.81025, "supported_languages": null}, "macro.dbt.default__drop_schema_named": {"name": "default__drop_schema_named", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/schema.sql", "original_file_path": "macros/relations/schema.sql", "unique_id": "macro.dbt.default__drop_schema_named", "macro_sql": "{% macro default__drop_schema_named(schema_name) %}\n {% set schema_relation = api.Relation.create(schema=schema_name) %}\n {{ adapter.drop_schema(schema_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.810434, "supported_languages": null}, "macro.dbt.get_drop_backup_sql": {"name": "get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.get_drop_backup_sql", "macro_sql": "{%- macro get_drop_backup_sql(relation) -%}\n {{- log('Applying DROP BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_drop_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.81071, "supported_languages": null}, "macro.dbt.default__get_drop_backup_sql": {"name": "default__get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.default__get_drop_backup_sql", "macro_sql": "{%- macro default__get_drop_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n {{ get_drop_sql(backup_relation) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.810894, "supported_languages": null}, "macro.dbt.get_rename_sql": {"name": "get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.get_rename_sql", "macro_sql": "{%- macro get_rename_sql(relation, new_name) -%}\n {{- log('Applying RENAME to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_sql', 'dbt')(relation, new_name) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.811506, "supported_languages": null}, "macro.dbt.default__get_rename_sql": {"name": "default__get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__get_rename_sql", "macro_sql": "{%- macro default__get_rename_sql(relation, new_name) -%}\n\n {%- if relation.is_view -%}\n {{ get_rename_view_sql(relation, new_name) }}\n\n {%- elif relation.is_table -%}\n {{ get_rename_table_sql(relation, new_name) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_rename_materialized_view_sql(relation, new_name) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_rename_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.get_rename_view_sql", "macro.dbt.get_rename_table_sql", "macro.dbt.get_rename_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.811878, "supported_languages": null}, "macro.dbt.rename_relation": {"name": "rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__rename_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812062, "supported_languages": null}, "macro.dbt.default__rename_relation": {"name": "default__rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation.render() }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812324, "supported_languages": null}, "macro.dbt.get_create_backup_sql": {"name": "get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.get_create_backup_sql", "macro_sql": "{%- macro get_create_backup_sql(relation) -%}\n {{- log('Applying CREATE BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_create_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812639, "supported_languages": null}, "macro.dbt.default__get_create_backup_sql": {"name": "default__get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.default__get_create_backup_sql", "macro_sql": "{%- macro default__get_create_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n -- drop any pre-existing backup\n {{ get_drop_sql(backup_relation) }};\n\n {{ get_rename_sql(relation, backup_relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812882, "supported_languages": null}, "macro.dbt.get_create_sql": {"name": "get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.get_create_sql", "macro_sql": "{%- macro get_create_sql(relation, sql) -%}\n {{- log('Applying CREATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8133051, "supported_languages": null}, "macro.dbt.default__get_create_sql": {"name": "default__get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.default__get_create_sql", "macro_sql": "{%- macro default__get_create_sql(relation, sql) -%}\n\n {%- if relation.is_view -%}\n {{ get_create_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_table -%}\n {{ get_create_table_as_sql(False, relation, sql) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_create_materialized_view_as_sql(relation, sql) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_create_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.get_create_view_as_sql", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.813689, "supported_languages": null}, "macro.dbt.get_rename_intermediate_sql": {"name": "get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.get_rename_intermediate_sql", "macro_sql": "{%- macro get_rename_intermediate_sql(relation) -%}\n {{- log('Applying RENAME INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_intermediate_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_rename_intermediate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.813972, "supported_languages": null}, "macro.dbt.default__get_rename_intermediate_sql": {"name": "default__get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.default__get_rename_intermediate_sql", "macro_sql": "{%- macro default__get_rename_intermediate_sql(relation) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n {{ get_rename_sql(intermediate_relation, relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8141482, "supported_languages": null}, "macro.dbt.drop_materialized_view": {"name": "drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.drop_materialized_view", "macro_sql": "{% macro drop_materialized_view(relation) -%}\n {{- adapter.dispatch('drop_materialized_view', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8143559, "supported_languages": null}, "macro.dbt.default__drop_materialized_view": {"name": "default__drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.default__drop_materialized_view", "macro_sql": "{% macro default__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.814456, "supported_languages": null}, "macro.dbt.get_replace_materialized_view_sql": {"name": "get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.get_replace_materialized_view_sql", "macro_sql": "{% macro get_replace_materialized_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_materialized_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8146849, "supported_languages": null}, "macro.dbt.default__get_replace_materialized_view_sql": {"name": "default__get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.default__get_replace_materialized_view_sql", "macro_sql": "{% macro default__get_replace_materialized_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.814875, "supported_languages": null}, "macro.dbt.refresh_materialized_view": {"name": "refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.refresh_materialized_view", "macro_sql": "{% macro refresh_materialized_view(relation) %}\n {{- log('Applying REFRESH to: ' ~ relation) -}}\n {{- adapter.dispatch('refresh_materialized_view', 'dbt')(relation) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__refresh_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.815148, "supported_languages": null}, "macro.dbt.default__refresh_materialized_view": {"name": "default__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.default__refresh_materialized_view", "macro_sql": "{% macro default__refresh_materialized_view(relation) %}\n {{ exceptions.raise_compiler_error(\"`refresh_materialized_view` has not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.815263, "supported_languages": null}, "macro.dbt.get_rename_materialized_view_sql": {"name": "get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.get_rename_materialized_view_sql", "macro_sql": "{% macro get_rename_materialized_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_rename_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8154979, "supported_languages": null}, "macro.dbt.default__get_rename_materialized_view_sql": {"name": "default__get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.default__get_rename_materialized_view_sql", "macro_sql": "{% macro default__get_rename_materialized_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8156219, "supported_languages": null}, "macro.dbt.get_alter_materialized_view_as_sql": {"name": "get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_alter_materialized_view_as_sql", "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {{- adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n ) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8161569, "supported_languages": null}, "macro.dbt.default__get_alter_materialized_view_as_sql": {"name": "default__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_alter_materialized_view_as_sql", "macro_sql": "{% macro default__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.816313, "supported_languages": null}, "macro.dbt.get_materialized_view_configuration_changes": {"name": "get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_materialized_view_configuration_changes", "macro_sql": "{% macro get_materialized_view_configuration_changes(existing_relation, new_config) %}\n /* {#\n It's recommended that configuration changes be formatted as follows:\n {\"\": [{\"action\": \"\", \"context\": ...}]}\n\n For example:\n {\n \"indexes\": [\n {\"action\": \"drop\", \"context\": \"index_abc\"},\n {\"action\": \"create\", \"context\": {\"columns\": [\"column_1\", \"column_2\"], \"type\": \"hash\", \"unique\": True}},\n ],\n }\n\n Either way, `get_materialized_view_configuration_changes` needs to align with `get_alter_materialized_view_as_sql`.\n #} */\n {{- log('Determining configuration changes on: ' ~ existing_relation) -}}\n {%- do return(adapter.dispatch('get_materialized_view_configuration_changes', 'dbt')(existing_relation, new_config)) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_materialized_view_configuration_changes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8165581, "supported_languages": null}, "macro.dbt.default__get_materialized_view_configuration_changes": {"name": "default__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_materialized_view_configuration_changes", "macro_sql": "{% macro default__get_materialized_view_configuration_changes(existing_relation, new_config) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.816685, "supported_languages": null}, "macro.dbt.get_create_materialized_view_as_sql": {"name": "get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.get_create_materialized_view_as_sql", "macro_sql": "{% macro get_create_materialized_view_as_sql(relation, sql) -%}\n {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.816923, "supported_languages": null}, "macro.dbt.default__get_create_materialized_view_as_sql": {"name": "default__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.default__get_create_materialized_view_as_sql", "macro_sql": "{% macro default__get_create_materialized_view_as_sql(relation, sql) -%}\n {{ exceptions.raise_compiler_error(\n \"`get_create_materialized_view_as_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.817044, "supported_languages": null}, "macro.dbt.get_table_columns_and_constraints": {"name": "get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_table_columns_and_constraints", "macro_sql": "{%- macro get_table_columns_and_constraints() -%}\n {{ adapter.dispatch('get_table_columns_and_constraints', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.817939, "supported_languages": null}, "macro.dbt.default__get_table_columns_and_constraints": {"name": "default__get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_table_columns_and_constraints", "macro_sql": "{% macro default__get_table_columns_and_constraints() -%}\n {{ return(table_columns_and_constraints()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.818042, "supported_languages": null}, "macro.dbt.table_columns_and_constraints": {"name": "table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.table_columns_and_constraints", "macro_sql": "{% macro table_columns_and_constraints() %}\n {# loop through user_provided_columns to create DDL with data types and constraints #}\n {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%}\n {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%}\n (\n {% for c in raw_column_constraints -%}\n {{ c }}{{ \",\" if not loop.last or raw_model_constraints }}\n {% endfor %}\n {% for c in raw_model_constraints -%}\n {{ c }}{{ \",\" if not loop.last }}\n {% endfor -%}\n )\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8184981, "supported_languages": null}, "macro.dbt.get_assert_columns_equivalent": {"name": "get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_assert_columns_equivalent", "macro_sql": "\n\n{%- macro get_assert_columns_equivalent(sql) -%}\n {{ adapter.dispatch('get_assert_columns_equivalent', 'dbt')(sql) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.818639, "supported_languages": null}, "macro.dbt.default__get_assert_columns_equivalent": {"name": "default__get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_assert_columns_equivalent", "macro_sql": "{% macro default__get_assert_columns_equivalent(sql) -%}\n {{ return(assert_columns_equivalent(sql)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.818753, "supported_languages": null}, "macro.dbt.assert_columns_equivalent": {"name": "assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.assert_columns_equivalent", "macro_sql": "{% macro assert_columns_equivalent(sql) %}\n\n {#-- First ensure the user has defined 'columns' in yaml specification --#}\n {%- set user_defined_columns = model['columns'] -%}\n {%- if not user_defined_columns -%}\n {{ exceptions.raise_contract_error([], []) }}\n {%- endif -%}\n\n {#-- Obtain the column schema provided by sql file. #}\n {%- set sql_file_provided_columns = get_column_schema_from_query(sql, config.get('sql_header', none)) -%}\n {#--Obtain the column schema provided by the schema file by generating an 'empty schema' query from the model's columns. #}\n {%- set schema_file_provided_columns = get_column_schema_from_query(get_empty_schema_sql(user_defined_columns)) -%}\n\n {#-- create dictionaries with name and formatted data type and strings for exception #}\n {%- set sql_columns = format_columns(sql_file_provided_columns) -%}\n {%- set yaml_columns = format_columns(schema_file_provided_columns) -%}\n\n {%- if sql_columns|length != yaml_columns|length -%}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n\n {%- for sql_col in sql_columns -%}\n {%- set yaml_col = [] -%}\n {%- for this_col in yaml_columns -%}\n {%- if this_col['name'] == sql_col['name'] -%}\n {%- do yaml_col.append(this_col) -%}\n {%- break -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if not yaml_col -%}\n {#-- Column with name not found in yaml #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- if sql_col['formatted'] != yaml_col[0]['formatted'] -%}\n {#-- Column data types don't match #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- endfor -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_column_schema_from_query", "macro.dbt.get_empty_schema_sql", "macro.dbt.format_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.819914, "supported_languages": null}, "macro.dbt.format_columns": {"name": "format_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.format_columns", "macro_sql": "{% macro format_columns(columns) %}\n {% set formatted_columns = [] %}\n {% for column in columns %}\n {%- set formatted_column = adapter.dispatch('format_column', 'dbt')(column) -%}\n {%- do formatted_columns.append(formatted_column) -%}\n {% endfor %}\n {{ return(formatted_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__format_column"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.820245, "supported_languages": null}, "macro.dbt.default__format_column": {"name": "default__format_column", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__format_column", "macro_sql": "{% macro default__format_column(column) -%}\n {% set data_type = column.dtype %}\n {% set formatted = column.column.lower() ~ \" \" ~ data_type %}\n {{ return({'name': column.name, 'data_type': data_type, 'formatted': formatted}) }}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8205361, "supported_languages": null}, "macro.dbt.drop_table": {"name": "drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.drop_table", "macro_sql": "{% macro drop_table(relation) -%}\n {{- adapter.dispatch('drop_table', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.820756, "supported_languages": null}, "macro.dbt.default__drop_table": {"name": "default__drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.default__drop_table", "macro_sql": "{% macro default__drop_table(relation) -%}\n drop table if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82086, "supported_languages": null}, "macro.dbt.get_replace_table_sql": {"name": "get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.get_replace_table_sql", "macro_sql": "{% macro get_replace_table_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_table_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_table_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8210938, "supported_languages": null}, "macro.dbt.default__get_replace_table_sql": {"name": "default__get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.default__get_replace_table_sql", "macro_sql": "{% macro default__get_replace_table_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8212202, "supported_languages": null}, "macro.dbt.get_rename_table_sql": {"name": "get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.get_rename_table_sql", "macro_sql": "{% macro get_rename_table_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_table_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_rename_table_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82146, "supported_languages": null}, "macro.dbt.default__get_rename_table_sql": {"name": "default__get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.default__get_rename_table_sql", "macro_sql": "{% macro default__get_rename_table_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82159, "supported_languages": null}, "macro.dbt.get_create_table_as_sql": {"name": "get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_create_table_as_sql", "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.822352, "supported_languages": null}, "macro.dbt.default__get_create_table_as_sql": {"name": "default__get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_create_table_as_sql", "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8225112, "supported_languages": null}, "macro.dbt.create_table_as": {"name": "create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {# backward compatibility for create_table_as that does not support language #}\n {% if language == \"sql\" %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code)}}\n {% else %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code, language) }}\n {% endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8228748, "supported_languages": null}, "macro.dbt.default__create_table_as": {"name": "default__create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced and (not temporary) %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_table_columns_and_constraints", "macro.dbt.get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.823435, "supported_languages": null}, "macro.dbt.default__get_column_names": {"name": "default__get_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_column_names", "macro_sql": "{% macro default__get_column_names() %}\n {#- loop through user_provided_columns to get column names -#}\n {%- set user_provided_columns = model['columns'] -%}\n {%- for i in user_provided_columns %}\n {%- set col = user_provided_columns[i] -%}\n {%- set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] -%}\n {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8238149, "supported_languages": null}, "macro.dbt.get_select_subquery": {"name": "get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_select_subquery", "macro_sql": "{% macro get_select_subquery(sql) %}\n {{ return(adapter.dispatch('get_select_subquery', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.823968, "supported_languages": null}, "macro.dbt.default__get_select_subquery": {"name": "default__get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_select_subquery", "macro_sql": "{% macro default__get_select_subquery(sql) %}\n select {{ adapter.dispatch('get_column_names', 'dbt')() }}\n from (\n {{ sql }}\n ) as model_subq\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.824113, "supported_languages": null}, "macro.dbt.drop_view": {"name": "drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.drop_view", "macro_sql": "{% macro drop_view(relation) -%}\n {{- adapter.dispatch('drop_view', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.824321, "supported_languages": null}, "macro.dbt.default__drop_view": {"name": "default__drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.default__drop_view", "macro_sql": "{% macro default__drop_view(relation) -%}\n drop view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.824422, "supported_languages": null}, "macro.dbt.get_replace_view_sql": {"name": "get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.get_replace_view_sql", "macro_sql": "{% macro get_replace_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.825161, "supported_languages": null}, "macro.dbt.default__get_replace_view_sql": {"name": "default__get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__get_replace_view_sql", "macro_sql": "{% macro default__get_replace_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82529, "supported_languages": null}, "macro.dbt.create_or_replace_view": {"name": "create_or_replace_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.create_or_replace_view", "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82631, "supported_languages": null}, "macro.dbt.handle_existing_table": {"name": "handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__handle_existing_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8264818, "supported_languages": null}, "macro.dbt.default__handle_existing_table": {"name": "default__handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation.render() ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8266811, "supported_languages": null}, "macro.dbt.get_rename_view_sql": {"name": "get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.get_rename_view_sql", "macro_sql": "{% macro get_rename_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_rename_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.826918, "supported_languages": null}, "macro.dbt.default__get_rename_view_sql": {"name": "default__get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.default__get_rename_view_sql", "macro_sql": "{% macro default__get_rename_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8270478, "supported_languages": null}, "macro.dbt.get_create_view_as_sql": {"name": "get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.get_create_view_as_sql", "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8274088, "supported_languages": null}, "macro.dbt.default__get_create_view_as_sql": {"name": "default__get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__get_create_view_as_sql", "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8275461, "supported_languages": null}, "macro.dbt.create_view_as": {"name": "create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.827704, "supported_languages": null}, "macro.dbt.default__create_view_as": {"name": "default__create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation.render() }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828066, "supported_languages": null}, "macro.dbt.default__test_relationships": {"name": "default__test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/relationships.sql", "original_file_path": "macros/generic_test_sql/relationships.sql", "unique_id": "macro.dbt.default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828371, "supported_languages": null}, "macro.dbt.default__test_not_null": {"name": "default__test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/not_null.sql", "original_file_path": "macros/generic_test_sql/not_null.sql", "unique_id": "macro.dbt.default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_store_failures"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828622, "supported_languages": null}, "macro.dbt.default__test_unique": {"name": "default__test_unique", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/unique.sql", "original_file_path": "macros/generic_test_sql/unique.sql", "unique_id": "macro.dbt.default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828834, "supported_languages": null}, "macro.dbt.default__test_accepted_values": {"name": "default__test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/accepted_values.sql", "original_file_path": "macros/generic_test_sql/accepted_values.sql", "unique_id": "macro.dbt.default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.829317, "supported_languages": null}, "macro.dbt.statement": {"name": "statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.statement", "macro_sql": "\n{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%}\n {%- if execute: -%}\n {%- set compiled_code = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime {} for node \"{}\"'.format(language, model['unique_id'])) }}\n {{ write(compiled_code) }}\n {%- endif -%}\n {%- if language == 'sql'-%}\n {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- elif language == 'python' -%}\n {%- set res = submit_python_job(model, compiled_code) -%}\n {#-- TODO: What should table be for python models? --#}\n {%- set table = None -%}\n {%- else -%}\n {% do exceptions.raise_compiler_error(\"statement macro didn't get supported language\") %}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.83059, "supported_languages": null}, "macro.dbt.noop_statement": {"name": "noop_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8313391, "supported_languages": null}, "macro.dbt.run_query": {"name": "run_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8315961, "supported_languages": null}, "macro.dbt.convert_datetime": {"name": "convert_datetime", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.833154, "supported_languages": null}, "macro.dbt.dates_in_range": {"name": "dates_in_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.convert_datetime"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.834102, "supported_languages": null}, "macro.dbt.partition_range": {"name": "partition_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.dates_in_range"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8347142, "supported_languages": null}, "macro.dbt.py_current_timestring": {"name": "py_current_timestring", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8349109, "supported_languages": null}, "macro.dbt.except": {"name": "except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.except", "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__except"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.835105, "supported_languages": null}, "macro.dbt.default__except": {"name": "default__except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.default__except", "macro_sql": "{% macro default__except() %}\n\n except\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.835169, "supported_languages": null}, "macro.dbt.get_intervals_between": {"name": "get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.get_intervals_between", "macro_sql": "{% macro get_intervals_between(start_date, end_date, datepart) -%}\n {{ return(adapter.dispatch('get_intervals_between', 'dbt')(start_date, end_date, datepart)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_intervals_between"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.835808, "supported_languages": null}, "macro.dbt.default__get_intervals_between": {"name": "default__get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__get_intervals_between", "macro_sql": "{% macro default__get_intervals_between(start_date, end_date, datepart) -%}\n {%- call statement('get_intervals_between', fetch_result=True) %}\n\n select {{ dbt.datediff(start_date, end_date, datepart) }}\n\n {%- endcall -%}\n\n {%- set value_list = load_result('get_intervals_between') -%}\n\n {%- if value_list and value_list['data'] -%}\n {%- set values = value_list['data'] | map(attribute=0) | list %}\n {{ return(values[0]) }}\n {%- else -%}\n {{ return(1) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8363059, "supported_languages": null}, "macro.dbt.date_spine": {"name": "date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.date_spine", "macro_sql": "{% macro date_spine(datepart, start_date, end_date) %}\n {{ return(adapter.dispatch('date_spine', 'dbt')(datepart, start_date, end_date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_spine"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.836503, "supported_languages": null}, "macro.dbt.default__date_spine": {"name": "default__date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__date_spine", "macro_sql": "{% macro default__date_spine(datepart, start_date, end_date) %}\n\n\n {# call as follows:\n\n date_spine(\n \"day\",\n \"to_date('01/01/2016', 'mm/dd/yyyy')\",\n \"dbt.dateadd(week, 1, current_date)\"\n ) #}\n\n\n with rawdata as (\n\n {{dbt.generate_series(\n dbt.get_intervals_between(start_date, end_date, datepart)\n )}}\n\n ),\n\n all_periods as (\n\n select (\n {{\n dbt.dateadd(\n datepart,\n \"row_number() over (order by 1) - 1\",\n start_date\n )\n }}\n ) as date_{{datepart}}\n from rawdata\n\n ),\n\n filtered as (\n\n select *\n from all_periods\n where date_{{datepart}} <= {{ end_date }}\n\n )\n\n select * from filtered\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.generate_series", "macro.dbt.get_intervals_between", "macro.dbt.dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.836822, "supported_languages": null}, "macro.dbt.date": {"name": "date", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt.date", "macro_sql": "{% macro date(year, month, day) %}\n {{ return(adapter.dispatch('date', 'dbt') (year, month, day)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837112, "supported_languages": null}, "macro.dbt.default__date": {"name": "default__date", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt.default__date", "macro_sql": "{% macro default__date(year, month, day) -%}\n {%- set dt = modules.datetime.date(year, month, day) -%}\n {%- set iso_8601_formatted_date = dt.strftime('%Y-%m-%d') -%}\n to_date('{{ iso_8601_formatted_date }}', 'YYYY-MM-DD')\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837353, "supported_languages": null}, "macro.dbt.replace": {"name": "replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.replace", "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__replace"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837631, "supported_languages": null}, "macro.dbt.default__replace": {"name": "default__replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.default__replace", "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837771, "supported_languages": null}, "macro.dbt.concat": {"name": "concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.concat", "macro_sql": "{% macro concat(fields) -%}\n {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837983, "supported_languages": null}, "macro.dbt.default__concat": {"name": "default__concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.default__concat", "macro_sql": "{% macro default__concat(fields) -%}\n {{ fields|join(' || ') }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.838089, "supported_languages": null}, "macro.dbt.get_powers_of_two": {"name": "get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.get_powers_of_two", "macro_sql": "{% macro get_powers_of_two(upper_bound) %}\n {{ return(adapter.dispatch('get_powers_of_two', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.838809, "supported_languages": null}, "macro.dbt.default__get_powers_of_two": {"name": "default__get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__get_powers_of_two", "macro_sql": "{% macro default__get_powers_of_two(upper_bound) %}\n\n {% if upper_bound <= 0 %}\n {{ exceptions.raise_compiler_error(\"upper bound must be positive\") }}\n {% endif %}\n\n {% for _ in range(1, 100) %}\n {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %}\n {% endfor %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.839253, "supported_languages": null}, "macro.dbt.generate_series": {"name": "generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.generate_series", "macro_sql": "{% macro generate_series(upper_bound) %}\n {{ return(adapter.dispatch('generate_series', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_series"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.839426, "supported_languages": null}, "macro.dbt.default__generate_series": {"name": "default__generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__generate_series", "macro_sql": "{% macro default__generate_series(upper_bound) %}\n\n {% set n = dbt.get_powers_of_two(upper_bound) %}\n\n with p as (\n select 0 as generated_number union all select 1\n ), unioned as (\n\n select\n\n {% for i in range(n) %}\n p{{i}}.generated_number * power(2, {{i}})\n {% if not loop.last %} + {% endif %}\n {% endfor %}\n + 1\n as generated_number\n\n from\n\n {% for i in range(n) %}\n p as p{{i}}\n {% if not loop.last %} cross join {% endif %}\n {% endfor %}\n\n )\n\n select *\n from unioned\n where generated_number <= {{upper_bound}}\n order by generated_number\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.839862, "supported_languages": null}, "macro.dbt.length": {"name": "length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.length", "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__length"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840083, "supported_languages": null}, "macro.dbt.default__length": {"name": "default__length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.default__length", "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8401742, "supported_languages": null}, "macro.dbt.dateadd": {"name": "dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.dateadd", "macro_sql": "{% macro dateadd(datepart, interval, from_date_or_timestamp) %}\n {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8404648, "supported_languages": null}, "macro.dbt.default__dateadd": {"name": "default__dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.default__dateadd", "macro_sql": "{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n dateadd(\n {{ datepart }},\n {{ interval }},\n {{ from_date_or_timestamp }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840607, "supported_languages": null}, "macro.dbt.intersect": {"name": "intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.intersect", "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__intersect"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840808, "supported_languages": null}, "macro.dbt.default__intersect": {"name": "default__intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.default__intersect", "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840875, "supported_languages": null}, "macro.dbt.escape_single_quotes": {"name": "escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.escape_single_quotes", "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__escape_single_quotes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.841106, "supported_languages": null}, "macro.dbt.default__escape_single_quotes": {"name": "default__escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.default__escape_single_quotes", "macro_sql": "{% macro default__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"''\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.841234, "supported_languages": null}, "macro.dbt.right": {"name": "right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.right", "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__right"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.84149, "supported_languages": null}, "macro.dbt.default__right": {"name": "default__right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.default__right", "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8416152, "supported_languages": null}, "macro.dbt.listagg": {"name": "listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.listagg", "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__listagg"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842151, "supported_languages": null}, "macro.dbt.default__listagg": {"name": "default__listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.default__listagg", "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842484, "supported_languages": null}, "macro.dbt.datediff": {"name": "datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.datediff", "macro_sql": "{% macro datediff(first_date, second_date, datepart) %}\n {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842758, "supported_languages": null}, "macro.dbt.default__datediff": {"name": "default__datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.default__datediff", "macro_sql": "{% macro default__datediff(first_date, second_date, datepart) -%}\n\n datediff(\n {{ datepart }},\n {{ first_date }},\n {{ second_date }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842896, "supported_languages": null}, "macro.dbt.safe_cast": {"name": "safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.safe_cast", "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843133, "supported_languages": null}, "macro.dbt.default__safe_cast": {"name": "default__safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.default__safe_cast", "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843246, "supported_languages": null}, "macro.dbt.hash": {"name": "hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.hash", "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__hash"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843461, "supported_languages": null}, "macro.dbt.default__hash": {"name": "default__hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.default__hash", "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843594, "supported_languages": null}, "macro.dbt.cast_bool_to_text": {"name": "cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.cast_bool_to_text", "macro_sql": "{% macro cast_bool_to_text(field) %}\n {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast_bool_to_text"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8438652, "supported_languages": null}, "macro.dbt.default__cast_bool_to_text": {"name": "default__cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.default__cast_bool_to_text", "macro_sql": "{% macro default__cast_bool_to_text(field) %}\n cast({{ field }} as {{ api.Column.translate_type('string') }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8439958, "supported_languages": null}, "macro.dbt.cast": {"name": "cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast.sql", "original_file_path": "macros/utils/cast.sql", "unique_id": "macro.dbt.cast", "macro_sql": "{% macro cast(field, type) %}\n {{ return(adapter.dispatch('cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844224, "supported_languages": null}, "macro.dbt.default__cast": {"name": "default__cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast.sql", "original_file_path": "macros/utils/cast.sql", "unique_id": "macro.dbt.default__cast", "macro_sql": "{% macro default__cast(field, type) %}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844336, "supported_languages": null}, "macro.dbt.any_value": {"name": "any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.any_value", "macro_sql": "{% macro any_value(expression) -%}\n {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__any_value"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8445451, "supported_languages": null}, "macro.dbt.default__any_value": {"name": "default__any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.default__any_value", "macro_sql": "{% macro default__any_value(expression) -%}\n\n any_value({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844633, "supported_languages": null}, "macro.dbt.position": {"name": "position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.position", "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__position"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8448782, "supported_languages": null}, "macro.dbt.default__position": {"name": "default__position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.default__position", "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844993, "supported_languages": null}, "macro.dbt.string_literal": {"name": "string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.string_literal", "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.845202, "supported_languages": null}, "macro.dbt.default__string_literal": {"name": "default__string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.default__string_literal", "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.845294, "supported_languages": null}, "macro.dbt.type_string": {"name": "type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_string", "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_string"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846126, "supported_languages": null}, "macro.dbt.default__type_string": {"name": "default__type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_string", "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846256, "supported_languages": null}, "macro.dbt.type_timestamp": {"name": "type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_timestamp", "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846394, "supported_languages": null}, "macro.dbt.default__type_timestamp": {"name": "default__type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_timestamp", "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8465211, "supported_languages": null}, "macro.dbt.type_float": {"name": "type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_float", "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_float"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846658, "supported_languages": null}, "macro.dbt.default__type_float": {"name": "default__type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_float", "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8467891, "supported_languages": null}, "macro.dbt.type_numeric": {"name": "type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_numeric", "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_numeric"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846924, "supported_languages": null}, "macro.dbt.default__type_numeric": {"name": "default__type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_numeric", "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8470762, "supported_languages": null}, "macro.dbt.type_bigint": {"name": "type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_bigint", "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_bigint"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.847214, "supported_languages": null}, "macro.dbt.default__type_bigint": {"name": "default__type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_bigint", "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.847407, "supported_languages": null}, "macro.dbt.type_int": {"name": "type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_int", "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_int"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8475451, "supported_languages": null}, "macro.dbt.default__type_int": {"name": "default__type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_int", "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8476732, "supported_languages": null}, "macro.dbt.type_boolean": {"name": "type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_boolean", "macro_sql": "\n\n{%- macro type_boolean() -%}\n {{ return(adapter.dispatch('type_boolean', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_boolean"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8478148, "supported_languages": null}, "macro.dbt.default__type_boolean": {"name": "default__type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_boolean", "macro_sql": "{%- macro default__type_boolean() -%}\n {{ return(api.Column.translate_type(\"boolean\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8479412, "supported_languages": null}, "macro.dbt.array_concat": {"name": "array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.array_concat", "macro_sql": "{% macro array_concat(array_1, array_2) -%}\n {{ return(adapter.dispatch('array_concat', 'dbt')(array_1, array_2)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.848181, "supported_languages": null}, "macro.dbt.default__array_concat": {"name": "default__array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.default__array_concat", "macro_sql": "{% macro default__array_concat(array_1, array_2) -%}\n array_cat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.848293, "supported_languages": null}, "macro.dbt.bool_or": {"name": "bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.bool_or", "macro_sql": "{% macro bool_or(expression) -%}\n {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__bool_or"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8485062, "supported_languages": null}, "macro.dbt.default__bool_or": {"name": "default__bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.default__bool_or", "macro_sql": "{% macro default__bool_or(expression) -%}\n\n bool_or({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8485959, "supported_languages": null}, "macro.dbt.last_day": {"name": "last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.last_day", "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.848897, "supported_languages": null}, "macro.dbt.default_last_day": {"name": "default_last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default_last_day", "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.84913, "supported_languages": null}, "macro.dbt.default__last_day": {"name": "default__last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default__last_day", "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.849261, "supported_languages": null}, "macro.dbt.split_part": {"name": "split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.split_part", "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__split_part"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8497112, "supported_languages": null}, "macro.dbt.default__split_part": {"name": "default__split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.default__split_part", "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8498502, "supported_languages": null}, "macro.dbt._split_part_negative": {"name": "_split_part_negative", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt._split_part_negative", "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 + {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8500328, "supported_languages": null}, "macro.dbt.date_trunc": {"name": "date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.date_trunc", "macro_sql": "{% macro date_trunc(datepart, date) -%}\n {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.850267, "supported_languages": null}, "macro.dbt.default__date_trunc": {"name": "default__date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.default__date_trunc", "macro_sql": "{% macro default__date_trunc(datepart, date) -%}\n date_trunc('{{datepart}}', {{date}})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.85038, "supported_languages": null}, "macro.dbt.array_construct": {"name": "array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.array_construct", "macro_sql": "{% macro array_construct(inputs=[], data_type=api.Column.translate_type('integer')) -%}\n {{ return(adapter.dispatch('array_construct', 'dbt')(inputs, data_type)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.850707, "supported_languages": null}, "macro.dbt.default__array_construct": {"name": "default__array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.default__array_construct", "macro_sql": "{% macro default__array_construct(inputs, data_type) -%}\n {% if inputs|length > 0 %}\n array[ {{ inputs|join(' , ') }} ]\n {% else %}\n array[]::{{data_type}}[]\n {% endif %}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.850914, "supported_languages": null}, "macro.dbt.array_append": {"name": "array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.array_append", "macro_sql": "{% macro array_append(array, new_element) -%}\n {{ return(adapter.dispatch('array_append', 'dbt')(array, new_element)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_append"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8511622, "supported_languages": null}, "macro.dbt.default__array_append": {"name": "default__array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.default__array_append", "macro_sql": "{% macro default__array_append(array, new_element) -%}\n array_append({{ array }}, {{ new_element }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851361, "supported_languages": null}, "macro.dbt.create_schema": {"name": "create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__create_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851688, "supported_languages": null}, "macro.dbt.default__create_schema": {"name": "default__create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851847, "supported_languages": null}, "macro.dbt.drop_schema": {"name": "drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__drop_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851985, "supported_languages": null}, "macro.dbt.default__drop_schema": {"name": "default__drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852136, "supported_languages": null}, "macro.dbt.current_timestamp": {"name": "current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp", "macro_sql": "{%- macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852708, "supported_languages": null}, "macro.dbt.default__current_timestamp": {"name": "default__current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter ' + adapter.type()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852832, "supported_languages": null}, "macro.dbt.snapshot_get_time": {"name": "snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.snapshot_get_time", "macro_sql": "\n\n{%- macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852951, "supported_languages": null}, "macro.dbt.default__snapshot_get_time": {"name": "default__snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() %}\n {{ current_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853037, "supported_languages": null}, "macro.dbt.get_snapshot_get_time_data_type": {"name": "get_snapshot_get_time_data_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.get_snapshot_get_time_data_type", "macro_sql": "{% macro get_snapshot_get_time_data_type() %}\n {% set snapshot_time = adapter.dispatch('snapshot_get_time', 'dbt')() %}\n {% set time_data_type_sql = 'select ' ~ snapshot_time ~ ' as dbt_snapshot_time' %}\n {% set snapshot_time_column_schema = get_column_schema_from_query(time_data_type_sql) %}\n {% set time_data_type = snapshot_time_column_schema[0].dtype %}\n {{ return(time_data_type or none) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.default__snapshot_get_time", "macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8533702, "supported_languages": null}, "macro.dbt.current_timestamp_backcompat": {"name": "current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_backcompat", "macro_sql": "{% macro current_timestamp_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853508, "supported_languages": null}, "macro.dbt.default__current_timestamp_backcompat": {"name": "default__current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_backcompat", "macro_sql": "{% macro default__current_timestamp_backcompat() %}\n current_timestamp::timestamp\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853577, "supported_languages": null}, "macro.dbt.current_timestamp_in_utc_backcompat": {"name": "current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_in_utc_backcompat", "macro_sql": "{% macro current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_in_utc_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__current_timestamp_in_utc_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853718, "supported_languages": null}, "macro.dbt.default__current_timestamp_in_utc_backcompat": {"name": "default__current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro default__current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp_backcompat", "macro.dbt.default__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853857, "supported_languages": null}, "macro.dbt.get_create_index_sql": {"name": "get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.854583, "supported_languages": null}, "macro.dbt.default__get_create_index_sql": {"name": "default__get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.854707, "supported_languages": null}, "macro.dbt.create_indexes": {"name": "create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8548422, "supported_languages": null}, "macro.dbt.default__create_indexes": {"name": "default__create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_create_index_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8552508, "supported_languages": null}, "macro.dbt.get_drop_index_sql": {"name": "get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_drop_index_sql", "macro_sql": "{% macro get_drop_index_sql(relation, index_name) -%}\n {{ adapter.dispatch('get_drop_index_sql', 'dbt')(relation, index_name) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_drop_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.855412, "supported_languages": null}, "macro.dbt.default__get_drop_index_sql": {"name": "default__get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_drop_index_sql", "macro_sql": "{% macro default__get_drop_index_sql(relation, index_name) -%}\n {{ exceptions.raise_compiler_error(\"`get_drop_index_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.855531, "supported_languages": null}, "macro.dbt.get_show_indexes_sql": {"name": "get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_show_indexes_sql", "macro_sql": "{% macro get_show_indexes_sql(relation) -%}\n {{ adapter.dispatch('get_show_indexes_sql', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_show_indexes_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8556669, "supported_languages": null}, "macro.dbt.default__get_show_indexes_sql": {"name": "default__get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_show_indexes_sql", "macro_sql": "{% macro default__get_show_indexes_sql(relation) -%}\n {{ exceptions.raise_compiler_error(\"`get_show_indexes_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.855777, "supported_languages": null}, "macro.dbt.make_intermediate_relation": {"name": "make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_intermediate_relation", "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_intermediate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.857485, "supported_languages": null}, "macro.dbt.default__make_intermediate_relation": {"name": "default__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_intermediate_relation", "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.857631, "supported_languages": null}, "macro.dbt.make_temp_relation": {"name": "make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.857822, "supported_languages": null}, "macro.dbt.default__make_temp_relation": {"name": "default__make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858082, "supported_languages": null}, "macro.dbt.make_backup_relation": {"name": "make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_backup_relation", "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_backup_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858318, "supported_languages": null}, "macro.dbt.default__make_backup_relation": {"name": "default__make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_backup_relation", "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8586, "supported_languages": null}, "macro.dbt.truncate_relation": {"name": "truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__truncate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858756, "supported_languages": null}, "macro.dbt.default__truncate_relation": {"name": "default__truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation.render() }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858906, "supported_languages": null}, "macro.dbt.get_or_create_relation": {"name": "get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_or_create_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859114, "supported_languages": null}, "macro.dbt.default__get_or_create_relation": {"name": "default__get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__get_or_create_relation", "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859583, "supported_languages": null}, "macro.dbt.load_cached_relation": {"name": "load_cached_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_cached_relation", "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859786, "supported_languages": null}, "macro.dbt.load_relation": {"name": "load_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859914, "supported_languages": null}, "macro.dbt.collect_freshness": {"name": "collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__collect_freshness"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.860286, "supported_languages": null}, "macro.dbt.default__collect_freshness": {"name": "default__collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86064, "supported_languages": null}, "macro.dbt.validate_sql": {"name": "validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.validate_sql", "macro_sql": "{% macro validate_sql(sql) -%}\n {{ return(adapter.dispatch('validate_sql', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__validate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8608792, "supported_languages": null}, "macro.dbt.default__validate_sql": {"name": "default__validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.default__validate_sql", "macro_sql": "{% macro default__validate_sql(sql) -%}\n {% call statement('validate_sql') -%}\n explain {{ sql }}\n {% endcall %}\n {{ return(load_result('validate_sql')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8611462, "supported_languages": null}, "macro.dbt.copy_grants": {"name": "copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.copy_grants", "macro_sql": "{% macro copy_grants() %}\n {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.862612, "supported_languages": null}, "macro.dbt.default__copy_grants": {"name": "default__copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__copy_grants", "macro_sql": "{% macro default__copy_grants() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.862713, "supported_languages": null}, "macro.dbt.support_multiple_grantees_per_dcl_statement": {"name": "support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement", "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8628511, "supported_languages": null}, "macro.dbt.default__support_multiple_grantees_per_dcl_statement": {"name": "default__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.862943, "supported_languages": null}, "macro.dbt.should_revoke": {"name": "should_revoke", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.should_revoke", "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863224, "supported_languages": null}, "macro.dbt.get_show_grant_sql": {"name": "get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_show_grant_sql", "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_show_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863383, "supported_languages": null}, "macro.dbt.default__get_show_grant_sql": {"name": "default__get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_show_grant_sql", "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863481, "supported_languages": null}, "macro.dbt.get_grant_sql": {"name": "get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_grant_sql", "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863672, "supported_languages": null}, "macro.dbt.default__get_grant_sql": {"name": "default__get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_grant_sql", "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation.render() }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863852, "supported_languages": null}, "macro.dbt.get_revoke_sql": {"name": "get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_revoke_sql", "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_revoke_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86405, "supported_languages": null}, "macro.dbt.default__get_revoke_sql": {"name": "default__get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_revoke_sql", "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation.render() }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.864309, "supported_languages": null}, "macro.dbt.get_dcl_statement_list": {"name": "get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_dcl_statement_list", "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_dcl_statement_list"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8645408, "supported_languages": null}, "macro.dbt.default__get_dcl_statement_list": {"name": "default__get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_dcl_statement_list", "macro_sql": "\n\n{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}\n {#\n -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.\n -- Depending on whether this database supports multiple grantees per statement, pass in the list of\n -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.\n -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`\n #}\n {%- set dcl_statements = [] -%}\n {%- for privilege, grantees in grant_config.items() %}\n {%- if support_multiple_grantees_per_dcl_statement() and grantees -%}\n {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}\n {%- do dcl_statements.append(dcl) -%}\n {%- else -%}\n {%- for grantee in grantees -%}\n {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}\n {%- do dcl_statements.append(dcl) -%}\n {% endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {{ return(dcl_statements) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865141, "supported_languages": null}, "macro.dbt.call_dcl_statements": {"name": "call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.call_dcl_statements", "macro_sql": "{% macro call_dcl_statements(dcl_statement_list) %}\n {{ return(adapter.dispatch(\"call_dcl_statements\", \"dbt\")(dcl_statement_list)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865317, "supported_languages": null}, "macro.dbt.default__call_dcl_statements": {"name": "default__call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__call_dcl_statements", "macro_sql": "{% macro default__call_dcl_statements(dcl_statement_list) %}\n {#\n -- By default, supply all grant + revoke statements in a single semicolon-separated block,\n -- so that they're all processed together.\n\n -- Some databases do not support this. Those adapters will need to override this macro\n -- to run each statement individually.\n #}\n {% call statement('grants') %}\n {% for dcl_statement in dcl_statement_list %}\n {{ dcl_statement }};\n {% endfor %}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865537, "supported_languages": null}, "macro.dbt.apply_grants": {"name": "apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.apply_grants", "macro_sql": "{% macro apply_grants(relation, grant_config, should_revoke) %}\n {{ return(adapter.dispatch(\"apply_grants\", \"dbt\")(relation, grant_config, should_revoke)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865737, "supported_languages": null}, "macro.dbt.default__apply_grants": {"name": "default__apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__apply_grants", "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation.render() ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.get_show_grant_sql", "macro.dbt.get_dcl_statement_list", "macro.dbt.call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8668082, "supported_languages": null}, "macro.dbt.get_show_sql": {"name": "get_show_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_show_sql", "macro_sql": "{% macro get_show_sql(compiled_code, sql_header, limit) -%}\n {%- if sql_header is not none -%}\n {{ sql_header }}\n {%- endif %}\n {{ get_limit_subquery_sql(compiled_code, limit) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_limit_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8671901, "supported_languages": null}, "macro.dbt.get_limit_subquery_sql": {"name": "get_limit_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_limit_subquery_sql", "macro_sql": "\n{%- macro get_limit_subquery_sql(sql, limit) -%}\n {{ adapter.dispatch('get_limit_sql', 'dbt')(sql, limit) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_limit_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8673549, "supported_languages": null}, "macro.dbt.default__get_limit_sql": {"name": "default__get_limit_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.default__get_limit_sql", "macro_sql": "{% macro default__get_limit_sql(sql, limit) %}\n {{ compiled_code }}\n {% if limit is not none %}\n limit {{ limit }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86752, "supported_languages": null}, "macro.dbt.alter_column_comment": {"name": "alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86812, "supported_languages": null}, "macro.dbt.default__alter_column_comment": {"name": "default__alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.868274, "supported_languages": null}, "macro.dbt.alter_relation_comment": {"name": "alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__alter_relation_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.868457, "supported_languages": null}, "macro.dbt.default__alter_relation_comment": {"name": "default__alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8686101, "supported_languages": null}, "macro.dbt.persist_docs": {"name": "persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86885, "supported_languages": null}, "macro.dbt.default__persist_docs": {"name": "default__persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8692548, "supported_languages": null}, "macro.dbt.get_catalog_relations": {"name": "get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog_relations", "macro_sql": "{% macro get_catalog_relations(information_schema, relations) -%}\n {{ return(adapter.dispatch('get_catalog_relations', 'dbt')(information_schema, relations)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872187, "supported_languages": null}, "macro.dbt.default__get_catalog_relations": {"name": "default__get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog_relations", "macro_sql": "{% macro default__get_catalog_relations(information_schema, relations) -%}\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog_relations not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872406, "supported_languages": null}, "macro.dbt.get_catalog": {"name": "get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872573, "supported_languages": null}, "macro.dbt.default__get_catalog": {"name": "default__get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8727858, "supported_languages": null}, "macro.dbt.information_schema_name": {"name": "information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__information_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872941, "supported_languages": null}, "macro.dbt.default__information_schema_name": {"name": "default__information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8730712, "supported_languages": null}, "macro.dbt.list_schemas": {"name": "list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__list_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8732271, "supported_languages": null}, "macro.dbt.default__list_schemas": {"name": "default__list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.information_schema_name", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.87344, "supported_languages": null}, "macro.dbt.check_schema_exists": {"name": "check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__check_schema_exists"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8736901, "supported_languages": null}, "macro.dbt.default__check_schema_exists": {"name": "default__check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.replace", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8739462, "supported_languages": null}, "macro.dbt.list_relations_without_caching": {"name": "list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874113, "supported_languages": null}, "macro.dbt.default__list_relations_without_caching": {"name": "default__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874256, "supported_languages": null}, "macro.dbt.get_catalog_for_single_relation": {"name": "get_catalog_for_single_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog_for_single_relation", "macro_sql": "{% macro get_catalog_for_single_relation(relation) %}\n {{ return(adapter.dispatch('get_catalog_for_single_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_catalog_for_single_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874414, "supported_languages": null}, "macro.dbt.default__get_catalog_for_single_relation": {"name": "default__get_catalog_for_single_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog_for_single_relation", "macro_sql": "{% macro default__get_catalog_for_single_relation(relation) %}\n {{ exceptions.raise_not_implemented(\n 'get_catalog_for_single_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8745549, "supported_languages": null}, "macro.dbt.get_relations": {"name": "get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relations", "macro_sql": "{% macro get_relations() %}\n {{ return(adapter.dispatch('get_relations', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874696, "supported_languages": null}, "macro.dbt.default__get_relations": {"name": "default__get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relations", "macro_sql": "{% macro default__get_relations() %}\n {{ exceptions.raise_not_implemented(\n 'get_relations macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8748312, "supported_languages": null}, "macro.dbt.get_relation_last_modified": {"name": "get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relation_last_modified", "macro_sql": "{% macro get_relation_last_modified(information_schema, relations) %}\n {{ return(adapter.dispatch('get_relation_last_modified', 'dbt')(information_schema, relations)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_relation_last_modified"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.87501, "supported_languages": null}, "macro.dbt.default__get_relation_last_modified": {"name": "default__get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relation_last_modified", "macro_sql": "{% macro default__get_relation_last_modified(information_schema, relations) %}\n {{ exceptions.raise_not_implemented(\n 'get_relation_last_modified macro not implemented for adapter ' + adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8751612, "supported_languages": null}, "macro.dbt.get_columns_in_relation": {"name": "get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877135, "supported_languages": null}, "macro.dbt.default__get_columns_in_relation": {"name": "default__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877271, "supported_languages": null}, "macro.dbt.sql_convert_columns_in_relation": {"name": "sql_convert_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877537, "supported_languages": null}, "macro.dbt.get_empty_subquery_sql": {"name": "get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_subquery_sql", "macro_sql": "{% macro get_empty_subquery_sql(select_sql, select_sql_header=none) -%}\n {{ return(adapter.dispatch('get_empty_subquery_sql', 'dbt')(select_sql, select_sql_header)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8777258, "supported_languages": null}, "macro.dbt.default__get_empty_subquery_sql": {"name": "default__get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_subquery_sql", "macro_sql": "{% macro default__get_empty_subquery_sql(select_sql, select_sql_header=none) %}\n {%- if select_sql_header is not none -%}\n {{ select_sql_header }}\n {%- endif -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877902, "supported_languages": null}, "macro.dbt.get_empty_schema_sql": {"name": "get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_schema_sql", "macro_sql": "{% macro get_empty_schema_sql(columns) -%}\n {{ return(adapter.dispatch('get_empty_schema_sql', 'dbt')(columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_schema_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8780658, "supported_languages": null}, "macro.dbt.default__get_empty_schema_sql": {"name": "default__get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_schema_sql", "macro_sql": "{% macro default__get_empty_schema_sql(columns) %}\n {%- set col_err = [] -%}\n {%- set col_naked_numeric = [] -%}\n select\n {% for i in columns %}\n {%- set col = columns[i] -%}\n {%- if col['data_type'] is not defined -%}\n {%- do col_err.append(col['name']) -%}\n {#-- If this column's type is just 'numeric' then it is missing precision/scale, raise a warning --#}\n {%- elif col['data_type'].strip().lower() in ('numeric', 'decimal', 'number') -%}\n {%- do col_naked_numeric.append(col['name']) -%}\n {%- endif -%}\n {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %}\n {{ cast('null', col['data_type']) }} as {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n {%- if (col_err | length) > 0 -%}\n {{ exceptions.column_type_missing(column_names=col_err) }}\n {%- elif (col_naked_numeric | length) > 0 -%}\n {{ exceptions.warn(\"Detected columns with numeric type and unspecified precision/scale, this can lead to unintended rounding: \" ~ col_naked_numeric ~ \"`\") }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879136, "supported_languages": null}, "macro.dbt.get_column_schema_from_query": {"name": "get_column_schema_from_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_column_schema_from_query", "macro_sql": "{% macro get_column_schema_from_query(select_sql, select_sql_header=none) -%}\n {% set columns = [] %}\n {# -- Using an 'empty subquery' here to get the same schema as the given select_sql statement, without necessitating a data scan.#}\n {% set sql = get_empty_subquery_sql(select_sql, select_sql_header) %}\n {% set column_schema = adapter.get_column_schema_from_query(sql) %}\n {{ return(column_schema) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879442, "supported_languages": null}, "macro.dbt.get_columns_in_query": {"name": "get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879605, "supported_languages": null}, "macro.dbt.default__get_columns_in_query": {"name": "default__get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n {{ get_empty_subquery_sql(select_sql) }}\n {% endcall %}\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879909, "supported_languages": null}, "macro.dbt.alter_column_type": {"name": "alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.880113, "supported_languages": null}, "macro.dbt.default__alter_column_type": {"name": "default__alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation.render() }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation.render() }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation.render() }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation.render() }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.880679, "supported_languages": null}, "macro.dbt.alter_relation_add_remove_columns": {"name": "alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_relation_add_remove_columns", "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__alter_relation_add_remove_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8809001, "supported_languages": null}, "macro.dbt.default__alter_relation_add_remove_columns": {"name": "default__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_relation_add_remove_columns", "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation.render() }}\n\n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.88154, "supported_languages": null}, "macro.dbt.get_fixture_sql": {"name": "get_fixture_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.get_fixture_sql", "macro_sql": "{% macro get_fixture_sql(rows, column_name_to_data_types) %}\n-- Fixture for {{ model.name }}\n{% set default_row = {} %}\n\n{%- if not column_name_to_data_types -%}\n{#-- Use defer_relation IFF it is available in the manifest and 'this' is missing from the database --#}\n{%- set this_or_defer_relation = defer_relation if (defer_relation and not load_relation(this)) else this -%}\n{%- set columns_in_relation = adapter.get_columns_in_relation(this_or_defer_relation) -%}\n\n{%- set column_name_to_data_types = {} -%}\n{%- for column in columns_in_relation -%}\n{#-- This needs to be a case-insensitive comparison --#}\n{%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n{%- endfor -%}\n{%- endif -%}\n\n{%- if not column_name_to_data_types -%}\n {{ exceptions.raise_compiler_error(\"Not able to get columns for unit test '\" ~ model.name ~ \"' from relation \" ~ this ~ \" because the relation doesn't exist\") }}\n{%- endif -%}\n\n{%- for column_name, column_type in column_name_to_data_types.items() -%}\n {%- do default_row.update({column_name: (safe_cast(\"null\", column_type) | trim )}) -%}\n{%- endfor -%}\n\n{{ validate_fixture_rows(rows, row_number) }}\n\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\n{%- set default_row_copy = default_row.copy() -%}\n{%- do default_row_copy.update(formatted_row) -%}\nselect\n{%- for column_name, column_value in default_row_copy.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n\n{%- if (rows | length) == 0 -%}\n select\n {%- for column_name, column_value in default_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%},{%- endif %}\n {%- endfor %}\n limit 0\n{%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_relation", "macro.dbt.safe_cast", "macro.dbt.validate_fixture_rows", "macro.dbt.format_row"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.884523, "supported_languages": null}, "macro.dbt.get_expected_sql": {"name": "get_expected_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.get_expected_sql", "macro_sql": "{% macro get_expected_sql(rows, column_name_to_data_types) %}\n\n{%- if (rows | length) == 0 -%}\n select * from dbt_internal_unit_test_actual\n limit 0\n{%- else -%}\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\nselect\n{%- for column_name, column_value in formatted_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n{%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.format_row"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8849862, "supported_languages": null}, "macro.dbt.format_row": {"name": "format_row", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.format_row", "macro_sql": "\n\n{%- macro format_row(row, column_name_to_data_types) -%}\n {#-- generate case-insensitive formatted row --#}\n {% set formatted_row = {} %}\n {%- for column_name, column_value in row.items() -%}\n {% set column_name = column_name|lower %}\n\n {%- if column_name not in column_name_to_data_types %}\n {#-- if user-provided row contains column name that relation does not contain, raise an error --#}\n {% set fixture_name = \"expected output\" if model.resource_type == 'unit_test' else (\"'\" ~ model.name ~ \"'\") %}\n {{ exceptions.raise_compiler_error(\n \"Invalid column name: '\" ~ column_name ~ \"' in unit test fixture for \" ~ fixture_name ~ \".\"\n \"\\nAccepted columns for \" ~ fixture_name ~ \" are: \" ~ (column_name_to_data_types.keys()|list)\n ) }}\n {%- endif -%}\n\n {%- set column_type = column_name_to_data_types[column_name] %}\n\n {#-- sanitize column_value: wrap yaml strings in quotes, apply cast --#}\n {%- set column_value_clean = column_value -%}\n {%- if column_value is string -%}\n {%- set column_value_clean = dbt.string_literal(dbt.escape_single_quotes(column_value)) -%}\n {%- elif column_value is none -%}\n {%- set column_value_clean = 'null' -%}\n {%- endif -%}\n\n {%- set row_update = {column_name: safe_cast(column_value_clean, column_type) } -%}\n {%- do formatted_row.update(row_update) -%}\n {%- endfor -%}\n {{ return(formatted_row) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.string_literal", "macro.dbt.escape_single_quotes", "macro.dbt.safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8859332, "supported_languages": null}, "macro.dbt.validate_fixture_rows": {"name": "validate_fixture_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.validate_fixture_rows", "macro_sql": "{%- macro validate_fixture_rows(rows, row_number) -%}\n {{ return(adapter.dispatch('validate_fixture_rows', 'dbt')(rows, row_number)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__validate_fixture_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8861172, "supported_languages": null}, "macro.dbt.default__validate_fixture_rows": {"name": "default__validate_fixture_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.default__validate_fixture_rows", "macro_sql": "{%- macro default__validate_fixture_rows(rows, row_number) -%}\n {# This is an abstract method for adapter overrides as needed #}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.886199, "supported_languages": null}, "macro.dbt.resolve_model_name": {"name": "resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.resolve_model_name", "macro_sql": "{% macro resolve_model_name(input_model_name) %}\n {{ return(adapter.dispatch('resolve_model_name', 'dbt')(input_model_name)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.887635, "supported_languages": null}, "macro.dbt.default__resolve_model_name": {"name": "default__resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.default__resolve_model_name", "macro_sql": "\n\n{%- macro default__resolve_model_name(input_model_name) -%}\n {{ input_model_name | string | replace('\"', '\\\"') }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.887774, "supported_languages": null}, "macro.dbt.build_ref_function": {"name": "build_ref_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_ref_function", "macro_sql": "{% macro build_ref_function(model) %}\n\n {%- set ref_dict = {} -%}\n {%- for _ref in model.refs -%}\n {% set _ref_args = [_ref.get('package'), _ref['name']] if _ref.get('package') else [_ref['name'],] %}\n {%- set resolved = ref(*_ref_args, v=_ref.get('version')) -%}\n {%- if _ref.get('version') -%}\n {% do _ref_args.extend([\"v\" ~ _ref['version']]) %}\n {%- endif -%}\n {%- do ref_dict.update({_ref_args | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef ref(*args, **kwargs):\n refs = {{ ref_dict | tojson }}\n key = '.'.join(args)\n version = kwargs.get(\"v\") or kwargs.get(\"version\")\n if version:\n key += f\".v{version}\"\n dbt_load_df_function = kwargs.get(\"dbt_load_df_function\")\n return dbt_load_df_function(refs[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.888478, "supported_languages": null}, "macro.dbt.build_source_function": {"name": "build_source_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_source_function", "macro_sql": "{% macro build_source_function(model) %}\n\n {%- set source_dict = {} -%}\n {%- for _source in model.sources -%}\n {%- set resolved = source(*_source) -%}\n {%- do source_dict.update({_source | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef source(*args, dbt_load_df_function):\n sources = {{ source_dict | tojson }}\n key = '.'.join(args)\n return dbt_load_df_function(sources[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8888261, "supported_languages": null}, "macro.dbt.build_config_dict": {"name": "build_config_dict", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_config_dict", "macro_sql": "{% macro build_config_dict(model) %}\n {%- set config_dict = {} -%}\n {% set config_dbt_used = zip(model.config.config_keys_used, model.config.config_keys_defaults) | list %}\n {%- for key, default in config_dbt_used -%}\n {# weird type testing with enum, would be much easier to write this logic in Python! #}\n {%- if key == \"language\" -%}\n {%- set value = \"python\" -%}\n {%- endif -%}\n {%- set value = model.config.get(key, default) -%}\n {%- do config_dict.update({key: value}) -%}\n {%- endfor -%}\nconfig_dict = {{ config_dict }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8893209, "supported_languages": null}, "macro.dbt.py_script_postfix": {"name": "py_script_postfix", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_postfix", "macro_sql": "{% macro py_script_postfix(model) %}\n# This part is user provided model code\n# you will need to copy the next section to run the code\n# COMMAND ----------\n# this part is dbt logic for get ref work, do not modify\n\n{{ build_ref_function(model ) }}\n{{ build_source_function(model ) }}\n{{ build_config_dict(model) }}\n\nclass config:\n def __init__(self, *args, **kwargs):\n pass\n\n @staticmethod\n def get(key, default=None):\n return config_dict.get(key, default)\n\nclass this:\n \"\"\"dbt.this() or dbt.this.identifier\"\"\"\n database = \"{{ this.database }}\"\n schema = \"{{ this.schema }}\"\n identifier = \"{{ this.identifier }}\"\n {% set this_relation_name = resolve_model_name(this) %}\n def __repr__(self):\n return '{{ this_relation_name }}'\n\n\nclass dbtObj:\n def __init__(self, load_df_function) -> None:\n self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function)\n self.ref = lambda *args, **kwargs: ref(*args, **kwargs, dbt_load_df_function=load_df_function)\n self.config = config\n self.this = this()\n self.is_incremental = {{ is_incremental() }}\n\n# COMMAND ----------\n{{py_script_comment()}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.build_ref_function", "macro.dbt.build_source_function", "macro.dbt.build_config_dict", "macro.dbt.resolve_model_name", "macro.dbt.is_incremental", "macro.dbt.py_script_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8898182, "supported_languages": null}, "macro.dbt.py_script_comment": {"name": "py_script_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n{%endmacro%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.889886, "supported_languages": null}, "macro.dbt.test_unique": {"name": "test_unique", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_unique"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.890347, "supported_languages": null}, "macro.dbt.test_not_null": {"name": "test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8905559, "supported_languages": null}, "macro.dbt.test_accepted_values": {"name": "test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.890809, "supported_languages": null}, "macro.dbt.test_relationships": {"name": "test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_relationships"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.891062, "supported_languages": null}}, "docs": {"doc.dbt.__overview__": {"name": "__overview__", "resource_type": "doc", "package_name": "dbt", "path": "overview.md", "original_file_path": "docs/overview.md", "unique_id": "doc.dbt.__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion"}}, "exposures": {}, "metrics": {}, "groups": {}, "selectors": {}, "disabled": {}, "parent_map": {"model.dbt_demo.diamonds_list_colors": ["model.dbt_demo.diamonds_four_cs"], "model.dbt_demo.diamonds_prices": [], "model.dbt_demo.diamonds_four_cs": ["source.dbt_demo.raw.diamonds"], "model.dbt_demo.zzz_game_details": [], "model.dbt_demo.second_dbt_model": ["model.dbt_demo.first_dbt_model"], "model.dbt_demo.first_dbt_model": [], "test.dbt_demo.zzz_game_details_check_scores": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.zzz_game_details_check_dates": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.zzz_win_loss_records_check_records": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.diamonds_list_colors_test": ["model.dbt_demo.diamonds_list_colors"], "seed.dbt_demo.country_codes": [], "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.unique_first_dbt_model_id.9dda05743e": ["model.dbt_demo.first_dbt_model"], "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061": ["model.dbt_demo.first_dbt_model"], "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4": ["model.dbt_demo.second_dbt_model"], "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2": ["model.dbt_demo.second_dbt_model"], "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f": ["model.dbt_demo.second_dbt_model"], "model.dbt_demo.changed_country": ["snapshot.dbt_demo.country_snapshot"], "model.dbt_demo.country": ["seed.dbt_demo.country_codes"], "snapshot.dbt_demo.country_snapshot": ["model.dbt_demo.country"], "model.dbt_demo.zzz_win_loss_records": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601": ["model.dbt_demo.zzz_win_loss_records"], "source.dbt_demo.raw.diamonds": [], "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors": ["model.dbt_demo.diamonds_list_colors"]}, "child_map": {"model.dbt_demo.diamonds_list_colors": ["test.dbt_demo.diamonds_list_colors_test", "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors"], "model.dbt_demo.diamonds_prices": [], "model.dbt_demo.diamonds_four_cs": ["model.dbt_demo.diamonds_list_colors"], "model.dbt_demo.zzz_game_details": ["model.dbt_demo.zzz_win_loss_records", "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40", "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d", "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c", "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68", "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905", "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5", "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39", "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e", "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f", "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be", "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218", "test.dbt_demo.zzz_game_details_check_dates", "test.dbt_demo.zzz_game_details_check_scores"], "model.dbt_demo.second_dbt_model": ["test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f", "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2", "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4"], "model.dbt_demo.first_dbt_model": ["model.dbt_demo.second_dbt_model", "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061", "test.dbt_demo.unique_first_dbt_model_id.9dda05743e"], "test.dbt_demo.zzz_game_details_check_scores": [], "test.dbt_demo.zzz_game_details_check_dates": [], "test.dbt_demo.zzz_win_loss_records_check_records": [], "test.dbt_demo.diamonds_list_colors_test": [], "seed.dbt_demo.country_codes": ["model.dbt_demo.country"], "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218": [], "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905": [], "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5": [], "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40": [], "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e": [], "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d": [], "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39": [], "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f": [], "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be": [], "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c": [], "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68": [], "test.dbt_demo.unique_first_dbt_model_id.9dda05743e": [], "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061": [], "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4": [], "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2": [], "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f": [], "model.dbt_demo.changed_country": [], "model.dbt_demo.country": ["snapshot.dbt_demo.country_snapshot"], "snapshot.dbt_demo.country_snapshot": ["model.dbt_demo.changed_country"], "model.dbt_demo.zzz_win_loss_records": ["test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601", "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86", "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe", "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105", "test.dbt_demo.zzz_win_loss_records_check_records"], "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105": [], "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86": [], "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe": [], "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601": [], "source.dbt_demo.raw.diamonds": ["model.dbt_demo.diamonds_four_cs"], "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors": []}, "group_map": {}, "saved_queries": {}, "semantic_models": {}, "unit_tests": {"unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors": {"model": "diamonds_list_colors", "given": [{"input": "ref('diamonds_four_cs')", "rows": [{"carat": 0.23, "cut": "Ideal", "color": "E", "clarity": "SI2"}, {"carat": 0.3, "cut": "Premium", "color": "E", "clarity": "SI2"}, {"carat": 0.3, "cut": "Premium", "color": "K", "clarity": "SI2"}], "format": "dict", "fixture": null}], "expect": {"rows": [{"color": "E"}, {"color": "K"}], "format": "dict", "fixture": null}, "name": "test_get_distinct_colors", "resource_type": "unit_test", "package_name": "dbt_demo", "path": "sql_model1/test_get_distinct_colors.yml", "original_file_path": "src/models/sql_model1/test_get_distinct_colors.yml", "unique_id": "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors", "fqn": ["dbt_demo", "models", "sql_model1", "diamonds_list_colors", "test_get_distinct_colors"], "description": "Check a distinct list of colors is returned.", "overrides": null, "depends_on": {"macros": [], "nodes": ["model.dbt_demo.diamonds_list_colors"]}, "config": {"tags": [], "meta": {}}, "checksum": "483c13eb849bf29c60173f6b4583def3c4a3b8c20e674a28c0db2c3e81834bb1", "schema": "dbt_dev", "created_at": 1733486794.4994261, "versions": null, "version": null}}} \ No newline at end of file diff --git a/contrib/dbt_factory/tests/test_dbt_factory.py b/contrib/dbt_factory/tests/test_dbt_factory.py new file mode 100644 index 00000000..9fb3db39 --- /dev/null +++ b/contrib/dbt_factory/tests/test_dbt_factory.py @@ -0,0 +1,419 @@ +import os +from tempfile import NamedTemporaryFile +from pathlib import Path +import pytest +import yaml + + +BASE_PATH = str(Path(__file__).resolve().parent) + + +def _model(package: str, name: str, depends_on: list[str] | None = None) -> tuple[str, dict]: + full_name = f"model.{package}.{name}" + return full_name, { + "resource_type": "model", + "name": name, + "package_name": package, + "depends_on": {"nodes": depends_on or []}, + } + + +def _test(package: str, name: str, depends_on: list[str], severity: str = "error") -> tuple[str, dict]: + full_name = f"test.{package}.{name}" + return full_name, { + "resource_type": "test", + "name": name, + "package_name": package, + "depends_on": {"nodes": depends_on}, + "config": {"severity": severity}, + } + + +def _seed(package: str, name: str) -> tuple[str, dict]: + full_name = f"seed.{package}.{name}" + return full_name, { + "resource_type": "seed", + "name": name, + "package_name": package, + "depends_on": {"nodes": []}, + } + + +def _snapshot(package: str, name: str, depends_on: list[str] | None = None) -> tuple[str, dict]: + full_name = f"snapshot.{package}.{name}" + return full_name, { + "resource_type": "snapshot", + "name": name, + "package_name": package, + "depends_on": {"nodes": depends_on or []}, + } + + +def _source(package: str, source_name: str, table: str) -> tuple[str, dict]: + full_name = f"source.{package}.{source_name}.{table}" + return full_name, { + "resource_type": "source", + "name": table, + "source_name": source_name, + "package_name": package, + } + + +def test_same_model_name_across_packages_produces_distinct_bundled_test_tasks(dbt_factory_bundled): + nodes = dict( + [ + _model("pkg_a", "customers"), + _model("pkg_b", "customers"), + _model("pkg_a", "orders", depends_on=["model.pkg_a.customers", "model.pkg_b.customers"]), + _test("pkg_a", "unique_customers_id", ["model.pkg_a.customers"]), + _test("pkg_b", "not_null_customers_id", ["model.pkg_b.customers"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "tests_model_pkg_a_customers" in by_key + assert "tests_model_pkg_b_customers" in by_key + assert by_key["tests_model_pkg_a_customers"]["dbt_task"]["commands"] == [ + "dbt test --select pkg_a.customers --indirect-selection cautious --target dev" + ] + assert by_key["tests_model_pkg_b_customers"]["dbt_task"]["commands"] == [ + "dbt test --select pkg_b.customers --indirect-selection cautious --target dev" + ] + assert by_key["tests_model_pkg_a_customers"]["depends_on"] == [{"task_key": "model_pkg_a_customers"}] + assert by_key["tests_model_pkg_b_customers"]["depends_on"] == [{"task_key": "model_pkg_b_customers"}] + + assert {dep["task_key"] for dep in by_key["model_pkg_a_orders"]["depends_on"]} == { + "tests_model_pkg_a_customers", + "tests_model_pkg_b_customers", + } + + +def test_tests_on_seed_produce_task_and_gate_downstream(dbt_factory_bundled): + nodes = dict( + [ + _seed("pkg", "countries"), + _model("pkg", "enriched", depends_on=["seed.pkg.countries"]), + _test("pkg", "unique_countries_code", ["seed.pkg.countries"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "tests_seed_pkg_countries" in by_key + assert by_key["tests_seed_pkg_countries"]["dbt_task"]["commands"] == [ + "dbt test --select pkg.countries --indirect-selection cautious --target dev" + ] + assert by_key["tests_seed_pkg_countries"]["depends_on"] == [{"task_key": "seed_pkg_countries"}] + assert by_key["model_pkg_enriched"]["depends_on"] == [{"task_key": "tests_seed_pkg_countries"}] + + +def test_tests_on_snapshot_produce_task_and_gate_downstream(dbt_factory_bundled): + nodes = dict( + [ + _snapshot("pkg", "orders_snap"), + _model("pkg", "orders_history", depends_on=["snapshot.pkg.orders_snap"]), + _test("pkg", "not_null_orders_snap_id", ["snapshot.pkg.orders_snap"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "tests_snapshot_pkg_orders_snap" in by_key + assert by_key["tests_snapshot_pkg_orders_snap"]["dbt_task"]["commands"] == [ + "dbt test --select pkg.orders_snap --indirect-selection cautious --target dev" + ] + assert by_key["tests_snapshot_pkg_orders_snap"]["depends_on"] == [{"task_key": "snapshot_pkg_orders_snap"}] + assert by_key["model_pkg_orders_history"]["depends_on"] == [{"task_key": "tests_snapshot_pkg_orders_snap"}] + + +def test_tests_on_source_produce_standalone_task(dbt_factory_bundled): + nodes = dict( + [ + _test("pkg", "unique_raw_customers_id", ["source.pkg.raw.customers"]), + ] + ) + sources = dict([_source("pkg", "raw", "customers")]) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes, "sources": sources}) + by_key = {t["task_key"]: t for t in tasks} + + assert "tests_source_pkg_raw_customers" in by_key + assert by_key["tests_source_pkg_raw_customers"]["dbt_task"]["commands"] == [ + "dbt test --select source:pkg.raw.customers --indirect-selection cautious --target dev" + ] + assert by_key["tests_source_pkg_raw_customers"]["depends_on"] == [] + + +def test_flat_mode_emits_one_task_per_test_node_and_gates_downstream(dbt_factory): + # Per-test mode mirrors `dbt build`: downstream models wait on upstream tests, so a + # failing `severity: error` test skips downstream via Databricks task failure. + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "orders", depends_on=["model.pkg.customers"]), + _test("pkg", "unique_customers_id", ["model.pkg.customers"]), + _test("pkg", "not_null_customers_id", ["model.pkg.customers"]), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "test_pkg_unique_customers_id" in by_key + assert "test_pkg_not_null_customers_id" in by_key + assert "tests_model_pkg_customers" not in by_key + + assert by_key["test_pkg_unique_customers_id"]["dbt_task"]["commands"] == [ + "dbt test --select unique_customers_id --target dev" + ] + assert by_key["test_pkg_unique_customers_id"]["depends_on"] == [{"task_key": "model_pkg_customers"}] + # orders depends on customers AND every test attached to customers + assert {dep["task_key"] for dep in by_key["model_pkg_orders"]["depends_on"]} == { + "model_pkg_customers", + "test_pkg_unique_customers_id", + "test_pkg_not_null_customers_id", + } + + +def test_flat_mode_cross_model_test_does_not_create_cycle(dbt_factory): + # Relationship test references BOTH `orders` and `customers`. Without care, extending + # `orders`'s deps with "tests of upstream (customers)" would pull in the relationship test, + # which itself depends on `orders` — a direct cycle. + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "orders", depends_on=["model.pkg.customers"]), + _model("pkg", "payments", depends_on=["model.pkg.orders"]), + _test("pkg", "unique_customers_id", ["model.pkg.customers"]), + _test( + "pkg", + "relationships_orders_customer_id__ref_customers", + ["model.pkg.orders", "model.pkg.customers"], + ), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + # orders depends on customers + unique_customers_id, but NOT on the relationship test + # (that test references orders itself — including it would cycle) + assert {dep["task_key"] for dep in by_key["model_pkg_orders"]["depends_on"]} == { + "model_pkg_customers", + "test_pkg_unique_customers_id", + } + + # payments (downstream of orders) picks up the relationship test — safe, payments + # transitively depends on both orders and customers (the test's refs) + payments_deps = {dep["task_key"] for dep in by_key["model_pkg_payments"]["depends_on"]} + assert "model_pkg_orders" in payments_deps + assert "test_pkg_relationships_orders_customer_id__ref_customers" in payments_deps + + +def test_flat_mode_transitive_cross_model_test_does_not_create_cycle(dbt_factory): + # Transitive cycle case: test T refs {A, C} where C is downstream of B which is downstream + # of A. Extending B's deps with "tests of upstream (A)" must NOT add T, because T depends + # on C and C depends on B → B → T → C → B cycle. Only nodes downstream of both A and C + # (i.e. downstream of C) should get T. + nodes = dict( + [ + _model("pkg", "a"), + _model("pkg", "b", depends_on=["model.pkg.a"]), + _model("pkg", "c", depends_on=["model.pkg.b"]), + _model("pkg", "d", depends_on=["model.pkg.c"]), + _test("pkg", "relationship_a_c", ["model.pkg.a", "model.pkg.c"]), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + # B's ancestors = {A}. Test T refs = {A, C}. C ∉ ancestors(B) → skip T. + assert by_key["model_pkg_b"]["depends_on"] == [{"task_key": "model_pkg_a"}] + # C's ancestors = {A, B}. C IS in T.refs → skip T (direct self-reference). + assert by_key["model_pkg_c"]["depends_on"] == [{"task_key": "model_pkg_b"}] + # D's ancestors = {A, B, C}. T.refs = {A, C} ⊆ ancestors(D) → add T. + d_deps = {dep["task_key"] for dep in by_key["model_pkg_d"]["depends_on"]} + assert d_deps == {"model_pkg_c", "test_pkg_relationship_a_c"} + + +def test_flat_mode_warn_severity_tests_do_not_gate_downstream(dbt_factory): + # Only error-severity tests gate downstream (matches `dbt build`: dbt exits 0 on warn + # so a gate wouldn't block anyway; we just keep the DAG cleaner). + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "orders", depends_on=["model.pkg.customers"]), + _test("pkg", "unique_customers_id", ["model.pkg.customers"], severity="warn"), + _test("pkg", "not_null_customers_id", ["model.pkg.customers"], severity="error"), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + # Both test tasks still exist (warn tests still run — they just don't gate anything) + assert "test_pkg_unique_customers_id" in by_key + assert "test_pkg_not_null_customers_id" in by_key + + # orders gates on customers + the error-severity test, but NOT the warn-severity one + assert {dep["task_key"] for dep in by_key["model_pkg_orders"]["depends_on"]} == { + "model_pkg_customers", + "test_pkg_not_null_customers_id", + } + + +def test_flat_mode_test_on_seed_gates_on_seed(dbt_factory): + nodes = dict( + [ + _seed("pkg", "countries"), + _test("pkg", "unique_countries_code", ["seed.pkg.countries"]), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert by_key["test_pkg_unique_countries_code"]["depends_on"] == [{"task_key": "seed_pkg_countries"}] + + +def test_bundled_task_factory_assembles_commands(dbt_factory_bundled): + test_factory = dbt_factory_bundled.task_factories["test"] + task = test_factory.create_bundled_task( + task_key="tests_model_pkg_customers", + select="pkg.customers", + deps_command_name="customers", + depends_on=["model_pkg_customers"], + ) + assert task.task_key == "tests_model_pkg_customers" + assert task.commands == ["dbt test --select pkg.customers --indirect-selection cautious --target dev"] + assert task.depends_on == ["model_pkg_customers"] + + +def test_cross_model_test_in_bundled_mode_is_emitted_as_standalone_task(dbt_factory_bundled): + # The relationship test spans two models, so it must NOT be collapsed into either model's + # bundled `_tests` task (dbt would hit a TABLE_OR_VIEW_NOT_FOUND on the un-built endpoint). + # It should emit its own task with deps on both referenced models. + nodes = dict( + [ + _model("pkg", "team_cities"), + _model("pkg", "game_details", depends_on=["model.pkg.team_cities"]), + _test("pkg", "not_null_team_cities_name", ["model.pkg.team_cities"]), + _test( + "pkg", + "relationships_game_details_winner__team_city__ref_team_cities_", + ["model.pkg.game_details", "model.pkg.team_cities"], + ), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + # Single-model test → bundled with cautious selection (relationship test is excluded by dbt) + assert "tests_model_pkg_team_cities" in by_key + assert by_key["tests_model_pkg_team_cities"]["dbt_task"]["commands"] == [ + "dbt test --select pkg.team_cities --indirect-selection cautious --target dev" + ] + + # Cross-model test → its own task, gated on BOTH referenced models + cross_test_key = "test_pkg_relationships_game_details_winner__team_city__ref_team_cities_" + assert cross_test_key in by_key + assert by_key[cross_test_key]["dbt_task"]["commands"] == [ + "dbt test --select relationships_game_details_winner__team_city__ref_team_cities_ --target dev" + ] + assert {dep["task_key"] for dep in by_key[cross_test_key]["depends_on"]} == { + "model_pkg_team_cities", + "model_pkg_game_details", + } + + # `game_details` has no single-model tests, so no bundled `game_details_tests` exists + assert "tests_model_pkg_game_details" not in by_key + + +def test_single_package_bundled_test_uses_qualified_select(dbt_factory_bundled): + nodes = dict( + [ + _model("pkg_a", "customers"), + _model("pkg_a", "orders", depends_on=["model.pkg_a.customers"]), + _test("pkg_a", "unique_customers_id", ["model.pkg_a.customers"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "tests_model_pkg_a_customers" in by_key + assert by_key["tests_model_pkg_a_customers"]["dbt_task"]["commands"] == [ + "dbt test --select pkg_a.customers --indirect-selection cautious --target dev" + ] + assert by_key["model_pkg_a_orders"]["depends_on"] == [{"task_key": "tests_model_pkg_a_customers"}] + + +def test_create_job_spec_and_update(dbt_factory): + run_job_spec_test( + dbt_factory, + BASE_PATH + "/test_data/job_definition_no_deps.yaml", + ) + + +def test_create_job_spec_and_update_with_dbt_deps(dbt_factory_with_deps): + run_job_spec_test( + dbt_factory_with_deps, + BASE_PATH + "/test_data/job_definition.yaml", + ) + + +def test_create_job_spec_and_update_with_selected_dbt_deps(dbt_factory_with_deps_selected): + run_job_spec_test( + dbt_factory_with_deps_selected, + BASE_PATH + "/test_data/job_definition_deps_selected.yaml", + ) + + +def test_notebook_task_generation(notebook_factory): + run_job_spec_test( + notebook_factory, + BASE_PATH + "/test_data/job_definition_notebook_no_deps.yaml", + ) + + +def run_job_spec_test(dbt_factory, expected_job_definition_path): + """Helper function to test databricks job definition generation.""" + dbt_manifest_path = BASE_PATH + "/test_data/manifest.json" + input_job_definition_path = BASE_PATH + "/test_data/job_definition_template.yaml" + + with NamedTemporaryFile(suffix=".yaml", delete=False) as temp_file: + actual_job_definition_path = temp_file.name + + try: + dbt_factory.create_tasks_and_update_job_spec( + dbt_manifest_path, input_job_definition_path, actual_job_definition_path + ) + + with open(expected_job_definition_path, "r", encoding="utf-8") as file: + expected_job_definition = yaml.safe_load(file) + + with open(actual_job_definition_path, "r", encoding="utf-8") as file: + job_definition = yaml.safe_load(file) + + assert job_definition == expected_job_definition + finally: + if os.path.exists(actual_job_definition_path): + os.remove(actual_job_definition_path) + + +@pytest.mark.skip("Manual testing") +def test_generate(databricks_dbt_factory): + """Test job definition generation and saving to file.""" + dbt_manifest_path = BASE_PATH + "/test_data/manifest.json" + job_definition_path = BASE_PATH + "/test_data/job_definition_template.yaml" + destination_job_definition_path = "job_definition.yaml" + + databricks_dbt_factory.create_tasks_and_update_job_spec( + dbt_manifest_path, job_definition_path, destination_job_definition_path, "new_job_name" + ) diff --git a/contrib/dbt_factory/tests/test_dbt_task.py b/contrib/dbt_factory/tests/test_dbt_task.py new file mode 100644 index 00000000..bdb40582 --- /dev/null +++ b/contrib/dbt_factory/tests/test_dbt_task.py @@ -0,0 +1,96 @@ +import pytest + +from databricks_dbt_factory.DbtTask import DbtTask, DbtTaskOptions, TaskType + + +def test_notebook_task_with_job_cluster_key(): + options = DbtTaskOptions( + task_type="notebook", + notebook_path="./notebooks/dbt_runner.py", + source="WORKSPACE", + project_directory="/project", + profiles_directory="/profiles", + job_cluster_key="dbt_cluster", + ) + task = DbtTask( + task_key="my_model", + commands=["dbt run --select my_model --target dev"], + options=options, + depends_on=["upstream_model"], + ) + + result = task.to_dict() + + assert result["job_cluster_key"] == "dbt_cluster" + assert "environment_key" not in result + assert result["task_key"] == "my_model" + assert result["depends_on"] == [{"task_key": "upstream_model"}] + assert result["notebook_task"]["source"] == "WORKSPACE" + assert result["notebook_task"]["base_parameters"]["project_directory"] == "/project" + assert result["notebook_task"]["base_parameters"]["profiles_directory"] == "/profiles" + + +def test_notebook_task_without_job_cluster_key_uses_environment(): + options = DbtTaskOptions( + task_type="notebook", + notebook_path="./notebooks/dbt_runner.py", + ) + task = DbtTask(task_key="my_model", commands=["dbt run --select my_model"], options=options) + + result = task.to_dict() + + assert result["environment_key"] == "Default" + assert "job_cluster_key" not in result + + +def test_dbt_task_without_job_cluster_key_uses_environment(): + options = DbtTaskOptions(environment_key="Default") + task = DbtTask(task_key="my_model", commands=["dbt run --select my_model"], options=options) + + result = task.to_dict() + + assert result["environment_key"] == "Default" + assert "job_cluster_key" not in result + + +def test_task_type_string_is_coerced_to_enum(): + options = DbtTaskOptions(task_type="notebook", notebook_path="./runner.py") + assert options.task_type is TaskType.NOTEBOOK + + +def test_task_type_invalid_value_raises(): + with pytest.raises(ValueError, match="not a valid TaskType"): + DbtTaskOptions(task_type="Notebook") + with pytest.raises(ValueError, match="not a valid TaskType"): + DbtTaskOptions(task_type="dbt_task") + + +def test_notebook_task_rejects_warehouse_schema_catalog(): + for kwargs in ( + {"warehouse_id": "wh123"}, + {"schema": "silver"}, + {"catalog": "main"}, + ): + with pytest.raises(ValueError, match="notebook tasks connect via profiles.yml"): + DbtTaskOptions(task_type=TaskType.NOTEBOOK, notebook_path="/n", **kwargs) + + +def test_notebook_task_rejects_multiple_incompatible_fields_at_once(): + with pytest.raises(ValueError, match=r"warehouse_id, schema, catalog"): + DbtTaskOptions( + task_type=TaskType.NOTEBOOK, + notebook_path="/n", + warehouse_id="wh123", + schema="silver", + catalog="main", + ) + + +def test_dbt_task_still_accepts_warehouse_schema_catalog(): + options = DbtTaskOptions( + task_type=TaskType.DBT, + warehouse_id="wh123", + schema="silver", + catalog="main", + ) + assert options.warehouse_id == "wh123" diff --git a/contrib/dbt_factory/tests/test_load_resources.py b/contrib/dbt_factory/tests/test_load_resources.py new file mode 100644 index 00000000..2d31c6db --- /dev/null +++ b/contrib/dbt_factory/tests/test_load_resources.py @@ -0,0 +1,70 @@ +""" +Offline test for the PyDABs integration in resources/__init__.py. + +Exercises the whole glue path against the committed dbt manifest (no Databricks workspace +required): reading the manifest, wiring the factory, generating notebook tasks, and registering +the job on a Resources object (which runs Job.from_dict, validating the task dict schema). +""" + +import json +from pathlib import Path + +import pytest +from databricks.bundles.core import Bundle + +import resources + +PROJECT_ROOT = Path(__file__).resolve().parent.parent + +# The manifest is generated by `make manifest` (dbt parse). In a freshly initialized project it +# does not exist yet, so skip these tests rather than erroring until the user has run it. +pytestmark = pytest.mark.skipif( + not (PROJECT_ROOT / resources.MANIFEST_PATH).exists(), + reason="dbt manifest not found; run `make manifest` (dbt parse) first", +) + + +@pytest.fixture(autouse=True) +def _chdir_to_project_root(monkeypatch): + # resources.MANIFEST_PATH is relative to the bundle root, as it is at deploy time. + monkeypatch.chdir(PROJECT_ROOT) + + +def test_build_tasks_covers_every_buildable_node(): + # Every model/seed/snapshot node gets its own task regardless of the BUNDLE_TESTS mode + # (tests may be collapsed into per-resource `tests_*` tasks when bundling is on). + tasks = resources._build_tasks("dev") + task_keys = {task["task_key"] for task in tasks} + + manifest = json.loads((PROJECT_ROOT / resources.MANIFEST_PATH).read_text()) + buildable = { + name + for name, node in manifest.get("nodes", {}).items() + if node.get("resource_type") in {"model", "seed", "snapshot"} + } + expected_keys = {name.replace(".", "_") for name in buildable} + + assert buildable, "committed manifest has no model/seed/snapshot nodes" + assert expected_keys <= task_keys + + +def test_generated_tasks_are_serverless_notebook_tasks(): + tasks = resources._build_tasks("dev") + + for task in tasks: + assert "notebook_task" in task, task + assert task["environment_key"] == resources.ENVIRONMENT_KEY + assert "job_cluster_key" not in task # serverless only + notebook_task = task["notebook_task"] + assert notebook_task["notebook_path"] == resources.RUNNER_NOTEBOOK_PATH + # The dbt target is injected into every command. + assert "--target dev" in notebook_task["base_parameters"]["dbt_commands"] + + +def test_load_resources_registers_the_job(): + result = resources.load_resources(Bundle(target="dev")) + + assert resources.JOB_NAME in result.jobs + job = result.jobs[resources.JOB_NAME] + # Job.from_dict succeeded and the environment carries the dbt-databricks dependency. + assert job.environments[0].environment_key == resources.ENVIRONMENT_KEY diff --git a/contrib/templates/dbt-factory/README.md b/contrib/templates/dbt-factory/README.md new file mode 100644 index 00000000..b2af16d0 --- /dev/null +++ b/contrib/templates/dbt-factory/README.md @@ -0,0 +1,95 @@ +# dbt-factory template + +A [Databricks Asset Bundle](https://docs.databricks.com/dev-tools/bundles/index.html) template +that generates a [dbt](https://docs.getdbt.com/) project whose Databricks Workflow is built +**from the dbt manifest at deploy time** — one Databricks task per dbt object (model, seed, +snapshot, test), running on serverless compute. + +It wires together two pieces: + +* **[databricks-dbt-factory](https://github.com/mwojtyczka/databricks-dbt-factory)** — expands a + dbt `manifest.json` into Databricks job tasks with their dependencies. Its source is included + in every generated project under `src/databricks_dbt_factory/`. +* **[PyDABs](https://docs.databricks.com/dev-tools/bundles/python)** — the `python.resources` + hook. At `databricks bundle deploy` time the Databricks CLI calls `load_resources`, which runs + the factory against the manifest and returns the generated job. No per-model job YAML is checked + in. + +Instead of running the whole dbt project as one opaque task, you get: + +* **Faster execution** — independent models run in parallel; the notebook task type keeps dbt's + dependencies pre-cached in the serverless environment, avoiding a per-task cold start. +* **Visibility & simplified troubleshooting** — pinpoint failures at the model level in the UI. +* **Enhanced logging & notifications** — per-task logs and precise, model-level error alerts. +* **Improved retriability** — retry only the failed model tasks, not the whole project. +* **Seamless testing** — dbt data tests run as their own tasks right after each model finishes. + +For a pre-initialized, ready-to-read version of what this template produces, see the +[`contrib/dbt_factory`](../../dbt_factory) example. + +## How it works + +`databricks bundle init` scaffolds a self-contained project; each `databricks bundle deploy` then +regenerates the Workflow from your current dbt manifest, so adding or removing a model just works +on the next deploy — no per-model YAML to maintain. + +```mermaid +flowchart TD + subgraph setup["One-time setup"] + T["dbt-factory bundle template"] -->|databricks bundle init| B["Scaffolded project:
dbt project + PyDABs hook + factory code"] + X["Existing dbt project
(optional)"] -.->|move models/seeds/... into src/| B + end + subgraph deploy["Every deploy"] + C["make manifest
(dbt parse)"] --> D["target/manifest.json"] + D --> E["databricks bundle deploy"] + E --> F["PyDABs load_resources reads the
manifest and generates the job"] + end + subgraph runtime["At run time — serverless"] + G["Databricks Workflow:
one task per model / seed / snapshot / test"] --> H["Each task triggers dbt
via the runner notebook"] + H --> I[("SQL warehouse")] + end + B --> C + F --> G + + classDef optional stroke:#999,stroke-dasharray:5 4,color:#888; + class X optional; +``` + +## Usage + +``` +$ databricks bundle init https://github.com/databricks/bundle-examples --template-dir contrib/templates/dbt-factory +``` + +Answer the prompts (project name, catalog, dev schema, warehouse HTTP path, and a few +factory options). Then: + +``` +$ cd +$ make setup # install dependencies into .venv +$ make manifest # generate the dbt manifest (dbt parse) — required before the first deploy +$ databricks bundle deploy --target dev +$ databricks bundle run _job +``` + +## Prompts + +| Prompt | Purpose | +|---|---| +| `project_name` | Bundle / dbt project name; also names the generated job `_job`. | +| `default_catalog` | Unity Catalog catalog dbt writes to. | +| `dev_schema` | Schema for the `dev` target (`prod` uses `default`). | +| `http_path` | HTTP path of the SQL warehouse dbt connects to. | +| `bundle_tests` | Bundle single-model tests per resource into one task (performance boost). | +| `environment_key` | Key of the serverless environment used by the generated job. | +| `extra_dbt_command_options` | Extra options appended to every generated dbt command. | + +## Already have a dbt project? + +This template scaffolds a new project. To reuse an **existing** dbt project, generate a project +from this template and move your dbt files into it — see the +["Migrating an existing dbt project"](../../dbt_factory/README.md#migrating-an-existing-dbt-project) +guide in the example. You won't need to bring any dependencies or edit paths. + +See https://github.com/databricks/bundle-examples/blob/main/contrib/README.md for more about +community contributions. diff --git a/contrib/templates/dbt-factory/databricks_template_schema.json b/contrib/templates/dbt-factory/databricks_template_schema.json new file mode 100644 index 00000000..0167fbc4 --- /dev/null +++ b/contrib/templates/dbt-factory/databricks_template_schema.json @@ -0,0 +1,57 @@ +{ + "welcome_message": "\nWelcome to the dbt-factory template for Databricks Asset Bundles!\nThis generates a dbt project whose Databricks Workflow is built from the dbt manifest at deploy time — one task per dbt object (model, seed, snapshot, test), on serverless.", + "properties": { + "project_name": { + "type": "string", + "default": "my_dbt_project", + "description": "\nPlease provide the following details to tailor the template to your preferences.\n\nUnique name for this project\nproject_name", + "order": 1, + "pattern": "^[A-Za-z0-9_]+$", + "pattern_match_failure_message": "Name must consist of letters, numbers, and underscores." + }, + "default_catalog": { + "type": "string", + "default": "{{default_catalog}}", + "pattern": "^\\w*$", + "pattern_match_failure_message": "Invalid catalog name.", + "description": "\nInitial catalog dbt writes to{{if eq (default_catalog) \"\"}} (leave blank when not using Unity Catalog){{end}}.\ndefault_catalog", + "order": 2 + }, + "dev_schema": { + "type": "string", + "default": "default", + "pattern": "^\\w+$", + "pattern_match_failure_message": "Invalid schema name.", + "description": "\nSchema dbt writes to for the 'dev' target (the 'prod' target uses 'default').\ndev_schema", + "order": 3 + }, + "http_path": { + "type": "string", + "default": "/sql/1.0/warehouses/abcdef1234567890", + "description": "\nHTTP path of the SQL warehouse dbt connects to.\nhttp_path", + "order": 4 + }, + "bundle_tests": { + "type": "string", + "default": "no", + "enum": ["no", "yes"], + "description": "\nBundle single-model tests per resource into one dbt test task? This is a performance boost for projects with many tests (fewer Databricks task startups), at the cost of per-test visibility in the UI.\nbundle_tests", + "order": 5 + }, + "environment_key": { + "type": "string", + "default": "Default", + "pattern": "^\\w+$", + "pattern_match_failure_message": "Invalid environment key.", + "description": "\nKey of the serverless environment used by the generated job.\nenvironment_key", + "order": 6 + }, + "extra_dbt_command_options": { + "type": "string", + "default": "", + "description": "\nAdditional options appended to every generated dbt command (e.g. \"--vars '{...}'\"). Leave blank if unsure.\nextra_dbt_command_options", + "order": 7 + } + }, + "success_message": "\n\nYour new project has been created in the '{{.project_name}}' directory!\n\nNext steps:\n\n $ cd {{.project_name}}\n $ make setup # install dependencies into .venv\n $ make manifest # generate the dbt manifest (dbt parse) — required before the first deploy\n $ databricks bundle deploy --target dev\n\nRefer to the README.md file for full instructions." +} diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/.gitignore b/contrib/templates/dbt-factory/template/{{.project_name}}/.gitignore new file mode 100644 index 00000000..0ea8c968 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/.gitignore @@ -0,0 +1,13 @@ +.venv/ +__pycache__/ +*.pyc +.databricks/ +logs/ +dbt_packages/ +dbt_profiles/.user.yml +uv.lock + +# dbt build artifacts, but keep the committed manifest that resources/__init__.py reads at +# deploy time. Regenerate it with `make manifest`. +target/* +!target/manifest.json diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/Makefile.tmpl b/contrib/templates/dbt-factory/template/{{.project_name}}/Makefile.tmpl new file mode 100644 index 00000000..7450a1ee --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/Makefile.tmpl @@ -0,0 +1,27 @@ +.PHONY: setup deps manifest validate deploy run test + +# Install dependencies into the .venv used by the bundle (databricks.yml -> python.venv_path). +setup: + uv sync --dev + +# Install dbt package dependencies declared in packages.yml / dependencies.yml (if any). +deps: + uv run dbt deps + +# Regenerate the dbt manifest that resources/__init__.py reads at deploy time. +# `dbt parse` does not connect to a warehouse; it only reads the project files. +manifest: deps + uv run dbt parse --profiles-dir dbt_profiles + +validate: + databricks bundle validate + +# Regenerate the manifest and deploy the generated job to the dev target. +deploy: manifest + databricks bundle deploy --target dev + +run: + databricks bundle run {{.project_name}}_job + +test: + uv run pytest tests diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/NOTICE b/contrib/templates/dbt-factory/template/{{.project_name}}/NOTICE new file mode 100644 index 00000000..703b9022 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/NOTICE @@ -0,0 +1,33 @@ +This project includes code adapted from the "databricks-dbt-factory" library. + + Source: https://github.com/mwojtyczka/databricks-dbt-factory + Full history: https://github.com/mwojtyczka/databricks-dbt-factory/commits/main + Adapted from: commit e767a9d865581226e4f144fb17b7a822df1ea1f4 (v0.2.1) + Location: src/databricks_dbt_factory/ + +The code under src/databricks_dbt_factory/ originates from that repository (reformatted to +this repository's style; otherwise functionally unchanged) and is provided under the MIT +license reproduced below, which this NOTICE preserves for attribution. All other files are +part of the bundle-examples repository and are covered by that repository's license. + +-------------------------------------------------------------------------------- + +MIT License + +Copyright (c) 2024-present mwojtyczka + +Permission is hereby granted, free of charge, to any person obtaining a copy of this +software and associated documentation files (the "Software"), to deal in the Software +without restriction, including without limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons +to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or +substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, +INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE +FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/README.md.tmpl b/contrib/templates/dbt-factory/template/{{.project_name}}/README.md.tmpl new file mode 100644 index 00000000..30c30f60 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/README.md.tmpl @@ -0,0 +1,81 @@ +# {{.project_name}} + +The '{{.project_name}}' project was generated using the **dbt-factory** template for +Databricks Asset Bundles. It runs a [dbt](https://docs.getdbt.com/) project on Databricks as a +Databricks Workflow with **one task per dbt object** (model, seed, snapshot, test), instead of +running the whole project as a single opaque task. + +It combines two pieces: + +* **[databricks-dbt-factory](https://github.com/mwojtyczka/databricks-dbt-factory)** — reads a + dbt `manifest.json` and expands it into Databricks job tasks with their dependencies wired up. + Vendored under `src/databricks_dbt_factory/` (see [`NOTICE`](NOTICE)). +* **[PyDABs](https://docs.databricks.com/dev-tools/bundles/python)** — at `bundle deploy` time + the Databricks CLI calls `load_resources` in [`resources/__init__.py`](resources/__init__.py), + which runs the factory against the manifest and returns the generated job. **No per-model job + YAML is checked in** — the task graph is generated on the fly from the manifest. + +This project uses **serverless compute** and the **notebook task type** for the fastest task +start times. + +## Setup + +1. Install the [Databricks CLI](https://docs.databricks.com/dev-tools/cli/databricks-cli.html) + and the [uv](https://docs.astral.sh/uv/) package manager. + +2. Authenticate to your Databricks workspace: + ``` + $ databricks configure + ``` + +3. Install dependencies into the `.venv` the bundle uses: + ``` + $ make setup # == uv sync --dev + ``` + +4. Check the SQL warehouse `http_path`, `catalog`, and `schema` in `dbt_profiles/profiles.yml`. + +## Generate the dbt manifest (required before the first deploy) + +`resources/__init__.py` reads the dbt manifest at deploy time. Generate it from your models: + +``` +$ make manifest # == uv run dbt deps && uv run dbt parse +``` + +`dbt parse` only reads your project files; it does not connect to a warehouse. Rerun it whenever +you change your models. The manifest location is configurable via the `DBT_MANIFEST_PATH` env var +or the `MANIFEST_PATH` constant in `resources/__init__.py`. + +## Deploy and run + +``` +$ databricks bundle deploy --target dev # or: make deploy (regenerates the manifest first) +$ databricks bundle run {{.project_name}}_job # or: make run +``` + +Open the run URL the CLI prints to watch the generated per-model task graph execute. + +## Configuring the generated job + +The knobs you chose at init are constants at the top of `resources/__init__.py` +(`BUNDLE_TESTS`, `ENVIRONMENT_KEY`, `EXTRA_DBT_COMMAND_OPTIONS`); edit them there to change +behavior. The dbt target, warehouse, catalog, and schema live in `dbt_profiles/profiles.yml` and +are selected per bundle target via `--target ${bundle.target}`. + +## Tests + +``` +$ make test # == uv run pytest tests (run `make manifest` first) +``` + +Runs the vendored factory's test suite plus an offline test of the PyDABs integration against the +generated manifest. + +## Folding in an existing dbt project + +Already have a dbt project you'd rather use? Just move your dbt sources into this project's `src/` +folders (`src/models`, `src/seeds`, …) in place of the starter models, then run `make manifest`. +See the +[Migrating an existing dbt project](https://github.com/databricks/bundle-examples/blob/main/contrib/dbt_factory/README.md#migrating-an-existing-dbt-project) +guide for the full steps. diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/databricks.yml.tmpl b/contrib/templates/dbt-factory/template/{{.project_name}}/databricks.yml.tmpl new file mode 100644 index 00000000..72dda0d1 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/databricks.yml.tmpl @@ -0,0 +1,38 @@ +# This is a Databricks Asset Bundle definition for {{.project_name}}. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +# +# The job for this bundle is NOT defined in YAML. It is generated at deploy time from the dbt +# manifest by resources/__init__.py (the `python.resources` hook below), producing one +# Databricks task per dbt object (model / seed / snapshot / test). +bundle: + name: {{.project_name}} + +# PyDABs: the Databricks CLI calls `load_resources` during `bundle deploy` to build resources +# defined in Python. See resources/__init__.py. +python: + venv_path: .venv + resources: + - "resources:load_resources" + +# Deployment targets. The dbt target is selected via `--target ${bundle.target}`, so these +# names match the dbt outputs in dbt_profiles/profiles.yml. +targets: + dev: + # The default target uses 'mode: development' to create a development copy. + # See https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. + mode: development + default: true + workspace: + host: {{workspace_host}} + + prod: + mode: production + workspace: + host: {{workspace_host}} + # We explicitly specify /Workspace/Users/{{user_name}} to make sure we only have a single copy. + root_path: /Workspace/Users/{{user_name}}/.bundle/${bundle.name}/${bundle.target} + permissions: + - {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}} + level: CAN_MANAGE + run_as: + {{if is_service_principal}}service_principal{{else}}user{{end}}_name: {{user_name}} diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/dbt_profiles/profiles.yml.tmpl b/contrib/templates/dbt-factory/template/{{.project_name}}/dbt_profiles/profiles.yml.tmpl new file mode 100644 index 00000000..7c52bfdd --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/dbt_profiles/profiles.yml.tmpl @@ -0,0 +1,37 @@ + +# This file defines dbt profiles for deployed dbt jobs. +# The generated Databricks job selects the target via `--target ${bundle.target}`, +# so the dbt target names below (dev / prod) match the bundle targets in databricks.yml. +{{.project_name}}: + target: dev # default target + outputs: + + # Doing local development with the dbt CLI? + # Then you should create your own profile in your .dbt/profiles.yml using 'dbt init' + # (See README.md) + + # The default target when deployed with the Databricks CLI + dev: + type: databricks + method: http + catalog: {{.default_catalog}} + schema: {{.dev_schema}} + + http_path: {{.http_path}} + + # The workspace host / token are provided by the runner notebook at runtime + # (see src/databricks_dbt_factory/notebook/run_dbt_command.py). + host: "{{`{{`}} env_var('DBT_HOST', '') {{`}}`}}" + token: "{{`{{`}} env_var('DBT_ACCESS_TOKEN', '') {{`}}`}}" + + # The production target when deployed with the Databricks CLI + prod: + type: databricks + method: http + catalog: {{.default_catalog}} + schema: default + + http_path: {{.http_path}} + + host: "{{`{{`}} env_var('DBT_HOST', '') {{`}}`}}" + token: "{{`{{`}} env_var('DBT_ACCESS_TOKEN', '') {{`}}`}}" diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/dbt_project.yml.tmpl b/contrib/templates/dbt-factory/template/{{.project_name}}/dbt_project.yml.tmpl new file mode 100644 index 00000000..a14419cc --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/dbt_project.yml.tmpl @@ -0,0 +1,28 @@ +name: '{{.project_name}}' +version: '1.0.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. +profile: '{{.project_name}}' + +# These configurations specify where dbt should look for different types of files. +# Everything dbt-related lives under src/ so the project can also hold non-dbt resources +# (such as the vendored databricks_dbt_factory library under src/databricks_dbt_factory). +model-paths: ["src/models"] +analysis-paths: ["src/analyses"] +test-paths: ["src/tests"] +seed-paths: ["src/seeds"] +macro-paths: ["src/macros"] +snapshot-paths: ["src/snapshots"] + +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + +# In this example config, we tell dbt to build all models in the example/ +# directory as views by default. These settings can be overridden in the +# individual model files using the `{{`{{`}} config(...) {{`}}`}}` macro. +models: + {{.project_name}}: + example: + +materialized: view diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/profile_template.yml b/contrib/templates/dbt-factory/template/{{.project_name}}/profile_template.yml new file mode 100644 index 00000000..f26ec0b3 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/profile_template.yml @@ -0,0 +1,23 @@ +# This file defines prompts with defaults for dbt initialization. +# It is used when the `dbt init` command is invoked. +# +fixed: + type: databricks +prompts: + host: + default: company.databricks.com + token: + hint: 'personal access token to use, dapiXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' + hide_input: true + http_path: + hint: 'HTTP path of SQL warehouse to use' + default: /sql/1.0/warehouses/abcdef1234567890 + catalog: + hint: 'initial catalog' + default: catalog + schema: + hint: 'personal schema where dbt will build objects during development, example: user_name' + threads: + hint: 'threads to use during development, 1 or more' + type: 'int' + default: 4 diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/pyproject.toml.tmpl b/contrib/templates/dbt-factory/template/{{.project_name}}/pyproject.toml.tmpl new file mode 100644 index 00000000..9874a373 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/pyproject.toml.tmpl @@ -0,0 +1,31 @@ +[project] +name = "{{.project_name}}" +version = "0.0.1" +requires-python = ">=3.10,<3.13" +# Runtime dependency of the vendored databricks_dbt_factory core (read by load_resources +# during `bundle deploy`). +dependencies = [ + "PyYAML~=6.0.1", +] + +[dependency-groups] +dev = [ + # Required by the PyDABs hook (resources/__init__.py) at `bundle deploy` time. + "databricks-bundles==0.295.0", + # Used locally to regenerate the dbt manifest (`make manifest`). At runtime dbt is + # installed into the serverless environment (see resources/__init__.py). + "dbt-databricks>=1.8.0,<2.0.0", + "pytest", + "ruff", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +# Only the vendored library is packaged; the dbt project (src/models, ...) is not a package. +[tool.hatch.build.targets.wheel] +packages = ["src/databricks_dbt_factory"] + +[tool.ruff] +line-length = 120 diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/resources/__init__.py.tmpl b/contrib/templates/dbt-factory/template/{{.project_name}}/resources/__init__.py.tmpl new file mode 100644 index 00000000..1a8979c3 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/resources/__init__.py.tmpl @@ -0,0 +1,123 @@ +""" +PyDABs integration for databricks-dbt-factory. + +At ``databricks bundle deploy`` time the Databricks CLI calls :func:`load_resources`, which +reads the dbt manifest and expands it into a Databricks job with one task per dbt object +(model / seed / snapshot / test) using the vendored ``databricks_dbt_factory`` core. No +per-model YAML is generated or checked in — the job graph is built on the fly from the manifest. + +This module is the only integration glue; the vendored core under ``src/databricks_dbt_factory`` +is unchanged. It mirrors the wiring in ``databricks_dbt_factory.main`` but is fixed to the +options this project exposes (serverless + notebook task type) and returns a ``Resources`` +object instead of writing a job-spec YAML file. +""" + +import os + +from databricks.bundles.core import Bundle, Resources +from databricks.bundles.jobs import Job + +from databricks_dbt_factory.DbtFactory import DbtFactory +from databricks_dbt_factory.SpecsHandler import SpecsHandler +from databricks_dbt_factory.DbtTask import DbtTaskOptions, TaskType +from databricks_dbt_factory.TaskFactory import ( + DbtDependencyResolver, + ModelTaskFactory, + SnapshotTaskFactory, + SeedTaskFactory, + TestTaskFactory, +) + +# --- Configuration ------------------------------------------------------------------------- + +# Path to the dbt manifest, read at deploy time. Regenerate with `make manifest` (dbt parse). +# Override the location without editing this file via the DBT_MANIFEST_PATH env var. +MANIFEST_PATH = os.environ.get("DBT_MANIFEST_PATH", "target/manifest.json") + +# Name of the generated Databricks job. +JOB_NAME = "{{.project_name}}_job" + +# Key of the serverless environment (defined on the job below). +ENVIRONMENT_KEY = "{{.environment_key}}" + +# Bundle single-model tests per resource into one `dbt test` task (fewer task startups, +# faster end-to-end runtime for projects with many tests). See the databricks-dbt-factory +# README "DBT Tests handling" section for the trade-offs. +BUNDLE_TESTS = {{if eq .bundle_tests "yes"}}True{{else}}False{{end}} + +# Extra options appended to every generated dbt command (e.g. "--vars '{...}'"). +# `printf "%q"` renders a safely-quoted string literal so values containing quotes stay valid Python. +EXTRA_DBT_COMMAND_OPTIONS = {{ printf "%q" .extra_dbt_command_options }} + +# dbt-databricks version installed into the serverless environment at runtime. +DBT_DATABRICKS_DEPENDENCY = "dbt-databricks>=1.8.0,<2.0.0" + +# The runner notebook shipped with the core, referenced in place (path relative to the bundle +# root). `PROJECT_DIRECTORY` is the path from the notebook's own directory back up to the dbt +# project root (== bundle root), where the runner changes directory to before running dbt. It is +# derived from the notebook path so the two stay consistent if the layout ever changes. +RUNNER_NOTEBOOK_PATH = "src/databricks_dbt_factory/notebook/run_dbt_command.py" +PROJECT_DIRECTORY = os.path.relpath(".", os.path.dirname(RUNNER_NOTEBOOK_PATH)) +PROFILES_DIRECTORY = "dbt_profiles" + + +def _build_tasks(target: str) -> list[dict]: + """Reads the dbt manifest and returns the list of Databricks task dicts (one per dbt node).""" + resolver = DbtDependencyResolver() + task_options = DbtTaskOptions( + task_type=TaskType.NOTEBOOK, + environment_key=ENVIRONMENT_KEY, + notebook_path=RUNNER_NOTEBOOK_PATH, + project_directory=PROJECT_DIRECTORY, + profiles_directory=PROFILES_DIRECTORY, + ) + dbt_options = f"--target {target} {EXTRA_DBT_COMMAND_OPTIONS}".strip() + + task_factories = { + "model": ModelTaskFactory(resolver, task_options, dbt_options), + "snapshot": SnapshotTaskFactory(resolver, task_options, dbt_options), + "seed": SeedTaskFactory(resolver, task_options, dbt_options), + "test": TestTaskFactory(resolver, task_options, dbt_options), + } + + handler = SpecsHandler() + factory = DbtFactory(handler, task_factories, bundle_tests=BUNDLE_TESTS) + manifest = handler.read_dbt_manifest(MANIFEST_PATH) + return factory.create_tasks(manifest) + + +def build_job(target: str) -> Job: + """Builds the Databricks job that runs the dbt project on serverless via notebook tasks.""" + return Job.from_dict( + { + "name": JOB_NAME, + "queue": {"enabled": True}, + "trigger": { + # Run this job every day, exactly one day from the last run; + # see https://docs.databricks.com/api/workspace/jobs/create#trigger + "periodic": {"interval": 1, "unit": "DAYS"}, + }, + "tasks": _build_tasks(target), + "environments": [ + { + "environment_key": ENVIRONMENT_KEY, + "spec": { + "environment_version": "4", + "dependencies": [DBT_DATABRICKS_DEPENDENCY], + }, + } + ], + } + ) + + +def load_resources(bundle: Bundle) -> Resources: + """ + Called by the Databricks CLI during `bundle deploy` to load resources defined in Python. + + Reads the dbt manifest and registers the generated job. After deployment this function is + not used. + """ + resources = Resources() + resources.add_job(JOB_NAME, build_job(bundle.target)) + return resources diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/analyses/.gitkeep b/contrib/templates/dbt-factory/template/{{.project_name}}/src/analyses/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/DbtFactory.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/DbtFactory.py new file mode 100644 index 00000000..3f9c9b53 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/DbtFactory.py @@ -0,0 +1,360 @@ +from dataclasses import replace + +from databricks_dbt_factory import TaskFactory +from databricks_dbt_factory.SpecsHandler import SpecsHandler +from databricks_dbt_factory.DbtTask import DbtTask +from databricks_dbt_factory.Utils import generate_task_key + + +class DbtFactory: + """A factory for generating Databricks job definitions from dbt manifests.""" + + def __init__( + self, + file_handler: SpecsHandler, + task_factories: dict[str, TaskFactory], + bundle_tests: bool = False, + ): + """ + Initializes the dbt factory. + + Args: + file_handler (SpecsHandler): Handles reading the dbt manifest and writing the job spec. + task_factories (dict[str, TaskFactory]): Maps dbt resource types (`model`, `seed`, + `snapshot`, `test`) to their respective `TaskFactory` instances. Omitting `test` + disables test-task generation entirely. + bundle_tests (bool): When True, emit one `tests_` task per tested resource + and rewire downstream models/seeds/snapshots to depend on the upstream's + `tests_` task so failing tests halt the DAG. When False, emit one task + per dbt test node. + """ + self.file_handler = file_handler + self.task_factories = task_factories + self.bundle_tests = bundle_tests + + def create_tasks_and_update_job_spec( + self, + dbt_manifest_path: str, + input_job_spec_path: str, + target_job_spec_path: str, + new_job_name: str | None = None, + dry_run: bool = False, + ): + """ + Generates tasks from a dbt manifest and writes them into a Databricks job spec. + + Args: + dbt_manifest_path (str): Path to the dbt `manifest.json`. + input_job_spec_path (str): Path to the input (template) job spec YAML. + target_job_spec_path (str): Path to write the updated job spec YAML. + new_job_name (str | None): Optional replacement job name. If provided, overrides the + name in the input spec. + dry_run (bool): When True, print the generated tasks instead of writing to disk. + """ + manifest = self.file_handler.read_dbt_manifest(dbt_manifest_path) + tasks = self.create_tasks(manifest) + if dry_run: + print(tasks) + else: + self.file_handler.replace_tasks_in_job_spec( + input_job_spec_path, tasks, target_job_spec_path, new_job_name + ) + + def create_tasks(self, dbt_manifest: dict) -> list[dict]: + """ + Generates the Databricks task dictionaries from a dbt manifest. + + Args: + dbt_manifest (dict): Parsed dbt manifest content. + + Returns: + list[dict]: Task dictionaries ready to be injected into the `tasks` list of a + Databricks job spec. + """ + tasks = self._create_tasks(dbt_manifest) + return [task.to_dict() for task in tasks] + + _GATEABLE_TYPES = frozenset({"model", "seed", "snapshot"}) + _DBT_TEST_TARGET_PREFIXES = ("model.", "seed.", "snapshot.", "source.") + + def _create_tasks(self, dbt_manifest: dict) -> list[DbtTask]: + """ + Builds `DbtTask` instances from the manifest, applying the bundling and gating policies. + + Args: + dbt_manifest (dict): Parsed dbt manifest content. + + Returns: + list[DbtTask]: `DbtTask` instances (not yet rendered to dicts). + """ + dbt_nodes = dbt_manifest.get("nodes", {}) + dbt_sources = dbt_manifest.get("sources", {}) + + bundle = "test" in self.task_factories and self.bundle_tests + single_model_tested: set[str] = set() + standalone_tests: list[tuple[str, dict]] = [] + tests_by_resource: dict[str, list[tuple[str, frozenset[str]]]] = {} + ancestors: dict[str, set[str]] = {} + if bundle: + single_model_tested, standalone_tests = self._classify_tests( + dbt_nodes, dbt_sources + ) + elif "test" in self.task_factories: + tests_by_resource = self._index_tests_by_resource(dbt_nodes, dbt_sources) + ancestors = self._compute_ancestors(dbt_nodes, dbt_sources) + task_keys_with_tests = {generate_task_key(fn) for fn in single_model_tested} + + tasks = self._build_resource_tasks( + dbt_nodes, bundle, task_keys_with_tests, tests_by_resource, ancestors + ) + + if bundle: + tasks.extend( + self._build_bundled_test_tasks( + dbt_nodes, dbt_sources, single_model_tested + ) + ) + tasks.extend(self._build_standalone_test_tasks(standalone_tests)) + + return tasks + + def _compute_ancestors( + self, dbt_nodes: dict, dbt_sources: dict + ) -> dict[str, set[str]]: + """ + Maps each testable resource's full name to the set of resources it transitively depends + on (not including itself). Used in per-test mode to decide whether a test can safely + gate a downstream node: a test `T` with refs `R` is only safe to add to node `N`'s + deps if `R ⊆ ancestors(N)` — i.e. `N` already waits for all of `T`'s endpoints, + transitively. Otherwise adding `T` would create a cycle (since `T` depends on each + ref, and some ref might depend on `N`). + """ + ancestors: dict[str, set[str]] = {} + + def visit(full_name: str) -> set[str]: + cached = ancestors.get(full_name) + if cached is not None: + return cached + result: set[str] = set() + info = dbt_nodes.get(full_name) or dbt_sources.get(full_name) + if info is not None: + for dep in info.get("depends_on", {}).get("nodes", []): + if dep in dbt_nodes or dep in dbt_sources: + result.add(dep) + result.update(visit(dep)) + ancestors[full_name] = result + return result + + for full_name in list(dbt_nodes.keys()) + list(dbt_sources.keys()): + visit(full_name) + return ancestors + + def _index_tests_by_resource( + self, dbt_nodes: dict, dbt_sources: dict + ) -> dict[str, list[tuple[str, frozenset[str]]]]: + """ + Maps each testable resource's full name to a list of (test_task_key, test_refs) pairs + for tests whose `severity` is `error` (the default). Warn-severity tests still run but + are NOT indexed here, so they do not appear in any downstream model's `depends_on` — + their job is to surface findings, not halt the DAG. This matches `dbt build` semantics: + dbt itself exits 0 on warn-severity failures, so even if we did gate on them the + Databricks task would succeed and downstream would run; keeping warn tests out of the + dep graph just avoids the extra DAG clutter. + + The refs set is carried alongside each test so `_extend_deps_with_upstream_tests` can + avoid cycles: a test with refs that aren't all ancestors of a candidate node would + create a cycle if added as that node's dep. + """ + index: dict[str, list[tuple[str, frozenset[str]]]] = {} + for node_full_name, node_info in dbt_nodes.items(): + if node_info["resource_type"] != "test": + continue + if self._test_severity(node_info) != "error": + continue + test_task_key = generate_task_key(node_full_name) + refs: set[str] = set() + for dep in node_info.get("depends_on", {}).get("nodes", []): + if dep.startswith(self._DBT_TEST_TARGET_PREFIXES) and ( + dep in dbt_nodes or dep in dbt_sources + ): + refs.add(dep) + frozen_refs = frozenset(refs) + for resource_full in refs: + index.setdefault(resource_full, []).append((test_task_key, frozen_refs)) + return index + + @staticmethod + def _test_severity(test_node_info: dict) -> str: + """Reads the test's severity from the manifest, defaulting to `error` when unset.""" + config = test_node_info.get("config") or {} + severity = config.get("severity") + if isinstance(severity, str): + return severity.lower() + return "error" + + @staticmethod + def _extend_deps_with_upstream_tests( + node_full_name: str, + existing_deps: list[str] | None, + tests_by_resource: dict[str, list[tuple[str, frozenset[str]]]], + ancestors_by_node: dict[str, set[str]], + ) -> list[str]: + """ + Appends task keys of tests that safely gate this node — i.e. tests whose refs are all + ancestors of the current node. This prevents both direct and transitive cycles: a test + `T` with refs `R` is added to node `N`'s deps only if `N` transitively depends on every + resource in `R`. If any ref of `T` is downstream of (or equal to) `N`, adding `T` would + cycle because `T` already depends on that ref, and the ref depends on `N`. + """ + extended: list[str] = list(existing_deps or []) + seen = set(extended) + node_ancestors = ancestors_by_node.get(node_full_name, set()) + for ancestor in node_ancestors: + for test_key, test_refs in tests_by_resource.get(ancestor, []): + if test_key in seen: + continue + if test_refs <= node_ancestors: + extended.append(test_key) + seen.add(test_key) + return extended + + def _classify_tests( + self, dbt_nodes: dict, dbt_sources: dict + ) -> tuple[set[str], list[tuple[str, dict]]]: + """ + Classifies test nodes for bundled mode so that no test is silently dropped. + + - Tests with exactly 1 testable dep: will be covered by their resource's bundled + `tests_` task under `--indirect-selection cautious`. + - Tests with >1 testable deps (cross-model, e.g. `relationships`): emitted as their own + tasks with multi-resource deps — `cautious` filters them out of bundles. + - Tests with 0 testable deps (singular/custom tests that don't `ref()` or `source()` + any resource): also emitted as their own tasks, since no bundle would pick them up. + + Returns: + (single_model_tested, standalone_tests): + - `single_model_tested`: full names of resources with at least one single-model + test — these become `tests_` bundled tasks. + - `standalone_tests`: list of `(test_full_name, test_node_info)` for tests + that must run as individual tasks (cross-model or zero-dep). + """ + single_model_tested: set[str] = set() + standalone_tests: list[tuple[str, dict]] = [] + for node_full_name, node_info in dbt_nodes.items(): + if node_info["resource_type"] != "test": + continue + testable_deps: list[str] = [] + for dep in node_info.get("depends_on", {}).get("nodes", []): + if dep.startswith(self._DBT_TEST_TARGET_PREFIXES) and ( + dep in dbt_nodes or dep in dbt_sources + ): + testable_deps.append(dep) + if len(testable_deps) == 1: + single_model_tested.add(testable_deps[0]) + else: + standalone_tests.append((node_full_name, node_info)) + return single_model_tested, standalone_tests + + def _build_resource_tasks( + self, + dbt_nodes: dict, + bundle: bool, + task_keys_with_tests: set[str], + tests_by_resource: dict[str, list[tuple[str, frozenset[str]]]], + ancestors_by_node: dict[str, set[str]], + ) -> list[DbtTask]: + """Builds tasks for every non-test resource (plus per-test tasks when not bundling).""" + tasks: list[DbtTask] = [] + for node_full_name, node_info in dbt_nodes.items(): + resource_type = node_info["resource_type"] + if resource_type not in self.task_factories: + continue + if bundle and resource_type == "test": + continue + + task_key = generate_task_key(node_full_name) + factory = self.task_factories[resource_type] + task = factory.create_task(node_info["name"], node_info, task_key) + + if resource_type in self._GATEABLE_TYPES: + if bundle: + task = replace( + task, + depends_on=self._rewire_deps( + task.depends_on, task_keys_with_tests + ), + ) + elif tests_by_resource: + task = replace( + task, + depends_on=self._extend_deps_with_upstream_tests( + node_full_name, + task.depends_on, + tests_by_resource, + ancestors_by_node, + ), + ) + + tasks.append(task) + return tasks + + @staticmethod + def _rewire_deps( + deps: list[str] | None, task_keys_with_tests: set[str] + ) -> list[str]: + """Rewrites dependencies that point at a tested resource to its `tests_` gating task.""" + rewired: list[str] = [] + for dep_key in deps or []: + rewired.append( + f"tests_{dep_key}" if dep_key in task_keys_with_tests else dep_key + ) + return rewired + + def _build_bundled_test_tasks( + self, + dbt_nodes: dict, + dbt_sources: dict, + nodes_with_tests: set[str], + ) -> list[DbtTask]: + """Emits one `tests_` task per tested resource using `TestTaskFactory.create_bundled_task`.""" + test_factory = self.task_factories["test"] + tasks: list[DbtTask] = [] + for full_name in sorted(nodes_with_tests): + is_source = full_name.startswith("source.") + info = dbt_sources[full_name] if is_source else dbt_nodes[full_name] + resource_task_key = generate_task_key(full_name) + bare_name = info["name"] + qualified = f"{info['package_name']}.{bare_name}" + select = ( + f"source:{info['package_name']}.{info['source_name']}.{bare_name}" + if is_source + else qualified + ) + tasks.append( + test_factory.create_bundled_task( + task_key=f"tests_{resource_task_key}", + select=select, + deps_command_name=bare_name, + depends_on=[] if is_source else [resource_task_key], + ) + ) + return tasks + + def _build_standalone_test_tasks( + self, + standalone_tests: list[tuple[str, dict]], + ) -> list[DbtTask]: + """ + Emits one task per standalone test — cross-model tests (e.g. `relationships`) gated on + every referenced resource, plus any zero-dep singular tests that bundles can't cover. + """ + test_factory = self.task_factories["test"] + tasks: list[DbtTask] = [] + for test_full_name, test_info in sorted( + standalone_tests, key=lambda item: item[0] + ): + test_task_key = generate_task_key(test_full_name) + tasks.append( + test_factory.create_task(test_info["name"], test_info, test_task_key) + ) + return tasks diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/DbtTask.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/DbtTask.py new file mode 100644 index 00000000..c0523720 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/DbtTask.py @@ -0,0 +1,150 @@ +import json +from dataclasses import dataclass, field +from enum import Enum +from typing import Any + + +class TaskType(Enum): + """Supported task types for generated Databricks tasks.""" + + DBT = "dbt" + NOTEBOOK = "notebook" + + +@dataclass(frozen=True) +class DbtTaskOptions: + environment_key: str = "Default" # serverless env + """The key of an environment. It has to be unique within a job.""" + + catalog: str | None = None + """Optional name of the catalog to use. The value is the top level in the 3-level namespace of + Unity Catalog (catalog / schema / relation). The catalog value can only be specified if a + warehouse_id is specified. Requires dbt-databricks >= 1.1.1.""" + + profiles_directory: str | None = None + """Optional (relative) path to the profiles directory. Can only be specified if no warehouse_id is + specified. If no warehouse_id is specified and this folder is unset, the root directory is used.""" + + project_directory: str | None = None + """Path to the project directory. Optional for Git sourced tasks, in which case if no value is + provided, the root of the Git repository is used.""" + + schema: str | None = None + """Optional schema to write to. This parameter is only used when a warehouse_id is also provided. + If not provided, the `default` schema is used.""" + + source: str | None = None + """Optional location type of the project directory. When set to `WORKSPACE`, the project will be + retrieved from the local Databricks workspace. When set to `GIT`, the project will be retrieved + from a Git repository defined in `git_source`. If the value is empty, the task will use `GIT` if + `git_source` is defined and `WORKSPACE` otherwise. + + * `WORKSPACE`: Project is located in Databricks workspace. * `GIT`: Project is located in cloud + Git provider.""" + + warehouse_id: str | None = None + """ID of the SQL warehouse to connect to. If provided, we automatically generate and provide the + profile and connection details to dbt. It can be overridden on a per-command basis by using the + `--profiles-dir` command line argument.""" + + dbt_deps_enabled: bool = False + """Optional flag to enable dbt deps to be run before each task. Defaults to False.""" + + dbt_tasks_deps: list[str] = field(default_factory=list) + """Optional comma separated list of tasks that requires dbt debs. Only in effect if dbt_deps_enabled is enabled.""" + + task_type: TaskType = TaskType.DBT + """Task type to generate: `TaskType.DBT` for native dbt_task, `TaskType.NOTEBOOK` for notebook_task + wrapper. Strings are accepted and coerced — any value outside the enum raises `ValueError`. + Notebook mode enables base environment support and environment variables on serverless.""" + + notebook_path: str | None = None + """Path to the dbt runner notebook. Required when task_type is `TaskType.NOTEBOOK`.""" + + job_cluster_key: str | None = None + """Job cluster key for running tasks on job compute instead of serverless.""" + + def __post_init__(self): + if not isinstance(self.task_type, TaskType): + object.__setattr__(self, "task_type", TaskType(self.task_type)) + if self.task_type is TaskType.NOTEBOOK: + unsupported = [] + for name, value in ( + ("warehouse_id", self.warehouse_id), + ("schema", self.schema), + ("catalog", self.catalog), + ): + if value: + unsupported.append(name) + if unsupported: + raise ValueError( + f"{', '.join(unsupported)} cannot be set with task_type=NOTEBOOK; " + "notebook tasks connect via profiles.yml." + ) + + +@dataclass(frozen=True) +class DbtTask: + """Represents a dbt task in the Databricks job definition.""" + + task_key: str + commands: list[str] + options: DbtTaskOptions + depends_on: list[str] | None = None + + def to_dict(self) -> dict: + """Converts the Task to a dictionary suitable for the job definition.""" + if self.options.task_type is TaskType.NOTEBOOK: + return self._to_notebook_dict() + return self._to_dbt_dict() + + def _base_spec(self) -> dict[str, Any]: + spec: dict[str, Any] = { + "task_key": self.task_key, + "depends_on": [{"task_key": dep} for dep in (self.depends_on or [])], + } + if self.options.job_cluster_key: + spec["job_cluster_key"] = self.options.job_cluster_key + else: + spec["environment_key"] = self.options.environment_key + return spec + + def _to_dbt_dict(self) -> dict[str, Any]: + spec = self._base_spec() + dbt_task: dict[str, Any] = {"commands": self.commands} + + if self.options.source: + dbt_task["source"] = self.options.source + if self.options.project_directory: + dbt_task["project_directory"] = self.options.project_directory + if self.options.schema: + dbt_task["schema"] = self.options.schema + if self.options.warehouse_id: + dbt_task["warehouse_id"] = self.options.warehouse_id + if self.options.catalog: + dbt_task["catalog"] = self.options.catalog + if self.options.profiles_directory: + dbt_task["profiles_directory"] = self.options.profiles_directory + + spec["dbt_task"] = dbt_task + return spec + + def _to_notebook_dict(self) -> dict[str, Any]: + base_parameters: dict[str, str] = { + "dbt_commands": json.dumps(self.commands), + } + if self.options.project_directory: + base_parameters["project_directory"] = self.options.project_directory + if self.options.profiles_directory: + base_parameters["profiles_directory"] = self.options.profiles_directory + + notebook_task: dict[str, Any] = { + "notebook_path": self.options.notebook_path, + "base_parameters": base_parameters, + } + if self.options.source: + notebook_task["source"] = self.options.source + + spec = self._base_spec() + spec["notebook_task"] = notebook_task + return spec diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/SpecsHandler.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/SpecsHandler.py new file mode 100644 index 00000000..c3a6bc8e --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/SpecsHandler.py @@ -0,0 +1,72 @@ +import json +import yaml + + +class SpecsHandler: + """Handles reading and writing files for dbt manifests and databricks job definitions.""" + + @staticmethod + def read_dbt_manifest(path: str) -> dict: + """ + Reads a JSON manifest file and returns its content as a dictionary. + + Args: + path (str): Path to the manifest file. + + Returns: + dict: Parsed content of the manifest file. + + Raises: + FileNotFoundError: If the file does not exist. + ValueError: If the file is not a valid manifest file. + """ + try: + with open(path, "r", encoding="utf-8") as file: + return json.load(file) + except FileNotFoundError as e: + raise FileNotFoundError( + f"Manifest file not found: {path}. Details: {e}" + ) from e + except json.JSONDecodeError as e: + raise ValueError( + f"Error parsing JSON from manifest file: {path}. Details: {e}" + ) from e + + @staticmethod + def replace_tasks_in_job_spec( + input_job_spec_path: str, + new_tasks: list[dict], + target_job_spec_path: str, + new_job_name: str | None = None, + ) -> None: + """Replace the tasks field in a Databricks job definition YAML file. The first job is only updated. + Args: + input_job_spec_path (str): Path to the job definition YAML file. + new_tasks (dict): New tasks to replace the existing tasks in the job definition file. + target_job_spec_path (str): Path to save the updated job definition file. + new_job_name (str, optional): The name of the job to update. Defaults to None. + + Raises: + KeyError: If no jobs are found in the provided YAML file. + """ + with open(input_job_spec_path, "r", encoding="utf-8") as file: + job_definition = yaml.safe_load(file) + + jobs = job_definition.get("resources", {}).get("jobs", {}) + + if jobs is None: + raise KeyError("No jobs found in the provided YAML file.") + + # replaces the first job only! + first_job_key = next(iter(jobs)) + if new_job_name: + jobs[new_job_name] = jobs.pop(first_job_key) + first_job_key = new_job_name + + first_job = jobs[first_job_key] + if new_job_name: + first_job["name"] = new_job_name + first_job["tasks"] = new_tasks # Replace tasks field + + with open(target_job_spec_path, "w", encoding="utf-8") as file: + yaml.dump(job_definition, file, sort_keys=False, width=1000) diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/TaskFactory.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/TaskFactory.py new file mode 100644 index 00000000..fe612941 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/TaskFactory.py @@ -0,0 +1,249 @@ +from abc import ABC, abstractmethod +from enum import Enum + +from databricks_dbt_factory.DbtTask import DbtTask, DbtTaskOptions +from databricks_dbt_factory.Utils import generate_task_key + + +class DbtNodeTypes(Enum): + """Enum class to represent dbt node types.""" + + MODEL = "model" + TEST = "test" + SEED = "seed" + SNAPSHOT = "snapshot" + + +class DbtDependencyResolver: + @staticmethod + def resolve(node_info: dict, valid_deps_types: list[str]) -> list[str]: + """ + Resolves dependencies for a given DBT node. + + Args: + node_info (dict): Information about the DBT node. + valid_deps_types (list[str]): List of valid DBT dependency types for the node. + + Returns: + list[str]: List of resolved dependencies. + """ + deps = node_info.get("depends_on", {}).get("nodes", []) + resolved_deps = [] + for node_full_name in deps: + if any( + node_full_name.startswith(dbt_type + ".") + for dbt_type in valid_deps_types + ): + resolved_deps.append(generate_task_key(node_full_name)) + return resolved_deps + + +class TaskFactory(ABC): + """Abstract base class for creating tasks.""" + + def __init__( + self, + resolver: DbtDependencyResolver, + task_options: DbtTaskOptions, + dbt_options: str = "", + ): + """ + Initializes the TaskFactory. + + Args: + resolver (DbtDependencyResolver): An instance of DbtDependencyResolver to resolve dependencies. + task_options (DbtTaskOptions): Options for the task. + dbt_options (str, optional): Additional DBT options. Defaults to "". + """ + self.resolver = resolver + self.task_options = task_options + self.dbt_options = dbt_options + + @abstractmethod + def create_task( + self, dbt_node_name: str, dbt_node_info: dict, task_key: str + ) -> DbtTask: + """ + Abstract method to create a task. + + Args: + dbt_node_name (str): Name of the DBT node. + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + + Returns: + DbtTask: An instance of Task. + """ + + def get_dbt_deps_command(self, dbt_task_name: str) -> str | None: + """Adds the dbt deps command if enabled and applicable. + Only return the command if enabled, and available in the task deps or not specific tasks provided. + + Args: + dbt_task_name (str): Name of the DBT task. + """ + if self.task_options.dbt_deps_enabled and ( + not self.task_options.dbt_tasks_deps + or dbt_task_name in self.task_options.dbt_tasks_deps + ): + return f"dbt deps {self.dbt_options}" + return None + + +class ModelTaskFactory(TaskFactory): + """Factory for creating model tasks.""" + + def create_task( + self, dbt_node_name: str, dbt_node_info: dict, task_key: str + ) -> DbtTask: + """ + Creates a model task. + + Args: + dbt_node_name (str): Name of the DBT node. + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + + Returns: + DbtTask: An instance of Task. + """ + valid_dbt_deps_types: list[str] = [ + DbtNodeTypes.MODEL.value, + DbtNodeTypes.SEED.value, + DbtNodeTypes.SNAPSHOT.value, + DbtNodeTypes.TEST.value, + ] + depends_on = self.resolver.resolve(dbt_node_info, valid_dbt_deps_types) + + dbt_deps = self.get_dbt_deps_command(dbt_node_name) + commands = [dbt_deps] if dbt_deps else [] + commands.append( + f"dbt run --select {dbt_node_name}" + + (f" {self.dbt_options}" if self.dbt_options else "") + ) + + return DbtTask(task_key, commands, self.task_options, depends_on) + + +class SnapshotTaskFactory(TaskFactory): + """Factory for creating snapshot tasks.""" + + def create_task( + self, dbt_node_name: str, dbt_node_info: dict, task_key: str + ) -> DbtTask: + """ + Creates a snapshot task. + + Args: + dbt_node_name (str): Name of the DBT node. + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + + Returns: + DbtTask: An instance of Task. + """ + valid_dbt_deps_types: list[str] = [DbtNodeTypes.MODEL.value] + depends_on = self.resolver.resolve(dbt_node_info, valid_dbt_deps_types) + + dbt_deps = self.get_dbt_deps_command(dbt_node_name) + commands = [dbt_deps] if dbt_deps else [] + commands.append( + f"dbt snapshot --select {dbt_node_name}" + + (f" {self.dbt_options}" if self.dbt_options else "") + ) + + return DbtTask(task_key, commands, self.task_options, depends_on) + + +class SeedTaskFactory(TaskFactory): + """Factory for creating seed tasks.""" + + def create_task( + self, dbt_node_name: str, dbt_node_info: dict, task_key: str + ) -> DbtTask: + """ + Creates a seed task. + + Args: + dbt_node_name (str): Name of the DBT node. + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + + Returns: + DbtTask: An instance of Task. + """ + valid_dbt_deps_types: list[str] = [] # Seeds don't have dependencies + + depends_on = self.resolver.resolve(dbt_node_info, valid_dbt_deps_types) + + dbt_deps = self.get_dbt_deps_command(dbt_node_name) + commands = [dbt_deps] if dbt_deps else [] + commands.append( + f"dbt seed --select {dbt_node_name}" + + (f" {self.dbt_options}" if self.dbt_options else "") + ) + + return DbtTask(task_key, commands, self.task_options, depends_on) + + +class TestTaskFactory(TaskFactory): + """Factory for creating test tasks.""" + + def create_task( + self, dbt_node_name: str, dbt_node_info: dict, task_key: str + ) -> DbtTask: + """ + Creates a test task for a single dbt test node. + + Args: + dbt_node_name (str): Name of the DBT node. + dbt_node_info (dict): Information about the DBT node. + task_key (str): Key for the task. + + Returns: + DbtTask: An instance of Task. + """ + valid_dbt_deps_types: list[str] = [ + DbtNodeTypes.MODEL.value, + DbtNodeTypes.SEED.value, + DbtNodeTypes.SNAPSHOT.value, + ] + + depends_on = self.resolver.resolve(dbt_node_info, valid_dbt_deps_types) + + dbt_deps = self.get_dbt_deps_command(dbt_node_name) + commands = [dbt_deps] if dbt_deps else [] + commands.append( + f"dbt test --select {dbt_node_name}" + + (f" {self.dbt_options}" if self.dbt_options else "") + ) + + return DbtTask(task_key, commands, self.task_options, depends_on) + + def create_bundled_task( + self, task_key: str, select: str, deps_command_name: str, depends_on: list[str] + ) -> DbtTask: + """ + Creates a single test task that runs the single-model tests for a given resource via + `dbt test --select --indirect-selection cautious`. The cautious selector + ensures only tests whose referenced resources are entirely within this bundle are + included; cross-model tests (e.g. `relationships`) are excluded and handled separately. + + Args: + task_key (str): Key for the bundled task. + select (str): Pre-computed dbt `--select` argument (qualified model name, or + `source:..` for sources). + deps_command_name (str): Name used by `get_dbt_deps_command` to decide whether to prepend `dbt deps`. + depends_on (list[str]): Upstream task keys this bundled task should gate on. + + Returns: + DbtTask: An instance of Task. + """ + dbt_deps = self.get_dbt_deps_command(deps_command_name) + commands = [dbt_deps] if dbt_deps else [] + commands.append( + f"dbt test --select {select} --indirect-selection cautious" + + (f" {self.dbt_options}" if self.dbt_options else "") + ) + + return DbtTask(task_key, commands, self.task_options, depends_on) diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/Utils.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/Utils.py new file mode 100644 index 00000000..a29cffdd --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/Utils.py @@ -0,0 +1,11 @@ +def generate_task_key(dbt_node_full_name: str) -> str: + """ + Generates a task key from a dbt node's fully qualified name by replacing dots with underscores. + + Args: + dbt_node_full_name (str): Fully qualified dbt node name, e.g. `model.my_project.customers`. + + Returns: + str: The generated task key (e.g. `model_my_project_customers`). + """ + return dbt_node_full_name.replace(".", "_") diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/__about__.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/__about__.py new file mode 100644 index 00000000..3ced3581 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/__about__.py @@ -0,0 +1 @@ +__version__ = "0.2.1" diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/__init__.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/main.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/main.py new file mode 100644 index 00000000..84422e0e --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/main.py @@ -0,0 +1,314 @@ +import argparse +import os +import shutil +from importlib import resources +from pathlib import Path + +from databricks_dbt_factory.DbtFactory import DbtFactory +from databricks_dbt_factory.SpecsHandler import SpecsHandler +from databricks_dbt_factory.DbtTask import DbtTaskOptions +from databricks_dbt_factory.TaskFactory import ( + ModelTaskFactory, + SnapshotTaskFactory, + SeedTaskFactory, + TestTaskFactory, + DbtDependencyResolver, +) + +_RUNNER_NOTEBOOK_FILENAME = "run_dbt_command.py" + + +def main(): + args = parse_args() + + file_handler = SpecsHandler() + resolver = DbtDependencyResolver() + + dbt_options = build_dbt_options(args) + dbt_tasks_deps = ( + [item.strip() for item in args.dbt_tasks_deps.split(",") if item.strip()] + if args.dbt_tasks_deps + else [] + ) + + notebook_path = args.notebook_path + effective_project_directory = args.project_directory + if args.task_type == "notebook" and notebook_path is None: + notebook_path, notebook_at_project_root = _copy_runner_notebook( + args.target_job_spec_path, args.project_directory + ) + # If the runner landed at the project root, CWD at task runtime already equals the + # project root. Pass `.` explicitly so the generated spec is self-documenting (and + # immune to any future change in dbt's default); the user's original `../` would + # resolve one level too high and is no longer correct. + if notebook_at_project_root: + effective_project_directory = "." + + task_options = DbtTaskOptions( + environment_key=args.environment_key + if args.environment_key is not None + else "Default", + warehouse_id=args.warehouse_id, + catalog=args.catalog, + schema=args.schema, + profiles_directory=args.profiles_directory, + project_directory=effective_project_directory, + source=args.source, + dbt_deps_enabled=args.enable_dbt_deps, + dbt_tasks_deps=dbt_tasks_deps, + task_type=args.task_type, + notebook_path=notebook_path, + job_cluster_key=args.job_cluster_key, + ) + task_factories = { + "model": ModelTaskFactory(resolver, task_options, dbt_options), + "snapshot": SnapshotTaskFactory(resolver, task_options, dbt_options), + "seed": SeedTaskFactory(resolver, task_options, dbt_options), + } + if args.run_tests: + task_factories["test"] = TestTaskFactory(resolver, task_options, dbt_options) + + factory = DbtFactory(file_handler, task_factories, bundle_tests=args.bundle_tests) + factory.create_tasks_and_update_job_spec( + args.dbt_manifest_path, + args.input_job_spec_path, + args.target_job_spec_path, + args.new_job_name, + args.dry_run, + ) + + +def _copy_runner_notebook( + target_job_spec_path: str, project_directory: str | None +) -> tuple[str, bool]: + """ + Copies the packaged dbt runner notebook into the bundle so `databricks bundle deploy` + uploads it automatically. + + When `project_directory` is a relative path (the common case, e.g. `../` when the spec + lives in a subdirectory), the notebook is placed at the computed project root. This way + the notebook sits next to `dbt_project.yml` / `profiles.yml`, and at task runtime CWD = + project root — dbt finds everything without any path gymnastics in the runner. + + When `project_directory` is absolute (typical for `--source WORKSPACE` with a pinned + workspace path we can't write to from local CLI) or missing, falls back to copying the + notebook next to the generated job spec. + + Returns `(notebook_path, notebook_at_project_root)`: + - `notebook_path`: relative path from the spec's directory to the copied notebook, + which DAB resolves at deploy time. + - `notebook_at_project_root`: True when the runner landed at the computed project root + (so the caller knows CWD = project root at runtime and can drop `--project-dir`). + + Overwrites any existing file at the destination. + """ + source = ( + resources.files("databricks_dbt_factory") + / "notebook" + / _RUNNER_NOTEBOOK_FILENAME + ) + spec_dir = Path(target_job_spec_path).resolve().parent + + if project_directory and not Path(project_directory).is_absolute(): + notebook_at_project_root = True + dest_dir = (spec_dir / project_directory).resolve() + else: + notebook_at_project_root = False + dest_dir = spec_dir + + dest = dest_dir / _RUNNER_NOTEBOOK_FILENAME + dest_dir.mkdir(parents=True, exist_ok=True) + with resources.as_file(source) as src_path: + shutil.copyfile(src_path, dest) + + relative = Path(os.path.relpath(dest, start=spec_dir)).as_posix() + notebook_path = relative if relative.startswith("..") else f"./{relative}" + return notebook_path, notebook_at_project_root + + +def build_dbt_options(args): + """Builds the dbt command options based on the provided arguments.""" + dbt_options = "" + + if args.target: + dbt_options += f"--target {args.target}" + + if args.extra_dbt_command_options: + dbt_options += f" {args.extra_dbt_command_options}" + + return dbt_options + + +def parse_args(): + parser = argparse.ArgumentParser( + description="Generate Databricks job definition from dbt manifest." + ) + parser.add_argument( + "--new-job-name", + type=str, + help="Optional job name. If provided the existing job name in job spec is updated", + required=False, + default=None, + ) + parser.add_argument( + "--dbt-manifest-path", type=str, help="Path to the manifest file", required=True + ) + parser.add_argument( + "--input-job-spec-path", + type=str, + help="Path to the input job spec file", + required=True, + ) + parser.add_argument( + "--target-job-spec-path", + type=str, + help="Path to the target job spec file.", + required=True, + ) + parser.add_argument( + "--target", type=str, help="Optional dbt target to use.", required=False + ) + parser.add_argument( + "--source", + type=str, + help="Optional project source. If not provided WORKSPACE will be used.", + required=False, + default=None, + ) + parser.add_argument( + "--warehouse_id", + type=str, + help="Optional SQL Warehouse to run dbt models on", + required=False, + default=None, + ) + parser.add_argument( + "--schema", + type=str, + help="Optional schema to write to.", + required=False, + default=None, + ) + parser.add_argument( + "--catalog", + type=str, + help="Optional catalog to write to.", + required=False, + default=None, + ) + parser.add_argument( + "--profiles-directory", + type=str, + help="Optional (relative) path to the profiles directory.", + required=False, + default=None, + ) + parser.add_argument( + "--project-directory", + type=str, + help="Optional (relative) path to the project directory.", + required=False, + default=None, + ) + parser.add_argument( + "--environment-key", + type=str, + help="Optional (relative) key of an environment. Defaults to 'Default' when unset.", + required=False, + default=None, + ) + parser.add_argument( + "--extra-dbt-command-options", + type=str, + help="Optional additional dbt command options", + required=False, + default="", + ) + parser.add_argument( + "--no-run-tests", + action="store_false", + dest="run_tests", + help="Skip generating dbt test tasks. Tests are included by default.", + ) + parser.add_argument( + "--bundle-tests", + action="store_true", + help=( + "Bundle single-model tests for a given resource into one " + "`dbt test --select . --indirect-selection cautious` task (default: " + "one task per test node). Cross-model tests (e.g. `relationships`) are detected " + "from the manifest and emitted as their own tasks gated on every referenced " + "resource, so no tests are silently dropped. Trade-off: fewer tasks and a smaller " + "DAG, but per-test failures show up as a single red `_tests` task — drill " + "into the logs to see which assertion failed." + ), + ) + parser.add_argument( + "--enable-dbt-deps", + action="store_true", + help="Run `dbt deps` before each task.", + ) + parser.add_argument( + "--dbt-tasks-deps", + type=str, + help="Optional list of tasks that require dbt deps. Only in effect if `--enable-dbt-deps` is enabled.", + required=False, + default=None, + ) + parser.add_argument( + "--task-type", + type=str, + help="Task type to generate: 'dbt' for native dbt_task (default), 'notebook' for notebook_task wrapper.", + required=False, + default="dbt", + choices=["dbt", "notebook"], + ) + parser.add_argument( + "--notebook-path", + type=str, + help=( + "Path to the dbt runner notebook (used when --task-type is 'notebook'). If omitted, " + "the factory copies the packaged runner notebook next to the generated job spec and " + "references it relatively, so `databricks bundle deploy` uploads it automatically. " + "Pass an explicit path to pin the notebook elsewhere and manage it yourself." + ), + required=False, + default=None, + ) + parser.add_argument( + "--job-cluster-key", + type=str, + help="Job cluster key for running tasks on job compute instead of serverless. Mutually exclusive with --environment-key.", + required=False, + default=None, + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print generated tasks without updating the job spec file.", + ) + args = parser.parse_args() + + if args.job_cluster_key and args.environment_key is not None: + parser.error("--job-cluster-key and --environment-key are mutually exclusive") + + if args.task_type == "notebook": + conflicting = [] + for flag, value in ( + ("--warehouse_id", args.warehouse_id), + ("--schema", args.schema), + ("--catalog", args.catalog), + ): + if value: + conflicting.append(flag) + if conflicting: + parser.error( + f"{', '.join(conflicting)} cannot be used with --task-type notebook; " + "notebook tasks connect via profiles.yml." + ) + + return args + + +if __name__ == "__main__": + main() diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/notebook/run_dbt_command.py b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/notebook/run_dbt_command.py new file mode 100644 index 00000000..4e250b66 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/databricks_dbt_factory/notebook/run_dbt_command.py @@ -0,0 +1,110 @@ +# Databricks notebook source + +import json +import os +import shlex +import shutil +import tempfile + +from dbt.cli.main import dbtRunner + +# COMMAND ---------- + +dbutils.widgets.text("dbt_commands", "") +dbutils.widgets.text("project_directory", "") +dbutils.widgets.text("profiles_directory", "") + +dbt_commands = dbutils.widgets.get("dbt_commands") +project_directory = dbutils.widgets.get("project_directory") +profiles_directory = dbutils.widgets.get("profiles_directory") + +if not dbt_commands: + raise ValueError("dbt_commands parameter is required") + +# COMMAND ---------- + +ctx = dbutils.notebook.entry_point.getDbutils().notebook().getContext() +os.environ["DBT_ACCESS_TOKEN"] = ctx.apiToken().get() +os.environ["DBT_HOST"] = ctx.apiUrl().get() + +# chdir to the dbt project so dbt runs from inside it. Relative `project_directory` is +# resolved against this notebook's own workspace location — the same anchor native +# `dbt_task` uses. Auto-copy mode sends `.` (resolves to the notebook's dir, which is +# project root by construction); user-pinned `--notebook-path` with relative +# `--project-directory` resolves against wherever the user placed the notebook; absolute +# `project_directory` is used as-is. +if project_directory: + notebook_dir = os.path.dirname("/Workspace" + ctx.notebookPath().get()) + target_dir = ( + project_directory + if os.path.isabs(project_directory) + else os.path.normpath(os.path.join(notebook_dir, project_directory)) + ) + os.chdir(target_dir) + +# dbt writes `logs/dbt.log` and `target/` inside CWD on every run. DAB sync only uploads +# files, not empty directories — pre-create them (idempotent). +os.makedirs("logs", exist_ok=True) +os.makedirs("target", exist_ok=True) + +# If a pre-built msgpack sits next to the project, deserialize it into a manifest and inject it into +# dbtRunner to skip dbt's parse phase (re-reading/hashing every file + DAG rebuild) on each task. Each +# task then writes artifacts to a private local dir (DBT_TARGET_PATH/DBT_LOG_PATH) to avoid contention +# on the shared workspace `target/`. Falls back to a normal parse if the msgpack is absent or unusable. +manifest = None +local_dir = None +prebuilt_manifest_path = os.path.join("target", "partial_parse.msgpack") +if os.path.exists(prebuilt_manifest_path): + try: + from dbt.contracts.graph.manifest import Manifest + + with open(prebuilt_manifest_path, "rb") as f: + manifest = Manifest.from_msgpack(f.read()) + manifest.build_flat_graph() + local_dir = tempfile.mkdtemp(prefix="dbt_local_") + os.environ["DBT_TARGET_PATH"] = local_dir + os.environ["DBT_LOG_PATH"] = local_dir + print( + f"[dbt-factory] injecting pre-built manifest from {prebuilt_manifest_path} (skipping dbt parse)" + ) + except Exception as e: + print( + f"[dbt-factory] manifest injection unavailable, falling back to dbt parse: {e}" + ) + manifest = None + +try: + runner = dbtRunner(manifest=manifest) + + for command_str in json.loads(dbt_commands): + command_str = command_str.strip() + if not command_str: + continue + + if command_str.startswith("dbt "): + command_str = command_str[4:] + + args = shlex.split(command_str) + + if profiles_directory: + args.extend(["--profiles-dir", profiles_directory]) + + print(f"Running: dbt {' '.join(args)}") + print("-" * 60) + + result = runner.invoke(args) + + if not result.success: + detail = result.exception or result.result or "(no further details)" + raise RuntimeError(f"dbt command failed: dbt {' '.join(args)}\n{detail}") + + print(f"Completed successfully: dbt {' '.join(args)}") +finally: + os.environ.pop("DBT_ACCESS_TOKEN", None) + os.environ.pop("DBT_HOST", None) + os.environ.pop("DBT_TARGET_PATH", None) + os.environ.pop("DBT_LOG_PATH", None) + # Remove the private per-task target/log dir; on reused (all-purpose) clusters these + # would otherwise accumulate under the system temp dir for the life of the cluster. + if local_dir: + shutil.rmtree(local_dir, ignore_errors=True) diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/macros/.gitkeep b/contrib/templates/dbt-factory/template/{{.project_name}}/src/macros/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/orders_daily.sql b/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/orders_daily.sql new file mode 100644 index 00000000..e32736ce --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/orders_daily.sql @@ -0,0 +1,17 @@ + +-- This model file defines a materialized view called 'orders_daily' +-- +-- Read more about materialized at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables +-- Current limitation: a "full refresh" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561. +{{ config(materialized = 'materialized_view') }} + +select order_date, count(*) AS number_of_orders + +from {{ ref('orders_raw') }} + +-- During development, only process a smaller range of data +{% if target.name != 'prod' %} +where order_date >= '2019-08-01' and order_date < '2019-09-01' +{% endif %} + +group by order_date diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/orders_raw.sql b/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/orders_raw.sql new file mode 100644 index 00000000..8faf8f38 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/orders_raw.sql @@ -0,0 +1,16 @@ +-- This model file defines a streaming table called 'orders_raw' +-- +-- The streaming table below ingests all JSON files in /databricks-datasets/retail-org/sales_orders/ +-- Read more about streaming tables at https://docs.getdbt.com/reference/resource-configs/databricks-configs#materialized-views-and-streaming-tables +-- Current limitation: a "full refresh" is needed in case the definition below is changed; see https://github.com/databricks/dbt-databricks/issues/561. +{{ config(materialized = 'streaming_table') }} + +select + customer_name, + date(timestamp(from_unixtime(try_cast(order_datetime as bigint)))) as order_date, + order_number +from stream read_files( + "/databricks-datasets/retail-org/sales_orders/", + format => "json", + header => true +) diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/schema.yml b/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/schema.yml new file mode 100644 index 00000000..c17c810e --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/src/models/example/schema.yml @@ -0,0 +1,22 @@ + +version: 2 + +models: + - name: orders_raw + description: "Raw ingested orders" + columns: + - name: customer_name + description: "The name of a customer" + # customer_name repeats across orders, so only a not_null test applies here. + data_tests: + - not_null + + - name: orders_daily + description: "Number of orders by day" + columns: + - name: order_date + description: "The date on which orders took place" + # orders_daily groups by order_date, so it is unique and not null. + data_tests: + - unique + - not_null diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/seeds/.gitkeep b/contrib/templates/dbt-factory/template/{{.project_name}}/src/seeds/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/snapshots/.gitkeep b/contrib/templates/dbt-factory/template/{{.project_name}}/src/snapshots/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/src/tests/.gitkeep b/contrib/templates/dbt-factory/template/{{.project_name}}/src/tests/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/__init__.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/conftest.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/conftest.py new file mode 100644 index 00000000..8f09a01c --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/conftest.py @@ -0,0 +1,76 @@ +import pytest +from databricks_dbt_factory.SpecsHandler import SpecsHandler +from databricks_dbt_factory.DbtFactory import DbtFactory +from databricks_dbt_factory.DbtTask import DbtTaskOptions +from databricks_dbt_factory.TaskFactory import ( + ModelTaskFactory, + SnapshotTaskFactory, + SeedTaskFactory, + TestTaskFactory, + DbtDependencyResolver, +) + + +@pytest.fixture +def file_handler() -> SpecsHandler: + return SpecsHandler() + + +@pytest.fixture +def dbt_factory(file_handler: SpecsHandler): + return create_dbt_factory(file_handler) + + +@pytest.fixture +def dbt_factory_with_deps(file_handler: SpecsHandler): + return create_dbt_factory(file_handler, dbt_deps_enabled=True) + + +@pytest.fixture +def dbt_factory_with_deps_selected(file_handler: SpecsHandler): + return create_dbt_factory( + file_handler, + dbt_deps_enabled=True, + dbt_tasks_deps=["diamonds_prices", "second_dbt_model"], + ) + + +@pytest.fixture +def notebook_factory(file_handler: SpecsHandler): + return create_dbt_factory( + file_handler, task_type="notebook", notebook_path="./notebooks/dbt_runner.py" + ) + + +@pytest.fixture +def dbt_factory_bundled(file_handler: SpecsHandler): + return create_dbt_factory(file_handler, bundle_tests=True) + + +def create_dbt_factory( + handler: SpecsHandler, + dbt_deps_enabled: bool = False, + dbt_tasks_deps: list[str] | None = None, + task_type: str = "dbt", + notebook_path: str | None = None, + bundle_tests: bool = False, +) -> DbtFactory: + resolver = DbtDependencyResolver() + task_options = DbtTaskOptions( + source="GIT", + environment_key="Default", + dbt_deps_enabled=dbt_deps_enabled, + dbt_tasks_deps=dbt_tasks_deps, + task_type=task_type, + notebook_path=notebook_path, + ) + dbt_options = "--target dev" + + task_factories = { + "model": ModelTaskFactory(resolver, task_options, dbt_options), + "snapshot": SnapshotTaskFactory(resolver, task_options, dbt_options), + "seed": SeedTaskFactory(resolver, task_options, dbt_options), + "test": TestTaskFactory(resolver, task_options, dbt_options), + } + + return DbtFactory(handler, task_factories, bundle_tests=bundle_tests) diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_app.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_app.py new file mode 100644 index 00000000..0caa8546 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_app.py @@ -0,0 +1,401 @@ +import os +from tempfile import NamedTemporaryFile +from pathlib import Path +import pytest +import yaml + +from databricks_dbt_factory.main import main, parse_args + +BASE_PATH = str(Path(__file__).resolve().parent) + + +def test_main_given_default_args(monkeypatch): + """Test the main function for job spec generation.""" + dbt_manifest_path = BASE_PATH + "/test_data/manifest.json" + input_job_spec_path = BASE_PATH + "/test_data/job_definition_template.yaml" + expected_job_definition_path = BASE_PATH + "/test_data/job_definition_default.yaml" + + with NamedTemporaryFile(suffix=".yaml", delete=False) as temp_file: + target_job_spec_path = temp_file.name + + monkeypatch.setattr( + "sys.argv", + [ + "main.py", + "--dbt-manifest-path", + dbt_manifest_path, + "--input-job-spec-path", + input_job_spec_path, + "--target-job-spec-path", + target_job_spec_path, + ], + ) + + try: + main() + + with open(expected_job_definition_path, "r", encoding="utf-8") as file: + expected_job_definition = yaml.safe_load(file) + + with open(target_job_spec_path, "r", encoding="utf-8") as file: + job_definition = yaml.safe_load(file) + + assert job_definition == expected_job_definition + finally: + if os.path.exists(target_job_spec_path): + os.remove(target_job_spec_path) + + +def test_main_notebook_mode_auto_copies_runner_notebook_next_to_spec( + monkeypatch, tmp_path +): + """Without --project-directory, the factory copies the runner notebook next to the + generated job spec and emits `notebook_path: ./run_dbt_command.py`.""" + target_job_spec_path = tmp_path / "job_definition.yaml" + + monkeypatch.setattr( + "sys.argv", + [ + "main.py", + "--dbt-manifest-path", + BASE_PATH + "/test_data/manifest.json", + "--input-job-spec-path", + BASE_PATH + "/test_data/job_definition_template.yaml", + "--target-job-spec-path", + str(target_job_spec_path), + "--task-type", + "notebook", + ], + ) + + main() + + copied_notebook = tmp_path / "run_dbt_command.py" + assert copied_notebook.exists(), ( + "runner notebook should have been copied next to the job spec" + ) + assert "dbtRunner" in copied_notebook.read_text(), ( + "copied file should be the packaged runner" + ) + + with open(target_job_spec_path, "r", encoding="utf-8") as file: + job_definition = yaml.safe_load(file) + + tasks = job_definition["resources"]["jobs"]["dbt_sql_job"]["tasks"] + for task in tasks: + assert task["notebook_task"]["notebook_path"] == "./run_dbt_command.py" + + +def test_main_notebook_mode_auto_copies_runner_notebook_to_project_root( + monkeypatch, tmp_path +): + """With a relative --project-directory (e.g. `../`), the factory copies the runner to + the computed project root and emits a matching relative notebook_path from the spec.""" + spec_dir = tmp_path / "resources" + spec_dir.mkdir() + target_job_spec_path = spec_dir / "job_definition.yaml" + + monkeypatch.setattr( + "sys.argv", + [ + "main.py", + "--dbt-manifest-path", + BASE_PATH + "/test_data/manifest.json", + "--input-job-spec-path", + BASE_PATH + "/test_data/job_definition_template.yaml", + "--target-job-spec-path", + str(target_job_spec_path), + "--task-type", + "notebook", + "--project-directory", + "../", + ], + ) + + main() + + copied_notebook = tmp_path / "run_dbt_command.py" + assert copied_notebook.exists(), ( + "runner should have been copied to the project root (one level up from the spec)" + ) + assert not (spec_dir / "run_dbt_command.py").exists(), ( + "runner should NOT be copied next to the spec in this case" + ) + + with open(target_job_spec_path, "r", encoding="utf-8") as file: + job_definition = yaml.safe_load(file) + + tasks = job_definition["resources"]["jobs"]["dbt_sql_job"]["tasks"] + for task in tasks: + assert task["notebook_task"]["notebook_path"] == "../run_dbt_command.py" + # With the runner at project root, CWD at runtime = project root. We explicitly + # pin project_directory to "." so the spec is self-documenting (the user's original + # "../" would resolve one level too high and has been rewritten). + assert task["notebook_task"]["base_parameters"]["project_directory"] == "." + + +def test_main_notebook_mode(monkeypatch): + """Test the main function for notebook task type generation.""" + dbt_manifest_path = BASE_PATH + "/test_data/manifest.json" + input_job_spec_path = BASE_PATH + "/test_data/job_definition_template.yaml" + expected_job_definition_path = ( + BASE_PATH + "/test_data/job_definition_notebook_default.yaml" + ) + + with NamedTemporaryFile(suffix=".yaml", delete=False) as temp_file: + target_job_spec_path = temp_file.name + + monkeypatch.setattr( + "sys.argv", + [ + "main.py", + "--dbt-manifest-path", + dbt_manifest_path, + "--input-job-spec-path", + input_job_spec_path, + "--target-job-spec-path", + target_job_spec_path, + "--task-type", + "notebook", + "--notebook-path", + "./notebooks/dbt_runner.py", + ], + ) + + try: + main() + + with open(expected_job_definition_path, "r", encoding="utf-8") as file: + expected_job_definition = yaml.safe_load(file) + + with open(target_job_spec_path, "r", encoding="utf-8") as file: + job_definition = yaml.safe_load(file) + + assert job_definition == expected_job_definition + finally: + if os.path.exists(target_job_spec_path): + os.remove(target_job_spec_path) + + +def test_main_all_args(monkeypatch): + """Test the main function for job spec generation.""" + dbt_manifest_path = BASE_PATH + "/test_data/manifest.json" + input_job_spec_path = BASE_PATH + "/test_data/job_definition_template.yaml" + expected_job_definition_path = ( + BASE_PATH + "/test_data/job_definition_deps_selected.yaml" + ) + + with NamedTemporaryFile(suffix=".yaml", delete=False) as temp_file: + target_job_spec_path = temp_file.name + + new_job_name = "test_job" + warehouse_id = "1234567890abcdef" + schema = "dqx_test" + catalog = "main" + profiles_dir = "profiles_dir" + project_dir = "/project_dir" + extra_dbt_command_options = '"--upgrade"' + + # Mock command-line arguments + monkeypatch.setattr( + "sys.argv", + [ + "main.py", + "--new-job-name", + new_job_name, + "--dbt-manifest-path", + dbt_manifest_path, + "--input-job-spec-path", + input_job_spec_path, + "--target-job-spec-path", + target_job_spec_path, + "--target", + "dev", + "--environment-key", + "Default", + "--source", + "GIT", + "--enable-dbt-deps", + "--dbt-tasks-deps", + "diamonds_prices,second_dbt_model", + "--warehouse_id", + warehouse_id, + "--schema", + schema, + "--catalog", + catalog, + "--profiles-directory", + profiles_dir, + "--project-directory", + project_dir, + "--extra-dbt-command-options", + extra_dbt_command_options, + ], + ) + + try: + main() + + with open(expected_job_definition_path, "r", encoding="utf-8") as file: + expected_job_definition = yaml.safe_load(file) + + with open(target_job_spec_path, "r", encoding="utf-8") as file: + job_definition = yaml.safe_load(file) + + expected_job_definition = update_spec( + expected_job_definition, + new_job_name, + warehouse_id, + schema, + catalog, + profiles_dir, + project_dir, + extra_dbt_command_options, + ) + + assert job_definition == expected_job_definition + finally: + if os.path.exists(target_job_spec_path): + os.remove(target_job_spec_path) + + +REQUIRED_ARGS = [ + "--dbt-manifest-path", + "manifest.json", + "--input-job-spec-path", + "in.yaml", + "--target-job-spec-path", + "out.yaml", +] + + +def test_explicit_environment_key_with_job_cluster_key_is_rejected(monkeypatch): + monkeypatch.setattr( + "sys.argv", + [ + "main.py", + *REQUIRED_ARGS, + "--job-cluster-key", + "foo", + "--environment-key", + "Default", + ], + ) + with pytest.raises(SystemExit): + parse_args() + + +def test_job_cluster_key_alone_parses(monkeypatch): + monkeypatch.setattr( + "sys.argv", ["main.py", *REQUIRED_ARGS, "--job-cluster-key", "foo"] + ) + args = parse_args() + assert args.job_cluster_key == "foo" + assert args.environment_key is None + + +def test_environment_key_alone_parses(monkeypatch): + monkeypatch.setattr( + "sys.argv", ["main.py", *REQUIRED_ARGS, "--environment-key", "Default"] + ) + args = parse_args() + assert args.environment_key == "Default" + assert args.job_cluster_key is None + + +def test_boolean_flags_default(monkeypatch): + monkeypatch.setattr("sys.argv", ["main.py", *REQUIRED_ARGS]) + args = parse_args() + assert args.run_tests is True # tests enabled by default + assert args.bundle_tests is False + assert args.enable_dbt_deps is False + assert args.dry_run is False + + +def test_boolean_flags_toggled(monkeypatch): + monkeypatch.setattr( + "sys.argv", + [ + "main.py", + *REQUIRED_ARGS, + "--no-run-tests", + "--bundle-tests", + "--enable-dbt-deps", + "--dry-run", + ], + ) + args = parse_args() + assert args.run_tests is False + assert args.bundle_tests is True + assert args.enable_dbt_deps is True + assert args.dry_run is True + + +def test_notebook_task_type_with_warehouse_id_is_rejected(monkeypatch): + monkeypatch.setattr( + "sys.argv", + [ + "main.py", + *REQUIRED_ARGS, + "--task-type", + "notebook", + "--notebook-path", + "/n", + "--warehouse_id", + "wh123", + ], + ) + with pytest.raises(SystemExit): + parse_args() + + +def remove_target_from_spec(expected_job_definition): + """Remove 'target' key from dbt_task in the expected job definition.""" + spec = dict(expected_job_definition) + + for task in expected_job_definition["resources"]["jobs"]["dbt_sql_job"]["tasks"]: + if "dbt_task" in task: + task["dbt_task"].pop("source", None) + + return spec + + +def update_spec( + expected_spec: dict, + new_job_name: str, + warehouse_id: str, + schema: str, + catalog: str, + profiles_dir: str, + project_dir: str, + extra_dbt_command_options: str, +) -> dict: + """Update the job specification with new parameters.""" + spec = dict(expected_spec) + + # Update job name + spec["resources"]["jobs"][new_job_name] = spec["resources"]["jobs"].pop( + "dbt_sql_job" + ) + spec["resources"]["jobs"][new_job_name]["name"] = new_job_name + + # Add warehouse_id under dbt_task + for task in spec["resources"]["jobs"][new_job_name]["tasks"]: + if "dbt_task" in task: + task["dbt_task"]["schema"] = schema + task["dbt_task"]["catalog"] = catalog + task["dbt_task"]["warehouse_id"] = warehouse_id + task["dbt_task"]["project_directory"] = project_dir + task["dbt_task"]["profiles_directory"] = profiles_dir + + # Update commands with extra dbt command options + updated_commands = [] + for command in task["dbt_task"]["commands"]: + if "--target dev" in command: + command = command.replace( + "--target dev", f"--target dev {extra_dbt_command_options}" + ) + updated_commands.append(command) + task["dbt_task"]["commands"] = updated_commands + + return spec diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition.yaml b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition.yaml new file mode 100644 index 00000000..88fe0908 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition.yaml @@ -0,0 +1,342 @@ +resources: + jobs: + dbt_sql_job: + name: dbt_job + git_source: + git_url: https://github.com/mwojtyczka/dbt-demo.git + git_provider: gitHub + git_branch: main + queue: + enabled: true + environments: + - environment_key: Default + spec: + client: '1' + dependencies: + - dbt-databricks + tasks: + - task_key: model_dbt_demo_diamonds_list_colors + depends_on: + - task_key: model_dbt_demo_diamonds_four_cs + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select diamonds_list_colors --target dev + source: GIT + - task_key: model_dbt_demo_diamonds_prices + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select diamonds_prices --target dev + source: GIT + - task_key: model_dbt_demo_diamonds_four_cs + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select diamonds_four_cs --target dev + source: GIT + - task_key: model_dbt_demo_zzz_game_details + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select zzz_game_details --target dev + source: GIT + - task_key: model_dbt_demo_second_dbt_model + depends_on: + - task_key: model_dbt_demo_first_dbt_model + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select second_dbt_model --target dev + source: GIT + - task_key: model_dbt_demo_first_dbt_model + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select first_dbt_model --target dev + source: GIT + - task_key: test_dbt_demo_zzz_game_details_check_scores + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select zzz_game_details_check_scores --target dev + source: GIT + - task_key: test_dbt_demo_zzz_game_details_check_dates + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select zzz_game_details_check_dates --target dev + source: GIT + - task_key: test_dbt_demo_zzz_win_loss_records_check_records + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select zzz_win_loss_records_check_records --target dev + source: GIT + - task_key: test_dbt_demo_diamonds_list_colors_test + depends_on: + - task_key: model_dbt_demo_diamonds_list_colors + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select diamonds_list_colors_test --target dev + source: GIT + - task_key: seed_dbt_demo_country_codes + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt seed --select country_codes --target dev + source: GIT + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select unique_zzz_game_details_game_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_game_details_game_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_game_details_home --target dev + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_game_details_visitor --target dev + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_game_details_home_score --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_game_details_visitor_score --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_game_details_winner --target dev + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_game_details_date --target dev + source: GIT + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select unique_first_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_first_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_unique_second_dbt_model_id_3d7f6966a4 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select unique_second_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_second_dbt_model_id_c791c712a2 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_second_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_second_dbt_model_amount_usd_28ae55e16f + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_second_dbt_model_amount_usd --target dev + source: GIT + - task_key: model_dbt_demo_changed_country + depends_on: + - task_key: snapshot_dbt_demo_country_snapshot + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select changed_country --target dev + source: GIT + - task_key: model_dbt_demo_country + depends_on: + - task_key: seed_dbt_demo_country_codes + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select country --target dev + source: GIT + - task_key: snapshot_dbt_demo_country_snapshot + depends_on: + - task_key: model_dbt_demo_country + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt snapshot --select country_snapshot --target dev + source: GIT + - task_key: model_dbt_demo_zzz_win_loss_records + depends_on: + - task_key: model_dbt_demo_zzz_game_details + - task_key: test_dbt_demo_zzz_game_details_check_scores + - task_key: test_dbt_demo_zzz_game_details_check_dates + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select zzz_win_loss_records --target dev + source: GIT + - task_key: test_dbt_demo_unique_zzz_win_loss_records_team_b989062105 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select unique_zzz_win_loss_records_team --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_team_9a18f33f86 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_win_loss_records_team --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_wins_f6815357fe + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_win_loss_records_wins --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_losses_82bf4ea601 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt test --select not_null_zzz_win_loss_records_losses --target dev + source: GIT diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition_default.yaml b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition_default.yaml new file mode 100644 index 00000000..8f836f24 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition_default.yaml @@ -0,0 +1,272 @@ +resources: + jobs: + dbt_sql_job: + name: dbt_job + git_source: + git_url: https://github.com/mwojtyczka/dbt-demo.git + git_provider: gitHub + git_branch: main + queue: + enabled: true + environments: + - environment_key: Default + spec: + client: '1' + dependencies: + - dbt-databricks + tasks: + - task_key: model_dbt_demo_diamonds_list_colors + depends_on: + - task_key: model_dbt_demo_diamonds_four_cs + environment_key: Default + dbt_task: + commands: + - dbt run --select diamonds_list_colors + - task_key: model_dbt_demo_diamonds_prices + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select diamonds_prices + - task_key: model_dbt_demo_diamonds_four_cs + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select diamonds_four_cs + - task_key: model_dbt_demo_zzz_game_details + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select zzz_game_details + - task_key: model_dbt_demo_second_dbt_model + depends_on: + - task_key: model_dbt_demo_first_dbt_model + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + environment_key: Default + dbt_task: + commands: + - dbt run --select second_dbt_model + - task_key: model_dbt_demo_first_dbt_model + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select first_dbt_model + - task_key: test_dbt_demo_zzz_game_details_check_scores + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select zzz_game_details_check_scores + - task_key: test_dbt_demo_zzz_game_details_check_dates + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select zzz_game_details_check_dates + - task_key: test_dbt_demo_zzz_win_loss_records_check_records + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select zzz_win_loss_records_check_records + - task_key: test_dbt_demo_diamonds_list_colors_test + depends_on: + - task_key: model_dbt_demo_diamonds_list_colors + environment_key: Default + dbt_task: + commands: + - dbt test --select diamonds_list_colors_test + - task_key: seed_dbt_demo_country_codes + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt seed --select country_codes + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_zzz_game_details_game_id + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_game_id + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_home + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_visitor + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_home_score + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_visitor_score + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_winner + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_date + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_first_dbt_model_id + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_first_dbt_model_id + - task_key: test_dbt_demo_unique_second_dbt_model_id_3d7f6966a4 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_second_dbt_model_id + - task_key: test_dbt_demo_not_null_second_dbt_model_id_c791c712a2 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_second_dbt_model_id + - task_key: test_dbt_demo_not_null_second_dbt_model_amount_usd_28ae55e16f + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_second_dbt_model_amount_usd + - task_key: model_dbt_demo_changed_country + depends_on: + - task_key: snapshot_dbt_demo_country_snapshot + environment_key: Default + dbt_task: + commands: + - dbt run --select changed_country + - task_key: model_dbt_demo_country + depends_on: + - task_key: seed_dbt_demo_country_codes + environment_key: Default + dbt_task: + commands: + - dbt run --select country + - task_key: snapshot_dbt_demo_country_snapshot + depends_on: + - task_key: model_dbt_demo_country + environment_key: Default + dbt_task: + commands: + - dbt snapshot --select country_snapshot + - task_key: model_dbt_demo_zzz_win_loss_records + depends_on: + - task_key: model_dbt_demo_zzz_game_details + - task_key: test_dbt_demo_zzz_game_details_check_scores + - task_key: test_dbt_demo_zzz_game_details_check_dates + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + environment_key: Default + dbt_task: + commands: + - dbt run --select zzz_win_loss_records + - task_key: test_dbt_demo_unique_zzz_win_loss_records_team_b989062105 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_zzz_win_loss_records_team + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_team_9a18f33f86 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_win_loss_records_team + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_wins_f6815357fe + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_win_loss_records_wins + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_losses_82bf4ea601 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_win_loss_records_losses diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition_deps_selected.yaml b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition_deps_selected.yaml new file mode 100644 index 00000000..3ab7d4ea --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition_deps_selected.yaml @@ -0,0 +1,309 @@ +resources: + jobs: + dbt_sql_job: + name: dbt_job + git_source: + git_url: https://github.com/mwojtyczka/dbt-demo.git + git_provider: gitHub + git_branch: main + queue: + enabled: true + environments: + - environment_key: Default + spec: + client: '1' + dependencies: + - dbt-databricks + tasks: + - task_key: model_dbt_demo_diamonds_list_colors + depends_on: + - task_key: model_dbt_demo_diamonds_four_cs + environment_key: Default + dbt_task: + commands: + - dbt run --select diamonds_list_colors --target dev + source: GIT + - task_key: model_dbt_demo_diamonds_prices + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select diamonds_prices --target dev + source: GIT + - task_key: model_dbt_demo_diamonds_four_cs + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select diamonds_four_cs --target dev + source: GIT + - task_key: model_dbt_demo_zzz_game_details + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select zzz_game_details --target dev + source: GIT + - task_key: model_dbt_demo_second_dbt_model + depends_on: + - task_key: model_dbt_demo_first_dbt_model + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + environment_key: Default + dbt_task: + commands: + - dbt deps --target dev + - dbt run --select second_dbt_model --target dev + source: GIT + - task_key: model_dbt_demo_first_dbt_model + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select first_dbt_model --target dev + source: GIT + - task_key: test_dbt_demo_zzz_game_details_check_scores + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select zzz_game_details_check_scores --target dev + source: GIT + - task_key: test_dbt_demo_zzz_game_details_check_dates + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select zzz_game_details_check_dates --target dev + source: GIT + - task_key: test_dbt_demo_zzz_win_loss_records_check_records + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select zzz_win_loss_records_check_records --target dev + source: GIT + - task_key: test_dbt_demo_diamonds_list_colors_test + depends_on: + - task_key: model_dbt_demo_diamonds_list_colors + environment_key: Default + dbt_task: + commands: + - dbt test --select diamonds_list_colors_test --target dev + source: GIT + - task_key: seed_dbt_demo_country_codes + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt seed --select country_codes --target dev + source: GIT + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_zzz_game_details_game_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_game_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_home --target dev + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_visitor --target dev + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_home_score --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_visitor_score --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_winner --target dev + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_date --target dev + source: GIT + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_first_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_first_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_unique_second_dbt_model_id_3d7f6966a4 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_second_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_second_dbt_model_id_c791c712a2 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_second_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_second_dbt_model_amount_usd_28ae55e16f + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_second_dbt_model_amount_usd --target dev + source: GIT + - task_key: model_dbt_demo_changed_country + depends_on: + - task_key: snapshot_dbt_demo_country_snapshot + environment_key: Default + dbt_task: + commands: + - dbt run --select changed_country --target dev + source: GIT + - task_key: model_dbt_demo_country + depends_on: + - task_key: seed_dbt_demo_country_codes + environment_key: Default + dbt_task: + commands: + - dbt run --select country --target dev + source: GIT + - task_key: snapshot_dbt_demo_country_snapshot + depends_on: + - task_key: model_dbt_demo_country + environment_key: Default + dbt_task: + commands: + - dbt snapshot --select country_snapshot --target dev + source: GIT + - task_key: model_dbt_demo_zzz_win_loss_records + depends_on: + - task_key: model_dbt_demo_zzz_game_details + - task_key: test_dbt_demo_zzz_game_details_check_scores + - task_key: test_dbt_demo_zzz_game_details_check_dates + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + environment_key: Default + dbt_task: + commands: + - dbt run --select zzz_win_loss_records --target dev + source: GIT + - task_key: test_dbt_demo_unique_zzz_win_loss_records_team_b989062105 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_zzz_win_loss_records_team --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_team_9a18f33f86 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_win_loss_records_team --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_wins_f6815357fe + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_win_loss_records_wins --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_losses_82bf4ea601 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_win_loss_records_losses --target dev + source: GIT diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition_no_deps.yaml b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition_no_deps.yaml new file mode 100644 index 00000000..0876d986 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition_no_deps.yaml @@ -0,0 +1,307 @@ +resources: + jobs: + dbt_sql_job: + name: dbt_job + git_source: + git_url: https://github.com/mwojtyczka/dbt-demo.git + git_provider: gitHub + git_branch: main + queue: + enabled: true + environments: + - environment_key: Default + spec: + client: '1' + dependencies: + - dbt-databricks + tasks: + - task_key: model_dbt_demo_diamonds_list_colors + depends_on: + - task_key: model_dbt_demo_diamonds_four_cs + environment_key: Default + dbt_task: + commands: + - dbt run --select diamonds_list_colors --target dev + source: GIT + - task_key: model_dbt_demo_diamonds_prices + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select diamonds_prices --target dev + source: GIT + - task_key: model_dbt_demo_diamonds_four_cs + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select diamonds_four_cs --target dev + source: GIT + - task_key: model_dbt_demo_zzz_game_details + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select zzz_game_details --target dev + source: GIT + - task_key: model_dbt_demo_second_dbt_model + depends_on: + - task_key: model_dbt_demo_first_dbt_model + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + environment_key: Default + dbt_task: + commands: + - dbt run --select second_dbt_model --target dev + source: GIT + - task_key: model_dbt_demo_first_dbt_model + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt run --select first_dbt_model --target dev + source: GIT + - task_key: test_dbt_demo_zzz_game_details_check_scores + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select zzz_game_details_check_scores --target dev + source: GIT + - task_key: test_dbt_demo_zzz_game_details_check_dates + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select zzz_game_details_check_dates --target dev + source: GIT + - task_key: test_dbt_demo_zzz_win_loss_records_check_records + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select zzz_win_loss_records_check_records --target dev + source: GIT + - task_key: test_dbt_demo_diamonds_list_colors_test + depends_on: + - task_key: model_dbt_demo_diamonds_list_colors + environment_key: Default + dbt_task: + commands: + - dbt test --select diamonds_list_colors_test --target dev + source: GIT + - task_key: seed_dbt_demo_country_codes + depends_on: [] + environment_key: Default + dbt_task: + commands: + - dbt seed --select country_codes --target dev + source: GIT + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_zzz_game_details_game_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_game_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_home --target dev + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_visitor --target dev + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_home_score --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_visitor_score --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_winner --target dev + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_game_details_date --target dev + source: GIT + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_first_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_first_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_unique_second_dbt_model_id_3d7f6966a4 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_second_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_second_dbt_model_id_c791c712a2 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_second_dbt_model_id --target dev + source: GIT + - task_key: test_dbt_demo_not_null_second_dbt_model_amount_usd_28ae55e16f + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_second_dbt_model_amount_usd --target dev + source: GIT + - task_key: model_dbt_demo_changed_country + depends_on: + - task_key: snapshot_dbt_demo_country_snapshot + environment_key: Default + dbt_task: + commands: + - dbt run --select changed_country --target dev + source: GIT + - task_key: model_dbt_demo_country + depends_on: + - task_key: seed_dbt_demo_country_codes + environment_key: Default + dbt_task: + commands: + - dbt run --select country --target dev + source: GIT + - task_key: snapshot_dbt_demo_country_snapshot + depends_on: + - task_key: model_dbt_demo_country + environment_key: Default + dbt_task: + commands: + - dbt snapshot --select country_snapshot --target dev + source: GIT + - task_key: model_dbt_demo_zzz_win_loss_records + depends_on: + - task_key: model_dbt_demo_zzz_game_details + - task_key: test_dbt_demo_zzz_game_details_check_scores + - task_key: test_dbt_demo_zzz_game_details_check_dates + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + environment_key: Default + dbt_task: + commands: + - dbt run --select zzz_win_loss_records --target dev + source: GIT + - task_key: test_dbt_demo_unique_zzz_win_loss_records_team_b989062105 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select unique_zzz_win_loss_records_team --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_team_9a18f33f86 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_win_loss_records_team --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_wins_f6815357fe + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_win_loss_records_wins --target dev + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_losses_82bf4ea601 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + dbt_task: + commands: + - dbt test --select not_null_zzz_win_loss_records_losses --target dev + source: GIT diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition_notebook_default.yaml b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition_notebook_default.yaml new file mode 100644 index 00000000..d3dcedb2 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition_notebook_default.yaml @@ -0,0 +1,307 @@ +resources: + jobs: + dbt_sql_job: + name: dbt_job + git_source: + git_url: https://github.com/mwojtyczka/dbt-demo.git + git_provider: gitHub + git_branch: main + queue: + enabled: true + environments: + - environment_key: Default + spec: + client: '1' + dependencies: + - dbt-databricks + tasks: + - task_key: model_dbt_demo_diamonds_list_colors + depends_on: + - task_key: model_dbt_demo_diamonds_four_cs + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select diamonds_list_colors"]' + - task_key: model_dbt_demo_diamonds_prices + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select diamonds_prices"]' + - task_key: model_dbt_demo_diamonds_four_cs + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select diamonds_four_cs"]' + - task_key: model_dbt_demo_zzz_game_details + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select zzz_game_details"]' + - task_key: model_dbt_demo_second_dbt_model + depends_on: + - task_key: model_dbt_demo_first_dbt_model + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select second_dbt_model"]' + - task_key: model_dbt_demo_first_dbt_model + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select first_dbt_model"]' + - task_key: test_dbt_demo_zzz_game_details_check_scores + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select zzz_game_details_check_scores"]' + - task_key: test_dbt_demo_zzz_game_details_check_dates + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select zzz_game_details_check_dates"]' + - task_key: test_dbt_demo_zzz_win_loss_records_check_records + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select zzz_win_loss_records_check_records"]' + - task_key: test_dbt_demo_diamonds_list_colors_test + depends_on: + - task_key: model_dbt_demo_diamonds_list_colors + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select diamonds_list_colors_test"]' + - task_key: seed_dbt_demo_country_codes + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt seed --select country_codes"]' + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select unique_zzz_game_details_game_id"]' + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_game_id"]' + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_home"]' + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle"]' + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_visitor"]' + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle"]' + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_home_score"]' + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_visitor_score"]' + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_winner"]' + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle"]' + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_date"]' + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select unique_first_dbt_model_id"]' + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_first_dbt_model_id"]' + - task_key: test_dbt_demo_unique_second_dbt_model_id_3d7f6966a4 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select unique_second_dbt_model_id"]' + - task_key: test_dbt_demo_not_null_second_dbt_model_id_c791c712a2 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_second_dbt_model_id"]' + - task_key: test_dbt_demo_not_null_second_dbt_model_amount_usd_28ae55e16f + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_second_dbt_model_amount_usd"]' + - task_key: model_dbt_demo_changed_country + depends_on: + - task_key: snapshot_dbt_demo_country_snapshot + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select changed_country"]' + - task_key: model_dbt_demo_country + depends_on: + - task_key: seed_dbt_demo_country_codes + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select country"]' + - task_key: snapshot_dbt_demo_country_snapshot + depends_on: + - task_key: model_dbt_demo_country + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt snapshot --select country_snapshot"]' + - task_key: model_dbt_demo_zzz_win_loss_records + depends_on: + - task_key: model_dbt_demo_zzz_game_details + - task_key: test_dbt_demo_zzz_game_details_check_scores + - task_key: test_dbt_demo_zzz_game_details_check_dates + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select zzz_win_loss_records"]' + - task_key: test_dbt_demo_unique_zzz_win_loss_records_team_b989062105 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select unique_zzz_win_loss_records_team"]' + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_team_9a18f33f86 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_win_loss_records_team"]' + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_wins_f6815357fe + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_win_loss_records_wins"]' + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_losses_82bf4ea601 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_win_loss_records_losses"]' diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition_notebook_no_deps.yaml b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition_notebook_no_deps.yaml new file mode 100644 index 00000000..0c9b453e --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition_notebook_no_deps.yaml @@ -0,0 +1,342 @@ +resources: + jobs: + dbt_sql_job: + name: dbt_job + git_source: + git_url: https://github.com/mwojtyczka/dbt-demo.git + git_provider: gitHub + git_branch: main + queue: + enabled: true + environments: + - environment_key: Default + spec: + client: '1' + dependencies: + - dbt-databricks + tasks: + - task_key: model_dbt_demo_diamonds_list_colors + depends_on: + - task_key: model_dbt_demo_diamonds_four_cs + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select diamonds_list_colors --target dev"]' + source: GIT + - task_key: model_dbt_demo_diamonds_prices + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select diamonds_prices --target dev"]' + source: GIT + - task_key: model_dbt_demo_diamonds_four_cs + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select diamonds_four_cs --target dev"]' + source: GIT + - task_key: model_dbt_demo_zzz_game_details + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select zzz_game_details --target dev"]' + source: GIT + - task_key: model_dbt_demo_second_dbt_model + depends_on: + - task_key: model_dbt_demo_first_dbt_model + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select second_dbt_model --target dev"]' + source: GIT + - task_key: model_dbt_demo_first_dbt_model + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select first_dbt_model --target dev"]' + source: GIT + - task_key: test_dbt_demo_zzz_game_details_check_scores + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select zzz_game_details_check_scores --target dev"]' + source: GIT + - task_key: test_dbt_demo_zzz_game_details_check_dates + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select zzz_game_details_check_dates --target dev"]' + source: GIT + - task_key: test_dbt_demo_zzz_win_loss_records_check_records + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select zzz_win_loss_records_check_records --target dev"]' + source: GIT + - task_key: test_dbt_demo_diamonds_list_colors_test + depends_on: + - task_key: model_dbt_demo_diamonds_list_colors + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select diamonds_list_colors_test --target dev"]' + source: GIT + - task_key: seed_dbt_demo_country_codes + depends_on: [] + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt seed --select country_codes --target dev"]' + source: GIT + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select unique_zzz_game_details_game_id --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_game_id --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_home --target dev"]' + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_visitor --target dev"]' + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_home_score --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_visitor_score --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_winner --target dev"]' + source: GIT + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + depends_on: + - task_key: model_dbt_demo_zzz_game_details + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_game_details_date --target dev"]' + source: GIT + - task_key: test_dbt_demo_unique_first_dbt_model_id_9dda05743e + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select unique_first_dbt_model_id --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_first_dbt_model_id_bd8d25d061 + depends_on: + - task_key: model_dbt_demo_first_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_first_dbt_model_id --target dev"]' + source: GIT + - task_key: test_dbt_demo_unique_second_dbt_model_id_3d7f6966a4 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select unique_second_dbt_model_id --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_second_dbt_model_id_c791c712a2 + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_second_dbt_model_id --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_second_dbt_model_amount_usd_28ae55e16f + depends_on: + - task_key: model_dbt_demo_second_dbt_model + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_second_dbt_model_amount_usd --target dev"]' + source: GIT + - task_key: model_dbt_demo_changed_country + depends_on: + - task_key: snapshot_dbt_demo_country_snapshot + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select changed_country --target dev"]' + source: GIT + - task_key: model_dbt_demo_country + depends_on: + - task_key: seed_dbt_demo_country_codes + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select country --target dev"]' + source: GIT + - task_key: snapshot_dbt_demo_country_snapshot + depends_on: + - task_key: model_dbt_demo_country + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt snapshot --select country_snapshot --target dev"]' + source: GIT + - task_key: model_dbt_demo_zzz_win_loss_records + depends_on: + - task_key: model_dbt_demo_zzz_game_details + - task_key: test_dbt_demo_zzz_game_details_check_scores + - task_key: test_dbt_demo_zzz_game_details_check_dates + - task_key: test_dbt_demo_unique_zzz_game_details_game_id_80431e7218 + - task_key: test_dbt_demo_not_null_zzz_game_details_game_id_3b5747f905 + - task_key: test_dbt_demo_not_null_zzz_game_details_home_a99d25edd5 + - task_key: test_dbt_demo_accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle_3771faef40 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_990a91372e + - task_key: test_dbt_demo_accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle_f0c81c502d + - task_key: test_dbt_demo_not_null_zzz_game_details_home_score_8c55a8fe39 + - task_key: test_dbt_demo_not_null_zzz_game_details_visitor_score_f31e89be7f + - task_key: test_dbt_demo_not_null_zzz_game_details_winner_7f08e737be + - task_key: test_dbt_demo_accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle_344bd88c4c + - task_key: test_dbt_demo_not_null_zzz_game_details_date_a9f3a56f68 + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt run --select zzz_win_loss_records --target dev"]' + source: GIT + - task_key: test_dbt_demo_unique_zzz_win_loss_records_team_b989062105 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select unique_zzz_win_loss_records_team --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_team_9a18f33f86 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_win_loss_records_team --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_wins_f6815357fe + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_win_loss_records_wins --target dev"]' + source: GIT + - task_key: test_dbt_demo_not_null_zzz_win_loss_records_losses_82bf4ea601 + depends_on: + - task_key: model_dbt_demo_zzz_win_loss_records + environment_key: Default + notebook_task: + notebook_path: ./notebooks/dbt_runner.py + base_parameters: + dbt_commands: '["dbt test --select not_null_zzz_win_loss_records_losses --target dev"]' + source: GIT diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition_template.yaml b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition_template.yaml new file mode 100644 index 00000000..6570068c --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/job_definition_template.yaml @@ -0,0 +1,16 @@ +resources: + jobs: + dbt_sql_job: + name: dbt_job + git_source: + git_url: https://github.com/mwojtyczka/dbt-demo.git + git_provider: gitHub + git_branch: main + queue: + enabled: true + environments: + - environment_key: Default + spec: + client: '1' + dependencies: + - dbt-databricks diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/manifest.json b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/manifest.json new file mode 100644 index 00000000..44282d07 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_data/manifest.json @@ -0,0 +1 @@ +{"metadata": {"dbt_schema_version": "https://schemas.getdbt.com/dbt/manifest/v12.json", "dbt_version": "1.8.7", "generated_at": "2024-12-06T13:55:35.349350Z", "invocation_id": "a2d9eb4c-6787-4ede-b5ce-44e61f3769b2", "env": {}, "project_name": "dbt_demo", "project_id": "f005e70093f1ddd3daa6316f5ec6f69e", "user_id": "b802ff80-7d9c-4428-8a39-5f56cf164bf8", "send_anonymous_usage_stats": true, "adapter_type": "databricks"}, "nodes": {"model.dbt_demo.diamonds_list_colors": {"database": "dbt_demo", "schema": "dbt_dev", "name": "diamonds_list_colors", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/diamonds_list_colors.sql", "original_file_path": "src/models/sql_model1/diamonds_list_colors.sql", "unique_id": "model.dbt_demo.diamonds_list_colors", "fqn": ["dbt_demo", "sql_model1", "diamonds_list_colors"], "alias": "diamonds_list_colors", "checksum": {"name": "sha256", "checksum": "fb8e076fbc40ae30c2441fbc660922d1d3428cf1557cc8f400dbd69caa78c266"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/diamonds_list_colors.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "created_at": 1733486794.194898, "relation_name": "`dbt_demo`.`dbt_dev`.`diamonds_list_colors`", "raw_code": "-- create a view\nselect distinct color\nfrom {{ ref('diamonds_four_cs') }}\nsort by color asc", "language": "sql", "refs": [{"name": "diamonds_four_cs", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.diamonds_four_cs"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/diamonds_list_colors.sql", "compiled": true, "compiled_code": "-- create a view\nselect distinct color\nfrom `dbt_demo`.`dbt_dev`.`diamonds_four_cs`\nsort by color asc", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.diamonds_prices": {"database": "dbt_demo", "schema": "dbt_dev", "name": "diamonds_prices", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/diamonds_prices.sql", "original_file_path": "src/models/sql_model1/diamonds_prices.sql", "unique_id": "model.dbt_demo.diamonds_prices", "fqn": ["dbt_demo", "sql_model1", "diamonds_prices"], "alias": "diamonds_prices", "checksum": {"name": "sha256", "checksum": "809b1318095c2871ab3e9d98af3cc11233e1fbb791e19682fbb38070074e53d2"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/diamonds_prices.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "created_at": 1733486794.207654, "relation_name": "`dbt_demo`.`dbt_dev`.`diamonds_prices`", "raw_code": "-- create a view\nselect\n account_id,\n workspace_id,\n usage_date,\n sum(usage_quantity) as quantity,\n sum({{ cents_to_dollars('usage_quantity') }}) as amount_usd\nfrom system.billing.usage\ngroup by account_id, workspace_id, usage_date", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt_demo.cents_to_dollars"], "nodes": []}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/diamonds_prices.sql", "compiled": true, "compiled_code": "-- create a view\nselect\n account_id,\n workspace_id,\n usage_date,\n sum(usage_quantity) as quantity,\n sum(\n (usage_quantity / 100)::numeric(16, 2)\n) as amount_usd\nfrom system.billing.usage\ngroup by account_id, workspace_id, usage_date", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.diamonds_four_cs": {"database": "dbt_demo", "schema": "dbt_dev", "name": "diamonds_four_cs", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/diamonds_four_cs.sql", "original_file_path": "src/models/sql_model1/diamonds_four_cs.sql", "unique_id": "model.dbt_demo.diamonds_four_cs", "fqn": ["dbt_demo", "sql_model1", "diamonds_four_cs"], "alias": "diamonds_four_cs", "checksum": {"name": "sha256", "checksum": "85a0d2493625371fdfbd2d8261dcc22335301c88e08791638ad760e6cb7dd614"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/diamonds_four_cs.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733486794.264364, "relation_name": "`dbt_demo`.`dbt_dev`.`diamonds_four_cs`", "raw_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n{{ config(\n materialized='table',\n file_format='delta'\n) }}\nselect\n carat,\n cut,\n color,\n clarity\n-- get table specification from schema.yml\nfrom {{ source('raw', 'diamonds') }} -- equivalent of: from hive_metastore.default.diamonds", "language": "sql", "refs": [], "sources": [["raw", "diamonds"]], "metrics": [], "depends_on": {"macros": [], "nodes": ["source.dbt_demo.raw.diamonds"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/diamonds_four_cs.sql", "compiled": true, "compiled_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n\nselect\n carat,\n cut,\n color,\n clarity\n-- get table specification from schema.yml\nfrom `hive_metastore`.`default`.`diamonds` -- equivalent of: from hive_metastore.default.diamonds", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.zzz_game_details": {"database": "dbt_demo", "schema": "dbt_dev", "name": "zzz_game_details", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/zzz_game_details.sql", "original_file_path": "src/models/sql_model1/zzz_game_details.sql", "unique_id": "model.dbt_demo.zzz_game_details", "fqn": ["dbt_demo", "sql_model1", "zzz_game_details"], "alias": "zzz_game_details", "checksum": {"name": "sha256", "checksum": "07a0ddce4e171f5838f27124b3574a77cbab23fac66bba7a1b8f87f6bfbd653f"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {"game_id": {"name": "game_id", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "home": {"name": "home", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "visitor": {"name": "visitor", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "home_score": {"name": "home_score", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "visitor_score": {"name": "visitor_score", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "winner": {"name": "winner", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "date": {"name": "date", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model1/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model1/zzz_game_details.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733486794.43343, "relation_name": "`dbt_demo`.`dbt_dev`.`zzz_game_details`", "raw_code": "-- Create a table that provides full details for each game, including\n-- the game ID, the home and visiting teams' city names and scores,\n-- the game winner's city name, and the game date.\n{{ config(\n materialized='table',\n file_format='delta'\n) }}\n-- Step 4 of 4: Replace the visitor team IDs with their city names.\nselect\n game_id,\n home,\n t.team_city as visitor,\n home_score,\n visitor_score,\n -- Step 3 of 4: Display the city name for each game's winner.\n game_date as date,\n case\n when\n home_score > visitor_score\n then\n home\n when\n visitor_score > home_score\n then\n t.team_city\n end as winner\nfrom (\n -- Step 2 of 4: Replace the home team IDs with their actual city names.\n select\n game_id,\n t.team_city as home,\n home_score,\n visitor_team_id,\n visitor_score,\n game_date\n from (\n -- Step 1 of 4: Combine data from various tables (for example, game and team IDs, scores, dates).\n select\n g.game_id,\n go.home_team_id,\n gs.home_team_score as home_score,\n go.visitor_team_id,\n gs.visitor_team_score as visitor_score,\n g.game_date\n from\n marcin_demo.dbt.{{ target.name }}_zzz_games as g,\n marcin_demo.dbt.zzz_game_opponents as go,\n marcin_demo.dbt.zzz_game_scores as gs\n where\n g.game_id = go.game_id\n and g.game_id = gs.game_id\n ) as all_ids,\n marcin_demo.dbt.zzz_teams as t\n where\n all_ids.home_team_id = t.team_id\n) as visitor_ids,\n marcin_demo.dbt.zzz_teams as t\nwhere\n visitor_ids.visitor_team_id = t.team_id\norder by game_date desc", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/zzz_game_details.sql", "compiled": true, "compiled_code": "-- Create a table that provides full details for each game, including\n-- the game ID, the home and visiting teams' city names and scores,\n-- the game winner's city name, and the game date.\n\n-- Step 4 of 4: Replace the visitor team IDs with their city names.\nselect\n game_id,\n home,\n t.team_city as visitor,\n home_score,\n visitor_score,\n -- Step 3 of 4: Display the city name for each game's winner.\n game_date as date,\n case\n when\n home_score > visitor_score\n then\n home\n when\n visitor_score > home_score\n then\n t.team_city\n end as winner\nfrom (\n -- Step 2 of 4: Replace the home team IDs with their actual city names.\n select\n game_id,\n t.team_city as home,\n home_score,\n visitor_team_id,\n visitor_score,\n game_date\n from (\n -- Step 1 of 4: Combine data from various tables (for example, game and team IDs, scores, dates).\n select\n g.game_id,\n go.home_team_id,\n gs.home_team_score as home_score,\n go.visitor_team_id,\n gs.visitor_team_score as visitor_score,\n g.game_date\n from\n marcin_demo.dbt.dev_zzz_games as g,\n marcin_demo.dbt.zzz_game_opponents as go,\n marcin_demo.dbt.zzz_game_scores as gs\n where\n g.game_id = go.game_id\n and g.game_id = gs.game_id\n ) as all_ids,\n marcin_demo.dbt.zzz_teams as t\n where\n all_ids.home_team_id = t.team_id\n) as visitor_ids,\n marcin_demo.dbt.zzz_teams as t\nwhere\n visitor_ids.visitor_team_id = t.team_id\norder by game_date desc", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.second_dbt_model": {"database": "dbt_demo", "schema": "dbt_dev", "name": "second_dbt_model", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model2/second_dbt_model.sql", "original_file_path": "src/models/sql_model2/second_dbt_model.sql", "unique_id": "model.dbt_demo.second_dbt_model", "fqn": ["dbt_demo", "sql_model2", "second_dbt_model"], "alias": "second_dbt_model", "checksum": {"name": "sha256", "checksum": "226c0e1d47e6f9bee7430c7acffadc0d239fa185558a2de703c5f0ab0b01d218"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": true, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "A starter dbt model", "columns": {"id": {"name": "id", "description": "The primary key for this table", "meta": {}, "data_type": "int", "constraints": [], "quote": null, "tags": []}, "amount_usd": {"name": "amount_usd", "description": "The primary key for this table", "meta": {}, "data_type": "decimal", "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model2/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model2/second_dbt_model.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "contract": {"enforced": true}}, "created_at": 1733486794.536133, "relation_name": "`dbt_demo`.`dbt_dev`.`second_dbt_model`", "raw_code": "-- Use the `ref` function to select from other models\n\nselect\n *, {{ cents_to_dollars('id') }} as amount_usd\nfrom {{ ref('first_dbt_model') }}\nwhere id = 1", "language": "sql", "refs": [{"name": "first_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt_demo.cents_to_dollars"], "nodes": ["model.dbt_demo.first_dbt_model"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model2/second_dbt_model.sql", "compiled": true, "compiled_code": "-- Use the `ref` function to select from other models\n\nselect\n *, \n (id / 100)::numeric(16, 2)\n as amount_usd\nfrom `dbt_demo`.`dbt_dev`.`first_dbt_model`\nwhere id = 1", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": true, "alias_types": true, "checksum": "d4b533efdb37067787094e25d1860ddcc473cb5cf92ebe2c358011ff2866658f"}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.first_dbt_model": {"database": "dbt_demo", "schema": "dbt_dev", "name": "first_dbt_model", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model2/first_dbt_model.sql", "original_file_path": "src/models/sql_model2/first_dbt_model.sql", "unique_id": "model.dbt_demo.first_dbt_model", "fqn": ["dbt_demo", "sql_model2", "first_dbt_model"], "alias": "first_dbt_model", "checksum": {"name": "sha256", "checksum": "91d9a8b1e5c899463e52c50492b7b3c7df99fa6635c868f9165ca83e5f3b13f7"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": true, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "A starter dbt model", "columns": {"id": {"name": "id", "description": "The primary key for this table", "meta": {}, "data_type": "int", "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model2/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model2/first_dbt_model.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "contract": {"enforced": true}, "materialized": "table"}, "created_at": 1733486794.534828, "relation_name": "`dbt_demo`.`dbt_dev`.`first_dbt_model`", "raw_code": "/*\n Welcome to your first dbt model!\n Did you know that you can also configure models directly within SQL files?\n This will override configurations stated in dbt_project.yml\n\n Try changing \"table\" to \"view\" below\n*/\n\n{{ config(materialized='table') }}\n\nwith source_data as (\n\n select 1 as id\n union all\n select 2 as id\n\n)\n\nselect *\nfrom source_data\n\n/*\n Uncomment the line below to remove records with null `id` values\n*/\n\n-- where id is not null", "language": "sql", "refs": [], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": []}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model2/first_dbt_model.sql", "compiled": true, "compiled_code": "/*\n Welcome to your first dbt model!\n Did you know that you can also configure models directly within SQL files?\n This will override configurations stated in dbt_project.yml\n\n Try changing \"table\" to \"view\" below\n*/\n\n\n\nwith source_data as (\n\n select 1 as id\n union all\n select 2 as id\n\n)\n\nselect *\nfrom source_data\n\n/*\n Uncomment the line below to remove records with null `id` values\n*/\n\n-- where id is not null", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": true, "alias_types": true, "checksum": "a31b0e38eb3a9f0c0ed2f45bd615732c39e48b7f698f50a36748a687fd6d7289"}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "test.dbt_demo.zzz_game_details_check_scores": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "zzz_game_details_check_scores", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/zzz_game_details_check_scores.sql", "original_file_path": "tests/data_tests/zzz_game_details_check_scores.sql", "unique_id": "test.dbt_demo.zzz_game_details_check_scores", "fqn": ["dbt_demo", "data_tests", "zzz_game_details_check_scores"], "alias": "zzz_game_details_check_scores", "checksum": {"name": "sha256", "checksum": "49cca9739f2738a4609833d267ade1a137756523f0659ec02f0bd31adb3c5789"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.31489, "relation_name": null, "raw_code": "-- This sport allows no negative scores or tie games.\n-- For this test to pass, this query must return no results.\n\nselect home_score, visitor_score\nfrom {{ ref('zzz_game_details') }}\nwhere home_score < 0\nor visitor_score < 0\nor home_score = visitor_score", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "test.dbt_demo.zzz_game_details_check_dates": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "zzz_game_details_check_dates", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/zzz_game_details_check_dates.sql", "original_file_path": "tests/data_tests/zzz_game_details_check_dates.sql", "unique_id": "test.dbt_demo.zzz_game_details_check_dates", "fqn": ["dbt_demo", "data_tests", "zzz_game_details_check_dates"], "alias": "zzz_game_details_check_dates", "checksum": {"name": "sha256", "checksum": "ac2ae3e40e5b734327e8a60a8c944e535c472314feee78d9320da1f7ed515918"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.3213592, "relation_name": null, "raw_code": "-- This season's games happened between 2020-12-12 and 2021-02-06.\n-- For this test to pass, this query must return no results.\n\nselect date\nfrom {{ ref('zzz_game_details') }}\nwhere date < '2020-12-12'\nor date > '2021-02-06'", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "test.dbt_demo.zzz_win_loss_records_check_records": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "zzz_win_loss_records_check_records", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/zzz_win_loss_records_check_records.sql", "original_file_path": "tests/data_tests/zzz_win_loss_records_check_records.sql", "unique_id": "test.dbt_demo.zzz_win_loss_records_check_records", "fqn": ["dbt_demo", "data_tests", "zzz_win_loss_records_check_records"], "alias": "zzz_win_loss_records_check_records", "checksum": {"name": "sha256", "checksum": "8840acce153171ad015abfe10626d1220cc9533e98c859a2b483f0d25d2e4903"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.323633, "relation_name": null, "raw_code": "-- Each team participated in 4 games this season.\n-- For this test to pass, this query must return no results.\n\nselect wins, losses\nfrom {{ ref('zzz_win_loss_records') }}\nwhere wins < 0 or wins > 4\nor losses < 0 or losses > 4\nor (wins + losses) > 4", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "test.dbt_demo.diamonds_list_colors_test": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "diamonds_list_colors_test", "resource_type": "test", "package_name": "dbt_demo", "path": "data_tests/diamonds_list_colors_test.sql", "original_file_path": "tests/data_tests/diamonds_list_colors_test.sql", "unique_id": "test.dbt_demo.diamonds_list_colors_test", "fqn": ["dbt_demo", "data_tests", "diamonds_list_colors_test"], "alias": "diamonds_list_colors_test", "checksum": {"name": "sha256", "checksum": "4a4d3ee2fa6aad9a7fc4b248111aa6432b67374e217793fbf29d92c2504f13be"}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.3264809, "relation_name": null, "raw_code": "SELECT color, COUNT(*)\nFROM {{ ref('diamonds_list_colors') }}\nGROUP BY color\nHAVING COUNT(*) > 1", "language": "sql", "refs": [{"name": "diamonds_list_colors", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.diamonds_list_colors"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "seed.dbt_demo.country_codes": {"database": "dbt_demo", "schema": "dbt_dev", "name": "country_codes", "resource_type": "seed", "package_name": "dbt_demo", "path": "country_codes.csv", "original_file_path": "src/seeds/country_codes.csv", "unique_id": "seed.dbt_demo.country_codes", "fqn": ["dbt_demo", "country_codes"], "alias": "country_codes", "checksum": {"name": "sha256", "checksum": "b54c38e4337b053b07cdc37fa5364e8f159b4e37b77d36281a8d2d11a2c09b8c"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "seed", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "delimiter": ",", "quote_columns": null}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.388313, "relation_name": "`dbt_demo`.`dbt_dev`.`country_codes`", "raw_code": "", "root_path": "/Users/marcin.wojtyczka/Documents/repos/dbt_demo", "depends_on": {"macros": []}}, "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_zzz_game_details_game_id", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_zzz_game_details_game_id.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218", "fqn": ["dbt_demo", "models", "sql_model1", "unique_zzz_game_details_game_id"], "alias": "unique_zzz_game_details_game_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.462498, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "game_id", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "unique", "kwargs": {"column_name": "game_id", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_game_id", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_game_id.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_game_id"], "alias": "not_null_zzz_game_details_game_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4634, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "game_id", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "game_id", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_home", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_home.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_home"], "alias": "not_null_zzz_game_details_home", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.464213, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "home", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "home", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle", "resource_type": "test", "package_name": "dbt_demo", "path": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40", "fqn": ["dbt_demo", "models", "sql_model1", "accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle"], "alias": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"alias": "accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3"}, "created_at": 1733486794.4649322, "relation_name": null, "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_zzz_game_detai_a03619ec6edbe71cee6bc995fd73e5a3\") }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_accepted_values", "macro.dbt.get_where_subquery"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "home", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "accepted_values", "kwargs": {"values": ["Amsterdam", "San Francisco", "Seattle"], "column_name": "home", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_visitor", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_visitor.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_visitor"], "alias": "not_null_zzz_game_details_visitor", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.474986, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "visitor", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "visitor", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle", "resource_type": "test", "package_name": "dbt_demo", "path": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d", "fqn": ["dbt_demo", "models", "sql_model1", "accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle"], "alias": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"alias": "accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272"}, "created_at": 1733486794.4757178, "relation_name": null, "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_zzz_game_detai_548b3e0a5a327f6865abfb6dba87b272\") }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_accepted_values", "macro.dbt.get_where_subquery"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "visitor", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "accepted_values", "kwargs": {"values": ["Amsterdam", "San Francisco", "Seattle"], "column_name": "visitor", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_home_score", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_home_score.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_home_score"], "alias": "not_null_zzz_game_details_home_score", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4776351, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "home_score", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "home_score", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_visitor_score", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_visitor_score.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_visitor_score"], "alias": "not_null_zzz_game_details_visitor_score", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4783318, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "visitor_score", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "visitor_score", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_winner", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_winner.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_winner"], "alias": "not_null_zzz_game_details_winner", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.479018, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "winner", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "winner", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle", "resource_type": "test", "package_name": "dbt_demo", "path": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c", "fqn": ["dbt_demo", "models", "sql_model1", "accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle"], "alias": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca", "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"alias": "accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca"}, "created_at": 1733486794.4797802, "relation_name": null, "raw_code": "{{ test_accepted_values(**_dbt_generic_test_kwargs) }}{{ config(alias=\"accepted_values_zzz_game_detai_b708d46a19b56a55e34c422000ef4dca\") }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_accepted_values", "macro.dbt.get_where_subquery"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "winner", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "accepted_values", "kwargs": {"values": ["Amsterdam", "San Francisco", "Seattle"], "column_name": "winner", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_game_details_date", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_game_details_date.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_game_details_date"], "alias": "not_null_zzz_game_details_date", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.4815848, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "date", "file_key_name": "models.zzz_game_details", "attached_node": "model.dbt_demo.zzz_game_details", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "date", "model": "{{ get_where_subquery(ref('zzz_game_details')) }}"}, "namespace": null}}, "test.dbt_demo.unique_first_dbt_model_id.9dda05743e": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_first_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_first_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.unique_first_dbt_model_id.9dda05743e", "fqn": ["dbt_demo", "models", "sql_model2", "unique_first_dbt_model_id"], "alias": "unique_first_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.536581, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "first_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.first_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.first_dbt_model", "attached_node": "model.dbt_demo.first_dbt_model", "test_metadata": {"name": "unique", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('first_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_first_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_first_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061", "fqn": ["dbt_demo", "models", "sql_model2", "not_null_first_dbt_model_id"], "alias": "not_null_first_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.537291, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "first_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.first_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.first_dbt_model", "attached_node": "model.dbt_demo.first_dbt_model", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('first_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_second_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_second_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4", "fqn": ["dbt_demo", "models", "sql_model2", "unique_second_dbt_model_id"], "alias": "unique_second_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.537977, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "second_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.second_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.second_dbt_model", "attached_node": "model.dbt_demo.second_dbt_model", "test_metadata": {"name": "unique", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('second_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_second_dbt_model_id", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_second_dbt_model_id.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2", "fqn": ["dbt_demo", "models", "sql_model2", "not_null_second_dbt_model_id"], "alias": "not_null_second_dbt_model_id", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.538719, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "second_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.second_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "id", "file_key_name": "models.second_dbt_model", "attached_node": "model.dbt_demo.second_dbt_model", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "id", "model": "{{ get_where_subquery(ref('second_dbt_model')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_second_dbt_model_amount_usd", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_second_dbt_model_amount_usd.sql", "original_file_path": "src/models/sql_model2/schemas.yml", "unique_id": "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f", "fqn": ["dbt_demo", "models", "sql_model2", "not_null_second_dbt_model_amount_usd"], "alias": "not_null_second_dbt_model_amount_usd", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733486794.539395, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "second_dbt_model", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.second_dbt_model"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "amount_usd", "file_key_name": "models.second_dbt_model", "attached_node": "model.dbt_demo.second_dbt_model", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "amount_usd", "model": "{{ get_where_subquery(ref('second_dbt_model')) }}"}, "namespace": null}}, "model.dbt_demo.changed_country": {"database": "dbt_demo", "schema": "dbt_dev", "name": "changed_country", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/changed_country.sql", "original_file_path": "src/models/sql_model1/changed_country.sql", "unique_id": "model.dbt_demo.changed_country", "fqn": ["dbt_demo", "sql_model1", "changed_country"], "alias": "changed_country", "checksum": {"name": "sha256", "checksum": "e1f9599d0e633856772d321bfc75b89ead550df1ba8fe3dc40a8102c9cc2eb9d"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/changed_country.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733490496.062186, "relation_name": "`dbt_demo`.`dbt_dev`.`changed_country`", "raw_code": "{{ config(\n materialized='table',\n file_format='delta'\n) }}\nselect * from {{ ref('country_snapshot') }}", "language": "sql", "refs": [{"name": "country_snapshot", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["snapshot.dbt_demo.country_snapshot"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/changed_country.sql", "compiled": true, "compiled_code": "\nselect * from `dbt_demo`.`dbt_dev`.`country_snapshot`", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "model.dbt_demo.country": {"database": "dbt_demo", "schema": "dbt_dev", "name": "country", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/country.sql", "original_file_path": "src/models/sql_model1/country.sql", "unique_id": "model.dbt_demo.country", "fqn": ["dbt_demo", "sql_model1", "country"], "alias": "country", "checksum": {"name": "sha256", "checksum": "be347ee69b8c27dec884b8bd692e1aa0ffdac4fb2578c410d048dc9ed16b0b08"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "table", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "file_format": "delta"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": "target/run/dbt_demo/src/models/sql_model1/country.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}, "materialized": "table", "file_format": "delta"}, "created_at": 1733492304.094096, "relation_name": "`dbt_demo`.`dbt_dev`.`country`", "raw_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n{{ config(\n materialized='table',\n file_format='delta'\n) }}\nselect\n country_code,\n country_name\nfrom {{ ref('country_codes') }} -- use seed data to populate the table", "language": "sql", "refs": [{"name": "country_codes", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["seed.dbt_demo.country_codes"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/country.sql", "compiled": true, "compiled_code": "-- create a table\n-- https://docs.getdbt.com/reference/resource-configs/databricks-configs\n\nselect\n country_code,\n country_name\nfrom `dbt_demo`.`dbt_dev`.`country_codes` -- use seed data to populate the table", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "snapshot.dbt_demo.country_snapshot": {"database": "dbt_demo", "schema": "dbt_dev", "name": "country_snapshot", "resource_type": "snapshot", "package_name": "dbt_demo", "path": "country_snapshot.sql", "original_file_path": "src/snapshots/country_snapshot.sql", "unique_id": "snapshot.dbt_demo.country_snapshot", "fqn": ["dbt_demo", "country_snapshot", "country_snapshot"], "alias": "country_snapshot", "checksum": {"name": "sha256", "checksum": "100b8c3c320732e98704fe520db0c0bcbf9f1fccf565fccd5807b87cd0d01984"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "snapshot", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": "country_code", "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "strategy": "check", "target_schema": "dbt_dev", "target_database": "dbt_demo", "updated_at": null, "check_cols": ["country_code"]}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {"target_database": "dbt_demo", "target_schema": "dbt_dev", "unique_key": "country_code", "strategy": "check", "check_cols": ["country_code"]}, "created_at": 1733493205.419168, "relation_name": "`dbt_demo`.`dbt_dev`.`country_snapshot`", "raw_code": "\n\n{% set unique_key = 'country_code' %}\n{% set strategy = 'check' %}\n{% set check_cols = ['country_code'] %}\n{% set target_database = 'dbt_demo' %}\n{% set target_schema = 'dbt_dev' %}\n\n{{\n config(\n target_database=target_database,\n target_schema=target_schema,\n unique_key=unique_key,\n strategy=strategy,\n check_cols=check_cols\n )\n}}\n\n-- SQL query for the snapshot\nSELECT *\nFROM {{ ref('country') }}\n\n", "language": "sql", "refs": [{"name": "country", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.country"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}}, "model.dbt_demo.zzz_win_loss_records": {"database": "dbt_demo", "schema": "dbt_dev", "name": "zzz_win_loss_records", "resource_type": "model", "package_name": "dbt_demo", "path": "sql_model1/zzz_win_loss_records.sql", "original_file_path": "src/models/sql_model1/zzz_win_loss_records.sql", "unique_id": "model.dbt_demo.zzz_win_loss_records", "fqn": ["dbt_demo", "sql_model1", "zzz_win_loss_records"], "alias": "zzz_win_loss_records", "checksum": {"name": "sha256", "checksum": "2af4334f4afcfc71dbb39dd2bb88e64cf6e49fb20ec401ef0abf345aa827eb69"}, "config": {"enabled": true, "alias": null, "schema": null, "database": null, "tags": [], "meta": {}, "group": null, "materialized": "view", "incremental_strategy": null, "persist_docs": {}, "post-hook": [], "pre-hook": [], "quoting": {}, "column_types": {}, "full_refresh": null, "unique_key": null, "on_schema_change": "ignore", "on_configuration_change": "apply", "grants": {}, "packages": [], "docs": {"show": true, "node_color": null}, "contract": {"enforced": false, "alias_types": true}, "access": "protected", "submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "tags": [], "description": "", "columns": {"team": {"name": "team", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "wins": {"name": "wins", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}, "losses": {"name": "losses", "description": "", "meta": {}, "data_type": null, "constraints": [], "quote": null, "tags": []}}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": "dbt_demo://src/models/sql_model1/schemas.yml", "build_path": "target/run/dbt_demo/src/models/sql_model1/zzz_win_loss_records.sql", "unrendered_config": {"submission_method": "job_cluster", "job_cluster_config": {"spark_version": "13.3.x-scala2.12", "node_type_id": "i3.xlarge", "data_security_mode": "SINGLE_USER", "num_workers": 0, "spark_conf": {"spark.master": "local[*, 4]", "spark.databricks.cluster.profile": "singleNode"}, "custom_tags": {"ResourceClass": "SingleNode"}}}, "created_at": 1733493328.3391528, "relation_name": "`dbt_demo`.`dbt_dev`.`zzz_win_loss_records`", "raw_code": "-- Step 2 of 2: Calculate the number of wins and losses for each team.\nselect\n winner as team,\n count(winner) as wins,\n -- Each team played in 4 games.\n (4 - count(winner)) as losses\nfrom (\n -- Step 1 of 2: Determine the winner and loser for each game.\n select\n game_id,\n winner,\n case\n when\n home = winner\n then\n visitor\n else\n home\n end as loser\n from {{ ref('zzz_game_details') }}\n)\ngroup by winner\norder by wins desc", "language": "sql", "refs": [{"name": "zzz_game_details", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": [], "nodes": ["model.dbt_demo.zzz_game_details"]}, "compiled_path": "target/compiled/dbt_demo/src/models/sql_model1/zzz_win_loss_records.sql", "compiled": true, "compiled_code": "-- Step 2 of 2: Calculate the number of wins and losses for each team.\nselect\n winner as team,\n count(winner) as wins,\n -- Each team played in 4 games.\n (4 - count(winner)) as losses\nfrom (\n -- Step 1 of 2: Determine the winner and loser for each game.\n select\n game_id,\n winner,\n case\n when\n home = winner\n then\n visitor\n else\n home\n end as loser\n from `dbt_demo`.`dbt_dev`.`zzz_game_details`\n)\ngroup by winner\norder by wins desc", "extra_ctes_injected": true, "extra_ctes": [], "contract": {"enforced": false, "alias_types": true, "checksum": null}, "access": "protected", "constraints": [], "version": null, "latest_version": null, "deprecation_date": null}, "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "unique_zzz_win_loss_records_team", "resource_type": "test", "package_name": "dbt_demo", "path": "unique_zzz_win_loss_records_team.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105", "fqn": ["dbt_demo", "models", "sql_model1", "unique_zzz_win_loss_records_team"], "alias": "unique_zzz_win_loss_records_team", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.3774579, "relation_name": null, "raw_code": "{{ test_unique(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_unique"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "team", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "unique", "kwargs": {"column_name": "team", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_win_loss_records_team", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_win_loss_records_team.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_win_loss_records_team"], "alias": "not_null_zzz_win_loss_records_team", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.378356, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "team", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "team", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_win_loss_records_wins", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_win_loss_records_wins.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_win_loss_records_wins"], "alias": "not_null_zzz_win_loss_records_wins", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.3790798, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "wins", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "wins", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}, "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601": {"database": "dbt_demo", "schema": "dbt_dev_dbt_test__audit", "name": "not_null_zzz_win_loss_records_losses", "resource_type": "test", "package_name": "dbt_demo", "path": "not_null_zzz_win_loss_records_losses.sql", "original_file_path": "src/models/sql_model1/schemas.yml", "unique_id": "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601", "fqn": ["dbt_demo", "models", "sql_model1", "not_null_zzz_win_loss_records_losses"], "alias": "not_null_zzz_win_loss_records_losses", "checksum": {"name": "none", "checksum": ""}, "config": {"enabled": true, "alias": null, "schema": "dbt_test__audit", "database": null, "tags": [], "meta": {}, "group": null, "materialized": "test", "severity": "ERROR", "store_failures": null, "store_failures_as": null, "where": null, "limit": null, "fail_calc": "count(*)", "warn_if": "!= 0", "error_if": "!= 0"}, "tags": [], "description": "", "columns": {}, "meta": {}, "group": null, "docs": {"show": true, "node_color": null}, "patch_path": null, "build_path": null, "unrendered_config": {}, "created_at": 1733493328.379801, "relation_name": null, "raw_code": "{{ test_not_null(**_dbt_generic_test_kwargs) }}", "language": "sql", "refs": [{"name": "zzz_win_loss_records", "package": null, "version": null}], "sources": [], "metrics": [], "depends_on": {"macros": ["macro.dbt.test_not_null"], "nodes": ["model.dbt_demo.zzz_win_loss_records"]}, "compiled_path": null, "contract": {"enforced": false, "alias_types": true, "checksum": null}, "column_name": "losses", "file_key_name": "models.zzz_win_loss_records", "attached_node": "model.dbt_demo.zzz_win_loss_records", "test_metadata": {"name": "not_null", "kwargs": {"column_name": "losses", "model": "{{ get_where_subquery(ref('zzz_win_loss_records')) }}"}, "namespace": null}}}, "sources": {"source.dbt_demo.raw.diamonds": {"database": "hive_metastore", "schema": "default", "name": "diamonds", "resource_type": "source", "package_name": "dbt_demo", "path": "src/models/sql_model1/sources.yml", "original_file_path": "src/models/sql_model1/sources.yml", "unique_id": "source.dbt_demo.raw.diamonds", "fqn": ["dbt_demo", "sql_model1", "raw", "diamonds"], "source_name": "raw", "source_description": "", "loader": "", "identifier": "diamonds", "quoting": {"database": null, "schema": null, "identifier": null, "column": null}, "loaded_at_field": null, "freshness": {"warn_after": {"count": null, "period": null}, "error_after": {"count": null, "period": null}, "filter": null}, "external": null, "description": "", "columns": {}, "meta": {}, "source_meta": {}, "tags": [], "config": {"enabled": true}, "patch_path": null, "unrendered_config": {}, "relation_name": "`hive_metastore`.`default`.`diamonds`", "created_at": 1733486794.541405}}, "macros": {"macro.dbt_demo.cents_to_dollars": {"name": "cents_to_dollars", "resource_type": "macro", "package_name": "dbt_demo", "path": "src/macros/centrs_to_dollars.sql", "original_file_path": "src/macros/centrs_to_dollars.sql", "unique_id": "macro.dbt_demo.cents_to_dollars", "macro_sql": "{% macro cents_to_dollars(column_name, scale=2) %}\n ({{ column_name }} / 100)::numeric(16, {{ scale }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.526016, "supported_languages": null}, "macro.dbt_databricks.materialization_materialized_view_databricks": {"name": "materialization_materialized_view_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialization_materialized_view_databricks", "macro_sql": "{% materialization materialized_view, adapter = 'databricks' %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation) %}\n\n {% if build_sql == '' %}\n {{ materialized_view_execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.run_hooks", "macro.dbt_databricks.materialized_view_get_build_sql", "macro.dbt_databricks.materialized_view_execute_no_op", "macro.dbt_databricks.materialized_view_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5290601, "supported_languages": ["sql"]}, "macro.dbt_databricks.materialized_view_get_build_sql": {"name": "materialized_view_get_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialized_view_get_build_sql", "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, None, None) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = \"\" %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt_databricks.get_replace_sql", "macro.dbt.get_materialized_view_configuration_changes", "macro.dbt.refresh_materialized_view", "macro.dbt_databricks.get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.530187, "supported_languages": null}, "macro.dbt_databricks.materialized_view_execute_no_op": {"name": "materialized_view_execute_no_op", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialized_view_execute_no_op", "macro_sql": "{% macro materialized_view_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5303931, "supported_languages": null}, "macro.dbt_databricks.materialized_view_execute_build_sql": {"name": "materialized_view_execute_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/materialized_view.sql", "original_file_path": "macros/materializations/materialized_view.sql", "unique_id": "macro.dbt_databricks.materialized_view_execute_build_sql", "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {%- if build_sql is string %}\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n {%- else %}\n {%- for sql in build_sql %}\n {% call statement(name=\"main\") %}\n {{ sql }}\n {% endcall %}\n {% endfor %}\n {% endif %}\n\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model, for_relation=False) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.531114, "supported_languages": null}, "macro.dbt_databricks.materialization_view_databricks": {"name": "materialization_view_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "unique_id": "macro.dbt_databricks.materialization_view_databricks", "macro_sql": "{% materialization view, adapter='databricks' -%}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n {% set tags = config.get('databricks_tags') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and not old_relation.is_view -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=True) %}\n\n {%- do apply_tags(target_relation, tags) -%}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.apply_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.532544, "supported_languages": ["sql"]}, "macro.dbt_databricks.materialization_table_databricks": {"name": "materialization_table_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_databricks.materialization_table_databricks", "macro_sql": "{% materialization table, adapter = 'databricks', supported_languages=['sql', 'python'] %}\n {{ log(\"MATERIALIZING TABLE\") }}\n {%- set language = model['language'] -%}\n {%- set identifier = model['alias'] -%}\n {%- set grant_config = config.get('grants') -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- set tags = config.get('databricks_tags') -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier, needs_information=True) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n\n {{ run_hooks(pre_hooks) }}\n\n -- setup: if the target relation already exists, drop it\n -- in case if the existing and future table is delta, we want to do a\n -- create or replace table instead of dropping, so we don't have the table unavailable\n {% if old_relation and (not (old_relation.is_delta and config.get('file_format', default='delta') == 'delta')) or (old_relation.is_materialized_view or old_relation.is_streaming_table) -%}\n {{ adapter.drop_relation(old_relation) }}\n {%- endif %}\n\n -- build model\n\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n {% if language==\"python\" %}\n {% do apply_tblproperties(target_relation, tblproperties) %}\n {% endif %}\n {%- do apply_tags(target_relation, tags) -%}\n\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {% do optimize(target_relation) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]})}}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.apply_tblproperties", "macro.dbt_databricks.apply_tags", "macro.dbt.persist_docs", "macro.dbt_databricks.persist_constraints", "macro.dbt_databricks.optimize"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5349078, "supported_languages": ["sql", "python"]}, "macro.dbt_databricks.databricks_build_snapshot_staging_table": {"name": "databricks_build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_databricks.databricks_build_snapshot_staging_table", "macro_sql": "{% macro databricks_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_identifier = target_relation.identifier ~ '__dbt_tmp' %}\n\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=target_relation.schema,\n database=target_relation.database,\n type='view') -%}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {# needs to be a non-temp view so that its columns can be ascertained via `describe` #}\n {% call statement('build_snapshot_staging_relation') %}\n create or replace view {{ tmp_relation }}\n as\n {{ select }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_staging_table", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5406778, "supported_languages": null}, "macro.dbt_databricks.materialization_snapshot_databricks": {"name": "materialization_snapshot_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_databricks.materialization_snapshot_databricks", "macro_sql": "{% materialization snapshot, adapter='databricks' %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n {%- set file_format = config.get('file_format', 'delta') -%}\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = databricks__get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table',\n needs_information=True) -%}\n\n {%- if file_format not in ['delta', 'hudi'] -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n Snapshot functionality requires file_format be set to 'delta' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n\n {%- if target_relation_exists -%}\n {%- if not target_relation.is_delta and not target_relation.is_hudi -%}\n {% set invalid_format_msg -%}\n The existing table {{ model.schema }}.{{ target_table }} is in another format than 'delta' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n {% endif %}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model, for_relation=False) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% if target_relation.database is none %}\n {% set staging_table = spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% else %}\n {% set staging_table = databricks_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% endif %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% call statement_with_staging_table('main', staging_table) %}\n {{ final_sql }}\n {% endcall %}\n\n {% do persist_docs(target_relation, model, for_relation=True) %}\n\n {% endif %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.statement", "macro.dbt.persist_docs", "macro.dbt_spark.spark_build_snapshot_staging_table", "macro.dbt_databricks.databricks_build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt_databricks.statement_with_staging_table", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.544019, "supported_languages": ["sql"]}, "macro.dbt_databricks.materialization_streaming_table_databricks": {"name": "materialization_streaming_table_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.materialization_streaming_table_databricks", "macro_sql": "{% materialization streaming_table, adapter='databricks' %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.StreamingTable) %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% set build_sql = streaming_table_get_build_sql(existing_relation, target_relation) %}\n\n {% if build_sql == '' %}\n {{ streaming_table_execute_no_op(target_relation) }}\n {% else %}\n {{ streaming_table_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.run_hooks", "macro.dbt_databricks.streaming_table_get_build_sql", "macro.dbt_databricks.streaming_table_execute_no_op", "macro.dbt_databricks.streaming_table_execute_build_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.546883, "supported_languages": ["sql"]}, "macro.dbt_databricks.streaming_table_get_build_sql": {"name": "streaming_table_get_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.streaming_table_get_build_sql", "macro_sql": "{% macro streaming_table_get_build_sql(existing_relation, target_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_streaming_table_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_streaming_table %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_streaming_table_configuration_changes(existing_relation, config) %}\n {% if configuration_changes is none %}\n {% set build_sql = refresh_streaming_table(target_relation, sql) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_streaming_table_as_sql(target_relation, configuration_changes, sql, existing_relation, None, None) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = \"\" %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt_databricks.get_create_streaming_table_as_sql", "macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_streaming_table_configuration_changes", "macro.dbt_databricks.refresh_streaming_table", "macro.dbt_databricks.get_alter_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5479271, "supported_languages": null}, "macro.dbt_databricks.streaming_table_execute_no_op": {"name": "streaming_table_execute_no_op", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.streaming_table_execute_no_op", "macro_sql": "{% macro streaming_table_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.548121, "supported_languages": null}, "macro.dbt_databricks.streaming_table_execute_build_sql": {"name": "streaming_table_execute_build_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/streaming_table.sql", "original_file_path": "macros/materializations/streaming_table.sql", "unique_id": "macro.dbt_databricks.streaming_table_execute_build_sql", "macro_sql": "{% macro streaming_table_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {%- if build_sql is string %}\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n {%- else %}\n {%- for sql in build_sql %}\n {% call statement(name=\"main\") %}\n {{ sql }}\n {% endcall %}\n {% endfor %}\n {% endif %}\n\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model, for_relation=False) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.548807, "supported_languages": null}, "macro.dbt_databricks.databricks__can_clone_table": {"name": "databricks__can_clone_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_databricks.databricks__can_clone_table", "macro_sql": "{% macro databricks__can_clone_table() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.550354, "supported_languages": null}, "macro.dbt_databricks.databricks__create_or_replace_clone": {"name": "databricks__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_databricks.databricks__create_or_replace_clone", "macro_sql": "{% macro databricks__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace\n table {{ this_relation }}\n shallow clone {{ defer_relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.550466, "supported_languages": null}, "macro.dbt_databricks.materialization_clone_databricks": {"name": "materialization_clone_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_databricks.materialization_clone_databricks", "macro_sql": "\n\n{%- materialization clone, adapter='databricks' -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n {% endif %}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.552143, "supported_languages": ["sql"]}, "macro.dbt_databricks.dbt_databricks_validate_get_file_format": {"name": "dbt_databricks_validate_get_file_format", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_databricks.dbt_databricks_validate_get_file_format", "macro_sql": "{% macro dbt_databricks_validate_get_file_format(raw_file_format) %}\n {#-- Validate the file format #}\n\n {% set accepted_formats = ['text', 'csv', 'json', 'jdbc', 'parquet', 'orc', 'hive', 'delta', 'libsvm', 'hudi'] %}\n\n {% set invalid_file_format_msg -%}\n Invalid file format provided: {{ raw_file_format }}\n Expected one of: {{ accepted_formats | join(', ') }}\n {%- endset %}\n\n {% if raw_file_format not in accepted_formats %}\n {% do exceptions.raise_compiler_error(invalid_file_format_msg) %}\n {% endif %}\n\n {% do return(raw_file_format) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.553088, "supported_languages": null}, "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy": {"name": "dbt_databricks_validate_get_incremental_strategy", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy", "macro_sql": "{% macro dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format) %}\n {#-- Validate the incremental strategy #}\n\n {% set invalid_strategy_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n Expected one of: 'merge', 'replace_where', 'append', 'insert_overwrite'\n {%- endset %}\n\n {% set invalid_delta_only_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You can only choose this strategy when file_format is set to 'delta'\n {%- endset %}\n\n {% set invalid_insert_overwrite_endpoint_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You cannot use this strategy when connecting via warehouse\n Use the 'merge' or 'replace_where' strategy instead\n {%- endset %}\n\n {% if raw_strategy not in ['append', 'merge', 'insert_overwrite', 'replace_where'] %}\n {% do exceptions.raise_compiler_error(invalid_strategy_msg) %}\n {%-else %}\n {% if raw_strategy == 'merge' and file_format not in ['delta', 'hudi'] %}\n {% do exceptions.raise_compiler_error(invalid_delta_only_msg) %}\n {% endif %}\n {% if raw_strategy == 'replace_where' and file_format not in ['delta'] %}\n {% do exceptions.raise_compiler_error(invalid_delta_only_msg) %}\n {% endif %}\n {% endif %}\n\n {% do return(raw_strategy) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.55381, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_default_sql": {"name": "databricks__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_default_sql", "macro_sql": "{% macro databricks__get_incremental_default_sql(arg_dict) %}\n {{ return(get_incremental_merge_sql(arg_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.55716, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_append_sql": {"name": "databricks__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_append_sql", "macro_sql": "{% macro databricks__get_incremental_append_sql(arg_dict) %}\n {% do return(get_insert_into_sql(arg_dict[\"temp_relation\"], arg_dict[\"target_relation\"])) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5573308, "supported_languages": null}, "macro.dbt_databricks.databricks__get_incremental_replace_where_sql": {"name": "databricks__get_incremental_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_incremental_replace_where_sql", "macro_sql": "{% macro databricks__get_incremental_replace_where_sql(arg_dict) %}\n {% do return(get_replace_where_sql(arg_dict)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_where_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.557461, "supported_languages": null}, "macro.dbt_databricks.get_incremental_replace_where_sql": {"name": "get_incremental_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_incremental_replace_where_sql", "macro_sql": "{% macro get_incremental_replace_where_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_replace_where_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_replace_where_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.557615, "supported_languages": null}, "macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql": {"name": "databricks__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql", "macro_sql": "{% macro databricks__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) %}\n {{ return(get_insert_overwrite_sql(source, target)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5577788, "supported_languages": null}, "macro.dbt_databricks.get_insert_overwrite_sql": {"name": "get_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_insert_overwrite_sql", "macro_sql": "{% macro get_insert_overwrite_sql(source_relation, target_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n insert overwrite table {{ target_relation }}\n {{ partition_cols(label=\"partition\") }}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.558079, "supported_languages": null}, "macro.dbt_databricks.get_replace_where_sql": {"name": "get_replace_where_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_replace_where_sql", "macro_sql": "{% macro get_replace_where_sql(args_dict) -%}\n {%- set predicates = args_dict['incremental_predicates'] -%}\n {%- set target_relation = args_dict['target_relation'] -%}\n {%- set temp_relation = args_dict['temp_relation'] -%}\n\n insert into {{ target_relation }}\n {% if predicates %}\n {% if predicates is sequence and predicates is not string %}\n replace where {{ predicates | join(' and ') }}\n {% else %}\n replace where {{ predicates }}\n {% endif %}\n {% endif %}\n table {{ temp_relation }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5584981, "supported_languages": null}, "macro.dbt_databricks.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(source_relation, target_relation) %}\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) | map(attribute=\"quoted\") | list -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) | map(attribute=\"quoted\") | list -%}\n {{ insert_into_sql_impl(target_relation, dest_columns, source_relation, source_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.insert_into_sql_impl"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.558819, "supported_languages": null}, "macro.dbt_databricks.insert_into_sql_impl": {"name": "insert_into_sql_impl", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.insert_into_sql_impl", "macro_sql": "{% macro insert_into_sql_impl(target_relation, dest_columns, source_relation, source_columns) %}\n {%- set common_columns = [] -%}\n {%- for dest_col in dest_columns -%}\n {%- if dest_col in source_columns -%}\n {%- do common_columns.append(dest_col) -%}\n {%- else -%}\n {%- do common_columns.append('DEFAULT') -%}\n {%- endif -%}\n {%- endfor -%}\n {%- set dest_cols_csv = dest_columns | join(', ') -%}\n {%- set source_cols_csv = common_columns | join(', ') -%}\ninsert into table {{ target_relation }} ({{ dest_cols_csv }})\nselect {{source_cols_csv}} from {{ source_relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5592902, "supported_languages": null}, "macro.dbt_databricks.databricks__get_merge_sql": {"name": "databricks__get_merge_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.databricks__get_merge_sql", "macro_sql": "{% macro databricks__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %}\n {# need dest_columns for merge_exclude_columns, default to use \"*\" #}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target) -%}\n {%- set source_columns = (adapter.get_columns_in_relation(source) | map(attribute='quoted') | list)-%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} <=> DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} <=> DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n when matched then update set {{ get_merge_update_set(update_columns, on_schema_change, source_columns) }}\n when not matched then insert {{ get_merge_insert(on_schema_change, source_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.incremental_validate_on_schema_change", "macro.dbt.get_merge_update_columns", "macro.dbt_databricks.get_merge_update_set", "macro.dbt_databricks.get_merge_insert"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5605829, "supported_languages": null}, "macro.dbt_databricks.get_merge_update_set": {"name": "get_merge_update_set", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_merge_update_set", "macro_sql": "{% macro get_merge_update_set(update_columns, on_schema_change, source_columns) %}\n {%- if update_columns -%}\n {%- for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n {%- elif on_schema_change == 'ignore' -%}\n *\n {%- else -%}\n {%- for column in source_columns -%}\n {{ column }} = DBT_INTERNAL_SOURCE.{{ column }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.560978, "supported_languages": null}, "macro.dbt_databricks.get_merge_insert": {"name": "get_merge_insert", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_databricks.get_merge_insert", "macro_sql": "{% macro get_merge_insert(on_schema_change, source_columns) %}\n {%- if on_schema_change == 'ignore' -%}\n *\n {%- else -%}\n ({{ source_columns | join(\", \") }}) VALUES (\n {%- for column in source_columns -%}\n DBT_INTERNAL_SOURCE.{{ column }}{%- if not loop.last %}, {% endif -%}\n {%- endfor %})\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.561265, "supported_languages": null}, "macro.dbt_databricks.materialization_incremental_databricks": {"name": "materialization_incremental_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_databricks.materialization_incremental_databricks", "macro_sql": "{% materialization incremental, adapter='databricks', supported_languages=['sql', 'python'] -%}\n {#-- Validate early so we don't run SQL if the file_format + strategy combo is invalid --#}\n {%- set raw_file_format = config.get('file_format', default='delta') -%}\n {%- set raw_strategy = config.get('incremental_strategy') or 'merge' -%}\n {%- set grant_config = config.get('grants') -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- set tags = config.get('databricks_tags') -%}\n {%- set unique_tmp_table_suffix = config.get('unique_tmp_table_suffix', False) | as_bool -%}\n\n {%- set file_format = dbt_databricks_validate_get_file_format(raw_file_format) -%}\n {%- set incremental_strategy = dbt_databricks_validate_get_incremental_strategy(raw_strategy, file_format) -%}\n\n {#-- Set vars --#}\n\n {%- set incremental_predicates = config.get('predicates', default=none) or config.get('incremental_predicates', default=none) -%}\n {%- set unique_key = config.get('unique_key', none) -%}\n {%- set partition_by = config.get('partition_by', none) -%}\n {%- set language = model['language'] -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set existing_relation = adapter.get_relation(database=this.database, schema=this.schema, identifier=this.identifier, needs_information=True) -%}\n {%- if unique_tmp_table_suffix == True and raw_strategy == 'replace_where' and raw_file_format == 'delta' -%}\n {%- set temp_relation_suffix = adapter.generate_unique_temporary_table_suffix() -%}\n {%- else -%}\n {%- set temp_relation_suffix = '__dbt_tmp' -%}\n {%- endif -%}\n\n {#-- Set Overwrite Mode to STATIC for initial replace --#}\n {%- if incremental_strategy == 'insert_overwrite' and should_full_refresh() -%}\n {%- call statement() -%}\n set spark.sql.sources.partitionOverwriteMode = STATIC\n {%- endcall -%}\n {%- endif -%}\n\n {#-- Run pre-hooks --#}\n {{ run_hooks(pre_hooks) }}\n\n {#-- Incremental run logic --#}\n {%- if existing_relation is none -%}\n {#-- Relation must be created --#}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {% do apply_tags(target_relation, tags) %}\n {%- if language == 'python' -%}\n {%- do apply_tblproperties(target_relation, tblproperties) %}\n {%- endif -%}\n\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n {%- elif existing_relation.is_view or existing_relation.is_materialized_view or existing_relation.is_streaming_table or should_full_refresh() -%}\n {#-- Relation must be dropped & recreated --#}\n {% set is_delta = (file_format == 'delta' and existing_relation.is_delta) %}\n {% if not is_delta %} {#-- If Delta, we will `create or replace` below, so no need to drop --#}\n {% do adapter.drop_relation(existing_relation) %}\n {% endif %}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% if not existing_relation.is_view %}\n {% do persist_constraints(target_relation, model) %}\n {% endif %}\n {% do apply_tags(target_relation, tags) %}\n {% do persist_docs(target_relation, model, for_relation=language=='python') %}\n {%- else -%}\n {#-- Set Overwrite Mode to DYNAMIC for subsequent incremental operations --#}\n {%- if incremental_strategy == 'insert_overwrite' and partition_by -%}\n {%- call statement() -%}\n set spark.sql.sources.partitionOverwriteMode = DYNAMIC\n {%- endcall -%}\n {%- endif -%}\n {#-- Relation must be merged --#}\n {%- set _existing_config = adapter.get_relation_config(existing_relation) -%}\n {%- set model_config = adapter.get_config_from_model(config.model) -%}\n {%- set _configuration_changes = model_config.get_changeset(_existing_config) -%}\n {%- set temp_relation = databricks__make_temp_relation(target_relation, suffix=temp_relation_suffix, as_table=language != 'sql') -%}\n {%- call statement('create_temp_relation', language=language) -%}\n {{ create_table_as(True, temp_relation, compiled_code, language) }}\n {%- endcall -%}\n {%- do process_schema_changes(on_schema_change, temp_relation, existing_relation) -%}\n {%- set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) -%}\n {%- set strategy_arg_dict = ({\n 'target_relation': target_relation,\n 'temp_relation': temp_relation,\n 'unique_key': unique_key,\n 'dest_columns': none,\n 'incremental_predicates': incremental_predicates}) -%}\n {%- set build_sql = strategy_sql_macro_func(strategy_arg_dict) -%}\n {%- if language == 'sql' -%}\n {%- call statement('main') -%}\n {{ build_sql }}\n {%- endcall -%}\n {%- elif language == 'python' -%}\n {%- call statement_with_staging_table('main', temp_relation) -%}\n {{ build_sql }}\n {%- endcall -%}\n {#--\n This is yucky.\n See note in dbt-spark/dbt/include/spark/macros/adapters.sql\n re: python models and temporary views.\n\n Also, why does not either drop_relation or adapter.drop_relation work here?!\n --#}\n {%- endif -%}\n {% do apply_liquid_clustered_cols(target_relation) %}\n {% if _configuration_changes is not none %}\n {% set tags = _configuration_changes.changes.get(\"tags\", None) %}\n {% set tblproperties = _configuration_changes.changes.get(\"tblproperties\", None) %}\n {% if tags is not none %}\n {% do apply_tags(target_relation, tags.set_tags, tags.unset_tags) %}\n {%- endif -%}\n {% if tblproperties is not none %}\n {% do apply_tblproperties(target_relation, tblproperties.tblproperties) %}\n {%- endif -%}\n {%- endif -%}\n {% do persist_docs(target_relation, model, for_relation=True) %}\n {%- endif -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n {% do optimize(target_relation) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt_databricks.dbt_databricks_validate_get_file_format", "macro.dbt_databricks.dbt_databricks_validate_get_incremental_strategy", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt_databricks.persist_constraints", "macro.dbt_databricks.apply_tags", "macro.dbt_databricks.apply_tblproperties", "macro.dbt.persist_docs", "macro.dbt_databricks.databricks__make_temp_relation", "macro.dbt.process_schema_changes", "macro.dbt_databricks.statement_with_staging_table", "macro.dbt_databricks.apply_liquid_clustered_cols", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt_databricks.optimize"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.566802, "supported_languages": ["sql", "python"]}, "macro.dbt_databricks.databricks__get_binding_char": {"name": "databricks__get_binding_char", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__get_binding_char", "macro_sql": "{% macro databricks__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5697222, "supported_languages": null}, "macro.dbt_databricks.databricks__load_csv_rows": {"name": "databricks__load_csv_rows", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__load_csv_rows", "macro_sql": "{% macro databricks__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n {% set column_override = model['config'].get('column_types', {}) %}\n {% set must_cast = model['config'].get(\"file_format\", \"delta\") == \"parquet\" %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert {% if loop.index0 == 0 -%} overwrite {% else -%} into {% endif -%} {{ this.render() }} values\n {% for row in chunk -%}\n ({%- for col_name in agate_table.column_names -%}\n {%- if must_cast -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n cast({{ get_binding_char() }} as {{type}})\n {%- else -%}\n {{ get_binding_char() }}\n {%- endif -%}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True, close_cursor=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.571152, "supported_languages": null}, "macro.dbt_databricks.databricks__reset_csv_table": {"name": "databricks__reset_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__reset_csv_table", "macro_sql": "{% macro databricks__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% if old_relation %}\n {% if old_relation.is_delta and config.get('file_format', default='delta') == 'delta' %}\n {% set sql = create_or_replace_csv_table(model, agate_table, True) %}\n {% else %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% endif %}\n {% else %}\n {% set sql = create_csv_table(model, agate_table) %}\n {% endif %}\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.create_or_replace_csv_table", "macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.57173, "supported_languages": null}, "macro.dbt_databricks.create_or_replace_csv_table": {"name": "create_or_replace_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.create_or_replace_csv_table", "macro_sql": "{% macro create_or_replace_csv_table(model, agate_table, replace=False) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {%- set column_comment = config.persist_column_docs() and model.columns %}\n {%- set identifier = model['alias'] -%}\n {%- set relation = api.Relation.create(database=database, schema=schema, identifier=identifier, type='table') -%}\n {%- set replace_clause = \"\" -%}\n {%- if replace -%}\n {%- set replace_clause = \"or replace\" -%}\n {%- endif -%}\n\n {% set sql %}\n create {{replace_clause}} table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {%- set column_comment_clause = \"\" -%}\n {%- if column_comment and col_name in model.columns.keys() -%} \n {%- set comment = model.columns[col_name]['description'] | replace(\"'\", \"\\\\'\") -%}\n {%- if comment and comment != \"\" -%}\n {%- set column_comment_clause = \"comment '\" ~ comment ~ \"'\" -%}\n {%- endif -%}\n {%- endif -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {{ column_comment_clause }}{%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {{ file_format_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause(relation) }}\n {{ comment_clause() }}\n {{ tblproperties_clause() }}\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.file_format_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_databricks.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_databricks.tblproperties_clause", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.57331, "supported_languages": null}, "macro.dbt_databricks.databricks__create_csv_table": {"name": "databricks__create_csv_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt_databricks.databricks__create_csv_table", "macro_sql": "{% macro databricks__create_csv_table(model, agate_table) %}\n {{ return(create_or_replace_csv_table(model, agate_table)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.create_or_replace_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.573471, "supported_languages": null}, "macro.dbt_databricks.materialization_seed_databricks": {"name": "materialization_seed_databricks", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/materializations/seeds/seeds.sql", "original_file_path": "macros/materializations/seeds/seeds.sql", "unique_id": "macro.dbt_databricks.materialization_seed_databricks", "macro_sql": "{% materialization seed, adapter='databricks' %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier, needs_information=True) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and (old_relation.is_view or old_relation.is_materialized_view)) -%}\n {%- set exists_as_streaming_table = (old_relation is not none and old_relation.is_streaming_table) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparision later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view or a materialized view\".format(old_relation)) }}\n {% elif exists_as_streaming_table %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a streaming table\".format(old_relation)) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n -- No need to persist docs, already handled in seed create\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.576501, "supported_languages": ["sql"]}, "macro.dbt_databricks.databricks__generate_database_name": {"name": "databricks__generate_database_name", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt_databricks.databricks__generate_database_name", "macro_sql": "{% macro databricks__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n {{ return(default_database) }}\n {%- else -%}\n {{ return(custom_database_name) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.576849, "supported_languages": null}, "macro.dbt_databricks.databricks__get_drop_sql": {"name": "databricks__get_drop_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt_databricks.databricks__get_drop_sql", "macro_sql": "{% macro databricks__get_drop_sql(relation) -%}\n {%- if relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n {%- elif relation.is_streaming_table-%}\n {{ drop_streaming_table(relation) }}\n {%- elif relation.is_view -%}\n {{ drop_view(relation) }}\n {%- else -%}\n {{ drop_table(relation) }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.drop_materialized_view", "macro.dbt_databricks.drop_streaming_table", "macro.dbt.drop_view", "macro.dbt.drop_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.577309, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_relation": {"name": "databricks__drop_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_relation", "macro_sql": "{% macro databricks__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {{ get_drop_sql(relation) }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.577478, "supported_languages": null}, "macro.dbt_databricks.get_replace_sql": {"name": "get_replace_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.get_replace_sql", "macro_sql": "{% macro get_replace_sql(existing_relation, target_relation, sql) %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {% do return(adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql)) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.57809, "supported_languages": null}, "macro.dbt_databricks.databricks__get_replace_sql": {"name": "databricks__get_replace_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt_databricks.databricks__get_replace_sql", "macro_sql": "{% macro databricks__get_replace_sql(existing_relation, target_relation, sql) %}\n {# /* use a create or replace statement if possible */ #}\n\n {% set is_replaceable = existing_relation.type == target_relation_type and existing_relation.can_be_replaced %}\n\n {% if is_replaceable and existing_relation.is_view %}\n {{ get_replace_view_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_table %}\n {{ get_replace_table_sql(target_relation, sql) }}\n\n {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}\n {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}\n {{ return([\n get_create_intermediate_sql(target_relation, sql),\n get_create_backup_sql(existing_relation),\n get_rename_intermediate_sql(target_relation),\n get_drop_backup_sql(existing_relation)\n ]) }}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}\n {%- elif target_relation.can_be_renamed -%}\n {{ return([\n get_create_intermediate_sql(target_relation, sql),\n get_drop_sql(existing_relation),\n get_rename_intermediate_sql(target_relation),\n ]) }}\n\n {# /* create target_relation in place by first backing up the existing relation */ #}\n {%- elif existing_relation.can_be_renamed -%}\n {{ return([\n get_create_backup_sql(existing_relation),\n get_create_sql(target_relation, sql),\n get_drop_backup_sql(existing_relation)\n ]) }}\n\n {# /* no renaming is allowed, so just drop and create */ #}\n {%- else -%}\n {{ return([\n get_drop_sql(existing_relation),\n get_create_sql(target_relation, sql)\n ]) }}\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_replace_view_sql", "macro.dbt.get_replace_table_sql", "macro.dbt.get_create_intermediate_sql", "macro.dbt.get_create_backup_sql", "macro.dbt.get_rename_intermediate_sql", "macro.dbt.get_drop_backup_sql", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.579058, "supported_languages": null}, "macro.dbt_databricks.fetch_tags": {"name": "fetch_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.fetch_tags", "macro_sql": "{% macro fetch_tags(relation) -%}\n {% if relation.is_hive_metastore() %}\n {{ exceptions.raise_compiler_error(\"Tags are only supported for Unity Catalog\") }}\n {%- endif %}\n {% call statement('list_tags', fetch_result=True) -%}\n {{ fetch_tags_sql(relation) }}\n {% endcall %}\n {% do return(load_result('list_tags').table) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.fetch_tags_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5802681, "supported_languages": null}, "macro.dbt_databricks.fetch_tags_sql": {"name": "fetch_tags_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.fetch_tags_sql", "macro_sql": "{% macro fetch_tags_sql(relation) -%}\n SELECT tag_name, tag_value\n FROM `system`.`information_schema`.`table_tags`\n WHERE catalog_name = '{{ relation.database|lower }}' \n AND schema_name = '{{ relation.schema|lower }}'\n AND table_name = '{{ relation.identifier|lower }}'\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5804422, "supported_languages": null}, "macro.dbt_databricks.apply_tags": {"name": "apply_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.apply_tags", "macro_sql": "{% macro apply_tags(relation, set_tags, unset_tags=[]) -%}\n {%- if (set_tags or unset_tags) and relation.is_hive_metastore() -%}\n {{ exceptions.raise_compiler_error(\"Tags are only supported for Unity Catalog\") }}\n {%- endif -%}\n {%- if set_tags %}\n {%- call statement('set_tags') -%}\n {{ alter_set_tags(relation, set_tags) }}\n {%- endcall -%}\n {%- endif %}\n {%- if unset_tags %}\n {%- call statement('unset_tags') -%}\n {{ alter_unset_tags(relation, unset_tags) }}\n {%- endcall -%}\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.alter_set_tags", "macro.dbt_databricks.alter_unset_tags"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.580887, "supported_languages": null}, "macro.dbt_databricks.alter_set_tags": {"name": "alter_set_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.alter_set_tags", "macro_sql": "{% macro alter_set_tags(relation, tags) -%}\n ALTER {{ relation.type }} {{ relation }} SET TAGS (\n {% for tag in tags -%}\n '{{ tag }}' = '{{ tags[tag] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n )\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.581147, "supported_languages": null}, "macro.dbt_databricks.alter_unset_tags": {"name": "alter_unset_tags", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tags.sql", "original_file_path": "macros/relations/tags.sql", "unique_id": "macro.dbt_databricks.alter_unset_tags", "macro_sql": "{% macro alter_unset_tags(relation, tags) -%}\n ALTER {{ relation.type }} {{ relation }} UNSET TAGS (\n {% for tag in tags -%}\n '{{ tag }}' {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n )\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.58137, "supported_languages": null}, "macro.dbt_databricks.location_clause": {"name": "location_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/location.sql", "original_file_path": "macros/relations/location.sql", "unique_id": "macro.dbt_databricks.location_clause", "macro_sql": "{% macro location_clause(relation) %}\n {%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n {%- set file_format = config.get('file_format', default='delta') -%}\n {%- set identifier = model['alias'] -%}\n {%- if location_root is not none %}\n location '{{ location_root }}/{{ identifier }}'\n {%- elif (not relation.is_hive_metastore()) and file_format != 'delta' -%}\n {{ exceptions.raise_compiler_error(\n 'Incompatible configuration: `location_root` must be set when using a non-delta file format with Unity Catalog'\n ) }}\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5819929, "supported_languages": null}, "macro.dbt_databricks.optimize": {"name": "optimize", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.optimize", "macro_sql": "{% macro optimize(relation) %}\n {{ return(adapter.dispatch('optimize', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__optimize"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.582578, "supported_languages": null}, "macro.dbt_databricks.databricks__optimize": {"name": "databricks__optimize", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.databricks__optimize", "macro_sql": "\n\n{%- macro databricks__optimize(relation) -%}\n {%- if var('DATABRICKS_SKIP_OPTIMIZE', 'false')|lower != 'true' and\n var('databricks_skip_optimize', 'false')|lower != 'true' and\n config.get('file_format', 'delta') == 'delta' -%}\n {%- if (config.get('zorder', False) or config.get('liquid_clustered_by', False)) -%}\n {%- call statement('run_optimize_stmt') -%}\n {{ get_optimize_sql(relation) }}\n {%- endcall -%}\n {%- endif -%}\n {%- endif -%}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt_databricks.get_optimize_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.583037, "supported_languages": null}, "macro.dbt_databricks.get_optimize_sql": {"name": "get_optimize_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/optimize.sql", "original_file_path": "macros/relations/optimize.sql", "unique_id": "macro.dbt_databricks.get_optimize_sql", "macro_sql": "{%- macro get_optimize_sql(relation) %}\n optimize {{ relation }}\n {%- if config.get('zorder', False) and config.get('file_format', 'delta') == 'delta' %}\n {%- if config.get('liquid_clustered_by', False) %}\n {{ exceptions.warn(\"Both zorder and liquid_clustered_by are set but they are incompatible. zorder will be ignored.\") }}\n {%- else %}\n {%- set zorder = config.get('zorder', none) %}\n {# TODO: predicates here? WHERE ... #}\n {%- if zorder is sequence and zorder is not string %}\n zorder by (\n {%- for col in zorder %}\n {{ col }}{% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- else %}\n zorder by ({{zorder}})\n {%- endif %}\n {%- endif %}\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.583699, "supported_languages": null}, "macro.dbt_databricks.file_format_clause": {"name": "file_format_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/file_format.sql", "original_file_path": "macros/relations/file_format.sql", "unique_id": "macro.dbt_databricks.file_format_clause", "macro_sql": "{% macro file_format_clause() %}\n {%- set file_format = config.get('file_format', default='delta') -%}\n using {{ file_format }}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.583901, "supported_languages": null}, "macro.dbt_databricks.liquid_clustered_cols": {"name": "liquid_clustered_cols", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/liquid_clustering.sql", "original_file_path": "macros/relations/liquid_clustering.sql", "unique_id": "macro.dbt_databricks.liquid_clustered_cols", "macro_sql": "{% macro liquid_clustered_cols() -%}\n {%- set cols = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n CLUSTER BY ({{ cols | join(', ') }})\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5845, "supported_languages": null}, "macro.dbt_databricks.apply_liquid_clustered_cols": {"name": "apply_liquid_clustered_cols", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/liquid_clustering.sql", "original_file_path": "macros/relations/liquid_clustering.sql", "unique_id": "macro.dbt_databricks.apply_liquid_clustered_cols", "macro_sql": "{% macro apply_liquid_clustered_cols(target_relation) -%}\n {%- set cols = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {%- call statement('set_cluster_by_columns') -%}\n ALTER {{ target_relation.type }} {{ target_relation }} CLUSTER BY ({{ cols | join(', ') }})\n {%- endcall -%}\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.584919, "supported_languages": null}, "macro.dbt_databricks.tblproperties_clause": {"name": "tblproperties_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.tblproperties_clause", "macro_sql": "{% macro tblproperties_clause() -%}\n {{ return(adapter.dispatch('tblproperties_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.58541, "supported_languages": null}, "macro.dbt_databricks.databricks__tblproperties_clause": {"name": "databricks__tblproperties_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.databricks__tblproperties_clause", "macro_sql": "{% macro databricks__tblproperties_clause() -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- if tblproperties is not none %}\n tblproperties (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}' {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.5857081, "supported_languages": null}, "macro.dbt_databricks.apply_tblproperties": {"name": "apply_tblproperties", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/tblproperties.sql", "original_file_path": "macros/relations/tblproperties.sql", "unique_id": "macro.dbt_databricks.apply_tblproperties", "macro_sql": "{% macro apply_tblproperties(relation, tblproperties) -%}\n {% if tblproperties %}\n {%- call statement('apply_tblproperties') -%}\n ALTER {{ relation.type }} {{ relation }} SET TBLPROPERTIES (\n {% for tblproperty in tblproperties -%}\n '{{ tblproperty }}' = '{{ tblproperties[tblproperty] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor %}\n )\n {%- endcall -%}\n {% endif %}\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.58604, "supported_languages": null}, "macro.dbt_databricks.persist_constraints": {"name": "persist_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.persist_constraints", "macro_sql": "{% macro persist_constraints(relation, model) %}\n {{ return(adapter.dispatch('persist_constraints', 'dbt')(relation, model)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.599794, "supported_languages": null}, "macro.dbt_databricks.databricks__persist_constraints": {"name": "databricks__persist_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__persist_constraints", "macro_sql": "{% macro databricks__persist_constraints(relation, model) %}\n {%- set contract_config = config.get('contract') -%}\n {% set has_model_contract = contract_config and contract_config.enforced %}\n {% set has_databricks_constraints = config.get('persist_constraints', False) %}\n\n {% if (has_model_contract or has_databricks_constraints) %}\n {% if config.get('file_format', 'delta') != 'delta' %}\n {# Constraints are only supported for delta tables #}\n {{ exceptions.warn(\"Constraints not supported for file format: \" ~ config.get('file_format')) }}\n {% elif relation.is_view %}\n {# Constraints are not supported for views. This point in the code should not have been reached. #}\n {{ exceptions.raise_compiler_error(\"Constraints not supported for views.\") }}\n {% elif is_incremental() %}\n {# Constraints are not applied for incremental updates. This point in the code should not have been reached #}\n {{ exceptions.raise_compiler_error(\"Constraints are not applied for incremental updates. Full refresh is required to update constraints.\") }}\n {% else %}\n {% do alter_column_set_constraints(relation, model) %}\n {% do alter_table_add_constraints(relation, model) %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.is_incremental", "macro.dbt_databricks.alter_column_set_constraints", "macro.dbt_databricks.alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.600513, "supported_languages": null}, "macro.dbt_databricks.alter_table_add_constraints": {"name": "alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.alter_table_add_constraints", "macro_sql": "{% macro alter_table_add_constraints(relation, constraints) %}\n {{ return(adapter.dispatch('alter_table_add_constraints', 'dbt')(relation, constraints)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.600693, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_table_add_constraints": {"name": "databricks__alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__alter_table_add_constraints", "macro_sql": "{% macro databricks__alter_table_add_constraints(relation, model) %}\n {% set constraints = get_model_constraints(model) %}\n {% set statements = get_constraints_sql(relation, constraints, model) %}\n {% for stmt in statements %}\n {% call statement() %}\n {{ stmt }}\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_model_constraints", "macro.dbt_databricks.get_constraints_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6010072, "supported_languages": null}, "macro.dbt_databricks.get_model_constraints": {"name": "get_model_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_model_constraints", "macro_sql": "{% macro get_model_constraints(model) %}\n {% set constraints = model.get('constraints', []) %}\n {% if config.get('persist_constraints', False) and model.get('meta', {}).get('constraints') is sequence %}\n {# Databricks constraints implementation. Constraints are in the meta property. #}\n {% set db_constraints = model.get('meta', {}).get('constraints', []) %}\n {% set constraints = databricks_constraints_to_dbt(db_constraints) %}\n {% endif %}\n {{ return(constraints) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks_constraints_to_dbt"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.601491, "supported_languages": null}, "macro.dbt_databricks.get_column_constraints": {"name": "get_column_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_column_constraints", "macro_sql": "{% macro get_column_constraints(column) %}\n {% set constraints = column.get('constraints', []) %}\n {% if config.get('persist_constraints', False) and column.get('meta', {}).get('constraint') %}\n {# Databricks constraints implementation. Constraint is in the meta property. #}\n {% set db_constraints = [column.get('meta', {}).get('constraint')] %}\n {% set constraints = databricks_constraints_to_dbt(db_constraints, column) %}\n {% endif %}\n {{ return(constraints) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks_constraints_to_dbt"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6019762, "supported_languages": null}, "macro.dbt_databricks.alter_column_set_constraints": {"name": "alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.alter_column_set_constraints", "macro_sql": "{% macro alter_column_set_constraints(relation, column_dict) %}\n {{ return(adapter.dispatch('alter_column_set_constraints', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.602155, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_column_set_constraints": {"name": "databricks__alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks__alter_column_set_constraints", "macro_sql": "{% macro databricks__alter_column_set_constraints(relation, model) %}\n {% set column_dict = model.columns %}\n {% for column_name in column_dict %}\n {% set column = column_dict[column_name] %}\n {% set constraints = get_column_constraints(column) %}\n {% set statements = get_constraints_sql(relation, constraints, model, column) %}\n {% for stmt in statements %}\n {% call statement() %}\n {{ stmt }}\n {% endcall %}\n {% endfor %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_column_constraints", "macro.dbt_databricks.get_constraints_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.602596, "supported_languages": null}, "macro.dbt_databricks.get_constraints_sql": {"name": "get_constraints_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_constraints_sql", "macro_sql": "{% macro get_constraints_sql(relation, constraints, model, column={}) %}\n {% set statements = [] %}\n -- Hack so that not null constraints will be applied before primary key constraints\n {% for constraint in constraints|sort(attribute='type') %}\n {% if constraint %}\n {% set constraint_statements = get_constraint_sql(relation, constraint, model, column) %}\n {% for statement in constraint_statements %}\n {% if statement %}\n {% do statements.append(statement) %}\n {% endif %}\n {% endfor %}\n {% endif %}\n {% endfor %}\n\n {{ return(statements) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_constraint_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.603168, "supported_languages": null}, "macro.dbt_databricks.get_constraint_sql": {"name": "get_constraint_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.get_constraint_sql", "macro_sql": "{% macro get_constraint_sql(relation, constraint, model, column={}) %}\n {% set statements = [] %}\n {% set type = constraint.get(\"type\", \"\") %}\n\n {% if type == 'check' %}\n {% set expression = constraint.get(\"expression\", \"\") %}\n {% if not expression %}\n {{ exceptions.raise_compiler_error('Invalid check constraint expression') }}\n {% endif %}\n\n {% set name = constraint.get(\"name\") %}\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5 (relation.identifier ~ \";\" ~ column.get(\"name\", \"\") ~ \";\" ~ expression ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %}\n {% endif %}\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" check (\" ~ expression ~ \");\" %}\n {% do statements.append(stmt) %}\n {% elif type == 'not_null' %}\n {% set column_names = constraint.get(\"columns\", []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if column %}\n {% set quoted_name = adapter.quote(column['name']) if column['quote'] else column['name'] %}\n {% set stmt = \"alter table \" ~ relation ~ \" change column \" ~ quoted_name ~ \" set not null \" ~ (constraint.expression or \"\") ~ \";\" %}\n {% do statements.append(stmt) %}\n {% else %}\n {{ exceptions.warn('not_null constraint on invalid column: ' ~ column_name) }}\n {% endif %}\n {% endfor %}\n {% elif type == 'primary_key' %}\n {% if constraint.get('warn_unenforced') %}\n {{ exceptions.warn(\"unenforced constraint type: \" ~ type)}}\n {% endif %}\n {% set column_names = constraint.get(\"columns\", []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% set quoted_names = [] %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if not column %}\n {{ exceptions.warn('Invalid primary key column: ' ~ column_name) }}\n {% else %}\n {% set quoted_name = adapter.quote(column['name']) if column['quote'] else column['name'] %}\n {% do quoted_names.append(quoted_name) %}\n {% endif %}\n {% endfor %}\n\n {% set joined_names = quoted_names|join(\", \") %}\n\n {% set name = constraint.get(\"name\") %}\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"primary_key;\" ~ relation.identifier ~ \";\" ~ column_names ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %}\n {% endif %}\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" primary key(\" ~ joined_names ~ \");\" %}\n {% do statements.append(stmt) %}\n {% elif type == 'foreign_key' %}\n\n {% if constraint.get('warn_unenforced') %}\n {{ exceptions.warn(\"unenforced constraint type: \" ~ constraint.type)}}\n {% endif %}\n\n {% set name = constraint.get(\"name\") %}\n \n {% if constraint.get('expression') %}\n\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"foreign_key;\" ~ relation.identifier ~ \";\" ~ constraint.get('expression') ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %} \n {% endif %}\n\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" foreign key\" ~ constraint.get('expression') %}\n {% else %}\n {% set column_names = constraint.get(\"columns\", []) %}\n {% if column and not column_names %}\n {% set column_names = [column['name']] %}\n {% endif %}\n {% set quoted_names = [] %}\n {% for column_name in column_names %}\n {% set column = model.get('columns', {}).get(column_name) %}\n {% if not column %}\n {{ exceptions.warn('Invalid foreign key column: ' ~ column_name) }}\n {% else %}\n {% set quoted_name = adapter.quote(column['name']) if column['quote'] else column['name'] %}\n {% do quoted_names.append(quoted_name) %}\n {% endif %}\n {% endfor %}\n\n {% set joined_names = quoted_names|join(\", \") %}\n\n {% set parent = constraint.get(\"to\") %}\n {% if not parent %}\n {{ exceptions.raise_compiler_error('No parent table defined for foreign key: ' ~ expression) }}\n {% endif %}\n {% if not \".\" in parent %}\n {% set parent = relation.schema ~ \".\" ~ parent%}\n {% endif %}\n\n {% if not name %}\n {% if local_md5 %}\n {{ exceptions.warn(\"Constraint of type \" ~ type ~ \" with no `name` provided. Generating hash instead for relation \" ~ relation.identifier) }}\n {%- set name = local_md5(\"foreign_key;\" ~ relation.identifier ~ \";\" ~ column_names ~ \";\" ~ parent ~ \";\") -%}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Constraint of type \" ~ type ~ \" with no `name` provided, and no md5 utility.\") }}\n {% endif %} \n {% endif %}\n\n {% set stmt = \"alter table \" ~ relation ~ \" add constraint \" ~ name ~ \" foreign key(\" ~ joined_names ~ \") references \" ~ parent %}\n {% set parent_columns = constraint.get(\"to_columns\") %}\n {% if parent_columns %}\n {% set stmt = stmt ~ \"(\" ~ parent_columns|join(\", \") ~ \")\"%}\n {% endif %}\n {% endif %}\n {% set stmt = stmt ~ \";\" %}\n {% do statements.append(stmt) %}\n {% elif constraint.get('warn_unsupported') %}\n {{ exceptions.warn(\"unsupported constraint type: \" ~ constraint.type)}}\n {% endif %}\n\n {{ return(statements) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6086242, "supported_languages": null}, "macro.dbt_databricks.databricks_constraints_to_dbt": {"name": "databricks_constraints_to_dbt", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/constraints.sql", "original_file_path": "macros/relations/constraints.sql", "unique_id": "macro.dbt_databricks.databricks_constraints_to_dbt", "macro_sql": "{% macro databricks_constraints_to_dbt(constraints, column) %}\n {# convert constraints defined using the original databricks format #}\n {% set dbt_constraints = [] %}\n {% for constraint in constraints %}\n {% if constraint.get and constraint.get(\"type\") %}\n {# already in model contract format #}\n {% do dbt_constraints.append(constraint) %}\n {% else %}\n {% if column %}\n {% if constraint == \"not_null\" %}\n {% do dbt_constraints.append({\"type\": \"not_null\", \"columns\": [column.get(\"name\")]}) %}\n {% else %}\n {{ exceptions.raise_compiler_error('Invalid constraint for column ' ~ column.get(\"name\", \"\") ~ '. Only `not_null` is supported.') }}\n {% endif %}\n {% else %}\n {% set name = constraint['name'] %}\n {% if not name %}\n {{ exceptions.raise_compiler_error('Invalid check constraint name') }}\n {% endif %}\n {% set condition = constraint['condition'] %}\n {% if not condition %}\n {{ exceptions.raise_compiler_error('Invalid check constraint condition') }}\n {% endif %}\n {% do dbt_constraints.append({\"name\": name, \"type\": \"check\", \"expression\": condition}) %}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(dbt_constraints) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.609767, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_sql": {"name": "databricks__get_create_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_sql", "macro_sql": "{%- macro databricks__get_create_sql(relation, sql) -%}\n {%- if relation.is_view -%}\n {{ get_create_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_table -%}\n {{ get_create_table_as_sql(False, relation, sql) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_create_materialized_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_streaming_table -%}\n {{ get_create_streaming_table_as_sql(relation, sql) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_create_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.get_create_view_as_sql", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt_databricks.get_create_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.610386, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_materialized_view": {"name": "databricks__drop_materialized_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_materialized_view", "macro_sql": "{% macro databricks__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.610508, "supported_languages": null}, "macro.dbt_databricks.databricks__refresh_materialized_view": {"name": "databricks__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt_databricks.databricks__refresh_materialized_view", "macro_sql": "{% macro databricks__refresh_materialized_view(relation) -%}\n refresh materialized view {{ relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.61062, "supported_languages": null}, "macro.dbt_databricks.get_alter_materialized_view_as_sql": {"name": "get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_materialized_view_as_sql", "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {%- do return(adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n )) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.611409, "supported_languages": null}, "macro.dbt_databricks.databricks__get_materialized_view_configuration_changes": {"name": "databricks__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_materialized_view_configuration_changes", "macro_sql": "\n\n\n{%- macro databricks__get_materialized_view_configuration_changes(existing_relation, new_config) -%}\n {%- set _existing_materialized_view = adapter.get_relation_config(existing_relation) -%}\n {%- set materialized_view = adapter.get_config_from_model(config.model) -%}\n {%- set _configuration_changes = materialized_view.get_changeset(_existing_materialized_view) -%}\n {% do return(_configuration_changes) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6116831, "supported_languages": null}, "macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql": {"name": "databricks__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql", "macro_sql": "{% macro databricks__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n {% do return(get_replace_sql(existing_relation, relation, sql)) %}\n\n -- otherwise apply individual changes as needed\n {% else %}\n {% do return(get_alter_mv_internal(relation, configuration_changes)) %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_alter_mv_internal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.611993, "supported_languages": null}, "macro.dbt_databricks.get_alter_mv_internal": {"name": "get_alter_mv_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_mv_internal", "macro_sql": "{% macro get_alter_mv_internal(relation, configuration_changes) %}\n {%- set refresh = configuration_changes.changes[\"refresh\"] -%}\n -- Currently only schedule can be altered\n ALTER MATERIALIZED VIEW {{ relation }}\n {{ get_alter_sql_refresh_schedule(refresh.cron, refresh.time_zone_value, refresh.is_altered) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_alter_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6122189, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_materialized_view_as_sql": {"name": "databricks__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_materialized_view_as_sql", "macro_sql": "{% macro databricks__get_create_materialized_view_as_sql(relation, sql) -%}\n {%- set materialized_view = adapter.get_config_from_model(config.model) -%}\n {%- set partition_by = materialized_view.config[\"partition_by\"].partition_by -%}\n {%- set tblproperties = materialized_view.config[\"tblproperties\"].tblproperties -%}\n {%- set comment = materialized_view.config[\"comment\"].comment -%}\n {%- set refresh = materialized_view.config[\"refresh\"] -%}\n create materialized view {{ relation }}\n {{ get_create_sql_partition_by(partition_by) }}\n {{ get_create_sql_comment(comment) }}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {{ get_create_sql_refresh_schedule(refresh.cron, refresh.time_zone_value) }}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties", "macro.dbt_databricks.get_create_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.612938, "supported_languages": null}, "macro.dbt_databricks.drop_streaming_table": {"name": "drop_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/drop.sql", "original_file_path": "macros/relations/streaming_table/drop.sql", "unique_id": "macro.dbt_databricks.drop_streaming_table", "macro_sql": "{% macro drop_streaming_table(relation) -%}\n {{ return(adapter.dispatch('drop_streaming_table', 'dbt')(relation)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.default__drop_streaming_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.613157, "supported_languages": null}, "macro.dbt_databricks.default__drop_streaming_table": {"name": "default__drop_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/drop.sql", "original_file_path": "macros/relations/streaming_table/drop.sql", "unique_id": "macro.dbt_databricks.default__drop_streaming_table", "macro_sql": "{% macro default__drop_streaming_table(relation) -%}\n drop table if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6132412, "supported_languages": null}, "macro.dbt_databricks.refresh_streaming_table": {"name": "refresh_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/refresh.sql", "original_file_path": "macros/relations/streaming_table/refresh.sql", "unique_id": "macro.dbt_databricks.refresh_streaming_table", "macro_sql": "{% macro refresh_streaming_table(relation, sql) -%}\n {{ adapter.dispatch('refresh_streaming_table', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__refresh_streaming_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.61346, "supported_languages": null}, "macro.dbt_databricks.databricks__refresh_streaming_table": {"name": "databricks__refresh_streaming_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/refresh.sql", "original_file_path": "macros/relations/streaming_table/refresh.sql", "unique_id": "macro.dbt_databricks.databricks__refresh_streaming_table", "macro_sql": "{% macro databricks__refresh_streaming_table(relation, sql) -%}\n create or refresh streaming table {{ relation }}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6135712, "supported_languages": null}, "macro.dbt_databricks.get_alter_streaming_table_as_sql": {"name": "get_alter_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_streaming_table_as_sql", "macro_sql": "{% macro get_alter_streaming_table_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {%- do return(adapter.dispatch('get_alter_streaming_table_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n )) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.615469, "supported_languages": null}, "macro.dbt_databricks.get_streaming_table_configuration_changes": {"name": "get_streaming_table_configuration_changes", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_streaming_table_configuration_changes", "macro_sql": "{% macro get_streaming_table_configuration_changes(existing_relation, new_config) -%}\n {{- log('Determining configuration changes on: ' ~ existing_relation) -}}\n {%- do return(adapter.dispatch('get_streaming_table_configuration_changes', 'dbt')(existing_relation, new_config)) -%}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_streaming_table_configuration_changes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6158211, "supported_languages": null}, "macro.dbt_databricks.databricks__get_streaming_table_configuration_changes": {"name": "databricks__get_streaming_table_configuration_changes", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_streaming_table_configuration_changes", "macro_sql": "\n\n{%- macro databricks__get_streaming_table_configuration_changes(existing_relation, new_config) -%}\n {%- set _existing_streaming_table = adapter.get_relation_config(existing_relation) -%}\n {%- set streaming_table = adapter.get_config_from_model(config.model) -%}\n {%- set _configuration_changes = streaming_table.get_changeset(_existing_streaming_table) -%}\n {% do return(_configuration_changes) %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6161752, "supported_languages": null}, "macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql": {"name": "databricks__get_alter_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.databricks__get_alter_streaming_table_as_sql", "macro_sql": "{% macro databricks__get_alter_streaming_table_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n -- apply a full refresh immediately if needed\n {% if configuration_changes.requires_full_refresh %}\n {% do return(get_replace_sql(existing_relation, relation, sql)) %}\n\n -- otherwise apply individual changes as needed\n {% else %}\n {%- set alter_statement = get_alter_st_internal(relation, configuration_changes) -%}\n {%- set create_statement = get_create_st_internal(relation, configuration_changes, sql) -%}\n {%- set return_statements = [] -%}\n {%- if create_statement -%}\n {{ return_statements.append(create_statement) }}\n {%- endif -%}\n {%- if alter_statement -%}\n {{ return_statements.append(alter_statement) }}\n {%- endif -%}\n {% do return(return_statements) %}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_replace_sql", "macro.dbt_databricks.get_alter_st_internal", "macro.dbt_databricks.get_create_st_internal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.616995, "supported_languages": null}, "macro.dbt_databricks.get_create_st_internal": {"name": "get_create_st_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_create_st_internal", "macro_sql": "{% macro get_create_st_internal(relation, configuration_changes, sql) %}\n {%- set partition_by = configuration_changes.changes[\"partition_by\"].partition_by -%}\n {%- set tblproperties = configuration_changes.changes[\"tblproperties\"].tblproperties -%}\n {%- set comment = configuration_changes.changes[\"comment\"].comment -%}\n CREATE OR REFRESH STREAMING TABLE {{ relation }}\n {% if partition_by -%}\n {{ get_create_sql_partition_by(partition_by) }}\n {%- endif %}\n {% if comment -%}\n {{ get_create_sql_comment(comment) }}\n {%- endif %}\n {% if tblproperties -%}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {%- endif %}\n AS {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6175182, "supported_languages": null}, "macro.dbt_databricks.get_alter_st_internal": {"name": "get_alter_st_internal", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/alter.sql", "original_file_path": "macros/relations/streaming_table/alter.sql", "unique_id": "macro.dbt_databricks.get_alter_st_internal", "macro_sql": "{% macro get_alter_st_internal(relation, configuration_changes) %}\n {%- set refresh = configuration_changes.changes[\"refresh\"] -%}\n {%- if refresh and refresh.cron -%}\n ALTER STREAMING TABLE {{ relation }}\n {{ get_alter_sql_refresh_schedule(refresh.cron, refresh.time_zone_value, False) -}}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_alter_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6177878, "supported_languages": null}, "macro.dbt_databricks.get_create_streaming_table_as_sql": {"name": "get_create_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/create.sql", "original_file_path": "macros/relations/streaming_table/create.sql", "unique_id": "macro.dbt_databricks.get_create_streaming_table_as_sql", "macro_sql": "{% macro get_create_streaming_table_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_streaming_table_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_streaming_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.618218, "supported_languages": null}, "macro.dbt_databricks.databricks__get_create_streaming_table_as_sql": {"name": "databricks__get_create_streaming_table_as_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/streaming_table/create.sql", "original_file_path": "macros/relations/streaming_table/create.sql", "unique_id": "macro.dbt_databricks.databricks__get_create_streaming_table_as_sql", "macro_sql": "{% macro databricks__get_create_streaming_table_as_sql(relation, sql) -%}\n {%- set streaming_table = adapter.get_config_from_model(config.model) -%}\n {%- set partition_by = streaming_table.config[\"partition_by\"].partition_by -%}\n {%- set tblproperties = streaming_table.config[\"tblproperties\"].tblproperties -%}\n {%- set comment = streaming_table.config[\"comment\"].comment -%}\n {%- set refresh = streaming_table.config[\"refresh\"] -%}\n\n CREATE STREAMING TABLE {{ relation }}\n {{ get_create_sql_partition_by(partition_by) }}\n {{ get_create_sql_comment(comment) }}\n {{ get_create_sql_tblproperties(tblproperties) }}\n {{ get_create_sql_refresh_schedule(refresh.cron, refresh.time_zone_value) }}\n AS {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_create_sql_partition_by", "macro.dbt_databricks.get_create_sql_comment", "macro.dbt_databricks.get_create_sql_tblproperties", "macro.dbt_databricks.get_create_sql_refresh_schedule"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.618762, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_comment": {"name": "get_create_sql_comment", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/comment.sql", "original_file_path": "macros/relations/components/comment.sql", "unique_id": "macro.dbt_databricks.get_create_sql_comment", "macro_sql": "{%- macro get_create_sql_comment(comment) -%}\n{% if comment is string -%}\n COMMENT '{{ comment }}'\n{%- endif -%}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6189408, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_refresh_schedule": {"name": "get_create_sql_refresh_schedule", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/refresh_schedule.sql", "original_file_path": "macros/relations/components/refresh_schedule.sql", "unique_id": "macro.dbt_databricks.get_create_sql_refresh_schedule", "macro_sql": "{% macro get_create_sql_refresh_schedule(cron, time_zone_value) %}\n {%- if cron -%}\n SCHEDULE CRON '{{ cron }}'{%- if time_zone_value %} AT TIME ZONE '{{ time_zone_value }}'{%- endif -%}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.619374, "supported_languages": null}, "macro.dbt_databricks.get_alter_sql_refresh_schedule": {"name": "get_alter_sql_refresh_schedule", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/refresh_schedule.sql", "original_file_path": "macros/relations/components/refresh_schedule.sql", "unique_id": "macro.dbt_databricks.get_alter_sql_refresh_schedule", "macro_sql": "{% macro get_alter_sql_refresh_schedule(cron, time_zone_value, is_altered) %}\n {%- if cron -%}\n {%- if is_altered -%}\n ALTER SCHEDULE CRON '{{ cron }}'{%- if time_zone_value %} AT TIME ZONE '{{ time_zone_value }}'{%- endif -%}\n {%- else -%}\n ADD SCHEDULE CRON '{{ cron }}'{%- if time_zone_value %} AT TIME ZONE '{{ time_zone_value }}'{%- endif -%}\n {%- endif -%}\n {%- else -%}\n DROP SCHEDULE\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.619678, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_partition_by": {"name": "get_create_sql_partition_by", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/partitioning.sql", "original_file_path": "macros/relations/components/partitioning.sql", "unique_id": "macro.dbt_databricks.get_create_sql_partition_by", "macro_sql": "{% macro get_create_sql_partition_by(partition_by) -%}\n{%- if partition_by -%}\n PARTITIONED BY ({%- for col in partition_by -%}{{ col }}{% if not loop.last %}, {% endif %}{%- endfor %})\n{%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.619945, "supported_languages": null}, "macro.dbt_databricks.get_create_sql_tblproperties": {"name": "get_create_sql_tblproperties", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/components/tblproperties.sql", "original_file_path": "macros/relations/components/tblproperties.sql", "unique_id": "macro.dbt_databricks.get_create_sql_tblproperties", "macro_sql": "{% macro get_create_sql_tblproperties(tblproperties) %}\n {%- if tblproperties and tblproperties|length>0 -%}\n TBLPROPERTIES (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}'{%- if not loop.last -%}, {% endif -%}\n {% endfor -%}\n )\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.62031, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_table": {"name": "databricks__drop_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_table", "macro_sql": "{% macro databricks__drop_table(relation) -%}\n drop table if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.620424, "supported_languages": null}, "macro.dbt_databricks.databricks__create_table_as": {"name": "databricks__create_table_as", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.databricks__create_table_as", "macro_sql": "{% macro databricks__create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {%- if language == 'sql' -%}\n {%- if temporary -%}\n {{ create_temporary_view(relation, compiled_code) }}\n {%- else -%}\n {%- set file_format = config.get('file_format', default='delta') -%}\n {% if file_format == 'delta' %}\n create or replace table {{ relation }}\n {% else %}\n create table {{ relation }}\n {% endif %}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config and contract_config.enforced %}\n {{ get_assert_columns_equivalent(compiled_code) }}\n {%- set compiled_code = get_select_subquery(compiled_code) %}\n {% endif %}\n {{ file_format_clause() }}\n {{ options_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ liquid_clustered_cols() }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause(relation) }}\n {{ comment_clause() }}\n {{ tblproperties_clause() }}\n as\n {{ compiled_code }}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {#--\n N.B. Python models _can_ write to temp views HOWEVER they use a different session\n and have already expired by the time they need to be used (I.E. in merges for incremental models)\n\n TODO: Deep dive into spark sessions to see if we can reuse a single session for an entire\n dbt invocation.\n --#}\n {{ databricks__py_write_table(compiled_code=compiled_code, target_relation=relation) }}\n {%- endif -%}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.create_temporary_view", "macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_select_subquery", "macro.dbt_databricks.file_format_clause", "macro.dbt_spark.options_clause", "macro.dbt_spark.partition_cols", "macro.dbt_databricks.liquid_clustered_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_databricks.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_databricks.tblproperties_clause", "macro.dbt_databricks.databricks__py_write_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6221719, "supported_languages": null}, "macro.dbt_databricks.databricks__options_clause": {"name": "databricks__options_clause", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt_databricks.databricks__options_clause", "macro_sql": "{% macro databricks__options_clause() -%}\n {%- set options = config.get('options') -%}\n {%- if config.get('file_format', default='delta') == 'hudi' -%}\n {%- set unique_key = config.get('unique_key') -%}\n {%- if unique_key is not none and options is none -%}\n {%- set options = {'primaryKey': config.get('unique_key')} -%}\n {%- elif unique_key is not none and options is not none and 'primaryKey' not in options -%}\n {%- set _ = options.update({'primaryKey': config.get('unique_key')}) -%}\n {%- elif options is not none and 'primaryKey' in options and options['primaryKey'] != unique_key -%}\n {{ exceptions.raise_compiler_error(\"unique_key and options('primaryKey') should be the same column(s).\") }}\n {%- endif %}\n {%- endif %}\n\n {%- if options is not none %}\n options (\n {%- for option in options -%}\n {{ option }} \"{{ options[option] }}\" {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.623204, "supported_languages": null}, "macro.dbt_databricks.databricks__drop_view": {"name": "databricks__drop_view", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt_databricks.databricks__drop_view", "macro_sql": "{% macro databricks__drop_view(relation) -%}\n drop view if exists {{ relation }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6233542, "supported_languages": null}, "macro.dbt_databricks.databricks__create_view_as": {"name": "databricks__create_view_as", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt_databricks.databricks__create_view_as", "macro_sql": "{% macro databricks__create_view_as(relation, sql) -%}\n create or replace view {{ relation }}\n {% if config.persist_column_docs() -%}\n {% set model_columns = model.columns %}\n {% set query_columns = get_columns_in_query(sql) %}\n {% if query_columns %}\n (\n {{ get_persist_docs_column_list(model_columns, query_columns) }}\n )\n {% endif %}\n {% endif %}\n {{ comment_clause() }}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config and contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n {{ tblproperties_clause() }}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt_databricks.get_persist_docs_column_list", "macro.dbt_spark.comment_clause", "macro.dbt.get_assert_columns_equivalent", "macro.dbt_databricks.tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6240902, "supported_languages": null}, "macro.dbt_databricks.statement_with_staging_table": {"name": "statement_with_staging_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt_databricks.statement_with_staging_table", "macro_sql": "{% macro statement_with_staging_table(name=None, staging_table=None, fetch_result=False, auto_begin=True) -%}\n {%- if execute: -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- set res, table = adapter.execute(sql, auto_begin=auto_begin, fetch=fetch_result, staging_table=staging_table) -%}\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6248162, "supported_languages": null}, "macro.dbt_databricks.databricks__dateadd": {"name": "databricks__dateadd", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_databricks.databricks__dateadd", "macro_sql": "{% macro databricks__dateadd(datepart, interval, from_date_or_timestamp) %}\n {%- if adapter.compare_dbr_version(10, 4) >= 0 -%}\n timestampadd({{datepart}}, {{interval}}, {{from_date_or_timestamp}})\n {%- else -%}\n {{ spark__dateadd(datepart, interval, from_date_or_timestamp) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6251829, "supported_languages": null}, "macro.dbt_databricks.databricks__datediff": {"name": "databricks__datediff", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_databricks.databricks__datediff", "macro_sql": "{% macro databricks__datediff(first_date, second_date, datepart) %}\n {%- if adapter.compare_dbr_version(10, 4) >= 0 -%}\n timestampdiff({{datepart}}, {{date_trunc(datepart, first_date)}}, {{date_trunc(datepart, second_date)}})\n {%- else -%}\n {{ spark__datediff(first_date, second_date, datepart) }}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.date_trunc", "macro.dbt_spark.spark__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6255841, "supported_languages": null}, "macro.dbt_databricks.databricks_copy_into": {"name": "databricks_copy_into", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/copy_into.sql", "original_file_path": "macros/adapters/copy_into.sql", "unique_id": "macro.dbt_databricks.databricks_copy_into", "macro_sql": "{% macro databricks_copy_into(\n target_table,\n source,\n file_format,\n expression_list=none,\n source_credential=none,\n source_encryption=none,\n validate=none,\n files=none,\n pattern=none,\n format_options=none,\n copy_options=none) -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=target.database,\n schema=target.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- set source_clause -%}\n {%- if expression_list -%}\n ( select {{ expression_list }} from '{{ source }}' )\n {%- else -%}\n '{{ source }}'\n {%- endif -%}\n {%- if source_credential or source_encryption %}\n WITH (\n {%- if source_credential %}\n credential (\n {%- for name in source_credential -%}\n '{{ name }}' = '{{ source_credential[name] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {%- if source_encryption %}\n encryption (\n {%- for name in source_encryption -%}\n '{{ name }}' = '{{ source_encryption[name] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n )\n {%- endif -%}\n {%- endset -%}\n\n {% set query %}\n copy into {{ target_relation }}\n from {{ source_clause }}\n fileformat = {{ file_format }}\n {% if validate -%} validate {{ validate }} {%- endif %}\n {% if files and pattern %}\n {{ exceptions.raise_compiler_error(\"You can only specify one of 'files' or 'pattern'\") }}\n {% endif %}\n {% if files -%}\n files = (\n {%- for file in files -%}\n '{{ file }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% if pattern -%}\n pattern = '{{ pattern }}'\n {%- endif %}\n {% if format_options -%}\n format_options (\n {%- for key in format_options -%}\n '{{ key }}' = '{{ format_options[key] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% if copy_options -%}\n copy_options (\n {%- for key in copy_options -%}\n '{{ key }}' = '{{ copy_options[key] }}' {%- if not loop.last %}, {% endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n {% endset %}\n\n {% do log(\"Running COPY INTO\" ~ adapter.redact_credentials(query), info=True) %}\n {% do run_query(query) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.62939, "supported_languages": null}, "macro.dbt_databricks.databricks__make_temp_relation": {"name": "databricks__make_temp_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.databricks__make_temp_relation", "macro_sql": "{% macro databricks__make_temp_relation(base_relation, suffix='__dbt_tmp', as_table=False) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {%- if as_table -%}\n {% set tmp_relation = api.Relation.create(\n identifier=tmp_identifier,\n schema=base_relation.schema,\n database=base_relation.database,\n type='table') %}\n {%- else -%}\n {% set tmp_relation = api.Relation.create(identifier=tmp_identifier, type='view') %}\n {%- endif -%}\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.630194, "supported_languages": null}, "macro.dbt_databricks.databricks__get_or_create_relation": {"name": "databricks__get_or_create_relation", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt_databricks.databricks__get_or_create_relation", "macro_sql": "{% macro databricks__get_or_create_relation(database, schema, identifier, type, needs_information=False) %}\n {%- set target_relation = adapter.get_relation(\n database=database,\n schema=schema,\n identifier=identifier,\n needs_information=needs_information) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6306949, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog": {"name": "databricks__get_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog", "macro_sql": "{% macro databricks__get_catalog(information_schema, schemas) -%}\n\n {% set query %}\n with tables as (\n {{ databricks__get_catalog_tables_sql(information_schema) }}\n {{ databricks__get_catalog_schemas_where_clause_sql(information_schema.database, schemas) }}\n ),\n columns as (\n {{ databricks__get_catalog_columns_sql(information_schema) }}\n {{ databricks__get_catalog_schemas_where_clause_sql(information_schema.database, schemas) }}\n )\n {{ databricks__get_catalog_results_sql() }}\n {%- endset -%}\n\n {{ return(run_query(query)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql", "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.633368, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_relations": {"name": "databricks__get_catalog_relations", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_relations", "macro_sql": "{% macro databricks__get_catalog_relations(information_schema, relations) -%}\n\n {% set query %}\n with tables as (\n {{ databricks__get_catalog_tables_sql(information_schema) }}\n {{ databricks__get_catalog_relations_where_clause_sql(relations) }}\n ),\n columns as (\n {{ databricks__get_catalog_columns_sql(information_schema) }}\n {{ databricks__get_catalog_relations_where_clause_sql(relations) }}\n )\n {{ databricks__get_catalog_results_sql() }}\n {%- endset -%}\n\n {{ return(run_query(query)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql", "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.63369, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_tables_sql": {"name": "databricks__get_catalog_tables_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_tables_sql", "macro_sql": "{% macro databricks__get_catalog_tables_sql(information_schema) -%}\n select\n table_catalog as table_database,\n table_schema,\n table_name,\n lower(table_type) as table_type,\n comment as table_comment,\n table_owner,\n 'Last Modified' as `stats:last_modified:label`,\n last_altered as `stats:last_modified:value`,\n 'The timestamp for last update/change' as `stats:last_modified:description`,\n (last_altered is not null and table_type not ilike '%VIEW%') as `stats:last_modified:include`\n from `system`.`information_schema`.`tables`\n where table_catalog = '{{ information_schema.database|lower }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6338282, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_columns_sql": {"name": "databricks__get_catalog_columns_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_columns_sql", "macro_sql": "{% macro databricks__get_catalog_columns_sql(information_schema) -%}\n select\n table_catalog as table_database,\n table_schema,\n table_name,\n column_name,\n ordinal_position as column_index,\n lower(data_type) as column_type,\n comment as column_comment\n from `system`.`information_schema`.`columns`\n where table_catalog = '{{ information_schema.database|lower }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.633953, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_results_sql": {"name": "databricks__get_catalog_results_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_results_sql", "macro_sql": "{% macro databricks__get_catalog_results_sql() -%}\n select *\n from tables\n join columns using (table_database, table_schema, table_name)\n order by column_index\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.634028, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql": {"name": "databricks__get_catalog_schemas_where_clause_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_schemas_where_clause_sql", "macro_sql": "{% macro databricks__get_catalog_schemas_where_clause_sql(catalog, schemas) -%}\n where table_catalog = '{{ catalog|lower }}' and ({%- for relation in schemas -%}\n table_schema = '{{ relation[1]|lower }}'{%- if not loop.last %} or {% endif -%}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.63427, "supported_languages": null}, "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql": {"name": "databricks__get_catalog_relations_where_clause_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/catalog.sql", "original_file_path": "macros/adapters/catalog.sql", "unique_id": "macro.dbt_databricks.databricks__get_catalog_relations_where_clause_sql", "macro_sql": "{% macro databricks__get_catalog_relations_where_clause_sql(catalog, relations) -%}\n where table_catalog = '{{ catalog|lower }}' and (\n {%- for relation in relations -%}\n {% if relation.schema and relation.identifier %}\n (\n table_schema = '{{ relation.schema|lower }}'\n and table_name = '{{ relation.identifier|lower }}'\n )\n {% elif relation.schema %}\n (\n table_schema = '{{ relation.schema|lower }}'\n )\n {% else %}\n {% do exceptions.raise_compiler_error(\n '`get_catalog_relations` requires a list of relations, each with a schema'\n ) %}\n {% endif %}\n\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6348171, "supported_languages": null}, "macro.dbt_databricks.current_catalog": {"name": "current_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.current_catalog", "macro_sql": "{% macro current_catalog() -%}\n {{ return(adapter.dispatch('current_catalog', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__current_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6351402, "supported_languages": null}, "macro.dbt_databricks.databricks__current_catalog": {"name": "databricks__current_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.databricks__current_catalog", "macro_sql": "{% macro databricks__current_catalog() -%}\n {% call statement('current_catalog', fetch_result=True) %}\n select current_catalog()\n {% endcall %}\n {% do return(load_result('current_catalog').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.635343, "supported_languages": null}, "macro.dbt_databricks.use_catalog": {"name": "use_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.use_catalog", "macro_sql": "{% macro use_catalog(catalog) -%}\n {{ return(adapter.dispatch('use_catalog', 'dbt')(catalog)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__use_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6355011, "supported_languages": null}, "macro.dbt_databricks.databricks__use_catalog": {"name": "databricks__use_catalog", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/databricks_catalog.sql", "original_file_path": "macros/adapters/databricks_catalog.sql", "unique_id": "macro.dbt_databricks.databricks__use_catalog", "macro_sql": "{% macro databricks__use_catalog(catalog) -%}\n {% call statement() %}\n use catalog {{ adapter.quote(catalog)|lower }}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.635657, "supported_languages": null}, "macro.dbt_databricks.databricks__py_write_table": {"name": "databricks__py_write_table", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.databricks__py_write_table", "macro_sql": "{% macro databricks__py_write_table(compiled_code, target_relation) %}\n{{ compiled_code }}\n# --- Autogenerated dbt materialization code. --- #\ndbt = dbtObj(spark.table)\ndf = model(dbt, spark)\n\nimport pyspark\n\n{{ py_try_import('pyspark.sql.connect.dataframe', 'newer_pyspark_available') }}\n{{ py_try_import('pandas', 'pandas_available') }}\n{{ py_try_import('pyspark.pandas', 'pyspark_pandas_api_available') }}\n{{ py_try_import('databricks.koalas', 'koalas_available') }}\n\n# preferentially convert pandas DataFrames to pandas-on-Spark or Koalas DataFrames first\n# since they know how to convert pandas DataFrames better than `spark.createDataFrame(df)`\n# and converting from pandas-on-Spark to Spark DataFrame has no overhead\n\nif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n if pyspark_pandas_api_available:\n df = pyspark.pandas.frame.DataFrame(df)\n elif koalas_available:\n df = databricks.koalas.frame.DataFrame(df)\n\n# convert to pyspark.sql.dataframe.DataFrame\nif isinstance(df, pyspark.sql.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif newer_pyspark_available and isinstance(df, pyspark.sql.connect.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif pyspark_pandas_api_available and isinstance(df, pyspark.pandas.frame.DataFrame):\n df = df.to_spark()\nelif koalas_available and isinstance(df, databricks.koalas.frame.DataFrame):\n df = df.to_spark()\nelif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = spark.createDataFrame(df)\nelse:\n msg = f\"{type(df)} is not a supported type for dbt Python materialization\"\n raise Exception(msg)\n\nwriter = (\n df.write\n .mode(\"overwrite\")\n .option(\"overwriteSchema\", \"true\")\n{{ py_get_writer_options()|indent(8, True) }}\n)\n\nwriter.saveAsTable(\"{{ target_relation }}\")\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.py_try_import", "macro.dbt_databricks.py_get_writer_options"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6376588, "supported_languages": null}, "macro.dbt_databricks.py_get_writer_options": {"name": "py_get_writer_options", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.py_get_writer_options", "macro_sql": "\n\n{%- macro py_get_writer_options() -%}\n{%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n{%- set file_format = config.get('file_format', validator=validation.any[basestring])|default('delta', true) -%}\n{%- set partition_by = config.get('partition_by', validator=validation.any[list, basestring]) -%}\n{%- set liquid_clustered_by = config.get('liquid_clustered_by', validator=validation.any[list, basestring]) -%}\n{%- set clustered_by = config.get('clustered_by', validator=validation.any[list, basestring]) -%}\n{%- set buckets = config.get('buckets', validator=validation.any[int]) -%}\n.format(\"{{ file_format }}\")\n{%- if location_root is not none %}\n{%- set identifier = model['alias'] %}\n{%- if is_incremental() %}\n{%- set identifier = identifier + '__dbt_tmp' %}\n{%- endif %}\n.option(\"path\", \"{{ location_root }}/{{ identifier }}\")\n{%- endif -%}\n{%- if partition_by is not none -%}\n {%- if partition_by is string -%}\n {%- set partition_by = [partition_by] -%}\n {%- endif %}\n.partitionBy({{ partition_by }})\n{%- endif -%}\n{%- if liquid_clustered_by and not is_incremental() -%}\n {%- if liquid_clustered_by is string -%}\n {%- set liquid_clustered_by = [liquid_clustered_by] -%}\n {%- endif %}\n.clusterBy({{ liquid_clustered_by }})\n{%- endif -%}\n{%- if (clustered_by is not none) and (buckets is not none) -%}\n {%- if clustered_by is string -%}\n {%- set clustered_by = [clustered_by] -%}\n {%- endif %}\n.bucketBy({{ buckets }}, {{ clustered_by }})\n{%- endif -%}\n{% endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.is_incremental"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6390772, "supported_languages": null}, "macro.dbt_databricks.py_try_import": {"name": "py_try_import", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/python.sql", "original_file_path": "macros/adapters/python.sql", "unique_id": "macro.dbt_databricks.py_try_import", "macro_sql": "{% macro py_try_import(library, var_name) -%}\n# make sure {{ library }} exists before using it\ntry:\n import {{ library }}\n {{ var_name }} = True\nexcept ImportError:\n {{ var_name }} = False\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.639234, "supported_languages": null}, "macro.dbt_databricks.databricks__alter_column_comment": {"name": "databricks__alter_column_comment", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.databricks__alter_column_comment", "macro_sql": "{% macro databricks__alter_column_comment(relation, column_dict) %}\n {% if config.get('file_format', default='delta') in ['delta', 'hudi'] %}\n {% for column_name in column_dict %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = comment | replace('\\'', '\\\\\\'') %}\n {% set comment_query %}\n alter table {{ relation }} change column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% endset %}\n {% do run_query(comment_query) %}\n {% endfor %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.641151, "supported_languages": null}, "macro.dbt_databricks.alter_table_comment": {"name": "alter_table_comment", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.alter_table_comment", "macro_sql": "{% macro alter_table_comment(relation, model) %}\n {% set comment_query %}\n comment on table {{ relation|lower }} is '{{ model.description | replace(\"'\", \"\\\\'\") }}'\n {% endset %}\n {% do run_query(comment_query) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.641404, "supported_languages": null}, "macro.dbt_databricks.get_columns_comments": {"name": "get_columns_comments", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.get_columns_comments", "macro_sql": "{% macro get_columns_comments(relation) -%}\n {% call statement('get_columns_comments', fetch_result=True) -%}\n describe table {{ relation|lower }}\n {% endcall %}\n\n {% do return(load_result('get_columns_comments').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6416378, "supported_languages": null}, "macro.dbt_databricks.databricks__persist_docs": {"name": "databricks__persist_docs", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.databricks__persist_docs", "macro_sql": "{% macro databricks__persist_docs(relation, model, for_relation, for_columns) -%}\n {%- if for_relation and config.persist_relation_docs() and model.description %}\n {% do alter_table_comment(relation, model) %}\n {% endif %}\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {%- set existing_columns = adapter.get_columns_in_relation(relation) -%}\n {%- set columns_to_persist_docs = adapter.get_persist_doc_columns(existing_columns, model.columns) -%}\n {% do alter_column_comment(relation, columns_to_persist_docs) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.alter_table_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.642109, "supported_languages": null}, "macro.dbt_databricks.get_column_comment_sql": {"name": "get_column_comment_sql", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.get_column_comment_sql", "macro_sql": "{% macro get_column_comment_sql(column_name, column_dict) -%}\n {% if column_name in column_dict and column_dict[column_name][\"description\"] -%}\n {% set escaped_description = column_dict[column_name][\"description\"] | replace(\"'\", \"\\\\'\") %}\n {% set column_comment_clause = \"comment '\" ~ escaped_description ~ \"'\" %}\n {%- endif -%}\n {{ adapter.quote(column_name) }} {{ column_comment_clause }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.642476, "supported_languages": null}, "macro.dbt_databricks.get_persist_docs_column_list": {"name": "get_persist_docs_column_list", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt_databricks.get_persist_docs_column_list", "macro_sql": "{% macro get_persist_docs_column_list(model_columns, query_columns) %}\n {% for column_name in query_columns %}\n {{ get_column_comment_sql(column_name, model_columns) }}\n {{- \", \" if not loop.last else \"\" }}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.get_column_comment_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.642708, "supported_languages": null}, "macro.dbt_databricks.databricks__list_relations_without_caching": {"name": "databricks__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__list_relations_without_caching", "macro_sql": "{% macro databricks__list_relations_without_caching(schema_relation) %}\n {{ return(adapter.get_relations_without_caching(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6461172, "supported_languages": null}, "macro.dbt_databricks.show_table_extended": {"name": "show_table_extended", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_table_extended", "macro_sql": "{% macro show_table_extended(schema_relation) %}\n {{ return(adapter.dispatch('show_table_extended', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_table_extended"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646267, "supported_languages": null}, "macro.dbt_databricks.databricks__show_table_extended": {"name": "databricks__show_table_extended", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_table_extended", "macro_sql": "{% macro databricks__show_table_extended(schema_relation) %}\n {% call statement('show_table_extended', fetch_result=True) -%}\n show table extended in {{ schema_relation.without_identifier()|lower }} like '{{ schema_relation.identifier|lower }}'\n {% endcall %}\n\n {% do return(load_result('show_table_extended').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646537, "supported_languages": null}, "macro.dbt_databricks.show_tables": {"name": "show_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_tables", "macro_sql": "{% macro show_tables(relation) %}\n {{ return(adapter.dispatch('show_tables', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_tables"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646763, "supported_languages": null}, "macro.dbt_databricks.databricks__show_tables": {"name": "databricks__show_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_tables", "macro_sql": "{% macro databricks__show_tables(relation) %}\n {% call statement('show_tables', fetch_result=True) -%}\n show tables in {{ relation|lower }}\n {% endcall %}\n\n {% do return(load_result('show_tables').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.646989, "supported_languages": null}, "macro.dbt_databricks.show_views": {"name": "show_views", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.show_views", "macro_sql": "{% macro show_views(relation) %}\n {{ return(adapter.dispatch('show_views', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__show_views"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.647151, "supported_languages": null}, "macro.dbt_databricks.databricks__show_views": {"name": "databricks__show_views", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__show_views", "macro_sql": "{% macro databricks__show_views(relation) %}\n {% call statement('show_views', fetch_result=True) -%}\n show views in {{ relation|lower }}\n {% endcall %}\n\n {% do return(load_result('show_views').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.647396, "supported_languages": null}, "macro.dbt_databricks.databricks__get_relation_last_modified": {"name": "databricks__get_relation_last_modified", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.databricks__get_relation_last_modified", "macro_sql": "{% macro databricks__get_relation_last_modified(information_schema, relations) -%}\n\n {%- call statement('last_modified', fetch_result=True) -%}\n {% if information_schema.is_hive_metastore() %}\n {%- for relation in relations -%}\n select '{{ relation.schema }}' as schema,\n '{{ relation.identifier }}' as identifier,\n max(timestamp) as last_modified,\n {{ current_timestamp() }} as snapshotted_at\n from (describe history {{ relation.schema|lower }}.{{ relation.identifier|lower }})\n {% if not loop.last %}\n union all\n {% endif %}\n {%- endfor -%}\n {% else %}\n select table_schema as schema,\n table_name as identifier,\n last_altered as last_modified,\n {{ current_timestamp() }} as snapshotted_at\n from `system`.`information_schema`.`tables`\n where table_catalog = '{{ information_schema.database|lower }}' and\n (\n {%- for relation in relations -%}\n (table_schema = '{{ relation.schema|lower }}' and\n table_name = '{{ relation.identifier|lower }}'){%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n )\n {% endif %}\n {%- endcall -%}\n\n {{ return(load_result('last_modified')) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6481018, "supported_languages": null}, "macro.dbt_databricks.get_view_description": {"name": "get_view_description", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_view_description", "macro_sql": "{% macro get_view_description(relation) %}\n {% call statement('get_view_description', fetch_result=True) -%}\n select *\n from `system`.`information_schema`.`views`\n where table_catalog = '{{ relation.database|lower }}'\n and table_schema = '{{ relation.schema|lower }}'\n and table_name = '{{ relation.identifier|lower }}'\n {%- endcall -%}\n\n {% do return(load_result('get_view_description').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.648402, "supported_languages": null}, "macro.dbt_databricks.get_uc_tables": {"name": "get_uc_tables", "resource_type": "macro", "package_name": "dbt_databricks", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt_databricks.get_uc_tables", "macro_sql": "{% macro get_uc_tables(relation) %}\n {% call statement('get_uc_tables', fetch_result=True) -%}\n select\n table_name,\n if(table_type in ('EXTERNAL', 'MANAGED', 'MANAGED_SHALLOW_CLONE', 'EXTERNAL_SHALLOW_CLONE'), 'table', lower(table_type)) as table_type,\n lower(data_source_format) as file_format,\n table_owner\n from `system`.`information_schema`.`tables`\n where table_catalog = '{{ relation.database|lower }}'\n and table_schema = '{{ relation.schema|lower }}'\n {% if relation.identifier %}\n and table_name = '{{ relation.identifier|lower }}'\n {% endif %}\n {% endcall %}\n\n {% do return(load_result('get_uc_tables').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.648753, "supported_languages": null}, "macro.dbt_spark.tblproperties_clause": {"name": "tblproperties_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.tblproperties_clause", "macro_sql": "{% macro tblproperties_clause() %}\n {{ return(adapter.dispatch('tblproperties_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__tblproperties_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.663456, "supported_languages": null}, "macro.dbt_spark.spark__tblproperties_clause": {"name": "spark__tblproperties_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__tblproperties_clause", "macro_sql": "{% macro spark__tblproperties_clause() -%}\n {%- set tblproperties = config.get('tblproperties') -%}\n {%- if tblproperties is not none %}\n tblproperties (\n {%- for prop in tblproperties -%}\n '{{ prop }}' = '{{ tblproperties[prop] }}' {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.663767, "supported_languages": null}, "macro.dbt_spark.file_format_clause": {"name": "file_format_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.file_format_clause", "macro_sql": "{% macro file_format_clause() %}\n {{ return(adapter.dispatch('file_format_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__file_format_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.663905, "supported_languages": null}, "macro.dbt_spark.spark__file_format_clause": {"name": "spark__file_format_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__file_format_clause", "macro_sql": "{% macro spark__file_format_clause() %}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n {%- if file_format is not none %}\n using {{ file_format }}\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6641312, "supported_languages": null}, "macro.dbt_spark.location_clause": {"name": "location_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.location_clause", "macro_sql": "{% macro location_clause() %}\n {{ return(adapter.dispatch('location_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__location_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.664276, "supported_languages": null}, "macro.dbt_spark.spark__location_clause": {"name": "spark__location_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__location_clause", "macro_sql": "{% macro spark__location_clause() %}\n {%- set location_root = config.get('location_root', validator=validation.any[basestring]) -%}\n {%- set identifier = model['alias'] -%}\n {%- if location_root is not none %}\n location '{{ location_root }}/{{ identifier }}'\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.664563, "supported_languages": null}, "macro.dbt_spark.options_clause": {"name": "options_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.options_clause", "macro_sql": "{% macro options_clause() -%}\n {{ return(adapter.dispatch('options_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__options_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6646988, "supported_languages": null}, "macro.dbt_spark.spark__options_clause": {"name": "spark__options_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__options_clause", "macro_sql": "{% macro spark__options_clause() -%}\n {%- set options = config.get('options') -%}\n {%- if config.get('file_format') == 'hudi' -%}\n {%- set unique_key = config.get('unique_key') -%}\n {%- if unique_key is not none and options is none -%}\n {%- set options = {'primaryKey': config.get('unique_key')} -%}\n {%- elif unique_key is not none and options is not none and 'primaryKey' not in options -%}\n {%- set _ = options.update({'primaryKey': config.get('unique_key')}) -%}\n {%- elif options is not none and 'primaryKey' in options and options['primaryKey'] != unique_key -%}\n {{ exceptions.raise_compiler_error(\"unique_key and options('primaryKey') should be the same column(s).\") }}\n {%- endif %}\n {%- endif %}\n\n {%- if options is not none %}\n options (\n {%- for option in options -%}\n {{ option }} \"{{ options[option] }}\" {% if not loop.last %}, {% endif %}\n {%- endfor %}\n )\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6656451, "supported_languages": null}, "macro.dbt_spark.comment_clause": {"name": "comment_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.comment_clause", "macro_sql": "{% macro comment_clause() %}\n {{ return(adapter.dispatch('comment_clause', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__comment_clause"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.665798, "supported_languages": null}, "macro.dbt_spark.spark__comment_clause": {"name": "spark__comment_clause", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__comment_clause", "macro_sql": "{% macro spark__comment_clause() %}\n {%- set raw_persist_docs = config.get('persist_docs', {}) -%}\n\n {%- if raw_persist_docs is mapping -%}\n {%- set raw_relation = raw_persist_docs.get('relation', false) -%}\n {%- if raw_relation -%}\n comment '{{ model.description | replace(\"'\", \"\\\\'\") }}'\n {% endif %}\n {%- elif raw_persist_docs -%}\n {{ exceptions.raise_compiler_error(\"Invalid value provided for 'persist_docs'. Expected dict but got value: \" ~ raw_persist_docs) }}\n {% endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.666194, "supported_languages": null}, "macro.dbt_spark.partition_cols": {"name": "partition_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.partition_cols", "macro_sql": "{% macro partition_cols(label, required=false) %}\n {{ return(adapter.dispatch('partition_cols', 'dbt')(label, required)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.666384, "supported_languages": null}, "macro.dbt_spark.spark__partition_cols": {"name": "spark__partition_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__partition_cols", "macro_sql": "{% macro spark__partition_cols(label, required=false) %}\n {%- set cols = config.get('partition_by', validator=validation.any[list, basestring]) -%}\n {%- if cols is not none %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {{ label }} (\n {%- for item in cols -%}\n {{ item }}\n {%- if not loop.last -%},{%- endif -%}\n {%- endfor -%}\n )\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.666821, "supported_languages": null}, "macro.dbt_spark.clustered_cols": {"name": "clustered_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.clustered_cols", "macro_sql": "{% macro clustered_cols(label, required=false) %}\n {{ return(adapter.dispatch('clustered_cols', 'dbt')(label, required)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__clustered_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.667007, "supported_languages": null}, "macro.dbt_spark.spark__clustered_cols": {"name": "spark__clustered_cols", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__clustered_cols", "macro_sql": "{% macro spark__clustered_cols(label, required=false) %}\n {%- set cols = config.get('clustered_by', validator=validation.any[list, basestring]) -%}\n {%- set buckets = config.get('buckets', validator=validation.any[int]) -%}\n {%- if (cols is not none) and (buckets is not none) %}\n {%- if cols is string -%}\n {%- set cols = [cols] -%}\n {%- endif -%}\n {{ label }} (\n {%- for item in cols -%}\n {{ item }}\n {%- if not loop.last -%},{%- endif -%}\n {%- endfor -%}\n ) into {{ buckets }} buckets\n {%- endif %}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6675909, "supported_languages": null}, "macro.dbt_spark.fetch_tbl_properties": {"name": "fetch_tbl_properties", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.fetch_tbl_properties", "macro_sql": "{% macro fetch_tbl_properties(relation) -%}\n {% call statement('list_properties', fetch_result=True) -%}\n SHOW TBLPROPERTIES {{ relation }}\n {% endcall %}\n {% do return(load_result('list_properties').table) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.667808, "supported_languages": null}, "macro.dbt_spark.create_temporary_view": {"name": "create_temporary_view", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.create_temporary_view", "macro_sql": "{% macro create_temporary_view(relation, compiled_code) -%}\n {{ return(adapter.dispatch('create_temporary_view', 'dbt')(relation, compiled_code)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__create_temporary_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.667983, "supported_languages": null}, "macro.dbt_spark.spark__create_temporary_view": {"name": "spark__create_temporary_view", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_temporary_view", "macro_sql": "{% macro spark__create_temporary_view(relation, compiled_code) -%}\n create or replace temporary view {{ relation }} as\n {{ compiled_code }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.668092, "supported_languages": null}, "macro.dbt_spark.spark__create_table_as": {"name": "spark__create_table_as", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_table_as", "macro_sql": "{%- macro spark__create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {%- if language == 'sql' -%}\n {%- if temporary -%}\n {{ create_temporary_view(relation, compiled_code) }}\n {%- else -%}\n {% if config.get('file_format', validator=validation.any[basestring]) in ['delta', 'iceberg'] %}\n create or replace table {{ relation }}\n {% else %}\n create table {{ relation }}\n {% endif %}\n {%- set contract_config = config.get('contract') -%}\n {%- if contract_config.enforced -%}\n {{ get_assert_columns_equivalent(compiled_code) }}\n {%- set compiled_code = get_select_subquery(compiled_code) %}\n {% endif %}\n {{ file_format_clause() }}\n {{ options_clause() }}\n {{ tblproperties_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause() }}\n {{ comment_clause() }}\n\n as\n {{ compiled_code }}\n {%- endif -%}\n {%- elif language == 'python' -%}\n {#--\n N.B. Python models _can_ write to temp views HOWEVER they use a different session\n and have already expired by the time they need to be used (I.E. in merges for incremental models)\n\n TODO: Deep dive into spark sessions to see if we can reuse a single session for an entire\n dbt invocation.\n --#}\n {{ py_write_table(compiled_code=compiled_code, target_relation=relation) }}\n {%- endif -%}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_spark.create_temporary_view", "macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_select_subquery", "macro.dbt_spark.file_format_clause", "macro.dbt_spark.options_clause", "macro.dbt_spark.tblproperties_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_spark.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt_spark.py_write_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.668963, "supported_languages": null}, "macro.dbt_spark.persist_constraints": {"name": "persist_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.persist_constraints", "macro_sql": "{% macro persist_constraints(relation, model) %}\n {{ return(adapter.dispatch('persist_constraints', 'dbt')(relation, model)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.669142, "supported_languages": null}, "macro.dbt_spark.spark__persist_constraints": {"name": "spark__persist_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__persist_constraints", "macro_sql": "{% macro spark__persist_constraints(relation, model) %}\n {%- set contract_config = config.get('contract') -%}\n {% if contract_config.enforced and config.get('file_format', 'delta') == 'delta' %}\n {% do alter_table_add_constraints(relation, model.constraints) %}\n {% do alter_column_set_constraints(relation, model.columns) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.alter_table_add_constraints", "macro.dbt_spark.alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.669481, "supported_languages": null}, "macro.dbt_spark.alter_table_add_constraints": {"name": "alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.alter_table_add_constraints", "macro_sql": "{% macro alter_table_add_constraints(relation, constraints) %}\n {{ return(adapter.dispatch('alter_table_add_constraints', 'dbt')(relation, constraints)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_table_add_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.669651, "supported_languages": null}, "macro.dbt_spark.spark__alter_table_add_constraints": {"name": "spark__alter_table_add_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_table_add_constraints", "macro_sql": "{% macro spark__alter_table_add_constraints(relation, constraints) %}\n {% for constraint in constraints %}\n {% if constraint.type == 'check' and not is_incremental() %}\n {%- set constraint_hash = local_md5(column_name ~ \";\" ~ constraint.expression ~ \";\" ~ loop.index) -%}\n {% call statement() %}\n alter table {{ relation }} add constraint {{ constraint.name if constraint.name else constraint_hash }} check ({{ constraint.expression }});\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.is_incremental", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.670092, "supported_languages": null}, "macro.dbt_spark.alter_column_set_constraints": {"name": "alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.alter_column_set_constraints", "macro_sql": "{% macro alter_column_set_constraints(relation, column_dict) %}\n {{ return(adapter.dispatch('alter_column_set_constraints', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_set_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.67027, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_set_constraints": {"name": "spark__alter_column_set_constraints", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_set_constraints", "macro_sql": "{% macro spark__alter_column_set_constraints(relation, column_dict) %}\n {% for column_name in column_dict %}\n {% set constraints = column_dict[column_name]['constraints'] %}\n {% for constraint in constraints %}\n {% if constraint.type != 'not_null' %}\n {{ exceptions.warn('Invalid constraint for column ' ~ column_name ~ '. Only `not_null` is supported.') }}\n {% else %}\n {% set quoted_name = adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name %}\n {% call statement() %}\n alter table {{ relation }} change column {{ quoted_name }} set not null {{ constraint.expression or \"\" }};\n {% endcall %}\n {% endif %}\n {% endfor %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.670861, "supported_languages": null}, "macro.dbt_spark.get_column_comment_sql": {"name": "get_column_comment_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_column_comment_sql", "macro_sql": "{% macro get_column_comment_sql(column_name, column_dict) -%}\n {% if column_name in column_dict and column_dict[column_name][\"description\"] -%}\n {% set escaped_description = column_dict[column_name][\"description\"] | replace(\"'\", \"\\\\'\") %}\n {% set column_comment_clause = \"comment '\" ~ escaped_description ~ \"'\" %}\n {%- endif -%}\n {{ adapter.quote(column_name) }} {{ column_comment_clause }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.671293, "supported_languages": null}, "macro.dbt_spark.get_persist_docs_column_list": {"name": "get_persist_docs_column_list", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_persist_docs_column_list", "macro_sql": "{% macro get_persist_docs_column_list(model_columns, query_columns) %}\n {% for column_name in query_columns %}\n {{ get_column_comment_sql(column_name, model_columns) }}\n {{- \", \" if not loop.last else \"\" }}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.get_column_comment_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.671527, "supported_languages": null}, "macro.dbt_spark.spark__create_view_as": {"name": "spark__create_view_as", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_view_as", "macro_sql": "{% macro spark__create_view_as(relation, sql) -%}\n create or replace view {{ relation }}\n {% if config.persist_column_docs() -%}\n {% set model_columns = model.columns %}\n {% set query_columns = get_columns_in_query(sql) %}\n (\n {{ get_persist_docs_column_list(model_columns, query_columns) }}\n )\n {% endif %}\n {{ comment_clause() }}\n {%- set contract_config = config.get('contract') -%}\n {%- if contract_config.enforced -%}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as\n {{ sql }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt_spark.get_persist_docs_column_list", "macro.dbt_spark.comment_clause", "macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6719759, "supported_languages": null}, "macro.dbt_spark.spark__create_schema": {"name": "spark__create_schema", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__create_schema", "macro_sql": "{% macro spark__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{relation}}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6721191, "supported_languages": null}, "macro.dbt_spark.spark__drop_schema": {"name": "spark__drop_schema", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__drop_schema", "macro_sql": "{% macro spark__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation }} cascade\n {%- endcall -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672259, "supported_languages": null}, "macro.dbt_spark.get_columns_in_relation_raw": {"name": "get_columns_in_relation_raw", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.get_columns_in_relation_raw", "macro_sql": "{% macro get_columns_in_relation_raw(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation_raw', 'dbt')(relation)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__get_columns_in_relation_raw"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672405, "supported_languages": null}, "macro.dbt_spark.spark__get_columns_in_relation_raw": {"name": "spark__get_columns_in_relation_raw", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__get_columns_in_relation_raw", "macro_sql": "{% macro spark__get_columns_in_relation_raw(relation) -%}\n {% call statement('get_columns_in_relation_raw', fetch_result=True) %}\n describe extended {{ relation }}\n {% endcall %}\n {% do return(load_result('get_columns_in_relation_raw').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672622, "supported_languages": null}, "macro.dbt_spark.spark__get_columns_in_relation": {"name": "spark__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__get_columns_in_relation", "macro_sql": "{% macro spark__get_columns_in_relation(relation) -%}\n {% call statement('get_columns_in_relation', fetch_result=True) %}\n describe extended {{ relation.include(schema=(schema is not none)) }}\n {% endcall %}\n {% do return(load_result('get_columns_in_relation').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.672888, "supported_languages": null}, "macro.dbt_spark.spark__list_relations_without_caching": {"name": "spark__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__list_relations_without_caching", "macro_sql": "{% macro spark__list_relations_without_caching(relation) %}\n {% call statement('list_relations_without_caching', fetch_result=True) -%}\n show table extended in {{ relation.schema }} like '*'\n {% endcall %}\n\n {% do return(load_result('list_relations_without_caching').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.673117, "supported_languages": null}, "macro.dbt_spark.list_relations_show_tables_without_caching": {"name": "list_relations_show_tables_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.list_relations_show_tables_without_caching", "macro_sql": "{% macro list_relations_show_tables_without_caching(schema_relation) %}\n {#-- Spark with iceberg tables don't work with show table extended for #}\n {#-- V2 iceberg tables #}\n {#-- https://issues.apache.org/jira/browse/SPARK-33393 #}\n {% call statement('list_relations_without_caching_show_tables', fetch_result=True) -%}\n show tables in {{ schema_relation.schema }} like '*'\n {% endcall %}\n\n {% do return(load_result('list_relations_without_caching_show_tables').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6733599, "supported_languages": null}, "macro.dbt_spark.describe_table_extended_without_caching": {"name": "describe_table_extended_without_caching", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.describe_table_extended_without_caching", "macro_sql": "{% macro describe_table_extended_without_caching(table_name) %}\n {#-- Spark with iceberg tables don't work with show table extended for #}\n {#-- V2 iceberg tables #}\n {#-- https://issues.apache.org/jira/browse/SPARK-33393 #}\n {% call statement('describe_table_extended_without_caching', fetch_result=True) -%}\n describe extended {{ table_name }}\n {% endcall %}\n {% do return(load_result('describe_table_extended_without_caching').table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6736, "supported_languages": null}, "macro.dbt_spark.spark__list_schemas": {"name": "spark__list_schemas", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__list_schemas", "macro_sql": "{% macro spark__list_schemas(database) -%}\n {% call statement('list_schemas', fetch_result=True, auto_begin=False) %}\n show databases\n {% endcall %}\n {{ return(load_result('list_schemas').table) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.673813, "supported_languages": null}, "macro.dbt_spark.spark__rename_relation": {"name": "spark__rename_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__rename_relation", "macro_sql": "{% macro spark__rename_relation(from_relation, to_relation) -%}\n {% call statement('rename_relation') -%}\n {% if not from_relation.type %}\n {% do exceptions.raise_database_error(\"Cannot rename a relation with a blank type: \" ~ from_relation.identifier) %}\n {% elif from_relation.type in ('table') %}\n alter table {{ from_relation }} rename to {{ to_relation }}\n {% elif from_relation.type == 'view' %}\n alter view {{ from_relation }} rename to {{ to_relation }}\n {% else %}\n {% do exceptions.raise_database_error(\"Unknown type '\" ~ from_relation.type ~ \"' for relation: \" ~ from_relation.identifier) %}\n {% endif %}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674293, "supported_languages": null}, "macro.dbt_spark.spark__drop_relation": {"name": "spark__drop_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__drop_relation", "macro_sql": "{% macro spark__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n drop {{ relation.type }} if exists {{ relation }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674477, "supported_languages": null}, "macro.dbt_spark.spark__generate_database_name": {"name": "spark__generate_database_name", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__generate_database_name", "macro_sql": "{% macro spark__generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(None) %}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674613, "supported_languages": null}, "macro.dbt_spark.spark__persist_docs": {"name": "spark__persist_docs", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__persist_docs", "macro_sql": "{% macro spark__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do alter_column_comment(relation, model.columns) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.674838, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_comment": {"name": "spark__alter_column_comment", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_comment", "macro_sql": "{% macro spark__alter_column_comment(relation, column_dict) %}\n {% if config.get('file_format', validator=validation.any[basestring]) in ['delta', 'hudi', 'iceberg'] %}\n {% for column_name in column_dict %}\n {% set comment = column_dict[column_name]['description'] %}\n {% set escaped_comment = comment | replace('\\'', '\\\\\\'') %}\n {% set comment_query %}\n {% if relation.is_iceberg %}\n alter table {{ relation }} alter column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% else %}\n alter table {{ relation }} change column\n {{ adapter.quote(column_name) if column_dict[column_name]['quote'] else column_name }}\n comment '{{ escaped_comment }}';\n {% endif %}\n {% endset %}\n {% do run_query(comment_query) %}\n {% endfor %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.675686, "supported_languages": null}, "macro.dbt_spark.spark__make_temp_relation": {"name": "spark__make_temp_relation", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__make_temp_relation", "macro_sql": "{% macro spark__make_temp_relation(base_relation, suffix) %}\n {% set tmp_identifier = base_relation.identifier ~ suffix %}\n {% set tmp_relation = base_relation.incorporate(path = {\n \"identifier\": tmp_identifier\n }) -%}\n\n {%- set tmp_relation = tmp_relation.include(database=false, schema=false) -%}\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6760252, "supported_languages": null}, "macro.dbt_spark.spark__alter_column_type": {"name": "spark__alter_column_type", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_column_type", "macro_sql": "{% macro spark__alter_column_type(relation, column_name, new_column_type) -%}\n {% call statement('alter_column_type') %}\n alter table {{ relation }} alter column {{ column_name }} type {{ new_column_type }};\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.67621, "supported_languages": null}, "macro.dbt_spark.spark__alter_relation_add_remove_columns": {"name": "spark__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/adapters.sql", "original_file_path": "macros/adapters.sql", "unique_id": "macro.dbt_spark.spark__alter_relation_add_remove_columns", "macro_sql": "{% macro spark__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if remove_columns %}\n {% if relation.is_delta %}\n {% set platform_name = 'Delta Lake' %}\n {% elif relation.is_iceberg %}\n {% set platform_name = 'Iceberg' %}\n {% else %}\n {% set platform_name = 'Apache Spark' %}\n {% endif %}\n {{ exceptions.raise_compiler_error(platform_name + ' does not support dropping columns from tables') }}\n {% endif %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation }}\n\n {% if add_columns %} add columns {% endif %}\n {% for column in add_columns %}\n {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.676926, "supported_languages": null}, "macro.dbt_spark.spark__copy_grants": {"name": "spark__copy_grants", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__copy_grants", "macro_sql": "{% macro spark__copy_grants() %}\n\n {% if config.materialized == 'view' %}\n {#-- Spark views don't copy grants when they're replaced --#}\n {{ return(False) }}\n\n {% else %}\n {#-- This depends on how we're replacing the table, which depends on its file format\n -- Just play it safe by assuming that grants have been copied over, and need to be checked / possibly revoked\n -- We can make this more efficient in the future\n #}\n {{ return(True) }}\n\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.677521, "supported_languages": null}, "macro.dbt_spark.spark__get_grant_sql": {"name": "spark__get_grant_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__get_grant_sql", "macro_sql": "\n\n\n{%- macro spark__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation }} to {{ adapter.quote(grantees[0]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6777, "supported_languages": null}, "macro.dbt_spark.spark__get_revoke_sql": {"name": "spark__get_revoke_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__get_revoke_sql", "macro_sql": "\n\n\n{%- macro spark__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation }} from {{ adapter.quote(grantees[0]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.677873, "supported_languages": null}, "macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement": {"name": "spark__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n\n{%- macro spark__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(False) }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.677966, "supported_languages": null}, "macro.dbt_spark.spark__call_dcl_statements": {"name": "spark__call_dcl_statements", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/apply_grants.sql", "original_file_path": "macros/apply_grants.sql", "unique_id": "macro.dbt_spark.spark__call_dcl_statements", "macro_sql": "{% macro spark__call_dcl_statements(dcl_statement_list) %}\n {% for dcl_statement in dcl_statement_list %}\n {% call statement('grant_or_revoke') %}\n {{ dcl_statement }}\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.678161, "supported_languages": null}, "macro.dbt_spark.spark__get_binding_char": {"name": "spark__get_binding_char", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__get_binding_char", "macro_sql": "{% macro spark__get_binding_char() %}\n {{ return('?' if target.method == 'odbc' else '%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.679692, "supported_languages": null}, "macro.dbt_spark.spark__reset_csv_table": {"name": "spark__reset_csv_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__reset_csv_table", "macro_sql": "{% macro spark__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% if old_relation %}\n {{ adapter.drop_relation(old_relation) }}\n {% endif %}\n {% set sql = create_csv_table(model, agate_table) %}\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6799822, "supported_languages": null}, "macro.dbt_spark.spark__load_csv_rows": {"name": "spark__load_csv_rows", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__load_csv_rows", "macro_sql": "{% macro spark__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n {% set column_override = model['config'].get('column_types', {}) %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} values\n {% for row in chunk -%}\n ({%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n cast({{ get_binding_char() }} as {{type}})\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.681087, "supported_languages": null}, "macro.dbt_spark.spark__create_csv_table": {"name": "spark__create_csv_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/seed.sql", "original_file_path": "macros/materializations/seed.sql", "unique_id": "macro.dbt_spark.spark__create_csv_table", "macro_sql": "{% macro spark__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {{ file_format_clause() }}\n {{ partition_cols(label=\"partitioned by\") }}\n {{ clustered_cols(label=\"clustered by\") }}\n {{ location_clause() }}\n {{ comment_clause() }}\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.file_format_clause", "macro.dbt_spark.partition_cols", "macro.dbt_spark.clustered_cols", "macro.dbt_spark.location_clause", "macro.dbt_spark.comment_clause", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6820521, "supported_languages": null}, "macro.dbt_spark.materialization_view_spark": {"name": "materialization_view_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/view.sql", "original_file_path": "macros/materializations/view.sql", "unique_id": "macro.dbt_spark.materialization_view_spark", "macro_sql": "{% materialization view, adapter='spark' -%}\n {{ return(create_or_replace_view()) }}\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.create_or_replace_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.682219, "supported_languages": ["sql"]}, "macro.dbt_spark.materialization_table_spark": {"name": "materialization_table_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.materialization_table_spark", "macro_sql": "{% materialization table, adapter = 'spark', supported_languages=['sql', 'python'] %}\n {%- set language = model['language'] -%}\n {%- set identifier = model['alias'] -%}\n {%- set grant_config = config.get('grants') -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set target_relation = api.Relation.create(identifier=identifier,\n schema=schema,\n database=database,\n type='table') -%}\n\n {{ run_hooks(pre_hooks) }}\n\n -- setup: if the target relation already exists, drop it\n -- in case if the existing and future table is delta or iceberg, we want to do a\n -- create or replace table instead of dropping, so we don't have the table unavailable\n {% if old_relation is not none %}\n {% set is_delta = (old_relation.is_delta and config.get('file_format', validator=validation.any[basestring]) == 'delta') %}\n {% set is_iceberg = (old_relation.is_iceberg and config.get('file_format', validator=validation.any[basestring]) == 'iceberg') %}\n {% set old_relation_type = old_relation.type %}\n {% else %}\n {% set is_delta = false %}\n {% set is_iceberg = false %}\n {% set old_relation_type = target_relation.type %}\n {% endif %}\n\n {% if not is_delta and not is_iceberg %}\n {% set existing_relation = target_relation %}\n {{ adapter.drop_relation(existing_relation.incorporate(type=old_relation_type)) }}\n {% endif %}\n\n -- build model\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% do persist_constraints(target_relation, model) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]})}}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.create_table_as", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt_spark.persist_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6857522, "supported_languages": ["sql", "python"]}, "macro.dbt_spark.py_write_table": {"name": "py_write_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.py_write_table", "macro_sql": "{% macro py_write_table(compiled_code, target_relation) %}\n{{ compiled_code }}\n# --- Autogenerated dbt materialization code. --- #\ndbt = dbtObj(spark.table)\ndf = model(dbt, spark)\n\n# make sure pyspark exists in the namepace, for 7.3.x-scala2.12 it does not exist\nimport pyspark\n# make sure pandas exists before using it\ntry:\n import pandas\n pandas_available = True\nexcept ImportError:\n pandas_available = False\n\n# make sure pyspark.pandas exists before using it\ntry:\n import pyspark.pandas\n pyspark_pandas_api_available = True\nexcept ImportError:\n pyspark_pandas_api_available = False\n\n# make sure databricks.koalas exists before using it\ntry:\n import databricks.koalas\n koalas_available = True\nexcept ImportError:\n koalas_available = False\n\n# preferentially convert pandas DataFrames to pandas-on-Spark or Koalas DataFrames first\n# since they know how to convert pandas DataFrames better than `spark.createDataFrame(df)`\n# and converting from pandas-on-Spark to Spark DataFrame has no overhead\nif pyspark_pandas_api_available and pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = pyspark.pandas.frame.DataFrame(df)\nelif koalas_available and pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = databricks.koalas.frame.DataFrame(df)\n\n# convert to pyspark.sql.dataframe.DataFrame\nif isinstance(df, pyspark.sql.dataframe.DataFrame):\n pass # since it is already a Spark DataFrame\nelif pyspark_pandas_api_available and isinstance(df, pyspark.pandas.frame.DataFrame):\n df = df.to_spark()\nelif koalas_available and isinstance(df, databricks.koalas.frame.DataFrame):\n df = df.to_spark()\nelif pandas_available and isinstance(df, pandas.core.frame.DataFrame):\n df = spark.createDataFrame(df)\nelse:\n msg = f\"{type(df)} is not a supported type for dbt Python materialization\"\n raise Exception(msg)\n\ndf.write.mode(\"overwrite\").format(\"{{ config.get('file_format', 'delta') }}\").option(\"overwriteSchema\", \"true\").saveAsTable(\"{{ target_relation }}\")\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.686034, "supported_languages": null}, "macro.dbt_spark.py_script_comment": {"name": "py_script_comment", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/table.sql", "original_file_path": "macros/materializations/table.sql", "unique_id": "macro.dbt_spark.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n# how to execute python model in notebook\n# dbt = dbtObj(spark.table)\n# df = model(dbt, spark)\n{%endmacro%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.686104, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_hash_arguments": {"name": "spark__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_hash_arguments", "macro_sql": "{% macro spark__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as string ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.692856, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_string_as_time": {"name": "spark__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_string_as_time", "macro_sql": "{% macro spark__snapshot_string_as_time(timestamp) -%}\n {%- set result = \"to_timestamp('\" ~ timestamp ~ \"')\" -%}\n {{ return(result) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.693005, "supported_languages": null}, "macro.dbt_spark.spark__snapshot_merge_sql": {"name": "spark__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__snapshot_merge_sql", "macro_sql": "{% macro spark__snapshot_merge_sql(target, source, insert_cols) -%}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n {% if target.is_iceberg %}\n {# create view only supports a name (no catalog, or schema) #}\n using {{ source.identifier }} as DBT_INTERNAL_SOURCE\n {% else %}\n using {{ source }} as DBT_INTERNAL_SOURCE\n {% endif %}\n on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id\n when matched\n and DBT_INTERNAL_DEST.dbt_valid_to is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set dbt_valid_to = DBT_INTERNAL_SOURCE.dbt_valid_to\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert *\n ;\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.693233, "supported_languages": null}, "macro.dbt_spark.spark_build_snapshot_staging_table": {"name": "spark_build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark_build_snapshot_staging_table", "macro_sql": "{% macro spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set tmp_identifier = target_relation.identifier ~ '__dbt_tmp' %}\n\n {% if target_relation.is_iceberg %}\n {# iceberg catalog does not support create view, but regular spark does. We removed the catalog and schema #}\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=none,\n database=none,\n type='view') -%}\n {% else %}\n {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier,\n schema=target_relation.schema,\n database=none,\n type='view') -%}\n {% endif %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {# needs to be a non-temp view so that its columns can be ascertained via `describe` #}\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_view_as(tmp_relation, select) }}\n {% endcall %}\n\n {% do return(tmp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.693888, "supported_languages": null}, "macro.dbt_spark.spark__post_snapshot": {"name": "spark__post_snapshot", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__post_snapshot", "macro_sql": "{% macro spark__post_snapshot(staging_relation) %}\n {% do adapter.drop_relation(staging_relation) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.694009, "supported_languages": null}, "macro.dbt_spark.spark__create_columns": {"name": "spark__create_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.spark__create_columns", "macro_sql": "{% macro spark__create_columns(relation, columns) %}\n {% if columns|length > 0 %}\n {% call statement() %}\n alter table {{ relation }} add columns (\n {% for column in columns %}\n `{{ column.name }}` {{ column.data_type }} {{- ',' if not loop.last -}}\n {% endfor %}\n );\n {% endcall %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6943328, "supported_languages": null}, "macro.dbt_spark.materialization_snapshot_spark": {"name": "materialization_snapshot_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/snapshot.sql", "original_file_path": "macros/materializations/snapshot.sql", "unique_id": "macro.dbt_spark.materialization_snapshot_spark", "macro_sql": "{% materialization snapshot, adapter='spark' %}\n {%- set config = model['config'] -%}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n {%- set file_format = config.get('file_format', 'parquet') -%}\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=none,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if file_format not in ['delta', 'iceberg', 'hudi'] -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n Snapshot functionality requires file_format be set to 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n\n {%- if target_relation_exists -%}\n {%- if not target_relation.is_delta and not target_relation.is_iceberg and not target_relation.is_hudi -%}\n {% set invalid_format_msg -%}\n The existing table {{ model.schema }}.{{ target_table }} is in another format than 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {% endif %}\n {% endif %}\n\n {% if not adapter.check_schema_exists(model.database, model.schema) %}\n {% do create_schema(model.schema) %}\n {% endif %}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", config, target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {{ adapter.valid_snapshot_target(target_relation) }}\n\n {% set staging_table = spark_build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.create_schema", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt_spark.spark_build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6975749, "supported_languages": ["sql"]}, "macro.dbt_spark.spark__can_clone_table": {"name": "spark__can_clone_table", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.spark__can_clone_table", "macro_sql": "{% macro spark__can_clone_table() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.6994078, "supported_languages": null}, "macro.dbt_spark.spark__create_or_replace_clone": {"name": "spark__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.spark__create_or_replace_clone", "macro_sql": "{% macro spark__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation }} shallow clone {{ defer_relation }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.699519, "supported_languages": null}, "macro.dbt_spark.materialization_clone_spark": {"name": "materialization_clone_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/clone.sql", "original_file_path": "macros/materializations/clone.sql", "unique_id": "macro.dbt_spark.materialization_clone_spark", "macro_sql": "\n\n{%- materialization clone, adapter='spark' -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n {%- set file_format = config.get('file_format', validator=validation.any[basestring]) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if file_format != 'delta' -%}\n {% set invalid_format_msg -%}\n Invalid file format: {{ file_format }}\n shallow clone requires file_format be set to 'delta'\n {%- endset %}\n {% do exceptions.raise_compiler_error(invalid_format_msg) %}\n {%- elif other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n {% endif %}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.70128, "supported_languages": ["sql"]}, "macro.dbt_spark.spark__get_merge_update_columns": {"name": "spark__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/column_helpers.sql", "original_file_path": "macros/materializations/incremental/column_helpers.sql", "unique_id": "macro.dbt_spark.spark__get_merge_update_columns", "macro_sql": "{% macro spark__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = None -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.702069, "supported_languages": null}, "macro.dbt_spark.dbt_spark_validate_get_file_format": {"name": "dbt_spark_validate_get_file_format", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_spark.dbt_spark_validate_get_file_format", "macro_sql": "{% macro dbt_spark_validate_get_file_format(raw_file_format) %}\n {#-- Validate the file format #}\n\n {% set accepted_formats = ['text', 'csv', 'json', 'jdbc', 'parquet', 'orc', 'hive', 'delta', 'iceberg', 'libsvm', 'hudi'] %}\n\n {% set invalid_file_format_msg -%}\n Invalid file format provided: {{ raw_file_format }}\n Expected one of: {{ accepted_formats | join(', ') }}\n {%- endset %}\n\n {% if raw_file_format not in accepted_formats %}\n {% do exceptions.raise_compiler_error(invalid_file_format_msg) %}\n {% endif %}\n\n {% do return(raw_file_format) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.703007, "supported_languages": null}, "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy": {"name": "dbt_spark_validate_get_incremental_strategy", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/validate.sql", "original_file_path": "macros/materializations/incremental/validate.sql", "unique_id": "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy", "macro_sql": "{% macro dbt_spark_validate_get_incremental_strategy(raw_strategy, file_format) %}\n {#-- Validate the incremental strategy #}\n\n {% set invalid_strategy_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n Expected one of: 'append', 'merge', 'insert_overwrite'\n {%- endset %}\n\n {% set invalid_merge_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You can only choose this strategy when file_format is set to 'delta' or 'iceberg' or 'hudi'\n {%- endset %}\n\n {% set invalid_insert_overwrite_endpoint_msg -%}\n Invalid incremental strategy provided: {{ raw_strategy }}\n You cannot use this strategy when connecting via endpoint\n Use the 'append' or 'merge' strategy instead\n {%- endset %}\n\n {% if raw_strategy not in ['append', 'merge', 'insert_overwrite'] %}\n {% do exceptions.raise_compiler_error(invalid_strategy_msg) %}\n {%-else %}\n {% if raw_strategy == 'merge' and file_format not in ['delta', 'iceberg', 'hudi'] %}\n {% do exceptions.raise_compiler_error(invalid_merge_msg) %}\n {% endif %}\n {% if raw_strategy == 'insert_overwrite' and target.endpoint %}\n {% do exceptions.raise_compiler_error(invalid_insert_overwrite_endpoint_msg) %}\n {% endif %}\n {% endif %}\n\n {% do return(raw_strategy) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.703701, "supported_languages": null}, "macro.dbt_spark.get_insert_overwrite_sql": {"name": "get_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.get_insert_overwrite_sql", "macro_sql": "{% macro get_insert_overwrite_sql(source_relation, target_relation, existing_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n {% if existing_relation.is_iceberg %}\n {# removed table from statement for iceberg #}\n insert overwrite {{ target_relation }}\n {# removed partition_cols for iceberg as well #}\n {% else %}\n insert overwrite table {{ target_relation }}\n {{ partition_cols(label=\"partition\") }}\n {% endif %}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.partition_cols"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.705375, "supported_languages": null}, "macro.dbt_spark.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(source_relation, target_relation) %}\n\n {%- set dest_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set dest_cols_csv = dest_columns | map(attribute='quoted') | join(', ') -%}\n insert into table {{ target_relation }}\n select {{dest_cols_csv}} from {{ source_relation }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.705639, "supported_languages": null}, "macro.dbt_spark.spark__get_merge_sql": {"name": "spark__get_merge_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.spark__get_merge_sql", "macro_sql": "{% macro spark__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) %}\n {# need dest_columns for merge_exclude_columns, default to use \"*\" #}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_columns = adapter.get_columns_in_relation(target) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{ predicates | join(' and ') }}\n\n when matched then update set\n {% if update_columns -%}{%- for column_name in update_columns %}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {%- else %} * {% endif %}\n\n when not matched then insert *\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.706816, "supported_languages": null}, "macro.dbt_spark.dbt_spark_get_incremental_sql": {"name": "dbt_spark_get_incremental_sql", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/strategies.sql", "original_file_path": "macros/materializations/incremental/strategies.sql", "unique_id": "macro.dbt_spark.dbt_spark_get_incremental_sql", "macro_sql": "{% macro dbt_spark_get_incremental_sql(strategy, source, target, existing, unique_key, incremental_predicates) %}\n {%- if strategy == 'append' -%}\n {#-- insert new records into existing table, without updating or overwriting #}\n {{ get_insert_into_sql(source, target) }}\n {%- elif strategy == 'insert_overwrite' -%}\n {#-- insert statements don't like CTEs, so support them via a temp view #}\n {{ get_insert_overwrite_sql(source, target, existing) }}\n {%- elif strategy == 'merge' -%}\n {#-- merge all columns for datasources which implement MERGE INTO (e.g. databricks, iceberg) - schema changes are handled for us #}\n {{ get_merge_sql(target, source, unique_key, dest_columns=none, incremental_predicates=incremental_predicates) }}\n {%- else -%}\n {% set no_sql_for_strategy_msg -%}\n No known SQL for the incremental strategy provided: {{ strategy }}\n {%- endset %}\n {%- do exceptions.raise_compiler_error(no_sql_for_strategy_msg) -%}\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.get_insert_into_sql", "macro.dbt_spark.get_insert_overwrite_sql", "macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.707412, "supported_languages": null}, "macro.dbt_spark.materialization_incremental_spark": {"name": "materialization_incremental_spark", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/materializations/incremental/incremental.sql", "original_file_path": "macros/materializations/incremental/incremental.sql", "unique_id": "macro.dbt_spark.materialization_incremental_spark", "macro_sql": "{% materialization incremental, adapter='spark', supported_languages=['sql', 'python'] -%}\n {#-- Validate early so we don't run SQL if the file_format + strategy combo is invalid --#}\n {%- set raw_file_format = config.get('file_format', default='parquet') -%}\n {%- set raw_strategy = config.get('incremental_strategy') or 'append' -%}\n {%- set grant_config = config.get('grants') -%}\n\n {%- set file_format = dbt_spark_validate_get_file_format(raw_file_format) -%}\n {%- set strategy = dbt_spark_validate_get_incremental_strategy(raw_strategy, file_format) -%}\n\n {#-- Set vars --#}\n\n {%- set unique_key = config.get('unique_key', none) -%}\n {%- set partition_by = config.get('partition_by', none) -%}\n {%- set language = model['language'] -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n {%- set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) -%}\n {%- set target_relation = this -%}\n {%- set existing_relation = load_relation(this) -%}\n {%- set tmp_relation = make_temp_relation(this) -%}\n\n {#-- for SQL model we will create temp view that doesn't have database and schema --#}\n {%- if language == 'sql'-%}\n {%- set tmp_relation = tmp_relation.include(database=false, schema=false) -%}\n {%- endif -%}\n\n {#-- Set Overwrite Mode --#}\n {%- if strategy == 'insert_overwrite' and partition_by -%}\n {%- call statement() -%}\n set spark.sql.sources.partitionOverwriteMode = DYNAMIC\n {%- endcall -%}\n {%- endif -%}\n\n {#-- Run pre-hooks --#}\n {{ run_hooks(pre_hooks) }}\n\n {#-- Incremental run logic --#}\n {%- if existing_relation is none -%}\n {#-- Relation must be created --#}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {%- elif existing_relation.is_view or should_full_refresh() -%}\n {#-- Relation must be dropped & recreated --#}\n {% set is_delta = (file_format == 'delta' and existing_relation.is_delta) %}\n {% if not is_delta %} {#-- If Delta, we will `create or replace` below, so no need to drop --#}\n {% do adapter.drop_relation(existing_relation) %}\n {% endif %}\n {%- call statement('main', language=language) -%}\n {{ create_table_as(False, target_relation, compiled_code, language) }}\n {%- endcall -%}\n {% do persist_constraints(target_relation, model) %}\n {%- else -%}\n {#-- Relation must be merged --#}\n {%- call statement('create_tmp_relation', language=language) -%}\n {{ create_table_as(True, tmp_relation, compiled_code, language) }}\n {%- endcall -%}\n {%- do process_schema_changes(on_schema_change, tmp_relation, existing_relation) -%}\n {%- call statement('main') -%}\n {{ dbt_spark_get_incremental_sql(strategy, tmp_relation, target_relation, existing_relation, unique_key, incremental_predicates) }}\n {%- endcall -%}\n {%- if language == 'python' -%}\n {#--\n This is yucky.\n See note in dbt-spark/dbt/include/spark/macros/adapters.sql\n re: python models and temporary views.\n\n Also, why do neither drop_relation or adapter.drop_relation work here?!\n --#}\n {% call statement('drop_relation') -%}\n drop table if exists {{ tmp_relation }}\n {%- endcall %}\n {%- endif -%}\n {%- endif -%}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt_spark.dbt_spark_validate_get_file_format", "macro.dbt_spark.dbt_spark_validate_get_incremental_strategy", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.load_relation", "macro.dbt.make_temp_relation", "macro.dbt.statement", "macro.dbt.run_hooks", "macro.dbt.create_table_as", "macro.dbt_spark.persist_constraints", "macro.dbt.should_full_refresh", "macro.dbt.process_schema_changes", "macro.dbt_spark.dbt_spark_get_incremental_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.710429, "supported_languages": ["sql", "python"]}, "macro.dbt_spark.spark__concat": {"name": "spark__concat", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt_spark.spark__concat", "macro_sql": "{% macro spark__concat(fields) -%}\n concat({{ fields|join(', ') }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.710591, "supported_languages": null}, "macro.dbt_spark.assert_not_null": {"name": "assert_not_null", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/assert_not_null.sql", "original_file_path": "macros/utils/assert_not_null.sql", "unique_id": "macro.dbt_spark.assert_not_null", "macro_sql": "{% macro assert_not_null(function, arg) -%}\n {{ return(adapter.dispatch('assert_not_null', 'dbt')(function, arg)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.710856, "supported_languages": null}, "macro.dbt_spark.spark__assert_not_null": {"name": "spark__assert_not_null", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/assert_not_null.sql", "original_file_path": "macros/utils/assert_not_null.sql", "unique_id": "macro.dbt_spark.spark__assert_not_null", "macro_sql": "{% macro spark__assert_not_null(function, arg) %}\n\n coalesce({{function}}({{arg}}), nvl2({{function}}({{arg}}), assert_true({{function}}({{arg}}) is not null), null))\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.711019, "supported_languages": null}, "macro.dbt_spark.spark__dateadd": {"name": "spark__dateadd", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt_spark.spark__dateadd", "macro_sql": "{% macro spark__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n {%- set clock_component -%}\n {# make sure the dates + timestamps are real, otherwise raise an error asap #}\n to_unix_timestamp({{ assert_not_null('to_timestamp', from_date_or_timestamp) }})\n - to_unix_timestamp({{ assert_not_null('date', from_date_or_timestamp) }})\n {%- endset -%}\n\n {%- if datepart in ['day', 'week'] -%}\n\n {%- set multiplier = 7 if datepart == 'week' else 1 -%}\n\n to_timestamp(\n to_unix_timestamp(\n date_add(\n {{ assert_not_null('date', from_date_or_timestamp) }},\n cast({{interval}} * {{multiplier}} as int)\n )\n ) + {{clock_component}}\n )\n\n {%- elif datepart in ['month', 'quarter', 'year'] -%}\n\n {%- set multiplier -%}\n {%- if datepart == 'month' -%} 1\n {%- elif datepart == 'quarter' -%} 3\n {%- elif datepart == 'year' -%} 12\n {%- endif -%}\n {%- endset -%}\n\n to_timestamp(\n to_unix_timestamp(\n add_months(\n {{ assert_not_null('date', from_date_or_timestamp) }},\n cast({{interval}} * {{multiplier}} as int)\n )\n ) + {{clock_component}}\n )\n\n {%- elif datepart in ('hour', 'minute', 'second', 'millisecond', 'microsecond') -%}\n\n {%- set multiplier -%}\n {%- if datepart == 'hour' -%} 3600\n {%- elif datepart == 'minute' -%} 60\n {%- elif datepart == 'second' -%} 1\n {%- elif datepart == 'millisecond' -%} (1/1000000)\n {%- elif datepart == 'microsecond' -%} (1/1000000)\n {%- endif -%}\n {%- endset -%}\n\n to_timestamp(\n {{ assert_not_null('to_unix_timestamp', from_date_or_timestamp) }}\n + cast({{interval}} * {{multiplier}} as int)\n )\n\n {%- else -%}\n\n {{ exceptions.raise_compiler_error(\"macro dateadd not implemented for datepart ~ '\" ~ datepart ~ \"' ~ on Spark\") }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.713269, "supported_languages": null}, "macro.dbt_spark.spark__current_timestamp": {"name": "spark__current_timestamp", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/timestamps.sql", "original_file_path": "macros/utils/timestamps.sql", "unique_id": "macro.dbt_spark.spark__current_timestamp", "macro_sql": "{% macro spark__current_timestamp() -%}\n current_timestamp()\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.713365, "supported_languages": null}, "macro.dbt_spark.spark__escape_single_quotes": {"name": "spark__escape_single_quotes", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt_spark.spark__escape_single_quotes", "macro_sql": "{% macro spark__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"\\\\'\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.713526, "supported_languages": null}, "macro.dbt_spark.spark__listagg": {"name": "spark__listagg", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt_spark.spark__listagg", "macro_sql": "{% macro spark__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if order_by_clause %}\n {{ exceptions.warn(\"order_by_clause is not supported for listagg on Spark/Databricks\") }}\n {% endif %}\n\n {% set collect_list %} collect_list({{ measure }}) {% endset %}\n\n {% set limited %} slice({{ collect_list }}, 1, {{ limit_num }}) {% endset %}\n\n {% set collected = limited if limit_num else collect_list %}\n\n {% set final %} array_join({{ collected }}, {{ delimiter_text }}) {% endset %}\n\n {% do return(final) %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.714139, "supported_languages": null}, "macro.dbt_spark.spark__datediff": {"name": "spark__datediff", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt_spark.spark__datediff", "macro_sql": "{% macro spark__datediff(first_date, second_date, datepart) %}\n\n {%- if datepart in ['day', 'week', 'month', 'quarter', 'year'] -%}\n\n {# make sure the dates are real, otherwise raise an error asap #}\n {% set first_date = assert_not_null('date', first_date) %}\n {% set second_date = assert_not_null('date', second_date) %}\n\n {%- endif -%}\n\n {%- if datepart == 'day' -%}\n\n datediff({{second_date}}, {{first_date}})\n\n {%- elif datepart == 'week' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(datediff({{second_date}}, {{first_date}})/7)\n else ceil(datediff({{second_date}}, {{first_date}})/7)\n end\n\n -- did we cross a week boundary (Sunday)?\n + case\n when {{first_date}} < {{second_date}} and dayofweek({{second_date}}) < dayofweek({{first_date}}) then 1\n when {{first_date}} > {{second_date}} and dayofweek({{second_date}}) > dayofweek({{first_date}}) then -1\n else 0 end\n\n {%- elif datepart == 'month' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(months_between(date({{second_date}}), date({{first_date}})))\n else ceil(months_between(date({{second_date}}), date({{first_date}})))\n end\n\n -- did we cross a month boundary?\n + case\n when {{first_date}} < {{second_date}} and dayofmonth({{second_date}}) < dayofmonth({{first_date}}) then 1\n when {{first_date}} > {{second_date}} and dayofmonth({{second_date}}) > dayofmonth({{first_date}}) then -1\n else 0 end\n\n {%- elif datepart == 'quarter' -%}\n\n case when {{first_date}} < {{second_date}}\n then floor(months_between(date({{second_date}}), date({{first_date}}))/3)\n else ceil(months_between(date({{second_date}}), date({{first_date}}))/3)\n end\n\n -- did we cross a quarter boundary?\n + case\n when {{first_date}} < {{second_date}} and (\n (dayofyear({{second_date}}) - (quarter({{second_date}}) * 365/4))\n < (dayofyear({{first_date}}) - (quarter({{first_date}}) * 365/4))\n ) then 1\n when {{first_date}} > {{second_date}} and (\n (dayofyear({{second_date}}) - (quarter({{second_date}}) * 365/4))\n > (dayofyear({{first_date}}) - (quarter({{first_date}}) * 365/4))\n ) then -1\n else 0 end\n\n {%- elif datepart == 'year' -%}\n\n year({{second_date}}) - year({{first_date}})\n\n {%- elif datepart in ('hour', 'minute', 'second', 'millisecond', 'microsecond') -%}\n\n {%- set divisor -%}\n {%- if datepart == 'hour' -%} 3600\n {%- elif datepart == 'minute' -%} 60\n {%- elif datepart == 'second' -%} 1\n {%- elif datepart == 'millisecond' -%} (1/1000)\n {%- elif datepart == 'microsecond' -%} (1/1000000)\n {%- endif -%}\n {%- endset -%}\n\n case when {{first_date}} < {{second_date}}\n then ceil((\n {# make sure the timestamps are real, otherwise raise an error asap #}\n {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', second_date)) }}\n - {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', first_date)) }}\n ) / {{divisor}})\n else floor((\n {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', second_date)) }}\n - {{ assert_not_null('to_unix_timestamp', assert_not_null('to_timestamp', first_date)) }}\n ) / {{divisor}})\n end\n\n {% if datepart == 'millisecond' %}\n + cast(date_format({{second_date}}, 'SSS') as int)\n - cast(date_format({{first_date}}, 'SSS') as int)\n {% endif %}\n\n {% if datepart == 'microsecond' %}\n {% set capture_str = '[0-9]{4}-[0-9]{2}-[0-9]{2}.[0-9]{2}:[0-9]{2}:[0-9]{2}.([0-9]{6})' %}\n -- Spark doesn't really support microseconds, so this is a massive hack!\n -- It will only work if the timestamp-string is of the format\n -- 'yyyy-MM-dd-HH mm.ss.SSSSSS'\n + cast(regexp_extract({{second_date}}, '{{capture_str}}', 1) as int)\n - cast(regexp_extract({{first_date}}, '{{capture_str}}', 1) as int)\n {% endif %}\n\n {%- else -%}\n\n {{ exceptions.raise_compiler_error(\"macro datediff not implemented for datepart ~ '\" ~ datepart ~ \"' ~ on Spark\") }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.assert_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7200809, "supported_languages": null}, "macro.dbt_spark.spark__safe_cast": {"name": "spark__safe_cast", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt_spark.spark__safe_cast", "macro_sql": "{% macro spark__safe_cast(field, type) %}\n{%- set field_clean = field.strip('\"').strip(\"'\") if (cast_from_string_unsupported_for(type) and field is string) else field -%}\ncast({{field_clean}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.cast_from_string_unsupported_for"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7204401, "supported_languages": null}, "macro.dbt_spark.cast_from_string_unsupported_for": {"name": "cast_from_string_unsupported_for", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt_spark.cast_from_string_unsupported_for", "macro_sql": "{% macro cast_from_string_unsupported_for(type) %}\n {{ return(type.lower().startswith('struct') or type.lower().startswith('array') or type.lower().startswith('map')) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.720673, "supported_languages": null}, "macro.dbt_spark.spark__any_value": {"name": "spark__any_value", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt_spark.spark__any_value", "macro_sql": "{% macro spark__any_value(expression) -%}\n {#-- return any value (non-deterministic) --#}\n first({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.720806, "supported_languages": null}, "macro.dbt_spark.spark__array_concat": {"name": "spark__array_concat", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt_spark.spark__array_concat", "macro_sql": "{% macro spark__array_concat(array_1, array_2) -%}\n concat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.720957, "supported_languages": null}, "macro.dbt_spark.spark__bool_or": {"name": "spark__bool_or", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt_spark.spark__bool_or", "macro_sql": "{% macro spark__bool_or(expression) -%}\n\n max({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.721088, "supported_languages": null}, "macro.dbt_spark.spark__split_part": {"name": "spark__split_part", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt_spark.spark__split_part", "macro_sql": "{% macro spark__split_part(string_text, delimiter_text, part_number) %}\n\n {% set delimiter_expr %}\n\n -- escape if starts with a special character\n case when regexp_extract({{ delimiter_text }}, '([^A-Za-z0-9])(.*)', 1) != '_'\n then concat('\\\\', {{ delimiter_text }})\n else {{ delimiter_text }} end\n\n {% endset %}\n\n {% if part_number >= 0 %}\n\n {% set split_part_expr %}\n\n split(\n {{ string_text }},\n {{ delimiter_expr }}\n )[({{ part_number - 1 if part_number > 0 else part_number }})]\n\n {% endset %}\n\n {% else %}\n\n {% set split_part_expr %}\n\n split(\n {{ string_text }},\n {{ delimiter_expr }}\n )[(\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 1 + {{ part_number }}\n )]\n\n {% endset %}\n\n {% endif %}\n\n {{ return(split_part_expr) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.722142, "supported_languages": null}, "macro.dbt_spark.spark__array_construct": {"name": "spark__array_construct", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt_spark.spark__array_construct", "macro_sql": "{% macro spark__array_construct(inputs, data_type) -%}\n array( {{ inputs|join(' , ') }} )\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.722311, "supported_languages": null}, "macro.dbt_spark.spark__array_append": {"name": "spark__array_append", "resource_type": "macro", "package_name": "dbt_spark", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt_spark.spark__array_append", "macro_sql": "{% macro spark__array_append(array, new_element) -%}\n {{ array_concat(array, array_construct([new_element])) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.array_concat", "macro.dbt.array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.722492, "supported_languages": null}, "macro.dbt.run_hooks": {"name": "run_hooks", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.run_hooks", "macro_sql": "{% macro run_hooks(hooks, inside_transaction=True) %}\n {% for hook in hooks | selectattr('transaction', 'equalto', inside_transaction) %}\n {% if not inside_transaction and loop.first %}\n {% call statement(auto_begin=inside_transaction) %}\n commit;\n {% endcall %}\n {% endif %}\n {% set rendered = render(hook.get('sql')) | trim %}\n {% if (rendered | length) > 0 %}\n {% call statement(auto_begin=inside_transaction) %}\n {{ rendered }}\n {% endcall %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.72342, "supported_languages": null}, "macro.dbt.make_hook_config": {"name": "make_hook_config", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.make_hook_config", "macro_sql": "{% macro make_hook_config(sql, inside_transaction) %}\n {{ tojson({\"sql\": sql, \"transaction\": inside_transaction}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723584, "supported_languages": null}, "macro.dbt.before_begin": {"name": "before_begin", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.before_begin", "macro_sql": "{% macro before_begin(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723713, "supported_languages": null}, "macro.dbt.in_transaction": {"name": "in_transaction", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.in_transaction", "macro_sql": "{% macro in_transaction(sql) %}\n {{ make_hook_config(sql, inside_transaction=True) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723829, "supported_languages": null}, "macro.dbt.after_commit": {"name": "after_commit", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/hooks.sql", "original_file_path": "macros/materializations/hooks.sql", "unique_id": "macro.dbt.after_commit", "macro_sql": "{% macro after_commit(sql) %}\n {{ make_hook_config(sql, inside_transaction=False) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_hook_config"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.723947, "supported_languages": null}, "macro.dbt.set_sql_header": {"name": "set_sql_header", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.set_sql_header", "macro_sql": "{% macro set_sql_header(config) -%}\n {{ config.set('sql_header', caller()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.724263, "supported_languages": null}, "macro.dbt.should_full_refresh": {"name": "should_full_refresh", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_full_refresh", "macro_sql": "{% macro should_full_refresh() %}\n {% set config_full_refresh = config.get('full_refresh') %}\n {% if config_full_refresh is none %}\n {% set config_full_refresh = flags.FULL_REFRESH %}\n {% endif %}\n {% do return(config_full_refresh) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.72451, "supported_languages": null}, "macro.dbt.should_store_failures": {"name": "should_store_failures", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/configs.sql", "original_file_path": "macros/materializations/configs.sql", "unique_id": "macro.dbt.should_store_failures", "macro_sql": "{% macro should_store_failures() %}\n {% set config_store_failures = config.get('store_failures') %}\n {% if config_store_failures is none %}\n {% set config_store_failures = flags.STORE_FAILURES %}\n {% endif %}\n {% do return(config_store_failures) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.724753, "supported_languages": null}, "macro.dbt.snapshot_merge_sql": {"name": "snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.snapshot_merge_sql", "macro_sql": "{% macro snapshot_merge_sql(target, source, insert_cols) -%}\n {{ adapter.dispatch('snapshot_merge_sql', 'dbt')(target, source, insert_cols) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__snapshot_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7252579, "supported_languages": null}, "macro.dbt.default__snapshot_merge_sql": {"name": "default__snapshot_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot_merge.sql", "original_file_path": "macros/materializations/snapshots/snapshot_merge.sql", "unique_id": "macro.dbt.default__snapshot_merge_sql", "macro_sql": "{% macro default__snapshot_merge_sql(target, source, insert_cols) -%}\n {%- set insert_cols_csv = insert_cols | join(', ') -%}\n\n {%- set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() -%}\n\n merge into {{ target.render() }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}\n\n when matched\n and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null\n and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')\n then update\n set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}\n\n when not matched\n and DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'\n then insert ({{ insert_cols_csv }})\n values ({{ insert_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.725742, "supported_languages": null}, "macro.dbt.strategy_dispatch": {"name": "strategy_dispatch", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.strategy_dispatch", "macro_sql": "{% macro strategy_dispatch(name) -%}\n{% set original_name = name %}\n {% if '.' in name %}\n {% set package_name, name = name.split(\".\", 1) %}\n {% else %}\n {% set package_name = none %}\n {% endif %}\n\n {% if package_name is none %}\n {% set package_context = context %}\n {% elif package_name in context %}\n {% set package_context = context[package_name] %}\n {% else %}\n {% set error_msg %}\n Could not find package '{{package_name}}', called with '{{original_name}}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n\n {%- set search_name = 'snapshot_' ~ name ~ '_strategy' -%}\n\n {% if search_name not in package_context %}\n {% set error_msg %}\n The specified strategy macro '{{name}}' was not found in package '{{ package_name }}'\n {% endset %}\n {{ exceptions.raise_compiler_error(error_msg | trim) }}\n {% endif %}\n {{ return(package_context[search_name]) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7284422, "supported_languages": null}, "macro.dbt.snapshot_hash_arguments": {"name": "snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_hash_arguments", "macro_sql": "{% macro snapshot_hash_arguments(args) -%}\n {{ adapter.dispatch('snapshot_hash_arguments', 'dbt')(args) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7285912, "supported_languages": null}, "macro.dbt.default__snapshot_hash_arguments": {"name": "default__snapshot_hash_arguments", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_hash_arguments", "macro_sql": "{% macro default__snapshot_hash_arguments(args) -%}\n md5({%- for arg in args -%}\n coalesce(cast({{ arg }} as varchar ), '')\n {% if not loop.last %} || '|' || {% endif %}\n {%- endfor -%})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.728775, "supported_languages": null}, "macro.dbt.snapshot_timestamp_strategy": {"name": "snapshot_timestamp_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_timestamp_strategy", "macro_sql": "{% macro snapshot_timestamp_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set primary_key = config.get('unique_key') %}\n {% set updated_at = config.get('updated_at') %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes') or false %}\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {#/*\n The snapshot relation might not have an {{ updated_at }} value if the\n snapshot strategy is changed from `check` to `timestamp`. We\n should use a dbt-created column for the comparison in the snapshot\n table instead of assuming that the user-supplied {{ updated_at }}\n will be present in the historical data.\n\n See https://github.com/dbt-labs/dbt-core/issues/2350\n */ #}\n {% set row_changed_expr -%}\n ({{ snapshotted_rel }}.{{ columns.dbt_valid_from }} < {{ current_rel }}.{{ updated_at }})\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7294762, "supported_languages": null}, "macro.dbt.snapshot_string_as_time": {"name": "snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_string_as_time", "macro_sql": "{% macro snapshot_string_as_time(timestamp) -%}\n {{ adapter.dispatch('snapshot_string_as_time', 'dbt')(timestamp) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__snapshot_string_as_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7296178, "supported_languages": null}, "macro.dbt.default__snapshot_string_as_time": {"name": "default__snapshot_string_as_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.default__snapshot_string_as_time", "macro_sql": "{% macro default__snapshot_string_as_time(timestamp) %}\n {% do exceptions.raise_not_implemented(\n 'snapshot_string_as_time macro not implemented for adapter '+adapter.type()\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.729759, "supported_languages": null}, "macro.dbt.snapshot_check_all_get_existing_columns": {"name": "snapshot_check_all_get_existing_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_all_get_existing_columns", "macro_sql": "{% macro snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) -%}\n {%- if not target_exists -%}\n {#-- no table yet -> return whatever the query does --#}\n {{ return((false, query_columns)) }}\n {%- endif -%}\n\n {#-- handle any schema changes --#}\n {%- set target_relation = adapter.get_relation(database=node.database, schema=node.schema, identifier=node.alias) -%}\n\n {% if check_cols_config == 'all' %}\n {%- set query_columns = get_columns_in_query(node['compiled_code']) -%}\n\n {% elif check_cols_config is iterable and (check_cols_config | length) > 0 %}\n {#-- query for proper casing/quoting, to support comparison below --#}\n {%- set select_check_cols_from_target -%}\n {#-- N.B. The whitespace below is necessary to avoid edge case issue with comments --#}\n {#-- See: https://github.com/dbt-labs/dbt-core/issues/6781 --#}\n select {{ check_cols_config | join(', ') }} from (\n {{ node['compiled_code'] }}\n ) subq\n {%- endset -%}\n {% set query_columns = get_columns_in_query(select_check_cols_from_target) %}\n\n {% else %}\n {% do exceptions.raise_compiler_error(\"Invalid value for 'check_cols': \" ~ check_cols_config) %}\n {% endif %}\n\n {%- set existing_cols = adapter.get_columns_in_relation(target_relation) | map(attribute = 'name') | list -%}\n {%- set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {%- set ns.column_added = false -%}\n\n {%- set intersection = [] -%}\n {%- for col in query_columns -%}\n {%- if col in existing_cols -%}\n {%- do intersection.append(adapter.quote(col)) -%}\n {%- else -%}\n {% set ns.column_added = true %}\n {%- endif -%}\n {%- endfor -%}\n {{ return((ns.column_added, intersection)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7308888, "supported_languages": null}, "macro.dbt.snapshot_check_strategy": {"name": "snapshot_check_strategy", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/strategies.sql", "original_file_path": "macros/materializations/snapshots/strategies.sql", "unique_id": "macro.dbt.snapshot_check_strategy", "macro_sql": "{% macro snapshot_check_strategy(node, snapshotted_rel, current_rel, model_config, target_exists) %}\n {# The model_config parameter is no longer used, but is passed in anyway for compatibility. #}\n {% set check_cols_config = config.get('check_cols') %}\n {% set primary_key = config.get('unique_key') %}\n {% set invalidate_hard_deletes = config.get('invalidate_hard_deletes') or false %}\n {% set updated_at = config.get('updated_at') or snapshot_get_time() %}\n\n {% set column_added = false %}\n\n {% set column_added, check_cols = snapshot_check_all_get_existing_columns(node, target_exists, check_cols_config) %}\n\n {%- set row_changed_expr -%}\n (\n {%- if column_added -%}\n {{ get_true_sql() }}\n {%- else -%}\n {%- for col in check_cols -%}\n {{ snapshotted_rel }}.{{ col }} != {{ current_rel }}.{{ col }}\n or\n (\n (({{ snapshotted_rel }}.{{ col }} is null) and not ({{ current_rel }}.{{ col }} is null))\n or\n ((not {{ snapshotted_rel }}.{{ col }} is null) and ({{ current_rel }}.{{ col }} is null))\n )\n {%- if not loop.last %} or {% endif -%}\n {%- endfor -%}\n {%- endif -%}\n )\n {%- endset %}\n\n {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %}\n\n {% do return({\n \"unique_key\": primary_key,\n \"updated_at\": updated_at,\n \"row_changed\": row_changed_expr,\n \"scd_id\": scd_id_expr,\n \"invalidate_hard_deletes\": invalidate_hard_deletes\n }) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.snapshot_check_all_get_existing_columns", "macro.dbt.get_true_sql", "macro.dbt.snapshot_hash_arguments"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7321742, "supported_languages": null}, "macro.dbt.create_columns": {"name": "create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.create_columns", "macro_sql": "{% macro create_columns(relation, columns) %}\n {{ adapter.dispatch('create_columns', 'dbt')(relation, columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__create_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.73771, "supported_languages": null}, "macro.dbt.default__create_columns": {"name": "default__create_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__create_columns", "macro_sql": "{% macro default__create_columns(relation, columns) %}\n {% for column in columns %}\n {% call statement() %}\n alter table {{ relation.render() }} add column \"{{ column.name }}\" {{ column.data_type }};\n {% endcall %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.737952, "supported_languages": null}, "macro.dbt.post_snapshot": {"name": "post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.post_snapshot", "macro_sql": "{% macro post_snapshot(staging_relation) %}\n {{ adapter.dispatch('post_snapshot', 'dbt')(staging_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.738092, "supported_languages": null}, "macro.dbt.default__post_snapshot": {"name": "default__post_snapshot", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__post_snapshot", "macro_sql": "{% macro default__post_snapshot(staging_relation) %}\n {# no-op #}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.738164, "supported_languages": null}, "macro.dbt.get_true_sql": {"name": "get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_true_sql", "macro_sql": "{% macro get_true_sql() %}\n {{ adapter.dispatch('get_true_sql', 'dbt')() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_true_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.738283, "supported_languages": null}, "macro.dbt.default__get_true_sql": {"name": "default__get_true_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__get_true_sql", "macro_sql": "{% macro default__get_true_sql() %}\n {{ return('TRUE') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7383811, "supported_languages": null}, "macro.dbt.snapshot_staging_table": {"name": "snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.snapshot_staging_table", "macro_sql": "{% macro snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {{ adapter.dispatch('snapshot_staging_table', 'dbt')(strategy, source_sql, target_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__snapshot_staging_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7385468, "supported_languages": null}, "macro.dbt.get_snapshot_table_column_names": {"name": "get_snapshot_table_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_snapshot_table_column_names", "macro_sql": "{% macro get_snapshot_table_column_names() %}\n {{ return({'dbt_valid_to': 'dbt_valid_to', 'dbt_valid_from': 'dbt_valid_from', 'dbt_scd_id': 'dbt_scd_id', 'dbt_updated_at': 'dbt_updated_at'}) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7387378, "supported_languages": null}, "macro.dbt.default__snapshot_staging_table": {"name": "default__snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__snapshot_staging_table", "macro_sql": "{% macro default__snapshot_staging_table(strategy, source_sql, target_relation) -%}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n\n with snapshot_query as (\n\n {{ source_sql }}\n\n ),\n\n snapshotted_data as (\n\n select *,\n {{ strategy.unique_key }} as dbt_unique_key\n\n from {{ target_relation }}\n where {{ columns.dbt_valid_to }} is null\n\n ),\n\n insertions_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }},\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }}\n\n from snapshot_query\n ),\n\n updates_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key,\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_to }}\n\n from snapshot_query\n ),\n\n {%- if strategy.invalidate_hard_deletes %}\n\n deletes_source_data as (\n\n select\n *,\n {{ strategy.unique_key }} as dbt_unique_key\n from snapshot_query\n ),\n {% endif %}\n\n insertions as (\n\n select\n 'insert' as dbt_change_type,\n source_data.*\n\n from insertions_source_data as source_data\n left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where snapshotted_data.dbt_unique_key is null\n or (\n snapshotted_data.dbt_unique_key is not null\n and (\n {{ strategy.row_changed }}\n )\n )\n\n ),\n\n updates as (\n\n select\n 'update' as dbt_change_type,\n source_data.*,\n snapshotted_data.{{ columns.dbt_scd_id }}\n\n from updates_source_data as source_data\n join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where (\n {{ strategy.row_changed }}\n )\n )\n\n {%- if strategy.invalidate_hard_deletes -%}\n ,\n\n deletes as (\n\n select\n 'delete' as dbt_change_type,\n source_data.*,\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_from }},\n {{ snapshot_get_time() }} as {{ columns.dbt_updated_at }},\n {{ snapshot_get_time() }} as {{ columns.dbt_valid_to }},\n snapshotted_data.{{ columns.dbt_scd_id }}\n\n from snapshotted_data\n left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key\n where source_data.dbt_unique_key is null\n )\n {%- endif %}\n\n select * from insertions\n union all\n select * from updates\n {%- if strategy.invalidate_hard_deletes %}\n union all\n select * from deletes\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.739788, "supported_languages": null}, "macro.dbt.build_snapshot_table": {"name": "build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_table", "macro_sql": "{% macro build_snapshot_table(strategy, sql) -%}\n {{ adapter.dispatch('build_snapshot_table', 'dbt')(strategy, sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__build_snapshot_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7399502, "supported_languages": null}, "macro.dbt.default__build_snapshot_table": {"name": "default__build_snapshot_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.default__build_snapshot_table", "macro_sql": "{% macro default__build_snapshot_table(strategy, sql) %}\n {% set columns = config.get('snapshot_table_column_names') or get_snapshot_table_column_names() %}\n\n select *,\n {{ strategy.scd_id }} as {{ columns.dbt_scd_id }},\n {{ strategy.updated_at }} as {{ columns.dbt_updated_at }},\n {{ strategy.updated_at }} as {{ columns.dbt_valid_from }},\n nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}) as {{ columns.dbt_valid_to }}\n from (\n {{ sql }}\n ) sbq\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_snapshot_table_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7403069, "supported_languages": null}, "macro.dbt.build_snapshot_staging_table": {"name": "build_snapshot_staging_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.build_snapshot_staging_table", "macro_sql": "{% macro build_snapshot_staging_table(strategy, sql, target_relation) %}\n {% set temp_relation = make_temp_relation(target_relation) %}\n\n {% set select = snapshot_staging_table(strategy, sql, target_relation) %}\n\n {% call statement('build_snapshot_staging_relation') %}\n {{ create_table_as(True, temp_relation, select) }}\n {% endcall %}\n\n {% do return(temp_relation) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.make_temp_relation", "macro.dbt.snapshot_staging_table", "macro.dbt.statement", "macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.740653, "supported_languages": null}, "macro.dbt.get_updated_at_column_data_type": {"name": "get_updated_at_column_data_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.get_updated_at_column_data_type", "macro_sql": "{% macro get_updated_at_column_data_type(snapshot_sql) %}\n {% set snapshot_sql_column_schema = get_column_schema_from_query(snapshot_sql) %}\n {% set dbt_updated_at_data_type = null %}\n {% set ns = namespace() -%} {#-- handle for-loop scoping with a namespace --#}\n {% set ns.dbt_updated_at_data_type = null -%}\n {% for column in snapshot_sql_column_schema %}\n {% if ((column.column == 'dbt_updated_at') or (column.column == 'DBT_UPDATED_AT')) %}\n {% set ns.dbt_updated_at_data_type = column.dtype %}\n {% endif %}\n {% endfor %}\n {{ return(ns.dbt_updated_at_data_type or none) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.741133, "supported_languages": null}, "macro.dbt.check_time_data_types": {"name": "check_time_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/helpers.sql", "original_file_path": "macros/materializations/snapshots/helpers.sql", "unique_id": "macro.dbt.check_time_data_types", "macro_sql": "{% macro check_time_data_types(sql) %}\n {% set dbt_updated_at_data_type = get_updated_at_column_data_type(sql) %}\n {% set snapshot_get_time_data_type = get_snapshot_get_time_data_type() %}\n {% if snapshot_get_time_data_type is not none and dbt_updated_at_data_type is not none and snapshot_get_time_data_type != dbt_updated_at_data_type %}\n {% if exceptions.warn_snapshot_timestamp_data_types %}\n {{ exceptions.warn_snapshot_timestamp_data_types(snapshot_get_time_data_type, dbt_updated_at_data_type) }}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_updated_at_column_data_type", "macro.dbt.get_snapshot_get_time_data_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.741546, "supported_languages": null}, "macro.dbt.materialization_snapshot_default": {"name": "materialization_snapshot_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/snapshots/snapshot.sql", "original_file_path": "macros/materializations/snapshots/snapshot.sql", "unique_id": "macro.dbt.materialization_snapshot_default", "macro_sql": "{% materialization snapshot, default %}\n\n {%- set target_table = model.get('alias', model.get('name')) -%}\n\n {%- set strategy_name = config.get('strategy') -%}\n {%- set unique_key = config.get('unique_key') %}\n -- grab current tables grants config for comparision later on\n {%- set grant_config = config.get('grants') -%}\n\n {% set target_relation_exists, target_relation = get_or_create_relation(\n database=model.database,\n schema=model.schema,\n identifier=target_table,\n type='table') -%}\n\n {%- if not target_relation.is_table -%}\n {% do exceptions.relation_wrong_type(target_relation, 'table') %}\n {%- endif -%}\n\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set strategy_macro = strategy_dispatch(strategy_name) %}\n {# The model['config'] parameter below is no longer used, but passing anyway for compatibility #}\n {# It was a dictionary of config, instead of the config object from the context #}\n {% set strategy = strategy_macro(model, \"snapshotted_data\", \"source_data\", model['config'], target_relation_exists) %}\n\n {% if not target_relation_exists %}\n\n {% set build_sql = build_snapshot_table(strategy, model['compiled_code']) %}\n {% set build_or_select_sql = build_sql %}\n {% set final_sql = create_table_as(False, target_relation, build_sql) %}\n\n {% else %}\n\n {% set columns = config.get(\"snapshot_table_column_names\") or get_snapshot_table_column_names() %}\n\n {{ adapter.valid_snapshot_target(target_relation, columns) }}\n\n {% set build_or_select_sql = snapshot_staging_table(strategy, sql, target_relation) %}\n {% set staging_table = build_snapshot_staging_table(strategy, sql, target_relation) %}\n\n -- this may no-op if the database does not require column expansion\n {% do adapter.expand_target_column_types(from_relation=staging_table,\n to_relation=target_relation) %}\n\n {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% do create_columns(target_relation, missing_columns) %}\n\n {% set source_columns = adapter.get_columns_in_relation(staging_table)\n | rejectattr('name', 'equalto', 'dbt_change_type')\n | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE')\n | rejectattr('name', 'equalto', 'dbt_unique_key')\n | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY')\n | list %}\n\n {% set quoted_source_columns = [] %}\n {% for column in source_columns %}\n {% do quoted_source_columns.append(adapter.quote(column.name)) %}\n {% endfor %}\n\n {% set final_sql = snapshot_merge_sql(\n target = target_relation,\n source = staging_table,\n insert_cols = quoted_source_columns\n )\n %}\n\n {% endif %}\n\n\n {{ check_time_data_types(build_or_select_sql) }}\n\n {% call statement('main') %}\n {{ final_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if not target_relation_exists %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {% if staging_table is defined %}\n {% do post_snapshot(staging_table) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.get_or_create_relation", "macro.dbt.run_hooks", "macro.dbt.strategy_dispatch", "macro.dbt.build_snapshot_table", "macro.dbt.create_table_as", "macro.dbt.get_snapshot_table_column_names", "macro.dbt.snapshot_staging_table", "macro.dbt.build_snapshot_staging_table", "macro.dbt.create_columns", "macro.dbt.snapshot_merge_sql", "macro.dbt.check_time_data_types", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes", "macro.dbt.post_snapshot"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.746002, "supported_languages": ["sql"]}, "macro.dbt.materialization_test_default": {"name": "materialization_test_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/test.sql", "original_file_path": "macros/materializations/tests/test.sql", "unique_id": "macro.dbt.materialization_test_default", "macro_sql": "{%- materialization test, default -%}\n\n {% set relations = [] %}\n\n {% if should_store_failures() %}\n\n {% set identifier = model['alias'] %}\n {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% set store_failures_as = config.get('store_failures_as') %}\n -- if `--store-failures` is invoked via command line and `store_failures_as` is not set,\n -- config.get('store_failures_as', 'table') returns None, not 'table'\n {% if store_failures_as == none %}{% set store_failures_as = 'table' %}{% endif %}\n {% if store_failures_as not in ['table', 'view'] %}\n {{ exceptions.raise_compiler_error(\n \"'\" ~ store_failures_as ~ \"' is not a valid value for `store_failures_as`. \"\n \"Accepted values are: ['ephemeral', 'table', 'view']\"\n ) }}\n {% endif %}\n\n {% set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database, type=store_failures_as) -%} %}\n\n {% if old_relation %}\n {% do adapter.drop_relation(old_relation) %}\n {% endif %}\n\n {% call statement(auto_begin=True) %}\n {{ get_create_sql(target_relation, sql) }}\n {% endcall %}\n\n {% do relations.append(target_relation) %}\n\n {% set main_sql %}\n select *\n from {{ target_relation }}\n {% endset %}\n\n {{ adapter.commit() }}\n\n {% else %}\n\n {% set main_sql = sql %}\n\n {% endif %}\n\n {% set limit = config.get('limit') %}\n {% set fail_calc = config.get('fail_calc') %}\n {% set warn_if = config.get('warn_if') %}\n {% set error_if = config.get('error_if') %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}\n\n {%- endcall %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.should_store_failures", "macro.dbt.statement", "macro.dbt.get_create_sql", "macro.dbt.get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7482162, "supported_languages": ["sql"]}, "macro.dbt.get_test_sql": {"name": "get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_test_sql", "macro_sql": "{% macro get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n {{ adapter.dispatch('get_test_sql', 'dbt')(main_sql, fail_calc, warn_if, error_if, limit) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.749175, "supported_languages": null}, "macro.dbt.default__get_test_sql": {"name": "default__get_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_test_sql", "macro_sql": "{% macro default__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}\n select\n {{ fail_calc }} as failures,\n {{ fail_calc }} {{ warn_if }} as should_warn,\n {{ fail_calc }} {{ error_if }} as should_error\n from (\n {{ main_sql }}\n {{ \"limit \" ~ limit if limit != none }}\n ) dbt_internal_test\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7494242, "supported_languages": null}, "macro.dbt.get_unit_test_sql": {"name": "get_unit_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.get_unit_test_sql", "macro_sql": "{% macro get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n {{ adapter.dispatch('get_unit_test_sql', 'dbt')(main_sql, expected_fixture_sql, expected_column_names) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_unit_test_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.749598, "supported_languages": null}, "macro.dbt.default__get_unit_test_sql": {"name": "default__get_unit_test_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/helpers.sql", "original_file_path": "macros/materializations/tests/helpers.sql", "unique_id": "macro.dbt.default__get_unit_test_sql", "macro_sql": "{% macro default__get_unit_test_sql(main_sql, expected_fixture_sql, expected_column_names) -%}\n-- Build actual result given inputs\nwith dbt_internal_unit_test_actual as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%},{% endif %}{%- endfor -%}, {{ dbt.string_literal(\"actual\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ main_sql }}\n ) _dbt_internal_unit_test_actual\n),\n-- Build expected result\ndbt_internal_unit_test_expected as (\n select\n {% for expected_column_name in expected_column_names %}{{expected_column_name}}{% if not loop.last -%}, {% endif %}{%- endfor -%}, {{ dbt.string_literal(\"expected\") }} as {{ adapter.quote(\"actual_or_expected\") }}\n from (\n {{ expected_fixture_sql }}\n ) _dbt_internal_unit_test_expected\n)\n-- Union actual and expected results\nselect * from dbt_internal_unit_test_actual\nunion all\nselect * from dbt_internal_unit_test_expected\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7500942, "supported_languages": null}, "macro.dbt.get_where_subquery": {"name": "get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.get_where_subquery", "macro_sql": "{% macro get_where_subquery(relation) -%}\n {% do return(adapter.dispatch('get_where_subquery', 'dbt')(relation)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_where_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7504158, "supported_languages": null}, "macro.dbt.default__get_where_subquery": {"name": "default__get_where_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/where_subquery.sql", "original_file_path": "macros/materializations/tests/where_subquery.sql", "unique_id": "macro.dbt.default__get_where_subquery", "macro_sql": "{% macro default__get_where_subquery(relation) -%}\n {% set where = config.get('where', '') %}\n {% if where %}\n {%- set filtered -%}\n (select * from {{ relation }} where {{ where }}) dbt_subquery\n {%- endset -%}\n {% do return(filtered) %}\n {%- else -%}\n {% do return(relation) %}\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7507288, "supported_languages": null}, "macro.dbt.materialization_unit_default": {"name": "materialization_unit_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/tests/unit.sql", "original_file_path": "macros/materializations/tests/unit.sql", "unique_id": "macro.dbt.materialization_unit_default", "macro_sql": "{%- materialization unit, default -%}\n\n {% set relations = [] %}\n\n {% set expected_rows = config.get('expected_rows') %}\n {% set expected_sql = config.get('expected_sql') %}\n {% set tested_expected_column_names = expected_rows[0].keys() if (expected_rows | length ) > 0 else get_columns_in_query(sql) %} %}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {% do run_query(get_create_table_as_sql(True, temp_relation, get_empty_subquery_sql(sql))) %}\n {%- set columns_in_relation = adapter.get_columns_in_relation(temp_relation) -%}\n {%- set column_name_to_data_types = {} -%}\n {%- for column in columns_in_relation -%}\n {%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n {%- endfor -%}\n\n {% if not expected_sql %}\n {% set expected_sql = get_expected_sql(expected_rows, column_name_to_data_types) %}\n {% endif %}\n {% set unit_test_sql = get_unit_test_sql(sql, expected_sql, tested_expected_column_names) %}\n\n {% call statement('main', fetch_result=True) -%}\n\n {{ unit_test_sql }}\n\n {%- endcall %}\n\n {% do adapter.drop_relation(temp_relation) %}\n\n {{ return({'relations': relations}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.get_columns_in_query", "macro.dbt.make_temp_relation", "macro.dbt.run_query", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_empty_subquery_sql", "macro.dbt.get_expected_sql", "macro.dbt.get_unit_test_sql", "macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7521741, "supported_languages": ["sql"]}, "macro.dbt.materialization_materialized_view_default": {"name": "materialization_materialized_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialization_materialized_view_default", "macro_sql": "{% materialization materialized_view, default %}\n {% set existing_relation = load_cached_relation(this) %}\n {% set target_relation = this.incorporate(type=this.MaterializedView) %}\n {% set intermediate_relation = make_intermediate_relation(target_relation) %}\n {% set backup_relation_type = target_relation.MaterializedView if existing_relation is none else existing_relation.type %}\n {% set backup_relation = make_backup_relation(target_relation, backup_relation_type) %}\n\n {{ materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) }}\n\n {% set build_sql = materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% if build_sql == '' %}\n {{ materialized_view_execute_no_op(target_relation) }}\n {% else %}\n {{ materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) }}\n {% endif %}\n\n {{ materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.materialized_view_setup", "macro.dbt.materialized_view_get_build_sql", "macro.dbt.materialized_view_execute_no_op", "macro.dbt.materialized_view_execute_build_sql", "macro.dbt.materialized_view_teardown"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.756201, "supported_languages": ["sql"]}, "macro.dbt.materialized_view_setup": {"name": "materialized_view_setup", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_setup", "macro_sql": "{% macro materialized_view_setup(backup_relation, intermediate_relation, pre_hooks) %}\n\n -- backup_relation and intermediate_relation should not already exist in the database\n -- it's possible these exist because of a previous run that exited unexpectedly\n {% set preexisting_backup_relation = load_cached_relation(backup_relation) %}\n {% set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.756525, "supported_languages": null}, "macro.dbt.materialized_view_teardown": {"name": "materialized_view_teardown", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_teardown", "macro_sql": "{% macro materialized_view_teardown(backup_relation, intermediate_relation, post_hooks) %}\n\n -- drop the temp relations if they exist to leave the database clean for the next run\n {{ drop_relation_if_exists(backup_relation) }}\n {{ drop_relation_if_exists(intermediate_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.756736, "supported_languages": null}, "macro.dbt.materialized_view_get_build_sql": {"name": "materialized_view_get_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_get_build_sql", "macro_sql": "{% macro materialized_view_get_build_sql(existing_relation, target_relation, backup_relation, intermediate_relation) %}\n\n {% set full_refresh_mode = should_full_refresh() %}\n\n -- determine the scenario we're in: create, full_refresh, alter, refresh data\n {% if existing_relation is none %}\n {% set build_sql = get_create_materialized_view_as_sql(target_relation, sql) %}\n {% elif full_refresh_mode or not existing_relation.is_materialized_view %}\n {% set build_sql = get_replace_sql(existing_relation, target_relation, sql) %}\n {% else %}\n\n -- get config options\n {% set on_configuration_change = config.get('on_configuration_change') %}\n {% set configuration_changes = get_materialized_view_configuration_changes(existing_relation, config) %}\n\n {% if configuration_changes is none %}\n {% set build_sql = refresh_materialized_view(target_relation) %}\n\n {% elif on_configuration_change == 'apply' %}\n {% set build_sql = get_alter_materialized_view_as_sql(target_relation, configuration_changes, sql, existing_relation, backup_relation, intermediate_relation) %}\n {% elif on_configuration_change == 'continue' %}\n {% set build_sql = '' %}\n {{ exceptions.warn(\"Configuration changes were identified and `on_configuration_change` was set to `continue` for `\" ~ target_relation.render() ~ \"`\") }}\n {% elif on_configuration_change == 'fail' %}\n {{ exceptions.raise_fail_fast_error(\"Configuration changes were identified and `on_configuration_change` was set to `fail` for `\" ~ target_relation.render() ~ \"`\") }}\n\n {% else %}\n -- this only happens if the user provides a value other than `apply`, 'skip', 'fail'\n {{ exceptions.raise_compiler_error(\"Unexpected configuration scenario\") }}\n\n {% endif %}\n\n {% endif %}\n\n {% do return(build_sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.get_create_materialized_view_as_sql", "macro.dbt.get_replace_sql", "macro.dbt.get_materialized_view_configuration_changes", "macro.dbt.refresh_materialized_view", "macro.dbt.get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.757851, "supported_languages": null}, "macro.dbt.materialized_view_execute_no_op": {"name": "materialized_view_execute_no_op", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_no_op", "macro_sql": "{% macro materialized_view_execute_no_op(target_relation) %}\n {% do store_raw_result(\n name=\"main\",\n message=\"skip \" ~ target_relation,\n code=\"skip\",\n rows_affected=\"-1\"\n ) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.75805, "supported_languages": null}, "macro.dbt.materialized_view_execute_build_sql": {"name": "materialized_view_execute_build_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/materialized_view.sql", "original_file_path": "macros/materializations/models/materialized_view.sql", "unique_id": "macro.dbt.materialized_view_execute_build_sql", "macro_sql": "{% macro materialized_view_execute_build_sql(build_sql, existing_relation, target_relation, post_hooks) %}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set grant_config = config.get('grants') %}\n\n {% call statement(name=\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7585862, "supported_languages": null}, "macro.dbt.materialization_view_default": {"name": "materialization_view_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/view.sql", "original_file_path": "macros/materializations/models/view.sql", "unique_id": "macro.dbt.materialization_view_default", "macro_sql": "{%- materialization view, default -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='view') -%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n This relation (probably) doesn't exist yet. If it does exist, it's a leftover from\n a previous run, and we're going to try to drop it immediately. At the end of this\n materialization, we're going to rename the \"existing_relation\" to this identifier,\n and then we're going to drop it. In order to make sure we run the correct one of:\n - drop view ...\n - drop table ...\n\n We need to set the type of this relation to be the type of the existing_relation, if it exists,\n or else \"view\" as a sane default if it does not. Note that if the existing_relation does not\n exist, then there is nothing to move out of the way and subsequentally drop. In that case,\n this relation will be effectively unused.\n */\n {%- set backup_relation_type = 'view' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n -- move the existing view out of the way\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {{ adapter.commit() }}\n\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.run_hooks", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.760961, "supported_languages": ["sql"]}, "macro.dbt.materialization_table_default": {"name": "materialization_table_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/table.sql", "original_file_path": "macros/materializations/models/table.sql", "unique_id": "macro.dbt.materialization_table_default", "macro_sql": "{% materialization table, default %}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') %}\n {%- set intermediate_relation = make_intermediate_relation(target_relation) -%}\n -- the intermediate_relation should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%}\n /*\n See ../view/view.sql for more information about this relation.\n */\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n -- as above, the backup_relation should not already exist\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n\n -- drop the temp relations if they exist already in the database\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_table_as_sql(False, intermediate_relation, sql) }}\n {%- endcall %}\n\n -- cleanup\n {% if existing_relation is not none %}\n /* Do the equivalent of rename_if_exists. 'existing_relation' could have been dropped\n since the variable was first set. */\n {% set existing_relation = load_cached_relation(existing_relation) %}\n {% if existing_relation is not none %}\n {{ adapter.rename_relation(existing_relation, backup_relation) }}\n {% endif %}\n {% endif %}\n\n {{ adapter.rename_relation(intermediate_relation, target_relation) }}\n\n {% do create_indexes(target_relation) %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n -- finally, drop the existing/backup relation after the commit\n {{ drop_relation_if_exists(backup_relation) }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.statement", "macro.dbt.get_create_table_as_sql", "macro.dbt.create_indexes", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.763325, "supported_languages": ["sql"]}, "macro.dbt.get_quoted_csv": {"name": "get_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_quoted_csv", "macro_sql": "{% macro get_quoted_csv(column_names) %}\n\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote(col)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7646668, "supported_languages": null}, "macro.dbt.diff_columns": {"name": "diff_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_columns", "macro_sql": "{% macro diff_columns(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% set source_names = source_columns | map(attribute = 'column') | list %}\n {% set target_names = target_columns | map(attribute = 'column') | list %}\n\n {# --check whether the name attribute exists in the target - this does not perform a data type check #}\n {% for sc in source_columns %}\n {% if sc.name not in target_names %}\n {{ result.append(sc) }}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.765106, "supported_languages": null}, "macro.dbt.diff_column_data_types": {"name": "diff_column_data_types", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.diff_column_data_types", "macro_sql": "{% macro diff_column_data_types(source_columns, target_columns) %}\n\n {% set result = [] %}\n {% for sc in source_columns %}\n {% set tc = target_columns | selectattr(\"name\", \"equalto\", sc.name) | list | first %}\n {% if tc %}\n {% if sc.data_type != tc.data_type and not sc.can_expand_to(other_column=tc) %}\n {{ result.append( { 'column_name': tc.name, 'new_type': sc.data_type } ) }}\n {% endif %}\n {% endif %}\n {% endfor %}\n\n {{ return(result) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.76564, "supported_languages": null}, "macro.dbt.get_merge_update_columns": {"name": "get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.get_merge_update_columns", "macro_sql": "{% macro get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {{ return(adapter.dispatch('get_merge_update_columns', 'dbt')(merge_update_columns, merge_exclude_columns, dest_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.765835, "supported_languages": null}, "macro.dbt.default__get_merge_update_columns": {"name": "default__get_merge_update_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/column_helpers.sql", "original_file_path": "macros/materializations/models/incremental/column_helpers.sql", "unique_id": "macro.dbt.default__get_merge_update_columns", "macro_sql": "{% macro default__get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) %}\n {%- set default_cols = dest_columns | map(attribute=\"quoted\") | list -%}\n\n {%- if merge_update_columns and merge_exclude_columns -%}\n {{ exceptions.raise_compiler_error(\n 'Model cannot specify merge_update_columns and merge_exclude_columns. Please update model to use only one config'\n )}}\n {%- elif merge_update_columns -%}\n {%- set update_columns = merge_update_columns -%}\n {%- elif merge_exclude_columns -%}\n {%- set update_columns = [] -%}\n {%- for column in dest_columns -%}\n {% if column.column | lower not in merge_exclude_columns | map(\"lower\") | list %}\n {%- do update_columns.append(column.quoted) -%}\n {% endif %}\n {%- endfor -%}\n {%- else -%}\n {%- set update_columns = default_cols -%}\n {%- endif -%}\n\n {{ return(update_columns) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7664142, "supported_languages": null}, "macro.dbt.get_merge_sql": {"name": "get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_merge_sql", "macro_sql": "{% macro get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n -- back compat for old kwarg name\n {% set incremental_predicates = kwargs.get('predicates', incremental_predicates) %}\n {{ adapter.dispatch('get_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.771425, "supported_languages": null}, "macro.dbt.default__get_merge_sql": {"name": "default__get_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_merge_sql", "macro_sql": "{% macro default__get_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) -%}\n {%- set predicates = [] if incremental_predicates is none else [] + incremental_predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set merge_update_columns = config.get('merge_update_columns') -%}\n {%- set merge_exclude_columns = config.get('merge_exclude_columns') -%}\n {%- set update_columns = get_merge_update_columns(merge_update_columns, merge_exclude_columns, dest_columns) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}\n {% for key in unique_key %}\n {% set this_key_match %}\n DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}\n {% endset %}\n {% do predicates.append(this_key_match) %}\n {% endfor %}\n {% else %}\n {% set unique_key_match %}\n DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}\n {% endset %}\n {% do predicates.append(unique_key_match) %}\n {% endif %}\n {% else %}\n {% do predicates.append('FALSE') %}\n {% endif %}\n\n {{ sql_header if sql_header is not none }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on {{\"(\" ~ predicates | join(\") and (\") ~ \")\"}}\n\n {% if unique_key %}\n when matched then update set\n {% for column_name in update_columns -%}\n {{ column_name }} = DBT_INTERNAL_SOURCE.{{ column_name }}\n {%- if not loop.last %}, {%- endif %}\n {%- endfor %}\n {% endif %}\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv", "macro.dbt.get_merge_update_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.772757, "supported_languages": null}, "macro.dbt.get_delete_insert_merge_sql": {"name": "get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_delete_insert_merge_sql", "macro_sql": "{% macro get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n {{ adapter.dispatch('get_delete_insert_merge_sql', 'dbt')(target, source, unique_key, dest_columns, incremental_predicates) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.77298, "supported_languages": null}, "macro.dbt.default__get_delete_insert_merge_sql": {"name": "default__get_delete_insert_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_delete_insert_merge_sql", "macro_sql": "{% macro default__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates) -%}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n {% if unique_key %}\n {% if unique_key is sequence and unique_key is not string %}\n delete from {{target }}\n using {{ source }}\n where (\n {% for key in unique_key %}\n {{ source }}.{{ key }} = {{ target }}.{{ key }}\n {{ \"and \" if not loop.last}}\n {% endfor %}\n {% if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {% endif %}\n );\n {% else %}\n delete from {{ target }}\n where (\n {{ unique_key }}) in (\n select ({{ unique_key }})\n from {{ source }}\n )\n {%- if incremental_predicates %}\n {% for predicate in incremental_predicates %}\n and {{ predicate }}\n {% endfor %}\n {%- endif -%};\n\n {% endif %}\n {% endif %}\n\n insert into {{ target }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ source }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.773788, "supported_languages": null}, "macro.dbt.get_insert_overwrite_merge_sql": {"name": "get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.get_insert_overwrite_merge_sql", "macro_sql": "{% macro get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header=false) -%}\n {{ adapter.dispatch('get_insert_overwrite_merge_sql', 'dbt')(target, source, dest_columns, predicates, include_sql_header) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.774013, "supported_languages": null}, "macro.dbt.default__get_insert_overwrite_merge_sql": {"name": "default__get_insert_overwrite_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/merge.sql", "original_file_path": "macros/materializations/models/incremental/merge.sql", "unique_id": "macro.dbt.default__get_insert_overwrite_merge_sql", "macro_sql": "{% macro default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) -%}\n {#-- The only time include_sql_header is True: --#}\n {#-- BigQuery + insert_overwrite strategy + \"static\" partitions config --#}\n {#-- We should consider including the sql header at the materialization level instead --#}\n\n {%- set predicates = [] if predicates is none else [] + predicates -%}\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none and include_sql_header }}\n\n merge into {{ target }} as DBT_INTERNAL_DEST\n using {{ source }} as DBT_INTERNAL_SOURCE\n on FALSE\n\n when not matched by source\n {% if predicates %} and {{ predicates | join(' and ') }} {% endif %}\n then delete\n\n when not matched then insert\n ({{ dest_cols_csv }})\n values\n ({{ dest_cols_csv }})\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.774538, "supported_languages": null}, "macro.dbt.is_incremental": {"name": "is_incremental", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/is_incremental.sql", "original_file_path": "macros/materializations/models/incremental/is_incremental.sql", "unique_id": "macro.dbt.is_incremental", "macro_sql": "{% macro is_incremental() %}\n {#-- do not run introspective queries in parsing #}\n {% if not execute %}\n {{ return(False) }}\n {% else %}\n {% set relation = adapter.get_relation(this.database, this.schema, this.table) %}\n {{ return(relation is not none\n and relation.type == 'table'\n and model.config.materialized == 'incremental'\n and not should_full_refresh()) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.775049, "supported_languages": null}, "macro.dbt.get_incremental_append_sql": {"name": "get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_append_sql", "macro_sql": "{% macro get_incremental_append_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_append_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.775977, "supported_languages": null}, "macro.dbt.default__get_incremental_append_sql": {"name": "default__get_incremental_append_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_append_sql", "macro_sql": "{% macro default__get_incremental_append_sql(arg_dict) %}\n\n {% do return(get_insert_into_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_into_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.776169, "supported_languages": null}, "macro.dbt.get_incremental_delete_insert_sql": {"name": "get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_delete_insert_sql", "macro_sql": "{% macro get_incremental_delete_insert_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_delete_insert_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_delete_insert_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.776327, "supported_languages": null}, "macro.dbt.default__get_incremental_delete_insert_sql": {"name": "default__get_incremental_delete_insert_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_delete_insert_sql", "macro_sql": "{% macro default__get_incremental_delete_insert_sql(arg_dict) %}\n\n {% do return(get_delete_insert_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_delete_insert_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7765691, "supported_languages": null}, "macro.dbt.get_incremental_merge_sql": {"name": "get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_merge_sql", "macro_sql": "{% macro get_incremental_merge_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_merge_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.776725, "supported_languages": null}, "macro.dbt.default__get_incremental_merge_sql": {"name": "default__get_incremental_merge_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_merge_sql", "macro_sql": "{% macro default__get_incremental_merge_sql(arg_dict) %}\n\n {% do return(get_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"unique_key\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7769668, "supported_languages": null}, "macro.dbt.get_incremental_insert_overwrite_sql": {"name": "get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_insert_overwrite_sql", "macro_sql": "{% macro get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_insert_overwrite_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_insert_overwrite_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7771251, "supported_languages": null}, "macro.dbt.default__get_incremental_insert_overwrite_sql": {"name": "default__get_incremental_insert_overwrite_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_insert_overwrite_sql", "macro_sql": "{% macro default__get_incremental_insert_overwrite_sql(arg_dict) %}\n\n {% do return(get_insert_overwrite_merge_sql(arg_dict[\"target_relation\"], arg_dict[\"temp_relation\"], arg_dict[\"dest_columns\"], arg_dict[\"incremental_predicates\"])) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_insert_overwrite_merge_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777345, "supported_languages": null}, "macro.dbt.get_incremental_default_sql": {"name": "get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_default_sql", "macro_sql": "{% macro get_incremental_default_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_default_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_incremental_default_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777499, "supported_languages": null}, "macro.dbt.default__get_incremental_default_sql": {"name": "default__get_incremental_default_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_default_sql", "macro_sql": "{% macro default__get_incremental_default_sql(arg_dict) %}\n\n {% do return(get_incremental_append_sql(arg_dict)) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_incremental_append_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777623, "supported_languages": null}, "macro.dbt.get_incremental_microbatch_sql": {"name": "get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_incremental_microbatch_sql", "macro_sql": "{% macro get_incremental_microbatch_sql(arg_dict) %}\n\n {{ return(adapter.dispatch('get_incremental_microbatch_sql', 'dbt')(arg_dict)) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_incremental_microbatch_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.777774, "supported_languages": null}, "macro.dbt.default__get_incremental_microbatch_sql": {"name": "default__get_incremental_microbatch_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.default__get_incremental_microbatch_sql", "macro_sql": "{% macro default__get_incremental_microbatch_sql(arg_dict) %}\n\n {{ exceptions.raise_not_implemented('microbatch materialization strategy not implemented for adapter ' + adapter.type()) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7779121, "supported_languages": null}, "macro.dbt.get_insert_into_sql": {"name": "get_insert_into_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/strategies.sql", "original_file_path": "macros/materializations/models/incremental/strategies.sql", "unique_id": "macro.dbt.get_insert_into_sql", "macro_sql": "{% macro get_insert_into_sql(target_relation, temp_relation, dest_columns) %}\n\n {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute=\"name\")) -%}\n\n insert into {{ target_relation }} ({{ dest_cols_csv }})\n (\n select {{ dest_cols_csv }}\n from {{ temp_relation }}\n )\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_quoted_csv"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7781398, "supported_languages": null}, "macro.dbt.materialization_incremental_default": {"name": "materialization_incremental_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/incremental.sql", "original_file_path": "macros/materializations/models/incremental/incremental.sql", "unique_id": "macro.dbt.materialization_incremental_default", "macro_sql": "{% materialization incremental, default -%}\n\n -- relations\n {%- set existing_relation = load_cached_relation(this) -%}\n {%- set target_relation = this.incorporate(type='table') -%}\n {%- set temp_relation = make_temp_relation(target_relation)-%}\n {%- set intermediate_relation = make_intermediate_relation(target_relation)-%}\n {%- set backup_relation_type = 'table' if existing_relation is none else existing_relation.type -%}\n {%- set backup_relation = make_backup_relation(target_relation, backup_relation_type) -%}\n\n -- configs\n {%- set unique_key = config.get('unique_key') -%}\n {%- set full_refresh_mode = (should_full_refresh() or existing_relation.is_view) -%}\n {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%}\n\n -- the temp_ and backup_ relations should not already exist in the database; get_relation\n -- will return None in that case. Otherwise, we get a relation that we can drop\n -- later, before we try to use this name for the current operation. This has to happen before\n -- BEGIN, in a separate transaction\n {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation)-%}\n {%- set preexisting_backup_relation = load_cached_relation(backup_relation) -%}\n -- grab current tables grants config for comparision later on\n {% set grant_config = config.get('grants') %}\n {{ drop_relation_if_exists(preexisting_intermediate_relation) }}\n {{ drop_relation_if_exists(preexisting_backup_relation) }}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n {% set to_drop = [] %}\n\n {% if existing_relation is none %}\n {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %}\n {% elif full_refresh_mode %}\n {% set build_sql = get_create_table_as_sql(False, intermediate_relation, sql) %}\n {% set need_swap = true %}\n {% else %}\n {% do run_query(get_create_table_as_sql(True, temp_relation, sql)) %}\n {% set contract_config = config.get('contract') %}\n {% if not contract_config or not contract_config.enforced %}\n {% do adapter.expand_target_column_types(\n from_relation=temp_relation,\n to_relation=target_relation) %}\n {% endif %}\n {#-- Process schema changes. Returns dict of changes if successful. Use source columns for upserting/merging --#}\n {% set dest_columns = process_schema_changes(on_schema_change, temp_relation, existing_relation) %}\n {% if not dest_columns %}\n {% set dest_columns = adapter.get_columns_in_relation(existing_relation) %}\n {% endif %}\n\n {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#}\n {% set incremental_strategy = config.get('incremental_strategy') or 'default' %}\n {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %}\n {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %}\n {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %}\n {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %}\n\n {% endif %}\n\n {% call statement(\"main\") %}\n {{ build_sql }}\n {% endcall %}\n\n {% if need_swap %}\n {% do adapter.rename_relation(target_relation, backup_relation) %}\n {% do adapter.rename_relation(intermediate_relation, target_relation) %}\n {% do to_drop.append(backup_relation) %}\n {% endif %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if existing_relation is none or existing_relation.is_view or should_full_refresh() %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {% do adapter.commit() %}\n\n {% for rel in to_drop %}\n {% do adapter.drop_relation(rel) %}\n {% endfor %}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{%- endmaterialization %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.make_temp_relation", "macro.dbt.make_intermediate_relation", "macro.dbt.make_backup_relation", "macro.dbt.should_full_refresh", "macro.dbt.incremental_validate_on_schema_change", "macro.dbt.drop_relation_if_exists", "macro.dbt.run_hooks", "macro.dbt.get_create_table_as_sql", "macro.dbt.run_query", "macro.dbt.process_schema_changes", "macro.dbt.statement", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.782465, "supported_languages": ["sql"]}, "macro.dbt.incremental_validate_on_schema_change": {"name": "incremental_validate_on_schema_change", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.incremental_validate_on_schema_change", "macro_sql": "{% macro incremental_validate_on_schema_change(on_schema_change, default='ignore') %}\n\n {% if on_schema_change not in ['sync_all_columns', 'append_new_columns', 'fail', 'ignore'] %}\n\n {% set log_message = 'Invalid value for on_schema_change (%s) specified. Setting default value of %s.' % (on_schema_change, default) %}\n {% do log(log_message) %}\n\n {{ return(default) }}\n\n {% else %}\n\n {{ return(on_schema_change) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7869802, "supported_languages": null}, "macro.dbt.check_for_schema_changes": {"name": "check_for_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.check_for_schema_changes", "macro_sql": "{% macro check_for_schema_changes(source_relation, target_relation) %}\n\n {% set schema_changed = False %}\n\n {%- set source_columns = adapter.get_columns_in_relation(source_relation) -%}\n {%- set target_columns = adapter.get_columns_in_relation(target_relation) -%}\n {%- set source_not_in_target = diff_columns(source_columns, target_columns) -%}\n {%- set target_not_in_source = diff_columns(target_columns, source_columns) -%}\n\n {% set new_target_types = diff_column_data_types(source_columns, target_columns) %}\n\n {% if source_not_in_target != [] %}\n {% set schema_changed = True %}\n {% elif target_not_in_source != [] or new_target_types != [] %}\n {% set schema_changed = True %}\n {% elif new_target_types != [] %}\n {% set schema_changed = True %}\n {% endif %}\n\n {% set changes_dict = {\n 'schema_changed': schema_changed,\n 'source_not_in_target': source_not_in_target,\n 'target_not_in_source': target_not_in_source,\n 'source_columns': source_columns,\n 'target_columns': target_columns,\n 'new_target_types': new_target_types\n } %}\n\n {% set msg %}\n In {{ target_relation }}:\n Schema changed: {{ schema_changed }}\n Source columns not in target: {{ source_not_in_target }}\n Target columns not in source: {{ target_not_in_source }}\n New column types: {{ new_target_types }}\n {% endset %}\n\n {% do log(msg) %}\n\n {{ return(changes_dict) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.diff_columns", "macro.dbt.diff_column_data_types"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.787961, "supported_languages": null}, "macro.dbt.sync_column_schemas": {"name": "sync_column_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.sync_column_schemas", "macro_sql": "{% macro sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {%- set add_to_target_arr = schema_changes_dict['source_not_in_target'] -%}\n\n {%- if on_schema_change == 'append_new_columns'-%}\n {%- if add_to_target_arr | length > 0 -%}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, none) -%}\n {%- endif -%}\n\n {% elif on_schema_change == 'sync_all_columns' %}\n {%- set remove_from_target_arr = schema_changes_dict['target_not_in_source'] -%}\n {%- set new_target_types = schema_changes_dict['new_target_types'] -%}\n\n {% if add_to_target_arr | length > 0 or remove_from_target_arr | length > 0 %}\n {%- do alter_relation_add_remove_columns(target_relation, add_to_target_arr, remove_from_target_arr) -%}\n {% endif %}\n\n {% if new_target_types != [] %}\n {% for ntt in new_target_types %}\n {% set column_name = ntt['column_name'] %}\n {% set new_type = ntt['new_type'] %}\n {% do alter_column_type(target_relation, column_name, new_type) %}\n {% endfor %}\n {% endif %}\n\n {% endif %}\n\n {% set schema_change_message %}\n In {{ target_relation }}:\n Schema change approach: {{ on_schema_change }}\n Columns added: {{ add_to_target_arr }}\n Columns removed: {{ remove_from_target_arr }}\n Data types changed: {{ new_target_types }}\n {% endset %}\n\n {% do log(schema_change_message) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.alter_relation_add_remove_columns", "macro.dbt.alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.788921, "supported_languages": null}, "macro.dbt.process_schema_changes": {"name": "process_schema_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/incremental/on_schema_change.sql", "original_file_path": "macros/materializations/models/incremental/on_schema_change.sql", "unique_id": "macro.dbt.process_schema_changes", "macro_sql": "{% macro process_schema_changes(on_schema_change, source_relation, target_relation) %}\n\n {% if on_schema_change == 'ignore' %}\n\n {{ return({}) }}\n\n {% else %}\n\n {% set schema_changes_dict = check_for_schema_changes(source_relation, target_relation) %}\n\n {% if schema_changes_dict['schema_changed'] %}\n\n {% if on_schema_change == 'fail' %}\n\n {% set fail_msg %}\n The source and target schemas on this incremental model are out of sync!\n They can be reconciled in several ways:\n - set the `on_schema_change` config to either append_new_columns or sync_all_columns, depending on your situation.\n - Re-run the incremental model with `full_refresh: True` to update the target schema.\n - update the schema manually and re-run the process.\n\n Additional troubleshooting context:\n Source columns not in target: {{ schema_changes_dict['source_not_in_target'] }}\n Target columns not in source: {{ schema_changes_dict['target_not_in_source'] }}\n New column types: {{ schema_changes_dict['new_target_types'] }}\n {% endset %}\n\n {% do exceptions.raise_compiler_error(fail_msg) %}\n\n {# -- unless we ignore, run the sync operation per the config #}\n {% else %}\n\n {% do sync_column_schemas(on_schema_change, target_relation, schema_changes_dict) %}\n\n {% endif %}\n\n {% endif %}\n\n {{ return(schema_changes_dict['source_columns']) }}\n\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.check_for_schema_changes", "macro.dbt.sync_column_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.789614, "supported_languages": null}, "macro.dbt.can_clone_table": {"name": "can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.can_clone_table", "macro_sql": "{% macro can_clone_table() %}\n {{ return(adapter.dispatch('can_clone_table', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__can_clone_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.789815, "supported_languages": null}, "macro.dbt.default__can_clone_table": {"name": "default__can_clone_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/can_clone_table.sql", "original_file_path": "macros/materializations/models/clone/can_clone_table.sql", "unique_id": "macro.dbt.default__can_clone_table", "macro_sql": "{% macro default__can_clone_table() %}\n {{ return(False) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.7899132, "supported_languages": null}, "macro.dbt.create_or_replace_clone": {"name": "create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.create_or_replace_clone", "macro_sql": "{% macro create_or_replace_clone(this_relation, defer_relation) %}\n {{ return(adapter.dispatch('create_or_replace_clone', 'dbt')(this_relation, defer_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_or_replace_clone"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.790163, "supported_languages": null}, "macro.dbt.default__create_or_replace_clone": {"name": "default__create_or_replace_clone", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/create_or_replace_clone.sql", "original_file_path": "macros/materializations/models/clone/create_or_replace_clone.sql", "unique_id": "macro.dbt.default__create_or_replace_clone", "macro_sql": "{% macro default__create_or_replace_clone(this_relation, defer_relation) %}\n create or replace table {{ this_relation.render() }} clone {{ defer_relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.790299, "supported_languages": null}, "macro.dbt.materialization_clone_default": {"name": "materialization_clone_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/models/clone/clone.sql", "original_file_path": "macros/materializations/models/clone/clone.sql", "unique_id": "macro.dbt.materialization_clone_default", "macro_sql": "{%- materialization clone, default -%}\n\n {%- set relations = {'relations': []} -%}\n\n {%- if not defer_relation -%}\n -- nothing to do\n {{ log(\"No relation found in state manifest for \" ~ model.unique_id, info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set existing_relation = load_cached_relation(this) -%}\n\n {%- if existing_relation and not flags.FULL_REFRESH -%}\n -- noop!\n {{ log(\"Relation \" ~ existing_relation ~ \" already exists\", info=True) }}\n {{ return(relations) }}\n {%- endif -%}\n\n {%- set other_existing_relation = load_cached_relation(defer_relation) -%}\n\n -- If this is a database that can do zero-copy cloning of tables, and the other relation is a table, then this will be a table\n -- Otherwise, this will be a view\n\n {% set can_clone_table = can_clone_table() %}\n\n {%- if other_existing_relation and other_existing_relation.type == 'table' and can_clone_table -%}\n\n {%- set target_relation = this.incorporate(type='table') -%}\n {% if existing_relation is not none and not existing_relation.is_table %}\n {{ log(\"Dropping relation \" ~ existing_relation.render() ~ \" because it is of type \" ~ existing_relation.type) }}\n {{ drop_relation_if_exists(existing_relation) }}\n {% endif %}\n\n -- as a general rule, data platforms that can clone tables can also do atomic 'create or replace'\n {% call statement('main') %}\n {% if target_relation and defer_relation and target_relation == defer_relation %}\n {{ log(\"Target relation and defer relation are the same, skipping clone for relation: \" ~ target_relation.render()) }}\n {% else %}\n {{ create_or_replace_clone(target_relation, defer_relation) }}\n {% endif %}\n\n {% endcall %}\n\n {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n {% do persist_docs(target_relation, model) %}\n\n {{ return({'relations': [target_relation]}) }}\n\n {%- else -%}\n\n {%- set target_relation = this.incorporate(type='view') -%}\n\n -- reuse the view materialization\n -- TODO: support actual dispatch for materialization macros\n -- Tracking ticket: https://github.com/dbt-labs/dbt-core/issues/7799\n {% set search_name = \"materialization_view_\" ~ adapter.type() %}\n {% if not search_name in context %}\n {% set search_name = \"materialization_view_default\" %}\n {% endif %}\n {% set materialization_macro = context[search_name] %}\n {% set relations = materialization_macro() %}\n {{ return(relations) }}\n\n {%- endif -%}\n\n{%- endmaterialization -%}", "depends_on": {"macros": ["macro.dbt.load_cached_relation", "macro.dbt.can_clone_table", "macro.dbt.drop_relation_if_exists", "macro.dbt.statement", "macro.dbt.create_or_replace_clone", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.793341, "supported_languages": ["sql"]}, "macro.dbt.materialization_seed_default": {"name": "materialization_seed_default", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/seed.sql", "original_file_path": "macros/materializations/seeds/seed.sql", "unique_id": "macro.dbt.materialization_seed_default", "macro_sql": "{% materialization seed, default %}\n\n {%- set identifier = model['alias'] -%}\n {%- set full_refresh_mode = (should_full_refresh()) -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n\n {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set grant_config = config.get('grants') -%}\n {%- set agate_table = load_agate_table() -%}\n -- grab current tables grants config for comparison later on\n\n {%- do store_result('agate_table', response='OK', agate_table=agate_table) -%}\n\n {{ run_hooks(pre_hooks, inside_transaction=False) }}\n\n -- `BEGIN` happens here:\n {{ run_hooks(pre_hooks, inside_transaction=True) }}\n\n -- build model\n {% set create_table_sql = \"\" %}\n {% if exists_as_view %}\n {{ exceptions.raise_compiler_error(\"Cannot seed to '{}', it is a view\".format(old_relation.render())) }}\n {% elif exists_as_table %}\n {% set create_table_sql = reset_csv_table(model, full_refresh_mode, old_relation, agate_table) %}\n {% else %}\n {% set create_table_sql = create_csv_table(model, agate_table) %}\n {% endif %}\n\n {% set code = 'CREATE' if full_refresh_mode else 'INSERT' %}\n {% set rows_affected = (agate_table.rows | length) %}\n {% set sql = load_csv_rows(model, agate_table) %}\n\n {% call noop_statement('main', code ~ ' ' ~ rows_affected, code, rows_affected) %}\n {{ get_csv_sql(create_table_sql, sql) }};\n {% endcall %}\n\n {% set target_relation = this.incorporate(type='table') %}\n\n {% set should_revoke = should_revoke(old_relation, full_refresh_mode) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {% do persist_docs(target_relation, model) %}\n\n {% if full_refresh_mode or not exists_as_table %}\n {% do create_indexes(target_relation) %}\n {% endif %}\n\n {{ run_hooks(post_hooks, inside_transaction=True) }}\n\n -- `COMMIT` happens here\n {{ adapter.commit() }}\n\n {{ run_hooks(post_hooks, inside_transaction=False) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmaterialization %}", "depends_on": {"macros": ["macro.dbt.should_full_refresh", "macro.dbt.run_hooks", "macro.dbt.reset_csv_table", "macro.dbt.create_csv_table", "macro.dbt.load_csv_rows", "macro.dbt.noop_statement", "macro.dbt.get_csv_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants", "macro.dbt.persist_docs", "macro.dbt.create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.796114, "supported_languages": ["sql"]}, "macro.dbt.create_csv_table": {"name": "create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.create_csv_table", "macro_sql": "{% macro create_csv_table(model, agate_table) -%}\n {{ adapter.dispatch('create_csv_table', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80035, "supported_languages": null}, "macro.dbt.default__create_csv_table": {"name": "default__create_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__create_csv_table", "macro_sql": "{% macro default__create_csv_table(model, agate_table) %}\n {%- set column_override = model['config'].get('column_types', {}) -%}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n\n {% set sql %}\n create table {{ this.render() }} (\n {%- for col_name in agate_table.column_names -%}\n {%- set inferred_type = adapter.convert_type(agate_table, loop.index0) -%}\n {%- set type = column_override.get(col_name, inferred_type) -%}\n {%- set column_name = (col_name | string) -%}\n {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }} {%- if not loop.last -%}, {%- endif -%}\n {%- endfor -%}\n )\n {% endset %}\n\n {% call statement('_') -%}\n {{ sql }}\n {%- endcall %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80112, "supported_languages": null}, "macro.dbt.reset_csv_table": {"name": "reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.reset_csv_table", "macro_sql": "{% macro reset_csv_table(model, full_refresh, old_relation, agate_table) -%}\n {{ adapter.dispatch('reset_csv_table', 'dbt')(model, full_refresh, old_relation, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__reset_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8013299, "supported_languages": null}, "macro.dbt.default__reset_csv_table": {"name": "default__reset_csv_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__reset_csv_table", "macro_sql": "{% macro default__reset_csv_table(model, full_refresh, old_relation, agate_table) %}\n {% set sql = \"\" %}\n {% if full_refresh %}\n {{ adapter.drop_relation(old_relation) }}\n {% set sql = create_csv_table(model, agate_table) %}\n {% else %}\n {{ adapter.truncate_relation(old_relation) }}\n {% set sql = \"truncate table \" ~ old_relation.render() %}\n {% endif %}\n\n {{ return(sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_csv_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.801737, "supported_languages": null}, "macro.dbt.get_csv_sql": {"name": "get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_csv_sql", "macro_sql": "{% macro get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ adapter.dispatch('get_csv_sql', 'dbt')(create_or_truncate_sql, insert_sql) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_csv_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.801903, "supported_languages": null}, "macro.dbt.default__get_csv_sql": {"name": "default__get_csv_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_csv_sql", "macro_sql": "{% macro default__get_csv_sql(create_or_truncate_sql, insert_sql) %}\n {{ create_or_truncate_sql }};\n -- dbt seed --\n {{ insert_sql }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80202, "supported_languages": null}, "macro.dbt.get_binding_char": {"name": "get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_binding_char", "macro_sql": "{% macro get_binding_char() -%}\n {{ adapter.dispatch('get_binding_char', 'dbt')() }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8021429, "supported_languages": null}, "macro.dbt.default__get_binding_char": {"name": "default__get_binding_char", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_binding_char", "macro_sql": "{% macro default__get_binding_char() %}\n {{ return('%s') }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.802243, "supported_languages": null}, "macro.dbt.get_batch_size": {"name": "get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_batch_size", "macro_sql": "{% macro get_batch_size() -%}\n {{ return(adapter.dispatch('get_batch_size', 'dbt')()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_batch_size"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8023849, "supported_languages": null}, "macro.dbt.default__get_batch_size": {"name": "default__get_batch_size", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__get_batch_size", "macro_sql": "{% macro default__get_batch_size() %}\n {{ return(10000) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.802489, "supported_languages": null}, "macro.dbt.get_seed_column_quoted_csv": {"name": "get_seed_column_quoted_csv", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.get_seed_column_quoted_csv", "macro_sql": "{% macro get_seed_column_quoted_csv(model, column_names) %}\n {%- set quote_seed_column = model['config'].get('quote_columns', None) -%}\n {% set quoted = [] %}\n {% for col in column_names -%}\n {%- do quoted.append(adapter.quote_seed_column(col, quote_seed_column)) -%}\n {%- endfor %}\n\n {%- set dest_cols_csv = quoted | join(', ') -%}\n {{ return(dest_cols_csv) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8028958, "supported_languages": null}, "macro.dbt.load_csv_rows": {"name": "load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.load_csv_rows", "macro_sql": "{% macro load_csv_rows(model, agate_table) -%}\n {{ adapter.dispatch('load_csv_rows', 'dbt')(model, agate_table) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__load_csv_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.803055, "supported_languages": null}, "macro.dbt.default__load_csv_rows": {"name": "default__load_csv_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/materializations/seeds/helpers.sql", "original_file_path": "macros/materializations/seeds/helpers.sql", "unique_id": "macro.dbt.default__load_csv_rows", "macro_sql": "{% macro default__load_csv_rows(model, agate_table) %}\n\n {% set batch_size = get_batch_size() %}\n\n {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}\n {% set bindings = [] %}\n\n {% set statements = [] %}\n\n {% for chunk in agate_table.rows | batch(batch_size) %}\n {% set bindings = [] %}\n\n {% for row in chunk %}\n {% do bindings.extend(row) %}\n {% endfor %}\n\n {% set sql %}\n insert into {{ this.render() }} ({{ cols_sql }}) values\n {% for row in chunk -%}\n ({%- for column in agate_table.column_names -%}\n {{ get_binding_char() }}\n {%- if not loop.last%},{%- endif %}\n {%- endfor -%})\n {%- if not loop.last%},{%- endif %}\n {%- endfor %}\n {% endset %}\n\n {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}\n\n {% if loop.index0 == 0 %}\n {% do statements.append(sql) %}\n {% endif %}\n {% endfor %}\n\n {# Return SQL so we can render it out into the compiled files #}\n {{ return(statements[0]) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_batch_size", "macro.dbt.get_seed_column_quoted_csv", "macro.dbt.get_binding_char"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8041549, "supported_languages": null}, "macro.dbt.generate_alias_name": {"name": "generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.generate_alias_name", "macro_sql": "{% macro generate_alias_name(custom_alias_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_alias_name', 'dbt')(custom_alias_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_alias_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.804568, "supported_languages": null}, "macro.dbt.default__generate_alias_name": {"name": "default__generate_alias_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_alias.sql", "original_file_path": "macros/get_custom_name/get_custom_alias.sql", "unique_id": "macro.dbt.default__generate_alias_name", "macro_sql": "{% macro default__generate_alias_name(custom_alias_name=none, node=none) -%}\n\n {%- if custom_alias_name -%}\n\n {{ custom_alias_name | trim }}\n\n {%- elif node.version -%}\n\n {{ return(node.name ~ \"_v\" ~ (node.version | replace(\".\", \"_\"))) }}\n\n {%- else -%}\n\n {{ node.name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.804885, "supported_languages": null}, "macro.dbt.generate_schema_name": {"name": "generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name", "macro_sql": "{% macro generate_schema_name(custom_schema_name=none, node=none) -%}\n {{ return(adapter.dispatch('generate_schema_name', 'dbt')(custom_schema_name, node)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8053389, "supported_languages": null}, "macro.dbt.default__generate_schema_name": {"name": "default__generate_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.default__generate_schema_name", "macro_sql": "{% macro default__generate_schema_name(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if custom_schema_name is none -%}\n\n {{ default_schema }}\n\n {%- else -%}\n\n {{ default_schema }}_{{ custom_schema_name | trim }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8055522, "supported_languages": null}, "macro.dbt.generate_schema_name_for_env": {"name": "generate_schema_name_for_env", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_schema.sql", "original_file_path": "macros/get_custom_name/get_custom_schema.sql", "unique_id": "macro.dbt.generate_schema_name_for_env", "macro_sql": "{% macro generate_schema_name_for_env(custom_schema_name, node) -%}\n\n {%- set default_schema = target.schema -%}\n {%- if target.name == 'prod' and custom_schema_name is not none -%}\n\n {{ custom_schema_name | trim }}\n\n {%- else -%}\n\n {{ default_schema }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8057861, "supported_languages": null}, "macro.dbt.generate_database_name": {"name": "generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.generate_database_name", "macro_sql": "{% macro generate_database_name(custom_database_name=none, node=none) -%}\n {% do return(adapter.dispatch('generate_database_name', 'dbt')(custom_database_name, node)) %}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__generate_database_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.806116, "supported_languages": null}, "macro.dbt.default__generate_database_name": {"name": "default__generate_database_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/get_custom_name/get_custom_database.sql", "original_file_path": "macros/get_custom_name/get_custom_database.sql", "unique_id": "macro.dbt.default__generate_database_name", "macro_sql": "{% macro default__generate_database_name(custom_database_name=none, node=none) -%}\n {%- set default_database = target.database -%}\n {%- if custom_database_name is none -%}\n\n {{ default_database }}\n\n {%- else -%}\n\n {{ custom_database_name }}\n\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.806323, "supported_languages": null}, "macro.dbt.get_drop_sql": {"name": "get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.get_drop_sql", "macro_sql": "{%- macro get_drop_sql(relation) -%}\n {{- log('Applying DROP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.806918, "supported_languages": null}, "macro.dbt.default__get_drop_sql": {"name": "default__get_drop_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__get_drop_sql", "macro_sql": "{%- macro default__get_drop_sql(relation) -%}\n\n {%- if relation.is_view -%}\n {{ drop_view(relation) }}\n\n {%- elif relation.is_table -%}\n {{ drop_table(relation) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ drop_materialized_view(relation) }}\n\n {%- else -%}\n drop {{ relation.type }} if exists {{ relation.render() }} cascade\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.drop_view", "macro.dbt.drop_table", "macro.dbt.drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.807232, "supported_languages": null}, "macro.dbt.drop_relation": {"name": "drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation", "macro_sql": "{% macro drop_relation(relation) -%}\n {{ return(adapter.dispatch('drop_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8073862, "supported_languages": null}, "macro.dbt.default__drop_relation": {"name": "default__drop_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.default__drop_relation", "macro_sql": "{% macro default__drop_relation(relation) -%}\n {% call statement('drop_relation', auto_begin=False) -%}\n {{ get_drop_sql(relation) }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.807553, "supported_languages": null}, "macro.dbt.drop_relation_if_exists": {"name": "drop_relation_if_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop.sql", "original_file_path": "macros/relations/drop.sql", "unique_id": "macro.dbt.drop_relation_if_exists", "macro_sql": "{% macro drop_relation_if_exists(relation) %}\n {% if relation is not none %}\n {{ adapter.drop_relation(relation) }}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.807711, "supported_languages": null}, "macro.dbt.get_replace_sql": {"name": "get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.get_replace_sql", "macro_sql": "{% macro get_replace_sql(existing_relation, target_relation, sql) %}\n {{- log('Applying REPLACE to: ' ~ existing_relation) -}}\n {{- adapter.dispatch('get_replace_sql', 'dbt')(existing_relation, target_relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_replace_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.808418, "supported_languages": null}, "macro.dbt.default__get_replace_sql": {"name": "default__get_replace_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/replace.sql", "original_file_path": "macros/relations/replace.sql", "unique_id": "macro.dbt.default__get_replace_sql", "macro_sql": "{% macro default__get_replace_sql(existing_relation, target_relation, sql) %}\n\n {# /* use a create or replace statement if possible */ #}\n\n {% set is_replaceable = existing_relation.type == target_relation_type and existing_relation.can_be_replaced %}\n\n {% if is_replaceable and existing_relation.is_view %}\n {{ get_replace_view_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_table %}\n {{ get_replace_table_sql(target_relation, sql) }}\n\n {% elif is_replaceable and existing_relation.is_materialized_view %}\n {{ get_replace_materialized_view_sql(target_relation, sql) }}\n\n {# /* a create or replace statement is not possible, so try to stage and/or backup to be safe */ #}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one using a backup */ #}\n {%- elif target_relation.can_be_renamed and existing_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* create target_relation as an intermediate relation, then swap it out with the existing one without using a backup */ #}\n {%- elif target_relation.can_be_renamed -%}\n {{ get_create_intermediate_sql(target_relation, sql) }};\n {{ get_drop_sql(existing_relation) }};\n {{ get_rename_intermediate_sql(target_relation) }}\n\n {# /* create target_relation in place by first backing up the existing relation */ #}\n {%- elif existing_relation.can_be_renamed -%}\n {{ get_create_backup_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }};\n {{ get_drop_backup_sql(existing_relation) }}\n\n {# /* no renaming is allowed, so just drop and create */ #}\n {%- else -%}\n {{ get_drop_sql(existing_relation) }};\n {{ get_create_sql(target_relation, sql) }}\n\n {%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_replace_view_sql", "macro.dbt.get_replace_table_sql", "macro.dbt.get_replace_materialized_view_sql", "macro.dbt.get_create_intermediate_sql", "macro.dbt.get_create_backup_sql", "macro.dbt.get_rename_intermediate_sql", "macro.dbt.get_drop_backup_sql", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.80947, "supported_languages": null}, "macro.dbt.get_create_intermediate_sql": {"name": "get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.get_create_intermediate_sql", "macro_sql": "{%- macro get_create_intermediate_sql(relation, sql) -%}\n {{- log('Applying CREATE INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_intermediate_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_create_intermediate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.809802, "supported_languages": null}, "macro.dbt.default__get_create_intermediate_sql": {"name": "default__get_create_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_intermediate.sql", "original_file_path": "macros/relations/create_intermediate.sql", "unique_id": "macro.dbt.default__get_create_intermediate_sql", "macro_sql": "{%- macro default__get_create_intermediate_sql(relation, sql) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n -- drop any pre-existing intermediate\n {{ get_drop_sql(intermediate_relation) }};\n\n {{ get_create_sql(intermediate_relation, sql) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.810016, "supported_languages": null}, "macro.dbt.drop_schema_named": {"name": "drop_schema_named", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/schema.sql", "original_file_path": "macros/relations/schema.sql", "unique_id": "macro.dbt.drop_schema_named", "macro_sql": "{% macro drop_schema_named(schema_name) %}\n {{ return(adapter.dispatch('drop_schema_named', 'dbt') (schema_name)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__drop_schema_named"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.81025, "supported_languages": null}, "macro.dbt.default__drop_schema_named": {"name": "default__drop_schema_named", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/schema.sql", "original_file_path": "macros/relations/schema.sql", "unique_id": "macro.dbt.default__drop_schema_named", "macro_sql": "{% macro default__drop_schema_named(schema_name) %}\n {% set schema_relation = api.Relation.create(schema=schema_name) %}\n {{ adapter.drop_schema(schema_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.810434, "supported_languages": null}, "macro.dbt.get_drop_backup_sql": {"name": "get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.get_drop_backup_sql", "macro_sql": "{%- macro get_drop_backup_sql(relation) -%}\n {{- log('Applying DROP BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_drop_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_drop_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.81071, "supported_languages": null}, "macro.dbt.default__get_drop_backup_sql": {"name": "default__get_drop_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/drop_backup.sql", "original_file_path": "macros/relations/drop_backup.sql", "unique_id": "macro.dbt.default__get_drop_backup_sql", "macro_sql": "{%- macro default__get_drop_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n {{ get_drop_sql(backup_relation) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.810894, "supported_languages": null}, "macro.dbt.get_rename_sql": {"name": "get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.get_rename_sql", "macro_sql": "{%- macro get_rename_sql(relation, new_name) -%}\n {{- log('Applying RENAME to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_sql', 'dbt')(relation, new_name) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.811506, "supported_languages": null}, "macro.dbt.default__get_rename_sql": {"name": "default__get_rename_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__get_rename_sql", "macro_sql": "{%- macro default__get_rename_sql(relation, new_name) -%}\n\n {%- if relation.is_view -%}\n {{ get_rename_view_sql(relation, new_name) }}\n\n {%- elif relation.is_table -%}\n {{ get_rename_table_sql(relation, new_name) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_rename_materialized_view_sql(relation, new_name) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_rename_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.get_rename_view_sql", "macro.dbt.get_rename_table_sql", "macro.dbt.get_rename_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.811878, "supported_languages": null}, "macro.dbt.rename_relation": {"name": "rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.rename_relation", "macro_sql": "{% macro rename_relation(from_relation, to_relation) -%}\n {{ return(adapter.dispatch('rename_relation', 'dbt')(from_relation, to_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__rename_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812062, "supported_languages": null}, "macro.dbt.default__rename_relation": {"name": "default__rename_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename.sql", "original_file_path": "macros/relations/rename.sql", "unique_id": "macro.dbt.default__rename_relation", "macro_sql": "{% macro default__rename_relation(from_relation, to_relation) -%}\n {% set target_name = adapter.quote_as_configured(to_relation.identifier, 'identifier') %}\n {% call statement('rename_relation') -%}\n alter table {{ from_relation.render() }} rename to {{ target_name }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812324, "supported_languages": null}, "macro.dbt.get_create_backup_sql": {"name": "get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.get_create_backup_sql", "macro_sql": "{%- macro get_create_backup_sql(relation) -%}\n {{- log('Applying CREATE BACKUP to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_backup_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_create_backup_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812639, "supported_languages": null}, "macro.dbt.default__get_create_backup_sql": {"name": "default__get_create_backup_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create_backup.sql", "original_file_path": "macros/relations/create_backup.sql", "unique_id": "macro.dbt.default__get_create_backup_sql", "macro_sql": "{%- macro default__get_create_backup_sql(relation) -%}\n\n -- get the standard backup name\n {% set backup_relation = make_backup_relation(relation, relation.type) %}\n\n -- drop any pre-existing backup\n {{ get_drop_sql(backup_relation) }};\n\n {{ get_rename_sql(relation, backup_relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_backup_relation", "macro.dbt.get_drop_sql", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.812882, "supported_languages": null}, "macro.dbt.get_create_sql": {"name": "get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.get_create_sql", "macro_sql": "{%- macro get_create_sql(relation, sql) -%}\n {{- log('Applying CREATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_create_sql', 'dbt')(relation, sql) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8133051, "supported_languages": null}, "macro.dbt.default__get_create_sql": {"name": "default__get_create_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/create.sql", "original_file_path": "macros/relations/create.sql", "unique_id": "macro.dbt.default__get_create_sql", "macro_sql": "{%- macro default__get_create_sql(relation, sql) -%}\n\n {%- if relation.is_view -%}\n {{ get_create_view_as_sql(relation, sql) }}\n\n {%- elif relation.is_table -%}\n {{ get_create_table_as_sql(False, relation, sql) }}\n\n {%- elif relation.is_materialized_view -%}\n {{ get_create_materialized_view_as_sql(relation, sql) }}\n\n {%- else -%}\n {{- exceptions.raise_compiler_error(\"`get_create_sql` has not been implemented for: \" ~ relation.type ) -}}\n\n {%- endif -%}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.get_create_view_as_sql", "macro.dbt.get_create_table_as_sql", "macro.dbt.get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.813689, "supported_languages": null}, "macro.dbt.get_rename_intermediate_sql": {"name": "get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.get_rename_intermediate_sql", "macro_sql": "{%- macro get_rename_intermediate_sql(relation) -%}\n {{- log('Applying RENAME INTERMEDIATE to: ' ~ relation) -}}\n {{- adapter.dispatch('get_rename_intermediate_sql', 'dbt')(relation) -}}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": ["macro.dbt.default__get_rename_intermediate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.813972, "supported_languages": null}, "macro.dbt.default__get_rename_intermediate_sql": {"name": "default__get_rename_intermediate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/rename_intermediate.sql", "original_file_path": "macros/relations/rename_intermediate.sql", "unique_id": "macro.dbt.default__get_rename_intermediate_sql", "macro_sql": "{%- macro default__get_rename_intermediate_sql(relation) -%}\n\n -- get the standard intermediate name\n {% set intermediate_relation = make_intermediate_relation(relation) %}\n\n {{ get_rename_sql(intermediate_relation, relation.identifier) }}\n\n{%- endmacro -%}", "depends_on": {"macros": ["macro.dbt.make_intermediate_relation", "macro.dbt.get_rename_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8141482, "supported_languages": null}, "macro.dbt.drop_materialized_view": {"name": "drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.drop_materialized_view", "macro_sql": "{% macro drop_materialized_view(relation) -%}\n {{- adapter.dispatch('drop_materialized_view', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8143559, "supported_languages": null}, "macro.dbt.default__drop_materialized_view": {"name": "default__drop_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/drop.sql", "original_file_path": "macros/relations/materialized_view/drop.sql", "unique_id": "macro.dbt.default__drop_materialized_view", "macro_sql": "{% macro default__drop_materialized_view(relation) -%}\n drop materialized view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.814456, "supported_languages": null}, "macro.dbt.get_replace_materialized_view_sql": {"name": "get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.get_replace_materialized_view_sql", "macro_sql": "{% macro get_replace_materialized_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_materialized_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8146849, "supported_languages": null}, "macro.dbt.default__get_replace_materialized_view_sql": {"name": "default__get_replace_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/replace.sql", "original_file_path": "macros/relations/materialized_view/replace.sql", "unique_id": "macro.dbt.default__get_replace_materialized_view_sql", "macro_sql": "{% macro default__get_replace_materialized_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.814875, "supported_languages": null}, "macro.dbt.refresh_materialized_view": {"name": "refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.refresh_materialized_view", "macro_sql": "{% macro refresh_materialized_view(relation) %}\n {{- log('Applying REFRESH to: ' ~ relation) -}}\n {{- adapter.dispatch('refresh_materialized_view', 'dbt')(relation) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__refresh_materialized_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.815148, "supported_languages": null}, "macro.dbt.default__refresh_materialized_view": {"name": "default__refresh_materialized_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/refresh.sql", "original_file_path": "macros/relations/materialized_view/refresh.sql", "unique_id": "macro.dbt.default__refresh_materialized_view", "macro_sql": "{% macro default__refresh_materialized_view(relation) %}\n {{ exceptions.raise_compiler_error(\"`refresh_materialized_view` has not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.815263, "supported_languages": null}, "macro.dbt.get_rename_materialized_view_sql": {"name": "get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.get_rename_materialized_view_sql", "macro_sql": "{% macro get_rename_materialized_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_materialized_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_rename_materialized_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8154979, "supported_languages": null}, "macro.dbt.default__get_rename_materialized_view_sql": {"name": "default__get_rename_materialized_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/rename.sql", "original_file_path": "macros/relations/materialized_view/rename.sql", "unique_id": "macro.dbt.default__get_rename_materialized_view_sql", "macro_sql": "{% macro default__get_rename_materialized_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_materialized_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8156219, "supported_languages": null}, "macro.dbt.get_alter_materialized_view_as_sql": {"name": "get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_alter_materialized_view_as_sql", "macro_sql": "{% macro get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{- log('Applying ALTER to: ' ~ relation) -}}\n {{- adapter.dispatch('get_alter_materialized_view_as_sql', 'dbt')(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n ) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_alter_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8161569, "supported_languages": null}, "macro.dbt.default__get_alter_materialized_view_as_sql": {"name": "default__get_alter_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_alter_materialized_view_as_sql", "macro_sql": "{% macro default__get_alter_materialized_view_as_sql(\n relation,\n configuration_changes,\n sql,\n existing_relation,\n backup_relation,\n intermediate_relation\n) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.816313, "supported_languages": null}, "macro.dbt.get_materialized_view_configuration_changes": {"name": "get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.get_materialized_view_configuration_changes", "macro_sql": "{% macro get_materialized_view_configuration_changes(existing_relation, new_config) %}\n /* {#\n It's recommended that configuration changes be formatted as follows:\n {\"\": [{\"action\": \"\", \"context\": ...}]}\n\n For example:\n {\n \"indexes\": [\n {\"action\": \"drop\", \"context\": \"index_abc\"},\n {\"action\": \"create\", \"context\": {\"columns\": [\"column_1\", \"column_2\"], \"type\": \"hash\", \"unique\": True}},\n ],\n }\n\n Either way, `get_materialized_view_configuration_changes` needs to align with `get_alter_materialized_view_as_sql`.\n #} */\n {{- log('Determining configuration changes on: ' ~ existing_relation) -}}\n {%- do return(adapter.dispatch('get_materialized_view_configuration_changes', 'dbt')(existing_relation, new_config)) -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_materialized_view_configuration_changes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8165581, "supported_languages": null}, "macro.dbt.default__get_materialized_view_configuration_changes": {"name": "default__get_materialized_view_configuration_changes", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/alter.sql", "original_file_path": "macros/relations/materialized_view/alter.sql", "unique_id": "macro.dbt.default__get_materialized_view_configuration_changes", "macro_sql": "{% macro default__get_materialized_view_configuration_changes(existing_relation, new_config) %}\n {{ exceptions.raise_compiler_error(\"Materialized views have not been implemented for this adapter.\") }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.816685, "supported_languages": null}, "macro.dbt.get_create_materialized_view_as_sql": {"name": "get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.get_create_materialized_view_as_sql", "macro_sql": "{% macro get_create_materialized_view_as_sql(relation, sql) -%}\n {{- adapter.dispatch('get_create_materialized_view_as_sql', 'dbt')(relation, sql) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_create_materialized_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.816923, "supported_languages": null}, "macro.dbt.default__get_create_materialized_view_as_sql": {"name": "default__get_create_materialized_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/materialized_view/create.sql", "original_file_path": "macros/relations/materialized_view/create.sql", "unique_id": "macro.dbt.default__get_create_materialized_view_as_sql", "macro_sql": "{% macro default__get_create_materialized_view_as_sql(relation, sql) -%}\n {{ exceptions.raise_compiler_error(\n \"`get_create_materialized_view_as_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.817044, "supported_languages": null}, "macro.dbt.get_table_columns_and_constraints": {"name": "get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_table_columns_and_constraints", "macro_sql": "{%- macro get_table_columns_and_constraints() -%}\n {{ adapter.dispatch('get_table_columns_and_constraints', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.817939, "supported_languages": null}, "macro.dbt.default__get_table_columns_and_constraints": {"name": "default__get_table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_table_columns_and_constraints", "macro_sql": "{% macro default__get_table_columns_and_constraints() -%}\n {{ return(table_columns_and_constraints()) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.table_columns_and_constraints"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.818042, "supported_languages": null}, "macro.dbt.table_columns_and_constraints": {"name": "table_columns_and_constraints", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.table_columns_and_constraints", "macro_sql": "{% macro table_columns_and_constraints() %}\n {# loop through user_provided_columns to create DDL with data types and constraints #}\n {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%}\n {%- set raw_model_constraints = adapter.render_raw_model_constraints(raw_constraints=model['constraints']) -%}\n (\n {% for c in raw_column_constraints -%}\n {{ c }}{{ \",\" if not loop.last or raw_model_constraints }}\n {% endfor %}\n {% for c in raw_model_constraints -%}\n {{ c }}{{ \",\" if not loop.last }}\n {% endfor -%}\n )\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8184981, "supported_languages": null}, "macro.dbt.get_assert_columns_equivalent": {"name": "get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.get_assert_columns_equivalent", "macro_sql": "\n\n{%- macro get_assert_columns_equivalent(sql) -%}\n {{ adapter.dispatch('get_assert_columns_equivalent', 'dbt')(sql) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.818639, "supported_languages": null}, "macro.dbt.default__get_assert_columns_equivalent": {"name": "default__get_assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__get_assert_columns_equivalent", "macro_sql": "{% macro default__get_assert_columns_equivalent(sql) -%}\n {{ return(assert_columns_equivalent(sql)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.818753, "supported_languages": null}, "macro.dbt.assert_columns_equivalent": {"name": "assert_columns_equivalent", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.assert_columns_equivalent", "macro_sql": "{% macro assert_columns_equivalent(sql) %}\n\n {#-- First ensure the user has defined 'columns' in yaml specification --#}\n {%- set user_defined_columns = model['columns'] -%}\n {%- if not user_defined_columns -%}\n {{ exceptions.raise_contract_error([], []) }}\n {%- endif -%}\n\n {#-- Obtain the column schema provided by sql file. #}\n {%- set sql_file_provided_columns = get_column_schema_from_query(sql, config.get('sql_header', none)) -%}\n {#--Obtain the column schema provided by the schema file by generating an 'empty schema' query from the model's columns. #}\n {%- set schema_file_provided_columns = get_column_schema_from_query(get_empty_schema_sql(user_defined_columns)) -%}\n\n {#-- create dictionaries with name and formatted data type and strings for exception #}\n {%- set sql_columns = format_columns(sql_file_provided_columns) -%}\n {%- set yaml_columns = format_columns(schema_file_provided_columns) -%}\n\n {%- if sql_columns|length != yaml_columns|length -%}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n\n {%- for sql_col in sql_columns -%}\n {%- set yaml_col = [] -%}\n {%- for this_col in yaml_columns -%}\n {%- if this_col['name'] == sql_col['name'] -%}\n {%- do yaml_col.append(this_col) -%}\n {%- break -%}\n {%- endif -%}\n {%- endfor -%}\n {%- if not yaml_col -%}\n {#-- Column with name not found in yaml #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- if sql_col['formatted'] != yaml_col[0]['formatted'] -%}\n {#-- Column data types don't match #}\n {%- do exceptions.raise_contract_error(yaml_columns, sql_columns) -%}\n {%- endif -%}\n {%- endfor -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_column_schema_from_query", "macro.dbt.get_empty_schema_sql", "macro.dbt.format_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.819914, "supported_languages": null}, "macro.dbt.format_columns": {"name": "format_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.format_columns", "macro_sql": "{% macro format_columns(columns) %}\n {% set formatted_columns = [] %}\n {% for column in columns %}\n {%- set formatted_column = adapter.dispatch('format_column', 'dbt')(column) -%}\n {%- do formatted_columns.append(formatted_column) -%}\n {% endfor %}\n {{ return(formatted_columns) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__format_column"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.820245, "supported_languages": null}, "macro.dbt.default__format_column": {"name": "default__format_column", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/column/columns_spec_ddl.sql", "original_file_path": "macros/relations/column/columns_spec_ddl.sql", "unique_id": "macro.dbt.default__format_column", "macro_sql": "{% macro default__format_column(column) -%}\n {% set data_type = column.dtype %}\n {% set formatted = column.column.lower() ~ \" \" ~ data_type %}\n {{ return({'name': column.name, 'data_type': data_type, 'formatted': formatted}) }}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8205361, "supported_languages": null}, "macro.dbt.drop_table": {"name": "drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.drop_table", "macro_sql": "{% macro drop_table(relation) -%}\n {{- adapter.dispatch('drop_table', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.820756, "supported_languages": null}, "macro.dbt.default__drop_table": {"name": "default__drop_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/drop.sql", "original_file_path": "macros/relations/table/drop.sql", "unique_id": "macro.dbt.default__drop_table", "macro_sql": "{% macro default__drop_table(relation) -%}\n drop table if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82086, "supported_languages": null}, "macro.dbt.get_replace_table_sql": {"name": "get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.get_replace_table_sql", "macro_sql": "{% macro get_replace_table_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_table_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_table_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8210938, "supported_languages": null}, "macro.dbt.default__get_replace_table_sql": {"name": "default__get_replace_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/replace.sql", "original_file_path": "macros/relations/table/replace.sql", "unique_id": "macro.dbt.default__get_replace_table_sql", "macro_sql": "{% macro default__get_replace_table_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8212202, "supported_languages": null}, "macro.dbt.get_rename_table_sql": {"name": "get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.get_rename_table_sql", "macro_sql": "{% macro get_rename_table_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_table_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_rename_table_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82146, "supported_languages": null}, "macro.dbt.default__get_rename_table_sql": {"name": "default__get_rename_table_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/rename.sql", "original_file_path": "macros/relations/table/rename.sql", "unique_id": "macro.dbt.default__get_rename_table_sql", "macro_sql": "{% macro default__get_rename_table_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_table_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82159, "supported_languages": null}, "macro.dbt.get_create_table_as_sql": {"name": "get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_create_table_as_sql", "macro_sql": "{% macro get_create_table_as_sql(temporary, relation, sql) -%}\n {{ adapter.dispatch('get_create_table_as_sql', 'dbt')(temporary, relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_table_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.822352, "supported_languages": null}, "macro.dbt.default__get_create_table_as_sql": {"name": "default__get_create_table_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_create_table_as_sql", "macro_sql": "{% macro default__get_create_table_as_sql(temporary, relation, sql) -%}\n {{ return(create_table_as(temporary, relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8225112, "supported_languages": null}, "macro.dbt.create_table_as": {"name": "create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.create_table_as", "macro_sql": "{% macro create_table_as(temporary, relation, compiled_code, language='sql') -%}\n {# backward compatibility for create_table_as that does not support language #}\n {% if language == \"sql\" %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code)}}\n {% else %}\n {{ adapter.dispatch('create_table_as', 'dbt')(temporary, relation, compiled_code, language) }}\n {% endif %}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_table_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8228748, "supported_languages": null}, "macro.dbt.default__create_table_as": {"name": "default__create_table_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__create_table_as", "macro_sql": "{% macro default__create_table_as(temporary, relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n\n create {% if temporary: -%}temporary{%- endif %} table\n {{ relation.include(database=(not temporary), schema=(not temporary)) }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced and (not temporary) %}\n {{ get_assert_columns_equivalent(sql) }}\n {{ get_table_columns_and_constraints() }}\n {%- set sql = get_select_subquery(sql) %}\n {% endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent", "macro.dbt.get_table_columns_and_constraints", "macro.dbt.get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.823435, "supported_languages": null}, "macro.dbt.default__get_column_names": {"name": "default__get_column_names", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_column_names", "macro_sql": "{% macro default__get_column_names() %}\n {#- loop through user_provided_columns to get column names -#}\n {%- set user_provided_columns = model['columns'] -%}\n {%- for i in user_provided_columns %}\n {%- set col = user_provided_columns[i] -%}\n {%- set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] -%}\n {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8238149, "supported_languages": null}, "macro.dbt.get_select_subquery": {"name": "get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.get_select_subquery", "macro_sql": "{% macro get_select_subquery(sql) %}\n {{ return(adapter.dispatch('get_select_subquery', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_select_subquery"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.823968, "supported_languages": null}, "macro.dbt.default__get_select_subquery": {"name": "default__get_select_subquery", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/table/create.sql", "original_file_path": "macros/relations/table/create.sql", "unique_id": "macro.dbt.default__get_select_subquery", "macro_sql": "{% macro default__get_select_subquery(sql) %}\n select {{ adapter.dispatch('get_column_names', 'dbt')() }}\n from (\n {{ sql }}\n ) as model_subq\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_column_names"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.824113, "supported_languages": null}, "macro.dbt.drop_view": {"name": "drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.drop_view", "macro_sql": "{% macro drop_view(relation) -%}\n {{- adapter.dispatch('drop_view', 'dbt')(relation) -}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__drop_view"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.824321, "supported_languages": null}, "macro.dbt.default__drop_view": {"name": "default__drop_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/drop.sql", "original_file_path": "macros/relations/view/drop.sql", "unique_id": "macro.dbt.default__drop_view", "macro_sql": "{% macro default__drop_view(relation) -%}\n drop view if exists {{ relation.render() }} cascade\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.824422, "supported_languages": null}, "macro.dbt.get_replace_view_sql": {"name": "get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.get_replace_view_sql", "macro_sql": "{% macro get_replace_view_sql(relation, sql) %}\n {{- adapter.dispatch('get_replace_view_sql', 'dbt')(relation, sql) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_replace_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.825161, "supported_languages": null}, "macro.dbt.default__get_replace_view_sql": {"name": "default__get_replace_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__get_replace_view_sql", "macro_sql": "{% macro default__get_replace_view_sql(relation, sql) %}\n {{ exceptions.raise_compiler_error(\n \"`get_replace_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82529, "supported_languages": null}, "macro.dbt.create_or_replace_view": {"name": "create_or_replace_view", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.create_or_replace_view", "macro_sql": "{% macro create_or_replace_view() %}\n {%- set identifier = model['alias'] -%}\n\n {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%}\n {%- set exists_as_view = (old_relation is not none and old_relation.is_view) -%}\n\n {%- set target_relation = api.Relation.create(\n identifier=identifier, schema=schema, database=database,\n type='view') -%}\n {% set grant_config = config.get('grants') %}\n\n {{ run_hooks(pre_hooks) }}\n\n -- If there's a table with the same name and we weren't told to full refresh,\n -- that's an error. If we were told to full refresh, drop it. This behavior differs\n -- for Snowflake and BigQuery, so multiple dispatch is used.\n {%- if old_relation is not none and old_relation.is_table -%}\n {{ handle_existing_table(should_full_refresh(), old_relation) }}\n {%- endif -%}\n\n -- build model\n {% call statement('main') -%}\n {{ get_create_view_as_sql(target_relation, sql) }}\n {%- endcall %}\n\n {% set should_revoke = should_revoke(exists_as_view, full_refresh_mode=True) %}\n {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %}\n\n {{ run_hooks(post_hooks) }}\n\n {{ return({'relations': [target_relation]}) }}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_hooks", "macro.dbt.handle_existing_table", "macro.dbt.should_full_refresh", "macro.dbt.statement", "macro.dbt.get_create_view_as_sql", "macro.dbt.should_revoke", "macro.dbt.apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.82631, "supported_languages": null}, "macro.dbt.handle_existing_table": {"name": "handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.handle_existing_table", "macro_sql": "{% macro handle_existing_table(full_refresh, old_relation) %}\n {{ adapter.dispatch('handle_existing_table', 'dbt')(full_refresh, old_relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__handle_existing_table"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8264818, "supported_languages": null}, "macro.dbt.default__handle_existing_table": {"name": "default__handle_existing_table", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/replace.sql", "original_file_path": "macros/relations/view/replace.sql", "unique_id": "macro.dbt.default__handle_existing_table", "macro_sql": "{% macro default__handle_existing_table(full_refresh, old_relation) %}\n {{ log(\"Dropping relation \" ~ old_relation.render() ~ \" because it is of type \" ~ old_relation.type) }}\n {{ adapter.drop_relation(old_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8266811, "supported_languages": null}, "macro.dbt.get_rename_view_sql": {"name": "get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.get_rename_view_sql", "macro_sql": "{% macro get_rename_view_sql(relation, new_name) %}\n {{- adapter.dispatch('get_rename_view_sql', 'dbt')(relation, new_name) -}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_rename_view_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.826918, "supported_languages": null}, "macro.dbt.default__get_rename_view_sql": {"name": "default__get_rename_view_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/rename.sql", "original_file_path": "macros/relations/view/rename.sql", "unique_id": "macro.dbt.default__get_rename_view_sql", "macro_sql": "{% macro default__get_rename_view_sql(relation, new_name) %}\n {{ exceptions.raise_compiler_error(\n \"`get_rename_view_sql` has not been implemented for this adapter.\"\n ) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8270478, "supported_languages": null}, "macro.dbt.get_create_view_as_sql": {"name": "get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.get_create_view_as_sql", "macro_sql": "{% macro get_create_view_as_sql(relation, sql) -%}\n {{ adapter.dispatch('get_create_view_as_sql', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_view_as_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8274088, "supported_languages": null}, "macro.dbt.default__get_create_view_as_sql": {"name": "default__get_create_view_as_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__get_create_view_as_sql", "macro_sql": "{% macro default__get_create_view_as_sql(relation, sql) -%}\n {{ return(create_view_as(relation, sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8275461, "supported_languages": null}, "macro.dbt.create_view_as": {"name": "create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.create_view_as", "macro_sql": "{% macro create_view_as(relation, sql) -%}\n {{ adapter.dispatch('create_view_as', 'dbt')(relation, sql) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__create_view_as"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.827704, "supported_languages": null}, "macro.dbt.default__create_view_as": {"name": "default__create_view_as", "resource_type": "macro", "package_name": "dbt", "path": "macros/relations/view/create.sql", "original_file_path": "macros/relations/view/create.sql", "unique_id": "macro.dbt.default__create_view_as", "macro_sql": "{% macro default__create_view_as(relation, sql) -%}\n {%- set sql_header = config.get('sql_header', none) -%}\n\n {{ sql_header if sql_header is not none }}\n create view {{ relation.render() }}\n {% set contract_config = config.get('contract') %}\n {% if contract_config.enforced %}\n {{ get_assert_columns_equivalent(sql) }}\n {%- endif %}\n as (\n {{ sql }}\n );\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.get_assert_columns_equivalent"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828066, "supported_languages": null}, "macro.dbt.default__test_relationships": {"name": "default__test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/relationships.sql", "original_file_path": "macros/generic_test_sql/relationships.sql", "unique_id": "macro.dbt.default__test_relationships", "macro_sql": "{% macro default__test_relationships(model, column_name, to, field) %}\n\nwith child as (\n select {{ column_name }} as from_field\n from {{ model }}\n where {{ column_name }} is not null\n),\n\nparent as (\n select {{ field }} as to_field\n from {{ to }}\n)\n\nselect\n from_field\n\nfrom child\nleft join parent\n on child.from_field = parent.to_field\n\nwhere parent.to_field is null\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828371, "supported_languages": null}, "macro.dbt.default__test_not_null": {"name": "default__test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/not_null.sql", "original_file_path": "macros/generic_test_sql/not_null.sql", "unique_id": "macro.dbt.default__test_not_null", "macro_sql": "{% macro default__test_not_null(model, column_name) %}\n\n{% set column_list = '*' if should_store_failures() else column_name %}\n\nselect {{ column_list }}\nfrom {{ model }}\nwhere {{ column_name }} is null\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.should_store_failures"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828622, "supported_languages": null}, "macro.dbt.default__test_unique": {"name": "default__test_unique", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/unique.sql", "original_file_path": "macros/generic_test_sql/unique.sql", "unique_id": "macro.dbt.default__test_unique", "macro_sql": "{% macro default__test_unique(model, column_name) %}\n\nselect\n {{ column_name }} as unique_field,\n count(*) as n_records\n\nfrom {{ model }}\nwhere {{ column_name }} is not null\ngroup by {{ column_name }}\nhaving count(*) > 1\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.828834, "supported_languages": null}, "macro.dbt.default__test_accepted_values": {"name": "default__test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "macros/generic_test_sql/accepted_values.sql", "original_file_path": "macros/generic_test_sql/accepted_values.sql", "unique_id": "macro.dbt.default__test_accepted_values", "macro_sql": "{% macro default__test_accepted_values(model, column_name, values, quote=True) %}\n\nwith all_values as (\n\n select\n {{ column_name }} as value_field,\n count(*) as n_records\n\n from {{ model }}\n group by {{ column_name }}\n\n)\n\nselect *\nfrom all_values\nwhere value_field not in (\n {% for value in values -%}\n {% if quote -%}\n '{{ value }}'\n {%- else -%}\n {{ value }}\n {%- endif -%}\n {%- if not loop.last -%},{%- endif %}\n {%- endfor %}\n)\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.829317, "supported_languages": null}, "macro.dbt.statement": {"name": "statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.statement", "macro_sql": "\n{%- macro statement(name=None, fetch_result=False, auto_begin=True, language='sql') -%}\n {%- if execute: -%}\n {%- set compiled_code = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime {} for node \"{}\"'.format(language, model['unique_id'])) }}\n {{ write(compiled_code) }}\n {%- endif -%}\n {%- if language == 'sql'-%}\n {%- set res, table = adapter.execute(compiled_code, auto_begin=auto_begin, fetch=fetch_result) -%}\n {%- elif language == 'python' -%}\n {%- set res = submit_python_job(model, compiled_code) -%}\n {#-- TODO: What should table be for python models? --#}\n {%- set table = None -%}\n {%- else -%}\n {% do exceptions.raise_compiler_error(\"statement macro didn't get supported language\") %}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_result(name, response=res, agate_table=table) }}\n {%- endif -%}\n\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.83059, "supported_languages": null}, "macro.dbt.noop_statement": {"name": "noop_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.noop_statement", "macro_sql": "{% macro noop_statement(name=None, message=None, code=None, rows_affected=None, res=None) -%}\n {%- set sql = caller() -%}\n\n {%- if name == 'main' -%}\n {{ log('Writing runtime SQL for node \"{}\"'.format(model['unique_id'])) }}\n {{ write(sql) }}\n {%- endif -%}\n\n {%- if name is not none -%}\n {{ store_raw_result(name, message=message, code=code, rows_affected=rows_affected, agate_table=res) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8313391, "supported_languages": null}, "macro.dbt.run_query": {"name": "run_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/statement.sql", "original_file_path": "macros/etc/statement.sql", "unique_id": "macro.dbt.run_query", "macro_sql": "{% macro run_query(sql) %}\n {% call statement(\"run_query_statement\", fetch_result=true, auto_begin=false) %}\n {{ sql }}\n {% endcall %}\n\n {% do return(load_result(\"run_query_statement\").table) %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8315961, "supported_languages": null}, "macro.dbt.convert_datetime": {"name": "convert_datetime", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.convert_datetime", "macro_sql": "{% macro convert_datetime(date_str, date_fmt) %}\n\n {% set error_msg -%}\n The provided partition date '{{ date_str }}' does not match the expected format '{{ date_fmt }}'\n {%- endset %}\n\n {% set res = try_or_compiler_error(error_msg, modules.datetime.datetime.strptime, date_str.strip(), date_fmt) %}\n {{ return(res) }}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.833154, "supported_languages": null}, "macro.dbt.dates_in_range": {"name": "dates_in_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.dates_in_range", "macro_sql": "{% macro dates_in_range(start_date_str, end_date_str=none, in_fmt=\"%Y%m%d\", out_fmt=\"%Y%m%d\") %}\n {% set end_date_str = start_date_str if end_date_str is none else end_date_str %}\n\n {% set start_date = convert_datetime(start_date_str, in_fmt) %}\n {% set end_date = convert_datetime(end_date_str, in_fmt) %}\n\n {% set day_count = (end_date - start_date).days %}\n {% if day_count < 0 %}\n {% set msg -%}\n Partition start date is after the end date ({{ start_date }}, {{ end_date }})\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg, model) }}\n {% endif %}\n\n {% set date_list = [] %}\n {% for i in range(0, day_count + 1) %}\n {% set the_date = (modules.datetime.timedelta(days=i) + start_date) %}\n {% if not out_fmt %}\n {% set _ = date_list.append(the_date) %}\n {% else %}\n {% set _ = date_list.append(the_date.strftime(out_fmt)) %}\n {% endif %}\n {% endfor %}\n\n {{ return(date_list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.convert_datetime"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.834102, "supported_languages": null}, "macro.dbt.partition_range": {"name": "partition_range", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.partition_range", "macro_sql": "{% macro partition_range(raw_partition_date, date_fmt='%Y%m%d') %}\n {% set partition_range = (raw_partition_date | string).split(\",\") %}\n\n {% if (partition_range | length) == 1 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = none %}\n {% elif (partition_range | length) == 2 %}\n {% set start_date = partition_range[0] %}\n {% set end_date = partition_range[1] %}\n {% else %}\n {{ exceptions.raise_compiler_error(\"Invalid partition time. Expected format: {Start Date}[,{End Date}]. Got: \" ~ raw_partition_date) }}\n {% endif %}\n\n {{ return(dates_in_range(start_date, end_date, in_fmt=date_fmt)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.dates_in_range"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8347142, "supported_languages": null}, "macro.dbt.py_current_timestring": {"name": "py_current_timestring", "resource_type": "macro", "package_name": "dbt", "path": "macros/etc/datetime.sql", "original_file_path": "macros/etc/datetime.sql", "unique_id": "macro.dbt.py_current_timestring", "macro_sql": "{% macro py_current_timestring() %}\n {% set dt = modules.datetime.datetime.now() %}\n {% do return(dt.strftime(\"%Y%m%d%H%M%S%f\")) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8349109, "supported_languages": null}, "macro.dbt.except": {"name": "except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.except", "macro_sql": "{% macro except() %}\n {{ return(adapter.dispatch('except', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__except"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.835105, "supported_languages": null}, "macro.dbt.default__except": {"name": "default__except", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/except.sql", "original_file_path": "macros/utils/except.sql", "unique_id": "macro.dbt.default__except", "macro_sql": "{% macro default__except() %}\n\n except\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.835169, "supported_languages": null}, "macro.dbt.get_intervals_between": {"name": "get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.get_intervals_between", "macro_sql": "{% macro get_intervals_between(start_date, end_date, datepart) -%}\n {{ return(adapter.dispatch('get_intervals_between', 'dbt')(start_date, end_date, datepart)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_intervals_between"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.835808, "supported_languages": null}, "macro.dbt.default__get_intervals_between": {"name": "default__get_intervals_between", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__get_intervals_between", "macro_sql": "{% macro default__get_intervals_between(start_date, end_date, datepart) -%}\n {%- call statement('get_intervals_between', fetch_result=True) %}\n\n select {{ dbt.datediff(start_date, end_date, datepart) }}\n\n {%- endcall -%}\n\n {%- set value_list = load_result('get_intervals_between') -%}\n\n {%- if value_list and value_list['data'] -%}\n {%- set values = value_list['data'] | map(attribute=0) | list %}\n {{ return(values[0]) }}\n {%- else -%}\n {{ return(1) }}\n {%- endif -%}\n\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8363059, "supported_languages": null}, "macro.dbt.date_spine": {"name": "date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.date_spine", "macro_sql": "{% macro date_spine(datepart, start_date, end_date) %}\n {{ return(adapter.dispatch('date_spine', 'dbt')(datepart, start_date, end_date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_spine"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.836503, "supported_languages": null}, "macro.dbt.default__date_spine": {"name": "default__date_spine", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_spine.sql", "original_file_path": "macros/utils/date_spine.sql", "unique_id": "macro.dbt.default__date_spine", "macro_sql": "{% macro default__date_spine(datepart, start_date, end_date) %}\n\n\n {# call as follows:\n\n date_spine(\n \"day\",\n \"to_date('01/01/2016', 'mm/dd/yyyy')\",\n \"dbt.dateadd(week, 1, current_date)\"\n ) #}\n\n\n with rawdata as (\n\n {{dbt.generate_series(\n dbt.get_intervals_between(start_date, end_date, datepart)\n )}}\n\n ),\n\n all_periods as (\n\n select (\n {{\n dbt.dateadd(\n datepart,\n \"row_number() over (order by 1) - 1\",\n start_date\n )\n }}\n ) as date_{{datepart}}\n from rawdata\n\n ),\n\n filtered as (\n\n select *\n from all_periods\n where date_{{datepart}} <= {{ end_date }}\n\n )\n\n select * from filtered\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.generate_series", "macro.dbt.get_intervals_between", "macro.dbt.dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.836822, "supported_languages": null}, "macro.dbt.date": {"name": "date", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt.date", "macro_sql": "{% macro date(year, month, day) %}\n {{ return(adapter.dispatch('date', 'dbt') (year, month, day)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837112, "supported_languages": null}, "macro.dbt.default__date": {"name": "default__date", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date.sql", "original_file_path": "macros/utils/date.sql", "unique_id": "macro.dbt.default__date", "macro_sql": "{% macro default__date(year, month, day) -%}\n {%- set dt = modules.datetime.date(year, month, day) -%}\n {%- set iso_8601_formatted_date = dt.strftime('%Y-%m-%d') -%}\n to_date('{{ iso_8601_formatted_date }}', 'YYYY-MM-DD')\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837353, "supported_languages": null}, "macro.dbt.replace": {"name": "replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.replace", "macro_sql": "{% macro replace(field, old_chars, new_chars) -%}\n {{ return(adapter.dispatch('replace', 'dbt') (field, old_chars, new_chars)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__replace"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837631, "supported_languages": null}, "macro.dbt.default__replace": {"name": "default__replace", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/replace.sql", "original_file_path": "macros/utils/replace.sql", "unique_id": "macro.dbt.default__replace", "macro_sql": "{% macro default__replace(field, old_chars, new_chars) %}\n\n replace(\n {{ field }},\n {{ old_chars }},\n {{ new_chars }}\n )\n\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837771, "supported_languages": null}, "macro.dbt.concat": {"name": "concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.concat", "macro_sql": "{% macro concat(fields) -%}\n {{ return(adapter.dispatch('concat', 'dbt')(fields)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.837983, "supported_languages": null}, "macro.dbt.default__concat": {"name": "default__concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/concat.sql", "original_file_path": "macros/utils/concat.sql", "unique_id": "macro.dbt.default__concat", "macro_sql": "{% macro default__concat(fields) -%}\n {{ fields|join(' || ') }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.838089, "supported_languages": null}, "macro.dbt.get_powers_of_two": {"name": "get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.get_powers_of_two", "macro_sql": "{% macro get_powers_of_two(upper_bound) %}\n {{ return(adapter.dispatch('get_powers_of_two', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.838809, "supported_languages": null}, "macro.dbt.default__get_powers_of_two": {"name": "default__get_powers_of_two", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__get_powers_of_two", "macro_sql": "{% macro default__get_powers_of_two(upper_bound) %}\n\n {% if upper_bound <= 0 %}\n {{ exceptions.raise_compiler_error(\"upper bound must be positive\") }}\n {% endif %}\n\n {% for _ in range(1, 100) %}\n {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %}\n {% endfor %}\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.839253, "supported_languages": null}, "macro.dbt.generate_series": {"name": "generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.generate_series", "macro_sql": "{% macro generate_series(upper_bound) %}\n {{ return(adapter.dispatch('generate_series', 'dbt')(upper_bound)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__generate_series"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.839426, "supported_languages": null}, "macro.dbt.default__generate_series": {"name": "default__generate_series", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/generate_series.sql", "original_file_path": "macros/utils/generate_series.sql", "unique_id": "macro.dbt.default__generate_series", "macro_sql": "{% macro default__generate_series(upper_bound) %}\n\n {% set n = dbt.get_powers_of_two(upper_bound) %}\n\n with p as (\n select 0 as generated_number union all select 1\n ), unioned as (\n\n select\n\n {% for i in range(n) %}\n p{{i}}.generated_number * power(2, {{i}})\n {% if not loop.last %} + {% endif %}\n {% endfor %}\n + 1\n as generated_number\n\n from\n\n {% for i in range(n) %}\n p as p{{i}}\n {% if not loop.last %} cross join {% endif %}\n {% endfor %}\n\n )\n\n select *\n from unioned\n where generated_number <= {{upper_bound}}\n order by generated_number\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_powers_of_two"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.839862, "supported_languages": null}, "macro.dbt.length": {"name": "length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.length", "macro_sql": "{% macro length(expression) -%}\n {{ return(adapter.dispatch('length', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__length"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840083, "supported_languages": null}, "macro.dbt.default__length": {"name": "default__length", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/length.sql", "original_file_path": "macros/utils/length.sql", "unique_id": "macro.dbt.default__length", "macro_sql": "{% macro default__length(expression) %}\n\n length(\n {{ expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8401742, "supported_languages": null}, "macro.dbt.dateadd": {"name": "dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.dateadd", "macro_sql": "{% macro dateadd(datepart, interval, from_date_or_timestamp) %}\n {{ return(adapter.dispatch('dateadd', 'dbt')(datepart, interval, from_date_or_timestamp)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__dateadd"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8404648, "supported_languages": null}, "macro.dbt.default__dateadd": {"name": "default__dateadd", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/dateadd.sql", "original_file_path": "macros/utils/dateadd.sql", "unique_id": "macro.dbt.default__dateadd", "macro_sql": "{% macro default__dateadd(datepart, interval, from_date_or_timestamp) %}\n\n dateadd(\n {{ datepart }},\n {{ interval }},\n {{ from_date_or_timestamp }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840607, "supported_languages": null}, "macro.dbt.intersect": {"name": "intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.intersect", "macro_sql": "{% macro intersect() %}\n {{ return(adapter.dispatch('intersect', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__intersect"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840808, "supported_languages": null}, "macro.dbt.default__intersect": {"name": "default__intersect", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/intersect.sql", "original_file_path": "macros/utils/intersect.sql", "unique_id": "macro.dbt.default__intersect", "macro_sql": "{% macro default__intersect() %}\n\n intersect\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.840875, "supported_languages": null}, "macro.dbt.escape_single_quotes": {"name": "escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.escape_single_quotes", "macro_sql": "{% macro escape_single_quotes(expression) %}\n {{ return(adapter.dispatch('escape_single_quotes', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__escape_single_quotes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.841106, "supported_languages": null}, "macro.dbt.default__escape_single_quotes": {"name": "default__escape_single_quotes", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/escape_single_quotes.sql", "original_file_path": "macros/utils/escape_single_quotes.sql", "unique_id": "macro.dbt.default__escape_single_quotes", "macro_sql": "{% macro default__escape_single_quotes(expression) -%}\n{{ expression | replace(\"'\",\"''\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.841234, "supported_languages": null}, "macro.dbt.right": {"name": "right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.right", "macro_sql": "{% macro right(string_text, length_expression) -%}\n {{ return(adapter.dispatch('right', 'dbt') (string_text, length_expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__right"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.84149, "supported_languages": null}, "macro.dbt.default__right": {"name": "default__right", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/right.sql", "original_file_path": "macros/utils/right.sql", "unique_id": "macro.dbt.default__right", "macro_sql": "{% macro default__right(string_text, length_expression) %}\n\n right(\n {{ string_text }},\n {{ length_expression }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8416152, "supported_languages": null}, "macro.dbt.listagg": {"name": "listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.listagg", "macro_sql": "{% macro listagg(measure, delimiter_text=\"','\", order_by_clause=none, limit_num=none) -%}\n {{ return(adapter.dispatch('listagg', 'dbt') (measure, delimiter_text, order_by_clause, limit_num)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__listagg"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842151, "supported_languages": null}, "macro.dbt.default__listagg": {"name": "default__listagg", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/listagg.sql", "original_file_path": "macros/utils/listagg.sql", "unique_id": "macro.dbt.default__listagg", "macro_sql": "{% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%}\n\n {% if limit_num -%}\n array_to_string(\n array_slice(\n array_agg(\n {{ measure }}\n ){% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n ,0\n ,{{ limit_num }}\n ),\n {{ delimiter_text }}\n )\n {%- else %}\n listagg(\n {{ measure }},\n {{ delimiter_text }}\n )\n {% if order_by_clause -%}\n within group ({{ order_by_clause }})\n {%- endif %}\n {%- endif %}\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842484, "supported_languages": null}, "macro.dbt.datediff": {"name": "datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.datediff", "macro_sql": "{% macro datediff(first_date, second_date, datepart) %}\n {{ return(adapter.dispatch('datediff', 'dbt')(first_date, second_date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__datediff"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842758, "supported_languages": null}, "macro.dbt.default__datediff": {"name": "default__datediff", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/datediff.sql", "original_file_path": "macros/utils/datediff.sql", "unique_id": "macro.dbt.default__datediff", "macro_sql": "{% macro default__datediff(first_date, second_date, datepart) -%}\n\n datediff(\n {{ datepart }},\n {{ first_date }},\n {{ second_date }}\n )\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.842896, "supported_languages": null}, "macro.dbt.safe_cast": {"name": "safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.safe_cast", "macro_sql": "{% macro safe_cast(field, type) %}\n {{ return(adapter.dispatch('safe_cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843133, "supported_languages": null}, "macro.dbt.default__safe_cast": {"name": "default__safe_cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/safe_cast.sql", "original_file_path": "macros/utils/safe_cast.sql", "unique_id": "macro.dbt.default__safe_cast", "macro_sql": "{% macro default__safe_cast(field, type) %}\n {# most databases don't support this function yet\n so we just need to use cast #}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843246, "supported_languages": null}, "macro.dbt.hash": {"name": "hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.hash", "macro_sql": "{% macro hash(field) -%}\n {{ return(adapter.dispatch('hash', 'dbt') (field)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__hash"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843461, "supported_languages": null}, "macro.dbt.default__hash": {"name": "default__hash", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/hash.sql", "original_file_path": "macros/utils/hash.sql", "unique_id": "macro.dbt.default__hash", "macro_sql": "{% macro default__hash(field) -%}\n md5(cast({{ field }} as {{ api.Column.translate_type('string') }}))\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.843594, "supported_languages": null}, "macro.dbt.cast_bool_to_text": {"name": "cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.cast_bool_to_text", "macro_sql": "{% macro cast_bool_to_text(field) %}\n {{ adapter.dispatch('cast_bool_to_text', 'dbt') (field) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast_bool_to_text"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8438652, "supported_languages": null}, "macro.dbt.default__cast_bool_to_text": {"name": "default__cast_bool_to_text", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast_bool_to_text.sql", "original_file_path": "macros/utils/cast_bool_to_text.sql", "unique_id": "macro.dbt.default__cast_bool_to_text", "macro_sql": "{% macro default__cast_bool_to_text(field) %}\n cast({{ field }} as {{ api.Column.translate_type('string') }})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8439958, "supported_languages": null}, "macro.dbt.cast": {"name": "cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast.sql", "original_file_path": "macros/utils/cast.sql", "unique_id": "macro.dbt.cast", "macro_sql": "{% macro cast(field, type) %}\n {{ return(adapter.dispatch('cast', 'dbt') (field, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844224, "supported_languages": null}, "macro.dbt.default__cast": {"name": "default__cast", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/cast.sql", "original_file_path": "macros/utils/cast.sql", "unique_id": "macro.dbt.default__cast", "macro_sql": "{% macro default__cast(field, type) %}\n cast({{field}} as {{type}})\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844336, "supported_languages": null}, "macro.dbt.any_value": {"name": "any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.any_value", "macro_sql": "{% macro any_value(expression) -%}\n {{ return(adapter.dispatch('any_value', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__any_value"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8445451, "supported_languages": null}, "macro.dbt.default__any_value": {"name": "default__any_value", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/any_value.sql", "original_file_path": "macros/utils/any_value.sql", "unique_id": "macro.dbt.default__any_value", "macro_sql": "{% macro default__any_value(expression) -%}\n\n any_value({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844633, "supported_languages": null}, "macro.dbt.position": {"name": "position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.position", "macro_sql": "{% macro position(substring_text, string_text) -%}\n {{ return(adapter.dispatch('position', 'dbt') (substring_text, string_text)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__position"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8448782, "supported_languages": null}, "macro.dbt.default__position": {"name": "default__position", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/position.sql", "original_file_path": "macros/utils/position.sql", "unique_id": "macro.dbt.default__position", "macro_sql": "{% macro default__position(substring_text, string_text) %}\n\n position(\n {{ substring_text }} in {{ string_text }}\n )\n\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.844993, "supported_languages": null}, "macro.dbt.string_literal": {"name": "string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.string_literal", "macro_sql": "{%- macro string_literal(value) -%}\n {{ return(adapter.dispatch('string_literal', 'dbt') (value)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__string_literal"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.845202, "supported_languages": null}, "macro.dbt.default__string_literal": {"name": "default__string_literal", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/literal.sql", "original_file_path": "macros/utils/literal.sql", "unique_id": "macro.dbt.default__string_literal", "macro_sql": "{% macro default__string_literal(value) -%}\n '{{ value }}'\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.845294, "supported_languages": null}, "macro.dbt.type_string": {"name": "type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_string", "macro_sql": "\n\n{%- macro type_string() -%}\n {{ return(adapter.dispatch('type_string', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_string"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846126, "supported_languages": null}, "macro.dbt.default__type_string": {"name": "default__type_string", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_string", "macro_sql": "{% macro default__type_string() %}\n {{ return(api.Column.translate_type(\"string\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846256, "supported_languages": null}, "macro.dbt.type_timestamp": {"name": "type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_timestamp", "macro_sql": "\n\n{%- macro type_timestamp() -%}\n {{ return(adapter.dispatch('type_timestamp', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846394, "supported_languages": null}, "macro.dbt.default__type_timestamp": {"name": "default__type_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_timestamp", "macro_sql": "{% macro default__type_timestamp() %}\n {{ return(api.Column.translate_type(\"timestamp\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8465211, "supported_languages": null}, "macro.dbt.type_float": {"name": "type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_float", "macro_sql": "\n\n{%- macro type_float() -%}\n {{ return(adapter.dispatch('type_float', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_float"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846658, "supported_languages": null}, "macro.dbt.default__type_float": {"name": "default__type_float", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_float", "macro_sql": "{% macro default__type_float() %}\n {{ return(api.Column.translate_type(\"float\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8467891, "supported_languages": null}, "macro.dbt.type_numeric": {"name": "type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_numeric", "macro_sql": "\n\n{%- macro type_numeric() -%}\n {{ return(adapter.dispatch('type_numeric', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_numeric"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.846924, "supported_languages": null}, "macro.dbt.default__type_numeric": {"name": "default__type_numeric", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_numeric", "macro_sql": "{% macro default__type_numeric() %}\n {{ return(api.Column.numeric_type(\"numeric\", 28, 6)) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8470762, "supported_languages": null}, "macro.dbt.type_bigint": {"name": "type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_bigint", "macro_sql": "\n\n{%- macro type_bigint() -%}\n {{ return(adapter.dispatch('type_bigint', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_bigint"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.847214, "supported_languages": null}, "macro.dbt.default__type_bigint": {"name": "default__type_bigint", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_bigint", "macro_sql": "{% macro default__type_bigint() %}\n {{ return(api.Column.translate_type(\"bigint\")) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.847407, "supported_languages": null}, "macro.dbt.type_int": {"name": "type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_int", "macro_sql": "\n\n{%- macro type_int() -%}\n {{ return(adapter.dispatch('type_int', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_int"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8475451, "supported_languages": null}, "macro.dbt.default__type_int": {"name": "default__type_int", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_int", "macro_sql": "{%- macro default__type_int() -%}\n {{ return(api.Column.translate_type(\"integer\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8476732, "supported_languages": null}, "macro.dbt.type_boolean": {"name": "type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.type_boolean", "macro_sql": "\n\n{%- macro type_boolean() -%}\n {{ return(adapter.dispatch('type_boolean', 'dbt')()) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__type_boolean"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8478148, "supported_languages": null}, "macro.dbt.default__type_boolean": {"name": "default__type_boolean", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/data_types.sql", "original_file_path": "macros/utils/data_types.sql", "unique_id": "macro.dbt.default__type_boolean", "macro_sql": "{%- macro default__type_boolean() -%}\n {{ return(api.Column.translate_type(\"boolean\")) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8479412, "supported_languages": null}, "macro.dbt.array_concat": {"name": "array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.array_concat", "macro_sql": "{% macro array_concat(array_1, array_2) -%}\n {{ return(adapter.dispatch('array_concat', 'dbt')(array_1, array_2)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_concat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.848181, "supported_languages": null}, "macro.dbt.default__array_concat": {"name": "default__array_concat", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_concat.sql", "original_file_path": "macros/utils/array_concat.sql", "unique_id": "macro.dbt.default__array_concat", "macro_sql": "{% macro default__array_concat(array_1, array_2) -%}\n array_cat({{ array_1 }}, {{ array_2 }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.848293, "supported_languages": null}, "macro.dbt.bool_or": {"name": "bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.bool_or", "macro_sql": "{% macro bool_or(expression) -%}\n {{ return(adapter.dispatch('bool_or', 'dbt') (expression)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__bool_or"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8485062, "supported_languages": null}, "macro.dbt.default__bool_or": {"name": "default__bool_or", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/bool_or.sql", "original_file_path": "macros/utils/bool_or.sql", "unique_id": "macro.dbt.default__bool_or", "macro_sql": "{% macro default__bool_or(expression) -%}\n\n bool_or({{ expression }})\n\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8485959, "supported_languages": null}, "macro.dbt.last_day": {"name": "last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.last_day", "macro_sql": "{% macro last_day(date, datepart) %}\n {{ return(adapter.dispatch('last_day', 'dbt') (date, datepart)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.848897, "supported_languages": null}, "macro.dbt.default_last_day": {"name": "default_last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default_last_day", "macro_sql": "\n\n{%- macro default_last_day(date, datepart) -%}\n cast(\n {{dbt.dateadd('day', '-1',\n dbt.dateadd(datepart, '1', dbt.date_trunc(datepart, date))\n )}}\n as date)\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.dateadd", "macro.dbt.date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.84913, "supported_languages": null}, "macro.dbt.default__last_day": {"name": "default__last_day", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/last_day.sql", "original_file_path": "macros/utils/last_day.sql", "unique_id": "macro.dbt.default__last_day", "macro_sql": "{% macro default__last_day(date, datepart) -%}\n {{dbt.default_last_day(date, datepart)}}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default_last_day"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.849261, "supported_languages": null}, "macro.dbt.split_part": {"name": "split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.split_part", "macro_sql": "{% macro split_part(string_text, delimiter_text, part_number) %}\n {{ return(adapter.dispatch('split_part', 'dbt') (string_text, delimiter_text, part_number)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__split_part"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8497112, "supported_languages": null}, "macro.dbt.default__split_part": {"name": "default__split_part", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt.default__split_part", "macro_sql": "{% macro default__split_part(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8498502, "supported_languages": null}, "macro.dbt._split_part_negative": {"name": "_split_part_negative", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/split_part.sql", "original_file_path": "macros/utils/split_part.sql", "unique_id": "macro.dbt._split_part_negative", "macro_sql": "{% macro _split_part_negative(string_text, delimiter_text, part_number) %}\n\n split_part(\n {{ string_text }},\n {{ delimiter_text }},\n length({{ string_text }})\n - length(\n replace({{ string_text }}, {{ delimiter_text }}, '')\n ) + 2 + {{ part_number }}\n )\n\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8500328, "supported_languages": null}, "macro.dbt.date_trunc": {"name": "date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.date_trunc", "macro_sql": "{% macro date_trunc(datepart, date) -%}\n {{ return(adapter.dispatch('date_trunc', 'dbt') (datepart, date)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__date_trunc"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.850267, "supported_languages": null}, "macro.dbt.default__date_trunc": {"name": "default__date_trunc", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/date_trunc.sql", "original_file_path": "macros/utils/date_trunc.sql", "unique_id": "macro.dbt.default__date_trunc", "macro_sql": "{% macro default__date_trunc(datepart, date) -%}\n date_trunc('{{datepart}}', {{date}})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.85038, "supported_languages": null}, "macro.dbt.array_construct": {"name": "array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.array_construct", "macro_sql": "{% macro array_construct(inputs=[], data_type=api.Column.translate_type('integer')) -%}\n {{ return(adapter.dispatch('array_construct', 'dbt')(inputs, data_type)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_construct"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.850707, "supported_languages": null}, "macro.dbt.default__array_construct": {"name": "default__array_construct", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_construct.sql", "original_file_path": "macros/utils/array_construct.sql", "unique_id": "macro.dbt.default__array_construct", "macro_sql": "{% macro default__array_construct(inputs, data_type) -%}\n {% if inputs|length > 0 %}\n array[ {{ inputs|join(' , ') }} ]\n {% else %}\n array[]::{{data_type}}[]\n {% endif %}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.850914, "supported_languages": null}, "macro.dbt.array_append": {"name": "array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.array_append", "macro_sql": "{% macro array_append(array, new_element) -%}\n {{ return(adapter.dispatch('array_append', 'dbt')(array, new_element)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__array_append"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8511622, "supported_languages": null}, "macro.dbt.default__array_append": {"name": "default__array_append", "resource_type": "macro", "package_name": "dbt", "path": "macros/utils/array_append.sql", "original_file_path": "macros/utils/array_append.sql", "unique_id": "macro.dbt.default__array_append", "macro_sql": "{% macro default__array_append(array, new_element) -%}\n array_append({{ array }}, {{ new_element }})\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851361, "supported_languages": null}, "macro.dbt.create_schema": {"name": "create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.create_schema", "macro_sql": "{% macro create_schema(relation) -%}\n {{ adapter.dispatch('create_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__create_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851688, "supported_languages": null}, "macro.dbt.default__create_schema": {"name": "default__create_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__create_schema", "macro_sql": "{% macro default__create_schema(relation) -%}\n {%- call statement('create_schema') -%}\n create schema if not exists {{ relation.without_identifier() }}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851847, "supported_languages": null}, "macro.dbt.drop_schema": {"name": "drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.drop_schema", "macro_sql": "{% macro drop_schema(relation) -%}\n {{ adapter.dispatch('drop_schema', 'dbt')(relation) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__drop_schema"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.851985, "supported_languages": null}, "macro.dbt.default__drop_schema": {"name": "default__drop_schema", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/schema.sql", "original_file_path": "macros/adapters/schema.sql", "unique_id": "macro.dbt.default__drop_schema", "macro_sql": "{% macro default__drop_schema(relation) -%}\n {%- call statement('drop_schema') -%}\n drop schema if exists {{ relation.without_identifier() }} cascade\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852136, "supported_languages": null}, "macro.dbt.current_timestamp": {"name": "current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp", "macro_sql": "{%- macro current_timestamp() -%}\n {{ adapter.dispatch('current_timestamp', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt_spark.spark__current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852708, "supported_languages": null}, "macro.dbt.default__current_timestamp": {"name": "default__current_timestamp", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp", "macro_sql": "{% macro default__current_timestamp() -%}\n {{ exceptions.raise_not_implemented(\n 'current_timestamp macro not implemented for adapter ' + adapter.type()) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852832, "supported_languages": null}, "macro.dbt.snapshot_get_time": {"name": "snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.snapshot_get_time", "macro_sql": "\n\n{%- macro snapshot_get_time() -%}\n {{ adapter.dispatch('snapshot_get_time', 'dbt')() }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__snapshot_get_time"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.852951, "supported_languages": null}, "macro.dbt.default__snapshot_get_time": {"name": "default__snapshot_get_time", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__snapshot_get_time", "macro_sql": "{% macro default__snapshot_get_time() %}\n {{ current_timestamp() }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853037, "supported_languages": null}, "macro.dbt.get_snapshot_get_time_data_type": {"name": "get_snapshot_get_time_data_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.get_snapshot_get_time_data_type", "macro_sql": "{% macro get_snapshot_get_time_data_type() %}\n {% set snapshot_time = adapter.dispatch('snapshot_get_time', 'dbt')() %}\n {% set time_data_type_sql = 'select ' ~ snapshot_time ~ ' as dbt_snapshot_time' %}\n {% set snapshot_time_column_schema = get_column_schema_from_query(time_data_type_sql) %}\n {% set time_data_type = snapshot_time_column_schema[0].dtype %}\n {{ return(time_data_type or none) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.snapshot_get_time", "macro.dbt.default__snapshot_get_time", "macro.dbt.get_column_schema_from_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8533702, "supported_languages": null}, "macro.dbt.current_timestamp_backcompat": {"name": "current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_backcompat", "macro_sql": "{% macro current_timestamp_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853508, "supported_languages": null}, "macro.dbt.default__current_timestamp_backcompat": {"name": "default__current_timestamp_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_backcompat", "macro_sql": "{% macro default__current_timestamp_backcompat() %}\n current_timestamp::timestamp\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853577, "supported_languages": null}, "macro.dbt.current_timestamp_in_utc_backcompat": {"name": "current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.current_timestamp_in_utc_backcompat", "macro_sql": "{% macro current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_in_utc_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__current_timestamp_in_utc_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853718, "supported_languages": null}, "macro.dbt.default__current_timestamp_in_utc_backcompat": {"name": "default__current_timestamp_in_utc_backcompat", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/timestamps.sql", "original_file_path": "macros/adapters/timestamps.sql", "unique_id": "macro.dbt.default__current_timestamp_in_utc_backcompat", "macro_sql": "{% macro default__current_timestamp_in_utc_backcompat() %}\n {{ return(adapter.dispatch('current_timestamp_backcompat', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.current_timestamp_backcompat", "macro.dbt.default__current_timestamp_backcompat"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.853857, "supported_languages": null}, "macro.dbt.get_create_index_sql": {"name": "get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_create_index_sql", "macro_sql": "{% macro get_create_index_sql(relation, index_dict) -%}\n {{ return(adapter.dispatch('get_create_index_sql', 'dbt')(relation, index_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_create_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.854583, "supported_languages": null}, "macro.dbt.default__get_create_index_sql": {"name": "default__get_create_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_create_index_sql", "macro_sql": "{% macro default__get_create_index_sql(relation, index_dict) -%}\n {% do return(None) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.854707, "supported_languages": null}, "macro.dbt.create_indexes": {"name": "create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.create_indexes", "macro_sql": "{% macro create_indexes(relation) -%}\n {{ adapter.dispatch('create_indexes', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__create_indexes"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8548422, "supported_languages": null}, "macro.dbt.default__create_indexes": {"name": "default__create_indexes", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__create_indexes", "macro_sql": "{% macro default__create_indexes(relation) -%}\n {%- set _indexes = config.get('indexes', default=[]) -%}\n\n {% for _index_dict in _indexes %}\n {% set create_index_sql = get_create_index_sql(relation, _index_dict) %}\n {% if create_index_sql %}\n {% do run_query(create_index_sql) %}\n {% endif %}\n {% endfor %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_create_index_sql", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8552508, "supported_languages": null}, "macro.dbt.get_drop_index_sql": {"name": "get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_drop_index_sql", "macro_sql": "{% macro get_drop_index_sql(relation, index_name) -%}\n {{ adapter.dispatch('get_drop_index_sql', 'dbt')(relation, index_name) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_drop_index_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.855412, "supported_languages": null}, "macro.dbt.default__get_drop_index_sql": {"name": "default__get_drop_index_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_drop_index_sql", "macro_sql": "{% macro default__get_drop_index_sql(relation, index_name) -%}\n {{ exceptions.raise_compiler_error(\"`get_drop_index_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.855531, "supported_languages": null}, "macro.dbt.get_show_indexes_sql": {"name": "get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.get_show_indexes_sql", "macro_sql": "{% macro get_show_indexes_sql(relation) -%}\n {{ adapter.dispatch('get_show_indexes_sql', 'dbt')(relation) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_show_indexes_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8556669, "supported_languages": null}, "macro.dbt.default__get_show_indexes_sql": {"name": "default__get_show_indexes_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/indexes.sql", "original_file_path": "macros/adapters/indexes.sql", "unique_id": "macro.dbt.default__get_show_indexes_sql", "macro_sql": "{% macro default__get_show_indexes_sql(relation) -%}\n {{ exceptions.raise_compiler_error(\"`get_show_indexes_sql has not been implemented for this adapter.\") }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.855777, "supported_languages": null}, "macro.dbt.make_intermediate_relation": {"name": "make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_intermediate_relation", "macro_sql": "{% macro make_intermediate_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_intermediate_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_intermediate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.857485, "supported_languages": null}, "macro.dbt.default__make_intermediate_relation": {"name": "default__make_intermediate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_intermediate_relation", "macro_sql": "{% macro default__make_intermediate_relation(base_relation, suffix) %}\n {{ return(default__make_temp_relation(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.857631, "supported_languages": null}, "macro.dbt.make_temp_relation": {"name": "make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_temp_relation", "macro_sql": "{% macro make_temp_relation(base_relation, suffix='__dbt_tmp') %}\n {{ return(adapter.dispatch('make_temp_relation', 'dbt')(base_relation, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__make_temp_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.857822, "supported_languages": null}, "macro.dbt.default__make_temp_relation": {"name": "default__make_temp_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_temp_relation", "macro_sql": "{% macro default__make_temp_relation(base_relation, suffix) %}\n {%- set temp_identifier = base_relation.identifier ~ suffix -%}\n {%- set temp_relation = base_relation.incorporate(\n path={\"identifier\": temp_identifier}) -%}\n\n {{ return(temp_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858082, "supported_languages": null}, "macro.dbt.make_backup_relation": {"name": "make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.make_backup_relation", "macro_sql": "{% macro make_backup_relation(base_relation, backup_relation_type, suffix='__dbt_backup') %}\n {{ return(adapter.dispatch('make_backup_relation', 'dbt')(base_relation, backup_relation_type, suffix)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__make_backup_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858318, "supported_languages": null}, "macro.dbt.default__make_backup_relation": {"name": "default__make_backup_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__make_backup_relation", "macro_sql": "{% macro default__make_backup_relation(base_relation, backup_relation_type, suffix) %}\n {%- set backup_identifier = base_relation.identifier ~ suffix -%}\n {%- set backup_relation = base_relation.incorporate(\n path={\"identifier\": backup_identifier},\n type=backup_relation_type\n ) -%}\n {{ return(backup_relation) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8586, "supported_languages": null}, "macro.dbt.truncate_relation": {"name": "truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.truncate_relation", "macro_sql": "{% macro truncate_relation(relation) -%}\n {{ return(adapter.dispatch('truncate_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__truncate_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858756, "supported_languages": null}, "macro.dbt.default__truncate_relation": {"name": "default__truncate_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__truncate_relation", "macro_sql": "{% macro default__truncate_relation(relation) -%}\n {% call statement('truncate_relation') -%}\n truncate table {{ relation.render() }}\n {%- endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.858906, "supported_languages": null}, "macro.dbt.get_or_create_relation": {"name": "get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.get_or_create_relation", "macro_sql": "{% macro get_or_create_relation(database, schema, identifier, type) -%}\n {{ return(adapter.dispatch('get_or_create_relation', 'dbt')(database, schema, identifier, type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_or_create_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859114, "supported_languages": null}, "macro.dbt.default__get_or_create_relation": {"name": "default__get_or_create_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.default__get_or_create_relation", "macro_sql": "{% macro default__get_or_create_relation(database, schema, identifier, type) %}\n {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}\n\n {% if target_relation %}\n {% do return([true, target_relation]) %}\n {% endif %}\n\n {%- set new_relation = api.Relation.create(\n database=database,\n schema=schema,\n identifier=identifier,\n type=type\n ) -%}\n {% do return([false, new_relation]) %}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859583, "supported_languages": null}, "macro.dbt.load_cached_relation": {"name": "load_cached_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_cached_relation", "macro_sql": "{% macro load_cached_relation(relation) %}\n {% do return(adapter.get_relation(\n database=relation.database,\n schema=relation.schema,\n identifier=relation.identifier\n )) -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859786, "supported_languages": null}, "macro.dbt.load_relation": {"name": "load_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/relation.sql", "original_file_path": "macros/adapters/relation.sql", "unique_id": "macro.dbt.load_relation", "macro_sql": "{% macro load_relation(relation) %}\n {{ return(load_cached_relation(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_cached_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.859914, "supported_languages": null}, "macro.dbt.collect_freshness": {"name": "collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.collect_freshness", "macro_sql": "{% macro collect_freshness(source, loaded_at_field, filter) %}\n {{ return(adapter.dispatch('collect_freshness', 'dbt')(source, loaded_at_field, filter))}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__collect_freshness"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.860286, "supported_languages": null}, "macro.dbt.default__collect_freshness": {"name": "default__collect_freshness", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/freshness.sql", "original_file_path": "macros/adapters/freshness.sql", "unique_id": "macro.dbt.default__collect_freshness", "macro_sql": "{% macro default__collect_freshness(source, loaded_at_field, filter) %}\n {% call statement('collect_freshness', fetch_result=True, auto_begin=False) -%}\n select\n max({{ loaded_at_field }}) as max_loaded_at,\n {{ current_timestamp() }} as snapshotted_at\n from {{ source }}\n {% if filter %}\n where {{ filter }}\n {% endif %}\n {% endcall %}\n {{ return(load_result('collect_freshness')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.current_timestamp"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86064, "supported_languages": null}, "macro.dbt.validate_sql": {"name": "validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.validate_sql", "macro_sql": "{% macro validate_sql(sql) -%}\n {{ return(adapter.dispatch('validate_sql', 'dbt')(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__validate_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8608792, "supported_languages": null}, "macro.dbt.default__validate_sql": {"name": "default__validate_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/validate_sql.sql", "original_file_path": "macros/adapters/validate_sql.sql", "unique_id": "macro.dbt.default__validate_sql", "macro_sql": "{% macro default__validate_sql(sql) -%}\n {% call statement('validate_sql') -%}\n explain {{ sql }}\n {% endcall %}\n {{ return(load_result('validate_sql')) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8611462, "supported_languages": null}, "macro.dbt.copy_grants": {"name": "copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.copy_grants", "macro_sql": "{% macro copy_grants() %}\n {{ return(adapter.dispatch('copy_grants', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.862612, "supported_languages": null}, "macro.dbt.default__copy_grants": {"name": "default__copy_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__copy_grants", "macro_sql": "{% macro default__copy_grants() %}\n {{ return(True) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.862713, "supported_languages": null}, "macro.dbt.support_multiple_grantees_per_dcl_statement": {"name": "support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.support_multiple_grantees_per_dcl_statement", "macro_sql": "{% macro support_multiple_grantees_per_dcl_statement() %}\n {{ return(adapter.dispatch('support_multiple_grantees_per_dcl_statement', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8628511, "supported_languages": null}, "macro.dbt.default__support_multiple_grantees_per_dcl_statement": {"name": "default__support_multiple_grantees_per_dcl_statement", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__support_multiple_grantees_per_dcl_statement", "macro_sql": "\n\n{%- macro default__support_multiple_grantees_per_dcl_statement() -%}\n {{ return(True) }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.862943, "supported_languages": null}, "macro.dbt.should_revoke": {"name": "should_revoke", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.should_revoke", "macro_sql": "{% macro should_revoke(existing_relation, full_refresh_mode=True) %}\n\n {% if not existing_relation %}\n {#-- The table doesn't already exist, so no grants to copy over --#}\n {{ return(False) }}\n {% elif full_refresh_mode %}\n {#-- The object is being REPLACED -- whether grants are copied over depends on the value of user config --#}\n {{ return(copy_grants()) }}\n {% else %}\n {#-- The table is being merged/upserted/inserted -- grants will be carried over --#}\n {{ return(True) }}\n {% endif %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.copy_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863224, "supported_languages": null}, "macro.dbt.get_show_grant_sql": {"name": "get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_show_grant_sql", "macro_sql": "{% macro get_show_grant_sql(relation) %}\n {{ return(adapter.dispatch(\"get_show_grant_sql\", \"dbt\")(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_show_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863383, "supported_languages": null}, "macro.dbt.default__get_show_grant_sql": {"name": "default__get_show_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_show_grant_sql", "macro_sql": "{% macro default__get_show_grant_sql(relation) %}\n show grants on {{ relation.render() }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863481, "supported_languages": null}, "macro.dbt.get_grant_sql": {"name": "get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_grant_sql", "macro_sql": "{% macro get_grant_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_grant_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_grant_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863672, "supported_languages": null}, "macro.dbt.default__get_grant_sql": {"name": "default__get_grant_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_grant_sql", "macro_sql": "\n\n{%- macro default__get_grant_sql(relation, privilege, grantees) -%}\n grant {{ privilege }} on {{ relation.render() }} to {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.863852, "supported_languages": null}, "macro.dbt.get_revoke_sql": {"name": "get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_revoke_sql", "macro_sql": "{% macro get_revoke_sql(relation, privilege, grantees) %}\n {{ return(adapter.dispatch('get_revoke_sql', 'dbt')(relation, privilege, grantees)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_revoke_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86405, "supported_languages": null}, "macro.dbt.default__get_revoke_sql": {"name": "default__get_revoke_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_revoke_sql", "macro_sql": "\n\n{%- macro default__get_revoke_sql(relation, privilege, grantees) -%}\n revoke {{ privilege }} on {{ relation.render() }} from {{ grantees | join(', ') }}\n{%- endmacro -%}\n\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.864309, "supported_languages": null}, "macro.dbt.get_dcl_statement_list": {"name": "get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.get_dcl_statement_list", "macro_sql": "{% macro get_dcl_statement_list(relation, grant_config, get_dcl_macro) %}\n {{ return(adapter.dispatch('get_dcl_statement_list', 'dbt')(relation, grant_config, get_dcl_macro)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_dcl_statement_list"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8645408, "supported_languages": null}, "macro.dbt.default__get_dcl_statement_list": {"name": "default__get_dcl_statement_list", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__get_dcl_statement_list", "macro_sql": "\n\n{%- macro default__get_dcl_statement_list(relation, grant_config, get_dcl_macro) -%}\n {#\n -- Unpack grant_config into specific privileges and the set of users who need them granted/revoked.\n -- Depending on whether this database supports multiple grantees per statement, pass in the list of\n -- all grantees per privilege, or (if not) template one statement per privilege-grantee pair.\n -- `get_dcl_macro` will be either `get_grant_sql` or `get_revoke_sql`\n #}\n {%- set dcl_statements = [] -%}\n {%- for privilege, grantees in grant_config.items() %}\n {%- if support_multiple_grantees_per_dcl_statement() and grantees -%}\n {%- set dcl = get_dcl_macro(relation, privilege, grantees) -%}\n {%- do dcl_statements.append(dcl) -%}\n {%- else -%}\n {%- for grantee in grantees -%}\n {% set dcl = get_dcl_macro(relation, privilege, [grantee]) %}\n {%- do dcl_statements.append(dcl) -%}\n {% endfor -%}\n {%- endif -%}\n {%- endfor -%}\n {{ return(dcl_statements) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt.support_multiple_grantees_per_dcl_statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865141, "supported_languages": null}, "macro.dbt.call_dcl_statements": {"name": "call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.call_dcl_statements", "macro_sql": "{% macro call_dcl_statements(dcl_statement_list) %}\n {{ return(adapter.dispatch(\"call_dcl_statements\", \"dbt\")(dcl_statement_list)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865317, "supported_languages": null}, "macro.dbt.default__call_dcl_statements": {"name": "default__call_dcl_statements", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__call_dcl_statements", "macro_sql": "{% macro default__call_dcl_statements(dcl_statement_list) %}\n {#\n -- By default, supply all grant + revoke statements in a single semicolon-separated block,\n -- so that they're all processed together.\n\n -- Some databases do not support this. Those adapters will need to override this macro\n -- to run each statement individually.\n #}\n {% call statement('grants') %}\n {% for dcl_statement in dcl_statement_list %}\n {{ dcl_statement }};\n {% endfor %}\n {% endcall %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865537, "supported_languages": null}, "macro.dbt.apply_grants": {"name": "apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.apply_grants", "macro_sql": "{% macro apply_grants(relation, grant_config, should_revoke) %}\n {{ return(adapter.dispatch(\"apply_grants\", \"dbt\")(relation, grant_config, should_revoke)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__apply_grants"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.865737, "supported_languages": null}, "macro.dbt.default__apply_grants": {"name": "default__apply_grants", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/apply_grants.sql", "original_file_path": "macros/adapters/apply_grants.sql", "unique_id": "macro.dbt.default__apply_grants", "macro_sql": "{% macro default__apply_grants(relation, grant_config, should_revoke=True) %}\n {#-- If grant_config is {} or None, this is a no-op --#}\n {% if grant_config %}\n {% if should_revoke %}\n {#-- We think previous grants may have carried over --#}\n {#-- Show current grants and calculate diffs --#}\n {% set current_grants_table = run_query(get_show_grant_sql(relation)) %}\n {% set current_grants_dict = adapter.standardize_grants_dict(current_grants_table) %}\n {% set needs_granting = diff_of_two_dicts(grant_config, current_grants_dict) %}\n {% set needs_revoking = diff_of_two_dicts(current_grants_dict, grant_config) %}\n {% if not (needs_granting or needs_revoking) %}\n {{ log('On ' ~ relation.render() ~': All grants are in place, no revocation or granting needed.')}}\n {% endif %}\n {% else %}\n {#-- We don't think there's any chance of previous grants having carried over. --#}\n {#-- Jump straight to granting what the user has configured. --#}\n {% set needs_revoking = {} %}\n {% set needs_granting = grant_config %}\n {% endif %}\n {% if needs_granting or needs_revoking %}\n {% set revoke_statement_list = get_dcl_statement_list(relation, needs_revoking, get_revoke_sql) %}\n {% set grant_statement_list = get_dcl_statement_list(relation, needs_granting, get_grant_sql) %}\n {% set dcl_statement_list = revoke_statement_list + grant_statement_list %}\n {% if dcl_statement_list %}\n {{ call_dcl_statements(dcl_statement_list) }}\n {% endif %}\n {% endif %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.get_show_grant_sql", "macro.dbt.get_dcl_statement_list", "macro.dbt.call_dcl_statements"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8668082, "supported_languages": null}, "macro.dbt.get_show_sql": {"name": "get_show_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_show_sql", "macro_sql": "{% macro get_show_sql(compiled_code, sql_header, limit) -%}\n {%- if sql_header is not none -%}\n {{ sql_header }}\n {%- endif %}\n {{ get_limit_subquery_sql(compiled_code, limit) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_limit_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8671901, "supported_languages": null}, "macro.dbt.get_limit_subquery_sql": {"name": "get_limit_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.get_limit_subquery_sql", "macro_sql": "\n{%- macro get_limit_subquery_sql(sql, limit) -%}\n {{ adapter.dispatch('get_limit_sql', 'dbt')(sql, limit) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__get_limit_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8673549, "supported_languages": null}, "macro.dbt.default__get_limit_sql": {"name": "default__get_limit_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/show.sql", "original_file_path": "macros/adapters/show.sql", "unique_id": "macro.dbt.default__get_limit_sql", "macro_sql": "{% macro default__get_limit_sql(sql, limit) %}\n {{ compiled_code }}\n {% if limit is not none %}\n limit {{ limit }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86752, "supported_languages": null}, "macro.dbt.alter_column_comment": {"name": "alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_column_comment", "macro_sql": "{% macro alter_column_comment(relation, column_dict) -%}\n {{ return(adapter.dispatch('alter_column_comment', 'dbt')(relation, column_dict)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86812, "supported_languages": null}, "macro.dbt.default__alter_column_comment": {"name": "default__alter_column_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_column_comment", "macro_sql": "{% macro default__alter_column_comment(relation, column_dict) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_column_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.868274, "supported_languages": null}, "macro.dbt.alter_relation_comment": {"name": "alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.alter_relation_comment", "macro_sql": "{% macro alter_relation_comment(relation, relation_comment) -%}\n {{ return(adapter.dispatch('alter_relation_comment', 'dbt')(relation, relation_comment)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__alter_relation_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.868457, "supported_languages": null}, "macro.dbt.default__alter_relation_comment": {"name": "default__alter_relation_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__alter_relation_comment", "macro_sql": "{% macro default__alter_relation_comment(relation, relation_comment) -%}\n {{ exceptions.raise_not_implemented(\n 'alter_relation_comment macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8686101, "supported_languages": null}, "macro.dbt.persist_docs": {"name": "persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.persist_docs", "macro_sql": "{% macro persist_docs(relation, model, for_relation=true, for_columns=true) -%}\n {{ return(adapter.dispatch('persist_docs', 'dbt')(relation, model, for_relation, for_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__persist_docs"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.86885, "supported_languages": null}, "macro.dbt.default__persist_docs": {"name": "default__persist_docs", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/persist_docs.sql", "original_file_path": "macros/adapters/persist_docs.sql", "unique_id": "macro.dbt.default__persist_docs", "macro_sql": "{% macro default__persist_docs(relation, model, for_relation, for_columns) -%}\n {% if for_relation and config.persist_relation_docs() and model.description %}\n {% do run_query(alter_relation_comment(relation, model.description)) %}\n {% endif %}\n\n {% if for_columns and config.persist_column_docs() and model.columns %}\n {% do run_query(alter_column_comment(relation, model.columns)) %}\n {% endif %}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query", "macro.dbt.alter_relation_comment", "macro.dbt.alter_column_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8692548, "supported_languages": null}, "macro.dbt.get_catalog_relations": {"name": "get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog_relations", "macro_sql": "{% macro get_catalog_relations(information_schema, relations) -%}\n {{ return(adapter.dispatch('get_catalog_relations', 'dbt')(information_schema, relations)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872187, "supported_languages": null}, "macro.dbt.default__get_catalog_relations": {"name": "default__get_catalog_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog_relations", "macro_sql": "{% macro default__get_catalog_relations(information_schema, relations) -%}\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog_relations not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872406, "supported_languages": null}, "macro.dbt.get_catalog": {"name": "get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog", "macro_sql": "{% macro get_catalog(information_schema, schemas) -%}\n {{ return(adapter.dispatch('get_catalog', 'dbt')(information_schema, schemas)) }}\n{%- endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_catalog"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872573, "supported_languages": null}, "macro.dbt.default__get_catalog": {"name": "default__get_catalog", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog", "macro_sql": "{% macro default__get_catalog(information_schema, schemas) -%}\n\n {% set typename = adapter.type() %}\n {% set msg -%}\n get_catalog not implemented for {{ typename }}\n {%- endset %}\n\n {{ exceptions.raise_compiler_error(msg) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8727858, "supported_languages": null}, "macro.dbt.information_schema_name": {"name": "information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.information_schema_name", "macro_sql": "{% macro information_schema_name(database) %}\n {{ return(adapter.dispatch('information_schema_name', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__information_schema_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.872941, "supported_languages": null}, "macro.dbt.default__information_schema_name": {"name": "default__information_schema_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__information_schema_name", "macro_sql": "{% macro default__information_schema_name(database) -%}\n {%- if database -%}\n {{ database }}.INFORMATION_SCHEMA\n {%- else -%}\n INFORMATION_SCHEMA\n {%- endif -%}\n{%- endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8730712, "supported_languages": null}, "macro.dbt.list_schemas": {"name": "list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_schemas", "macro_sql": "{% macro list_schemas(database) -%}\n {{ return(adapter.dispatch('list_schemas', 'dbt')(database)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__list_schemas"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8732271, "supported_languages": null}, "macro.dbt.default__list_schemas": {"name": "default__list_schemas", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_schemas", "macro_sql": "{% macro default__list_schemas(database) -%}\n {% set sql %}\n select distinct schema_name\n from {{ information_schema_name(database) }}.SCHEMATA\n where catalog_name ilike '{{ database }}'\n {% endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.information_schema_name", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.87344, "supported_languages": null}, "macro.dbt.check_schema_exists": {"name": "check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.check_schema_exists", "macro_sql": "{% macro check_schema_exists(information_schema, schema) -%}\n {{ return(adapter.dispatch('check_schema_exists', 'dbt')(information_schema, schema)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__check_schema_exists"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8736901, "supported_languages": null}, "macro.dbt.default__check_schema_exists": {"name": "default__check_schema_exists", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__check_schema_exists", "macro_sql": "{% macro default__check_schema_exists(information_schema, schema) -%}\n {% set sql -%}\n select count(*)\n from {{ information_schema.replace(information_schema_view='SCHEMATA') }}\n where catalog_name='{{ information_schema.database }}'\n and schema_name='{{ schema }}'\n {%- endset %}\n {{ return(run_query(sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.replace", "macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8739462, "supported_languages": null}, "macro.dbt.list_relations_without_caching": {"name": "list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.list_relations_without_caching", "macro_sql": "{% macro list_relations_without_caching(schema_relation) %}\n {{ return(adapter.dispatch('list_relations_without_caching', 'dbt')(schema_relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__list_relations_without_caching"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874113, "supported_languages": null}, "macro.dbt.default__list_relations_without_caching": {"name": "default__list_relations_without_caching", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__list_relations_without_caching", "macro_sql": "{% macro default__list_relations_without_caching(schema_relation) %}\n {{ exceptions.raise_not_implemented(\n 'list_relations_without_caching macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874256, "supported_languages": null}, "macro.dbt.get_catalog_for_single_relation": {"name": "get_catalog_for_single_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_catalog_for_single_relation", "macro_sql": "{% macro get_catalog_for_single_relation(relation) %}\n {{ return(adapter.dispatch('get_catalog_for_single_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_catalog_for_single_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874414, "supported_languages": null}, "macro.dbt.default__get_catalog_for_single_relation": {"name": "default__get_catalog_for_single_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_catalog_for_single_relation", "macro_sql": "{% macro default__get_catalog_for_single_relation(relation) %}\n {{ exceptions.raise_not_implemented(\n 'get_catalog_for_single_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8745549, "supported_languages": null}, "macro.dbt.get_relations": {"name": "get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relations", "macro_sql": "{% macro get_relations() %}\n {{ return(adapter.dispatch('get_relations', 'dbt')()) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_relations"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.874696, "supported_languages": null}, "macro.dbt.default__get_relations": {"name": "default__get_relations", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relations", "macro_sql": "{% macro default__get_relations() %}\n {{ exceptions.raise_not_implemented(\n 'get_relations macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8748312, "supported_languages": null}, "macro.dbt.get_relation_last_modified": {"name": "get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.get_relation_last_modified", "macro_sql": "{% macro get_relation_last_modified(information_schema, relations) %}\n {{ return(adapter.dispatch('get_relation_last_modified', 'dbt')(information_schema, relations)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_databricks.databricks__get_relation_last_modified"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.87501, "supported_languages": null}, "macro.dbt.default__get_relation_last_modified": {"name": "default__get_relation_last_modified", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/metadata.sql", "original_file_path": "macros/adapters/metadata.sql", "unique_id": "macro.dbt.default__get_relation_last_modified", "macro_sql": "{% macro default__get_relation_last_modified(information_schema, relations) %}\n {{ exceptions.raise_not_implemented(\n 'get_relation_last_modified macro not implemented for adapter ' + adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8751612, "supported_languages": null}, "macro.dbt.get_columns_in_relation": {"name": "get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_relation", "macro_sql": "{% macro get_columns_in_relation(relation) -%}\n {{ return(adapter.dispatch('get_columns_in_relation', 'dbt')(relation)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__get_columns_in_relation"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877135, "supported_languages": null}, "macro.dbt.default__get_columns_in_relation": {"name": "default__get_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_relation", "macro_sql": "{% macro default__get_columns_in_relation(relation) -%}\n {{ exceptions.raise_not_implemented(\n 'get_columns_in_relation macro not implemented for adapter '+adapter.type()) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877271, "supported_languages": null}, "macro.dbt.sql_convert_columns_in_relation": {"name": "sql_convert_columns_in_relation", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.sql_convert_columns_in_relation", "macro_sql": "{% macro sql_convert_columns_in_relation(table) -%}\n {% set columns = [] %}\n {% for row in table %}\n {% do columns.append(api.Column(*row)) %}\n {% endfor %}\n {{ return(columns) }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877537, "supported_languages": null}, "macro.dbt.get_empty_subquery_sql": {"name": "get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_subquery_sql", "macro_sql": "{% macro get_empty_subquery_sql(select_sql, select_sql_header=none) -%}\n {{ return(adapter.dispatch('get_empty_subquery_sql', 'dbt')(select_sql, select_sql_header)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8777258, "supported_languages": null}, "macro.dbt.default__get_empty_subquery_sql": {"name": "default__get_empty_subquery_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_subquery_sql", "macro_sql": "{% macro default__get_empty_subquery_sql(select_sql, select_sql_header=none) %}\n {%- if select_sql_header is not none -%}\n {{ select_sql_header }}\n {%- endif -%}\n select * from (\n {{ select_sql }}\n ) as __dbt_sbq\n where false\n limit 0\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.877902, "supported_languages": null}, "macro.dbt.get_empty_schema_sql": {"name": "get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_empty_schema_sql", "macro_sql": "{% macro get_empty_schema_sql(columns) -%}\n {{ return(adapter.dispatch('get_empty_schema_sql', 'dbt')(columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_empty_schema_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8780658, "supported_languages": null}, "macro.dbt.default__get_empty_schema_sql": {"name": "default__get_empty_schema_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_empty_schema_sql", "macro_sql": "{% macro default__get_empty_schema_sql(columns) %}\n {%- set col_err = [] -%}\n {%- set col_naked_numeric = [] -%}\n select\n {% for i in columns %}\n {%- set col = columns[i] -%}\n {%- if col['data_type'] is not defined -%}\n {%- do col_err.append(col['name']) -%}\n {#-- If this column's type is just 'numeric' then it is missing precision/scale, raise a warning --#}\n {%- elif col['data_type'].strip().lower() in ('numeric', 'decimal', 'number') -%}\n {%- do col_naked_numeric.append(col['name']) -%}\n {%- endif -%}\n {% set col_name = adapter.quote(col['name']) if col.get('quote') else col['name'] %}\n {{ cast('null', col['data_type']) }} as {{ col_name }}{{ \", \" if not loop.last }}\n {%- endfor -%}\n {%- if (col_err | length) > 0 -%}\n {{ exceptions.column_type_missing(column_names=col_err) }}\n {%- elif (col_naked_numeric | length) > 0 -%}\n {{ exceptions.warn(\"Detected columns with numeric type and unspecified precision/scale, this can lead to unintended rounding: \" ~ col_naked_numeric ~ \"`\") }}\n {%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879136, "supported_languages": null}, "macro.dbt.get_column_schema_from_query": {"name": "get_column_schema_from_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_column_schema_from_query", "macro_sql": "{% macro get_column_schema_from_query(select_sql, select_sql_header=none) -%}\n {% set columns = [] %}\n {# -- Using an 'empty subquery' here to get the same schema as the given select_sql statement, without necessitating a data scan.#}\n {% set sql = get_empty_subquery_sql(select_sql, select_sql_header) %}\n {% set column_schema = adapter.get_column_schema_from_query(sql) %}\n {{ return(column_schema) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879442, "supported_languages": null}, "macro.dbt.get_columns_in_query": {"name": "get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.get_columns_in_query", "macro_sql": "{% macro get_columns_in_query(select_sql) -%}\n {{ return(adapter.dispatch('get_columns_in_query', 'dbt')(select_sql)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__get_columns_in_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879605, "supported_languages": null}, "macro.dbt.default__get_columns_in_query": {"name": "default__get_columns_in_query", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__get_columns_in_query", "macro_sql": "{% macro default__get_columns_in_query(select_sql) %}\n {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%}\n {{ get_empty_subquery_sql(select_sql) }}\n {% endcall %}\n {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement", "macro.dbt.get_empty_subquery_sql"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.879909, "supported_languages": null}, "macro.dbt.alter_column_type": {"name": "alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_column_type", "macro_sql": "{% macro alter_column_type(relation, column_name, new_column_type) -%}\n {{ return(adapter.dispatch('alter_column_type', 'dbt')(relation, column_name, new_column_type)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__alter_column_type"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.880113, "supported_languages": null}, "macro.dbt.default__alter_column_type": {"name": "default__alter_column_type", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_column_type", "macro_sql": "{% macro default__alter_column_type(relation, column_name, new_column_type) -%}\n {#\n 1. Create a new column (w/ temp name and correct type)\n 2. Copy data over to it\n 3. Drop the existing column (cascade!)\n 4. Rename the new column to existing column\n #}\n {%- set tmp_column = column_name + \"__dbt_alter\" -%}\n\n {% call statement('alter_column_type') %}\n alter table {{ relation.render() }} add column {{ adapter.quote(tmp_column) }} {{ new_column_type }};\n update {{ relation.render() }} set {{ adapter.quote(tmp_column) }} = {{ adapter.quote(column_name) }};\n alter table {{ relation.render() }} drop column {{ adapter.quote(column_name) }} cascade;\n alter table {{ relation.render() }} rename column {{ adapter.quote(tmp_column) }} to {{ adapter.quote(column_name) }}\n {% endcall %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.statement"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.880679, "supported_languages": null}, "macro.dbt.alter_relation_add_remove_columns": {"name": "alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.alter_relation_add_remove_columns", "macro_sql": "{% macro alter_relation_add_remove_columns(relation, add_columns = none, remove_columns = none) -%}\n {{ return(adapter.dispatch('alter_relation_add_remove_columns', 'dbt')(relation, add_columns, remove_columns)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt_spark.spark__alter_relation_add_remove_columns"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8809001, "supported_languages": null}, "macro.dbt.default__alter_relation_add_remove_columns": {"name": "default__alter_relation_add_remove_columns", "resource_type": "macro", "package_name": "dbt", "path": "macros/adapters/columns.sql", "original_file_path": "macros/adapters/columns.sql", "unique_id": "macro.dbt.default__alter_relation_add_remove_columns", "macro_sql": "{% macro default__alter_relation_add_remove_columns(relation, add_columns, remove_columns) %}\n\n {% if add_columns is none %}\n {% set add_columns = [] %}\n {% endif %}\n {% if remove_columns is none %}\n {% set remove_columns = [] %}\n {% endif %}\n\n {% set sql -%}\n\n alter {{ relation.type }} {{ relation.render() }}\n\n {% for column in add_columns %}\n add column {{ column.name }} {{ column.data_type }}{{ ',' if not loop.last }}\n {% endfor %}{{ ',' if add_columns and remove_columns }}\n\n {% for column in remove_columns %}\n drop column {{ column.name }}{{ ',' if not loop.last }}\n {% endfor %}\n\n {%- endset -%}\n\n {% do run_query(sql) %}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.run_query"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.88154, "supported_languages": null}, "macro.dbt.get_fixture_sql": {"name": "get_fixture_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.get_fixture_sql", "macro_sql": "{% macro get_fixture_sql(rows, column_name_to_data_types) %}\n-- Fixture for {{ model.name }}\n{% set default_row = {} %}\n\n{%- if not column_name_to_data_types -%}\n{#-- Use defer_relation IFF it is available in the manifest and 'this' is missing from the database --#}\n{%- set this_or_defer_relation = defer_relation if (defer_relation and not load_relation(this)) else this -%}\n{%- set columns_in_relation = adapter.get_columns_in_relation(this_or_defer_relation) -%}\n\n{%- set column_name_to_data_types = {} -%}\n{%- for column in columns_in_relation -%}\n{#-- This needs to be a case-insensitive comparison --#}\n{%- do column_name_to_data_types.update({column.name|lower: column.data_type}) -%}\n{%- endfor -%}\n{%- endif -%}\n\n{%- if not column_name_to_data_types -%}\n {{ exceptions.raise_compiler_error(\"Not able to get columns for unit test '\" ~ model.name ~ \"' from relation \" ~ this ~ \" because the relation doesn't exist\") }}\n{%- endif -%}\n\n{%- for column_name, column_type in column_name_to_data_types.items() -%}\n {%- do default_row.update({column_name: (safe_cast(\"null\", column_type) | trim )}) -%}\n{%- endfor -%}\n\n{{ validate_fixture_rows(rows, row_number) }}\n\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\n{%- set default_row_copy = default_row.copy() -%}\n{%- do default_row_copy.update(formatted_row) -%}\nselect\n{%- for column_name, column_value in default_row_copy.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n\n{%- if (rows | length) == 0 -%}\n select\n {%- for column_name, column_value in default_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%},{%- endif %}\n {%- endfor %}\n limit 0\n{%- endif -%}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.load_relation", "macro.dbt.safe_cast", "macro.dbt.validate_fixture_rows", "macro.dbt.format_row"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.884523, "supported_languages": null}, "macro.dbt.get_expected_sql": {"name": "get_expected_sql", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.get_expected_sql", "macro_sql": "{% macro get_expected_sql(rows, column_name_to_data_types) %}\n\n{%- if (rows | length) == 0 -%}\n select * from dbt_internal_unit_test_actual\n limit 0\n{%- else -%}\n{%- for row in rows -%}\n{%- set formatted_row = format_row(row, column_name_to_data_types) -%}\nselect\n{%- for column_name, column_value in formatted_row.items() %} {{ column_value }} as {{ column_name }}{% if not loop.last -%}, {%- endif %}\n{%- endfor %}\n{%- if not loop.last %}\nunion all\n{% endif %}\n{%- endfor -%}\n{%- endif -%}\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.format_row"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8849862, "supported_languages": null}, "macro.dbt.format_row": {"name": "format_row", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.format_row", "macro_sql": "\n\n{%- macro format_row(row, column_name_to_data_types) -%}\n {#-- generate case-insensitive formatted row --#}\n {% set formatted_row = {} %}\n {%- for column_name, column_value in row.items() -%}\n {% set column_name = column_name|lower %}\n\n {%- if column_name not in column_name_to_data_types %}\n {#-- if user-provided row contains column name that relation does not contain, raise an error --#}\n {% set fixture_name = \"expected output\" if model.resource_type == 'unit_test' else (\"'\" ~ model.name ~ \"'\") %}\n {{ exceptions.raise_compiler_error(\n \"Invalid column name: '\" ~ column_name ~ \"' in unit test fixture for \" ~ fixture_name ~ \".\"\n \"\\nAccepted columns for \" ~ fixture_name ~ \" are: \" ~ (column_name_to_data_types.keys()|list)\n ) }}\n {%- endif -%}\n\n {%- set column_type = column_name_to_data_types[column_name] %}\n\n {#-- sanitize column_value: wrap yaml strings in quotes, apply cast --#}\n {%- set column_value_clean = column_value -%}\n {%- if column_value is string -%}\n {%- set column_value_clean = dbt.string_literal(dbt.escape_single_quotes(column_value)) -%}\n {%- elif column_value is none -%}\n {%- set column_value_clean = 'null' -%}\n {%- endif -%}\n\n {%- set row_update = {column_name: safe_cast(column_value_clean, column_type) } -%}\n {%- do formatted_row.update(row_update) -%}\n {%- endfor -%}\n {{ return(formatted_row) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.string_literal", "macro.dbt.escape_single_quotes", "macro.dbt.safe_cast"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8859332, "supported_languages": null}, "macro.dbt.validate_fixture_rows": {"name": "validate_fixture_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.validate_fixture_rows", "macro_sql": "{%- macro validate_fixture_rows(rows, row_number) -%}\n {{ return(adapter.dispatch('validate_fixture_rows', 'dbt')(rows, row_number)) }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": ["macro.dbt.default__validate_fixture_rows"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8861172, "supported_languages": null}, "macro.dbt.default__validate_fixture_rows": {"name": "default__validate_fixture_rows", "resource_type": "macro", "package_name": "dbt", "path": "macros/unit_test_sql/get_fixture_sql.sql", "original_file_path": "macros/unit_test_sql/get_fixture_sql.sql", "unique_id": "macro.dbt.default__validate_fixture_rows", "macro_sql": "{%- macro default__validate_fixture_rows(rows, row_number) -%}\n {# This is an abstract method for adapter overrides as needed #}\n{%- endmacro -%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.886199, "supported_languages": null}, "macro.dbt.resolve_model_name": {"name": "resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.resolve_model_name", "macro_sql": "{% macro resolve_model_name(input_model_name) %}\n {{ return(adapter.dispatch('resolve_model_name', 'dbt')(input_model_name)) }}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.default__resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.887635, "supported_languages": null}, "macro.dbt.default__resolve_model_name": {"name": "default__resolve_model_name", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.default__resolve_model_name", "macro_sql": "\n\n{%- macro default__resolve_model_name(input_model_name) -%}\n {{ input_model_name | string | replace('\"', '\\\"') }}\n{%- endmacro -%}\n\n", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.887774, "supported_languages": null}, "macro.dbt.build_ref_function": {"name": "build_ref_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_ref_function", "macro_sql": "{% macro build_ref_function(model) %}\n\n {%- set ref_dict = {} -%}\n {%- for _ref in model.refs -%}\n {% set _ref_args = [_ref.get('package'), _ref['name']] if _ref.get('package') else [_ref['name'],] %}\n {%- set resolved = ref(*_ref_args, v=_ref.get('version')) -%}\n {%- if _ref.get('version') -%}\n {% do _ref_args.extend([\"v\" ~ _ref['version']]) %}\n {%- endif -%}\n {%- do ref_dict.update({_ref_args | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef ref(*args, **kwargs):\n refs = {{ ref_dict | tojson }}\n key = '.'.join(args)\n version = kwargs.get(\"v\") or kwargs.get(\"version\")\n if version:\n key += f\".v{version}\"\n dbt_load_df_function = kwargs.get(\"dbt_load_df_function\")\n return dbt_load_df_function(refs[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.888478, "supported_languages": null}, "macro.dbt.build_source_function": {"name": "build_source_function", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_source_function", "macro_sql": "{% macro build_source_function(model) %}\n\n {%- set source_dict = {} -%}\n {%- for _source in model.sources -%}\n {%- set resolved = source(*_source) -%}\n {%- do source_dict.update({_source | join('.'): resolve_model_name(resolved)}) -%}\n {%- endfor -%}\n\ndef source(*args, dbt_load_df_function):\n sources = {{ source_dict | tojson }}\n key = '.'.join(args)\n return dbt_load_df_function(sources[key])\n\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.resolve_model_name"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8888261, "supported_languages": null}, "macro.dbt.build_config_dict": {"name": "build_config_dict", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.build_config_dict", "macro_sql": "{% macro build_config_dict(model) %}\n {%- set config_dict = {} -%}\n {% set config_dbt_used = zip(model.config.config_keys_used, model.config.config_keys_defaults) | list %}\n {%- for key, default in config_dbt_used -%}\n {# weird type testing with enum, would be much easier to write this logic in Python! #}\n {%- if key == \"language\" -%}\n {%- set value = \"python\" -%}\n {%- endif -%}\n {%- set value = model.config.get(key, default) -%}\n {%- do config_dict.update({key: value}) -%}\n {%- endfor -%}\nconfig_dict = {{ config_dict }}\n{% endmacro %}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8893209, "supported_languages": null}, "macro.dbt.py_script_postfix": {"name": "py_script_postfix", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_postfix", "macro_sql": "{% macro py_script_postfix(model) %}\n# This part is user provided model code\n# you will need to copy the next section to run the code\n# COMMAND ----------\n# this part is dbt logic for get ref work, do not modify\n\n{{ build_ref_function(model ) }}\n{{ build_source_function(model ) }}\n{{ build_config_dict(model) }}\n\nclass config:\n def __init__(self, *args, **kwargs):\n pass\n\n @staticmethod\n def get(key, default=None):\n return config_dict.get(key, default)\n\nclass this:\n \"\"\"dbt.this() or dbt.this.identifier\"\"\"\n database = \"{{ this.database }}\"\n schema = \"{{ this.schema }}\"\n identifier = \"{{ this.identifier }}\"\n {% set this_relation_name = resolve_model_name(this) %}\n def __repr__(self):\n return '{{ this_relation_name }}'\n\n\nclass dbtObj:\n def __init__(self, load_df_function) -> None:\n self.source = lambda *args: source(*args, dbt_load_df_function=load_df_function)\n self.ref = lambda *args, **kwargs: ref(*args, **kwargs, dbt_load_df_function=load_df_function)\n self.config = config\n self.this = this()\n self.is_incremental = {{ is_incremental() }}\n\n# COMMAND ----------\n{{py_script_comment()}}\n{% endmacro %}", "depends_on": {"macros": ["macro.dbt.build_ref_function", "macro.dbt.build_source_function", "macro.dbt.build_config_dict", "macro.dbt.resolve_model_name", "macro.dbt.is_incremental", "macro.dbt.py_script_comment"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8898182, "supported_languages": null}, "macro.dbt.py_script_comment": {"name": "py_script_comment", "resource_type": "macro", "package_name": "dbt", "path": "macros/python_model/python.sql", "original_file_path": "macros/python_model/python.sql", "unique_id": "macro.dbt.py_script_comment", "macro_sql": "{%macro py_script_comment()%}\n{%endmacro%}", "depends_on": {"macros": []}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.889886, "supported_languages": null}, "macro.dbt.test_unique": {"name": "test_unique", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_unique", "macro_sql": "{% test unique(model, column_name) %}\n {% set macro = adapter.dispatch('test_unique', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_unique"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.890347, "supported_languages": null}, "macro.dbt.test_not_null": {"name": "test_not_null", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_not_null", "macro_sql": "{% test not_null(model, column_name) %}\n {% set macro = adapter.dispatch('test_not_null', 'dbt') %}\n {{ macro(model, column_name) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_not_null"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.8905559, "supported_languages": null}, "macro.dbt.test_accepted_values": {"name": "test_accepted_values", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_accepted_values", "macro_sql": "{% test accepted_values(model, column_name, values, quote=True) %}\n {% set macro = adapter.dispatch('test_accepted_values', 'dbt') %}\n {{ macro(model, column_name, values, quote) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_accepted_values"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.890809, "supported_languages": null}, "macro.dbt.test_relationships": {"name": "test_relationships", "resource_type": "macro", "package_name": "dbt", "path": "tests/generic/builtin.sql", "original_file_path": "tests/generic/builtin.sql", "unique_id": "macro.dbt.test_relationships", "macro_sql": "{% test relationships(model, column_name, to, field) %}\n {% set macro = adapter.dispatch('test_relationships', 'dbt') %}\n {{ macro(model, column_name, to, field) }}\n{% endtest %}", "depends_on": {"macros": ["macro.dbt.default__test_relationships"]}, "description": "", "meta": {}, "docs": {"show": true, "node_color": null}, "patch_path": null, "arguments": [], "created_at": 1733486793.891062, "supported_languages": null}}, "docs": {"doc.dbt.__overview__": {"name": "__overview__", "resource_type": "doc", "package_name": "dbt", "path": "overview.md", "original_file_path": "docs/overview.md", "unique_id": "doc.dbt.__overview__", "block_contents": "### Welcome!\n\nWelcome to the auto-generated documentation for your dbt project!\n\n### Navigation\n\nYou can use the `Project` and `Database` navigation tabs on the left side of the window to explore the models\nin your project.\n\n#### Project Tab\nThe `Project` tab mirrors the directory structure of your dbt project. In this tab, you can see all of the\nmodels defined in your dbt project, as well as models imported from dbt packages.\n\n#### Database Tab\nThe `Database` tab also exposes your models, but in a format that looks more like a database explorer. This view\nshows relations (tables and views) grouped into database schemas. Note that ephemeral models are _not_ shown\nin this interface, as they do not exist in the database.\n\n### Graph Exploration\nYou can click the blue icon on the bottom-right corner of the page to view the lineage graph of your models.\n\nOn model pages, you'll see the immediate parents and children of the model you're exploring. By clicking the `Expand`\nbutton at the top-right of this lineage pane, you'll be able to see all of the models that are used to build,\nor are built from, the model you're exploring.\n\nOnce expanded, you'll be able to use the `--select` and `--exclude` model selection syntax to filter the\nmodels in the graph. For more information on model selection, check out the [dbt docs](https://docs.getdbt.com/docs/model-selection-syntax).\n\nNote that you can also right-click on models to interactively filter and explore the graph.\n\n---\n\n### More information\n\n- [What is dbt](https://docs.getdbt.com/docs/introduction)?\n- Read the [dbt viewpoint](https://docs.getdbt.com/docs/viewpoint)\n- [Installation](https://docs.getdbt.com/docs/installation)\n- Join the [dbt Community](https://www.getdbt.com/community/) for questions and discussion"}}, "exposures": {}, "metrics": {}, "groups": {}, "selectors": {}, "disabled": {}, "parent_map": {"model.dbt_demo.diamonds_list_colors": ["model.dbt_demo.diamonds_four_cs"], "model.dbt_demo.diamonds_prices": [], "model.dbt_demo.diamonds_four_cs": ["source.dbt_demo.raw.diamonds"], "model.dbt_demo.zzz_game_details": [], "model.dbt_demo.second_dbt_model": ["model.dbt_demo.first_dbt_model"], "model.dbt_demo.first_dbt_model": [], "test.dbt_demo.zzz_game_details_check_scores": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.zzz_game_details_check_dates": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.zzz_win_loss_records_check_records": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.diamonds_list_colors_test": ["model.dbt_demo.diamonds_list_colors"], "seed.dbt_demo.country_codes": [], "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.unique_first_dbt_model_id.9dda05743e": ["model.dbt_demo.first_dbt_model"], "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061": ["model.dbt_demo.first_dbt_model"], "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4": ["model.dbt_demo.second_dbt_model"], "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2": ["model.dbt_demo.second_dbt_model"], "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f": ["model.dbt_demo.second_dbt_model"], "model.dbt_demo.changed_country": ["snapshot.dbt_demo.country_snapshot"], "model.dbt_demo.country": ["seed.dbt_demo.country_codes"], "snapshot.dbt_demo.country_snapshot": ["model.dbt_demo.country"], "model.dbt_demo.zzz_win_loss_records": ["model.dbt_demo.zzz_game_details"], "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe": ["model.dbt_demo.zzz_win_loss_records"], "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601": ["model.dbt_demo.zzz_win_loss_records"], "source.dbt_demo.raw.diamonds": [], "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors": ["model.dbt_demo.diamonds_list_colors"]}, "child_map": {"model.dbt_demo.diamonds_list_colors": ["test.dbt_demo.diamonds_list_colors_test", "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors"], "model.dbt_demo.diamonds_prices": [], "model.dbt_demo.diamonds_four_cs": ["model.dbt_demo.diamonds_list_colors"], "model.dbt_demo.zzz_game_details": ["model.dbt_demo.zzz_win_loss_records", "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40", "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d", "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c", "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68", "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905", "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5", "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39", "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e", "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f", "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be", "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218", "test.dbt_demo.zzz_game_details_check_dates", "test.dbt_demo.zzz_game_details_check_scores"], "model.dbt_demo.second_dbt_model": ["test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f", "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2", "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4"], "model.dbt_demo.first_dbt_model": ["model.dbt_demo.second_dbt_model", "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061", "test.dbt_demo.unique_first_dbt_model_id.9dda05743e"], "test.dbt_demo.zzz_game_details_check_scores": [], "test.dbt_demo.zzz_game_details_check_dates": [], "test.dbt_demo.zzz_win_loss_records_check_records": [], "test.dbt_demo.diamonds_list_colors_test": [], "seed.dbt_demo.country_codes": ["model.dbt_demo.country"], "test.dbt_demo.unique_zzz_game_details_game_id.80431e7218": [], "test.dbt_demo.not_null_zzz_game_details_game_id.3b5747f905": [], "test.dbt_demo.not_null_zzz_game_details_home.a99d25edd5": [], "test.dbt_demo.accepted_values_zzz_game_details_home__Amsterdam__San_Francisco__Seattle.3771faef40": [], "test.dbt_demo.not_null_zzz_game_details_visitor.990a91372e": [], "test.dbt_demo.accepted_values_zzz_game_details_visitor__Amsterdam__San_Francisco__Seattle.f0c81c502d": [], "test.dbt_demo.not_null_zzz_game_details_home_score.8c55a8fe39": [], "test.dbt_demo.not_null_zzz_game_details_visitor_score.f31e89be7f": [], "test.dbt_demo.not_null_zzz_game_details_winner.7f08e737be": [], "test.dbt_demo.accepted_values_zzz_game_details_winner__Amsterdam__San_Francisco__Seattle.344bd88c4c": [], "test.dbt_demo.not_null_zzz_game_details_date.a9f3a56f68": [], "test.dbt_demo.unique_first_dbt_model_id.9dda05743e": [], "test.dbt_demo.not_null_first_dbt_model_id.bd8d25d061": [], "test.dbt_demo.unique_second_dbt_model_id.3d7f6966a4": [], "test.dbt_demo.not_null_second_dbt_model_id.c791c712a2": [], "test.dbt_demo.not_null_second_dbt_model_amount_usd.28ae55e16f": [], "model.dbt_demo.changed_country": [], "model.dbt_demo.country": ["snapshot.dbt_demo.country_snapshot"], "snapshot.dbt_demo.country_snapshot": ["model.dbt_demo.changed_country"], "model.dbt_demo.zzz_win_loss_records": ["test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601", "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86", "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe", "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105", "test.dbt_demo.zzz_win_loss_records_check_records"], "test.dbt_demo.unique_zzz_win_loss_records_team.b989062105": [], "test.dbt_demo.not_null_zzz_win_loss_records_team.9a18f33f86": [], "test.dbt_demo.not_null_zzz_win_loss_records_wins.f6815357fe": [], "test.dbt_demo.not_null_zzz_win_loss_records_losses.82bf4ea601": [], "source.dbt_demo.raw.diamonds": ["model.dbt_demo.diamonds_four_cs"], "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors": []}, "group_map": {}, "saved_queries": {}, "semantic_models": {}, "unit_tests": {"unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors": {"model": "diamonds_list_colors", "given": [{"input": "ref('diamonds_four_cs')", "rows": [{"carat": 0.23, "cut": "Ideal", "color": "E", "clarity": "SI2"}, {"carat": 0.3, "cut": "Premium", "color": "E", "clarity": "SI2"}, {"carat": 0.3, "cut": "Premium", "color": "K", "clarity": "SI2"}], "format": "dict", "fixture": null}], "expect": {"rows": [{"color": "E"}, {"color": "K"}], "format": "dict", "fixture": null}, "name": "test_get_distinct_colors", "resource_type": "unit_test", "package_name": "dbt_demo", "path": "sql_model1/test_get_distinct_colors.yml", "original_file_path": "src/models/sql_model1/test_get_distinct_colors.yml", "unique_id": "unit_test.dbt_demo.diamonds_list_colors.test_get_distinct_colors", "fqn": ["dbt_demo", "models", "sql_model1", "diamonds_list_colors", "test_get_distinct_colors"], "description": "Check a distinct list of colors is returned.", "overrides": null, "depends_on": {"macros": [], "nodes": ["model.dbt_demo.diamonds_list_colors"]}, "config": {"tags": [], "meta": {}}, "checksum": "483c13eb849bf29c60173f6b4583def3c4a3b8c20e674a28c0db2c3e81834bb1", "schema": "dbt_dev", "created_at": 1733486794.4994261, "versions": null, "version": null}}} \ No newline at end of file diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_dbt_factory.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_dbt_factory.py new file mode 100644 index 00000000..8acbcbbe --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_dbt_factory.py @@ -0,0 +1,469 @@ +import os +from tempfile import NamedTemporaryFile +from pathlib import Path +import pytest +import yaml + + +BASE_PATH = str(Path(__file__).resolve().parent) + + +def _model( + package: str, name: str, depends_on: list[str] | None = None +) -> tuple[str, dict]: + full_name = f"model.{package}.{name}" + return full_name, { + "resource_type": "model", + "name": name, + "package_name": package, + "depends_on": {"nodes": depends_on or []}, + } + + +def _test( + package: str, name: str, depends_on: list[str], severity: str = "error" +) -> tuple[str, dict]: + full_name = f"test.{package}.{name}" + return full_name, { + "resource_type": "test", + "name": name, + "package_name": package, + "depends_on": {"nodes": depends_on}, + "config": {"severity": severity}, + } + + +def _seed(package: str, name: str) -> tuple[str, dict]: + full_name = f"seed.{package}.{name}" + return full_name, { + "resource_type": "seed", + "name": name, + "package_name": package, + "depends_on": {"nodes": []}, + } + + +def _snapshot( + package: str, name: str, depends_on: list[str] | None = None +) -> tuple[str, dict]: + full_name = f"snapshot.{package}.{name}" + return full_name, { + "resource_type": "snapshot", + "name": name, + "package_name": package, + "depends_on": {"nodes": depends_on or []}, + } + + +def _source(package: str, source_name: str, table: str) -> tuple[str, dict]: + full_name = f"source.{package}.{source_name}.{table}" + return full_name, { + "resource_type": "source", + "name": table, + "source_name": source_name, + "package_name": package, + } + + +def test_same_model_name_across_packages_produces_distinct_bundled_test_tasks( + dbt_factory_bundled, +): + nodes = dict( + [ + _model("pkg_a", "customers"), + _model("pkg_b", "customers"), + _model( + "pkg_a", + "orders", + depends_on=["model.pkg_a.customers", "model.pkg_b.customers"], + ), + _test("pkg_a", "unique_customers_id", ["model.pkg_a.customers"]), + _test("pkg_b", "not_null_customers_id", ["model.pkg_b.customers"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "tests_model_pkg_a_customers" in by_key + assert "tests_model_pkg_b_customers" in by_key + assert by_key["tests_model_pkg_a_customers"]["dbt_task"]["commands"] == [ + "dbt test --select pkg_a.customers --indirect-selection cautious --target dev" + ] + assert by_key["tests_model_pkg_b_customers"]["dbt_task"]["commands"] == [ + "dbt test --select pkg_b.customers --indirect-selection cautious --target dev" + ] + assert by_key["tests_model_pkg_a_customers"]["depends_on"] == [ + {"task_key": "model_pkg_a_customers"} + ] + assert by_key["tests_model_pkg_b_customers"]["depends_on"] == [ + {"task_key": "model_pkg_b_customers"} + ] + + assert {dep["task_key"] for dep in by_key["model_pkg_a_orders"]["depends_on"]} == { + "tests_model_pkg_a_customers", + "tests_model_pkg_b_customers", + } + + +def test_tests_on_seed_produce_task_and_gate_downstream(dbt_factory_bundled): + nodes = dict( + [ + _seed("pkg", "countries"), + _model("pkg", "enriched", depends_on=["seed.pkg.countries"]), + _test("pkg", "unique_countries_code", ["seed.pkg.countries"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "tests_seed_pkg_countries" in by_key + assert by_key["tests_seed_pkg_countries"]["dbt_task"]["commands"] == [ + "dbt test --select pkg.countries --indirect-selection cautious --target dev" + ] + assert by_key["tests_seed_pkg_countries"]["depends_on"] == [ + {"task_key": "seed_pkg_countries"} + ] + assert by_key["model_pkg_enriched"]["depends_on"] == [ + {"task_key": "tests_seed_pkg_countries"} + ] + + +def test_tests_on_snapshot_produce_task_and_gate_downstream(dbt_factory_bundled): + nodes = dict( + [ + _snapshot("pkg", "orders_snap"), + _model("pkg", "orders_history", depends_on=["snapshot.pkg.orders_snap"]), + _test("pkg", "not_null_orders_snap_id", ["snapshot.pkg.orders_snap"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "tests_snapshot_pkg_orders_snap" in by_key + assert by_key["tests_snapshot_pkg_orders_snap"]["dbt_task"]["commands"] == [ + "dbt test --select pkg.orders_snap --indirect-selection cautious --target dev" + ] + assert by_key["tests_snapshot_pkg_orders_snap"]["depends_on"] == [ + {"task_key": "snapshot_pkg_orders_snap"} + ] + assert by_key["model_pkg_orders_history"]["depends_on"] == [ + {"task_key": "tests_snapshot_pkg_orders_snap"} + ] + + +def test_tests_on_source_produce_standalone_task(dbt_factory_bundled): + nodes = dict( + [ + _test("pkg", "unique_raw_customers_id", ["source.pkg.raw.customers"]), + ] + ) + sources = dict([_source("pkg", "raw", "customers")]) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes, "sources": sources}) + by_key = {t["task_key"]: t for t in tasks} + + assert "tests_source_pkg_raw_customers" in by_key + assert by_key["tests_source_pkg_raw_customers"]["dbt_task"]["commands"] == [ + "dbt test --select source:pkg.raw.customers --indirect-selection cautious --target dev" + ] + assert by_key["tests_source_pkg_raw_customers"]["depends_on"] == [] + + +def test_flat_mode_emits_one_task_per_test_node_and_gates_downstream(dbt_factory): + # Per-test mode mirrors `dbt build`: downstream models wait on upstream tests, so a + # failing `severity: error` test skips downstream via Databricks task failure. + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "orders", depends_on=["model.pkg.customers"]), + _test("pkg", "unique_customers_id", ["model.pkg.customers"]), + _test("pkg", "not_null_customers_id", ["model.pkg.customers"]), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "test_pkg_unique_customers_id" in by_key + assert "test_pkg_not_null_customers_id" in by_key + assert "tests_model_pkg_customers" not in by_key + + assert by_key["test_pkg_unique_customers_id"]["dbt_task"]["commands"] == [ + "dbt test --select unique_customers_id --target dev" + ] + assert by_key["test_pkg_unique_customers_id"]["depends_on"] == [ + {"task_key": "model_pkg_customers"} + ] + # orders depends on customers AND every test attached to customers + assert {dep["task_key"] for dep in by_key["model_pkg_orders"]["depends_on"]} == { + "model_pkg_customers", + "test_pkg_unique_customers_id", + "test_pkg_not_null_customers_id", + } + + +def test_flat_mode_cross_model_test_does_not_create_cycle(dbt_factory): + # Relationship test references BOTH `orders` and `customers`. Without care, extending + # `orders`'s deps with "tests of upstream (customers)" would pull in the relationship test, + # which itself depends on `orders` — a direct cycle. + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "orders", depends_on=["model.pkg.customers"]), + _model("pkg", "payments", depends_on=["model.pkg.orders"]), + _test("pkg", "unique_customers_id", ["model.pkg.customers"]), + _test( + "pkg", + "relationships_orders_customer_id__ref_customers", + ["model.pkg.orders", "model.pkg.customers"], + ), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + # orders depends on customers + unique_customers_id, but NOT on the relationship test + # (that test references orders itself — including it would cycle) + assert {dep["task_key"] for dep in by_key["model_pkg_orders"]["depends_on"]} == { + "model_pkg_customers", + "test_pkg_unique_customers_id", + } + + # payments (downstream of orders) picks up the relationship test — safe, payments + # transitively depends on both orders and customers (the test's refs) + payments_deps = { + dep["task_key"] for dep in by_key["model_pkg_payments"]["depends_on"] + } + assert "model_pkg_orders" in payments_deps + assert "test_pkg_relationships_orders_customer_id__ref_customers" in payments_deps + + +def test_flat_mode_transitive_cross_model_test_does_not_create_cycle(dbt_factory): + # Transitive cycle case: test T refs {A, C} where C is downstream of B which is downstream + # of A. Extending B's deps with "tests of upstream (A)" must NOT add T, because T depends + # on C and C depends on B → B → T → C → B cycle. Only nodes downstream of both A and C + # (i.e. downstream of C) should get T. + nodes = dict( + [ + _model("pkg", "a"), + _model("pkg", "b", depends_on=["model.pkg.a"]), + _model("pkg", "c", depends_on=["model.pkg.b"]), + _model("pkg", "d", depends_on=["model.pkg.c"]), + _test("pkg", "relationship_a_c", ["model.pkg.a", "model.pkg.c"]), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + # B's ancestors = {A}. Test T refs = {A, C}. C ∉ ancestors(B) → skip T. + assert by_key["model_pkg_b"]["depends_on"] == [{"task_key": "model_pkg_a"}] + # C's ancestors = {A, B}. C IS in T.refs → skip T (direct self-reference). + assert by_key["model_pkg_c"]["depends_on"] == [{"task_key": "model_pkg_b"}] + # D's ancestors = {A, B, C}. T.refs = {A, C} ⊆ ancestors(D) → add T. + d_deps = {dep["task_key"] for dep in by_key["model_pkg_d"]["depends_on"]} + assert d_deps == {"model_pkg_c", "test_pkg_relationship_a_c"} + + +def test_flat_mode_warn_severity_tests_do_not_gate_downstream(dbt_factory): + # Only error-severity tests gate downstream (matches `dbt build`: dbt exits 0 on warn + # so a gate wouldn't block anyway; we just keep the DAG cleaner). + nodes = dict( + [ + _model("pkg", "customers"), + _model("pkg", "orders", depends_on=["model.pkg.customers"]), + _test( + "pkg", "unique_customers_id", ["model.pkg.customers"], severity="warn" + ), + _test( + "pkg", + "not_null_customers_id", + ["model.pkg.customers"], + severity="error", + ), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + # Both test tasks still exist (warn tests still run — they just don't gate anything) + assert "test_pkg_unique_customers_id" in by_key + assert "test_pkg_not_null_customers_id" in by_key + + # orders gates on customers + the error-severity test, but NOT the warn-severity one + assert {dep["task_key"] for dep in by_key["model_pkg_orders"]["depends_on"]} == { + "model_pkg_customers", + "test_pkg_not_null_customers_id", + } + + +def test_flat_mode_test_on_seed_gates_on_seed(dbt_factory): + nodes = dict( + [ + _seed("pkg", "countries"), + _test("pkg", "unique_countries_code", ["seed.pkg.countries"]), + ] + ) + + tasks = dbt_factory.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert by_key["test_pkg_unique_countries_code"]["depends_on"] == [ + {"task_key": "seed_pkg_countries"} + ] + + +def test_bundled_task_factory_assembles_commands(dbt_factory_bundled): + test_factory = dbt_factory_bundled.task_factories["test"] + task = test_factory.create_bundled_task( + task_key="tests_model_pkg_customers", + select="pkg.customers", + deps_command_name="customers", + depends_on=["model_pkg_customers"], + ) + assert task.task_key == "tests_model_pkg_customers" + assert task.commands == [ + "dbt test --select pkg.customers --indirect-selection cautious --target dev" + ] + assert task.depends_on == ["model_pkg_customers"] + + +def test_cross_model_test_in_bundled_mode_is_emitted_as_standalone_task( + dbt_factory_bundled, +): + # The relationship test spans two models, so it must NOT be collapsed into either model's + # bundled `_tests` task (dbt would hit a TABLE_OR_VIEW_NOT_FOUND on the un-built endpoint). + # It should emit its own task with deps on both referenced models. + nodes = dict( + [ + _model("pkg", "team_cities"), + _model("pkg", "game_details", depends_on=["model.pkg.team_cities"]), + _test("pkg", "not_null_team_cities_name", ["model.pkg.team_cities"]), + _test( + "pkg", + "relationships_game_details_winner__team_city__ref_team_cities_", + ["model.pkg.game_details", "model.pkg.team_cities"], + ), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + # Single-model test → bundled with cautious selection (relationship test is excluded by dbt) + assert "tests_model_pkg_team_cities" in by_key + assert by_key["tests_model_pkg_team_cities"]["dbt_task"]["commands"] == [ + "dbt test --select pkg.team_cities --indirect-selection cautious --target dev" + ] + + # Cross-model test → its own task, gated on BOTH referenced models + cross_test_key = ( + "test_pkg_relationships_game_details_winner__team_city__ref_team_cities_" + ) + assert cross_test_key in by_key + assert by_key[cross_test_key]["dbt_task"]["commands"] == [ + "dbt test --select relationships_game_details_winner__team_city__ref_team_cities_ --target dev" + ] + assert {dep["task_key"] for dep in by_key[cross_test_key]["depends_on"]} == { + "model_pkg_team_cities", + "model_pkg_game_details", + } + + # `game_details` has no single-model tests, so no bundled `game_details_tests` exists + assert "tests_model_pkg_game_details" not in by_key + + +def test_single_package_bundled_test_uses_qualified_select(dbt_factory_bundled): + nodes = dict( + [ + _model("pkg_a", "customers"), + _model("pkg_a", "orders", depends_on=["model.pkg_a.customers"]), + _test("pkg_a", "unique_customers_id", ["model.pkg_a.customers"]), + ] + ) + + tasks = dbt_factory_bundled.create_tasks({"nodes": nodes}) + by_key = {t["task_key"]: t for t in tasks} + + assert "tests_model_pkg_a_customers" in by_key + assert by_key["tests_model_pkg_a_customers"]["dbt_task"]["commands"] == [ + "dbt test --select pkg_a.customers --indirect-selection cautious --target dev" + ] + assert by_key["model_pkg_a_orders"]["depends_on"] == [ + {"task_key": "tests_model_pkg_a_customers"} + ] + + +def test_create_job_spec_and_update(dbt_factory): + run_job_spec_test( + dbt_factory, + BASE_PATH + "/test_data/job_definition_no_deps.yaml", + ) + + +def test_create_job_spec_and_update_with_dbt_deps(dbt_factory_with_deps): + run_job_spec_test( + dbt_factory_with_deps, + BASE_PATH + "/test_data/job_definition.yaml", + ) + + +def test_create_job_spec_and_update_with_selected_dbt_deps( + dbt_factory_with_deps_selected, +): + run_job_spec_test( + dbt_factory_with_deps_selected, + BASE_PATH + "/test_data/job_definition_deps_selected.yaml", + ) + + +def test_notebook_task_generation(notebook_factory): + run_job_spec_test( + notebook_factory, + BASE_PATH + "/test_data/job_definition_notebook_no_deps.yaml", + ) + + +def run_job_spec_test(dbt_factory, expected_job_definition_path): + """Helper function to test databricks job definition generation.""" + dbt_manifest_path = BASE_PATH + "/test_data/manifest.json" + input_job_definition_path = BASE_PATH + "/test_data/job_definition_template.yaml" + + with NamedTemporaryFile(suffix=".yaml", delete=False) as temp_file: + actual_job_definition_path = temp_file.name + + try: + dbt_factory.create_tasks_and_update_job_spec( + dbt_manifest_path, input_job_definition_path, actual_job_definition_path + ) + + with open(expected_job_definition_path, "r", encoding="utf-8") as file: + expected_job_definition = yaml.safe_load(file) + + with open(actual_job_definition_path, "r", encoding="utf-8") as file: + job_definition = yaml.safe_load(file) + + assert job_definition == expected_job_definition + finally: + if os.path.exists(actual_job_definition_path): + os.remove(actual_job_definition_path) + + +@pytest.mark.skip("Manual testing") +def test_generate(databricks_dbt_factory): + """Test job definition generation and saving to file.""" + dbt_manifest_path = BASE_PATH + "/test_data/manifest.json" + job_definition_path = BASE_PATH + "/test_data/job_definition_template.yaml" + destination_job_definition_path = "job_definition.yaml" + + databricks_dbt_factory.create_tasks_and_update_job_spec( + dbt_manifest_path, + job_definition_path, + destination_job_definition_path, + "new_job_name", + ) diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_dbt_task.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_dbt_task.py new file mode 100644 index 00000000..0975fb41 --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_dbt_task.py @@ -0,0 +1,102 @@ +import pytest + +from databricks_dbt_factory.DbtTask import DbtTask, DbtTaskOptions, TaskType + + +def test_notebook_task_with_job_cluster_key(): + options = DbtTaskOptions( + task_type="notebook", + notebook_path="./notebooks/dbt_runner.py", + source="WORKSPACE", + project_directory="/project", + profiles_directory="/profiles", + job_cluster_key="dbt_cluster", + ) + task = DbtTask( + task_key="my_model", + commands=["dbt run --select my_model --target dev"], + options=options, + depends_on=["upstream_model"], + ) + + result = task.to_dict() + + assert result["job_cluster_key"] == "dbt_cluster" + assert "environment_key" not in result + assert result["task_key"] == "my_model" + assert result["depends_on"] == [{"task_key": "upstream_model"}] + assert result["notebook_task"]["source"] == "WORKSPACE" + assert result["notebook_task"]["base_parameters"]["project_directory"] == "/project" + assert ( + result["notebook_task"]["base_parameters"]["profiles_directory"] == "/profiles" + ) + + +def test_notebook_task_without_job_cluster_key_uses_environment(): + options = DbtTaskOptions( + task_type="notebook", + notebook_path="./notebooks/dbt_runner.py", + ) + task = DbtTask( + task_key="my_model", commands=["dbt run --select my_model"], options=options + ) + + result = task.to_dict() + + assert result["environment_key"] == "Default" + assert "job_cluster_key" not in result + + +def test_dbt_task_without_job_cluster_key_uses_environment(): + options = DbtTaskOptions(environment_key="Default") + task = DbtTask( + task_key="my_model", commands=["dbt run --select my_model"], options=options + ) + + result = task.to_dict() + + assert result["environment_key"] == "Default" + assert "job_cluster_key" not in result + + +def test_task_type_string_is_coerced_to_enum(): + options = DbtTaskOptions(task_type="notebook", notebook_path="./runner.py") + assert options.task_type is TaskType.NOTEBOOK + + +def test_task_type_invalid_value_raises(): + with pytest.raises(ValueError, match="not a valid TaskType"): + DbtTaskOptions(task_type="Notebook") + with pytest.raises(ValueError, match="not a valid TaskType"): + DbtTaskOptions(task_type="dbt_task") + + +def test_notebook_task_rejects_warehouse_schema_catalog(): + for kwargs in ( + {"warehouse_id": "wh123"}, + {"schema": "silver"}, + {"catalog": "main"}, + ): + with pytest.raises(ValueError, match="notebook tasks connect via profiles.yml"): + DbtTaskOptions(task_type=TaskType.NOTEBOOK, notebook_path="/n", **kwargs) + + +def test_notebook_task_rejects_multiple_incompatible_fields_at_once(): + with pytest.raises(ValueError, match=r"warehouse_id, schema, catalog"): + DbtTaskOptions( + task_type=TaskType.NOTEBOOK, + notebook_path="/n", + warehouse_id="wh123", + schema="silver", + catalog="main", + ) + + +def test_dbt_task_still_accepts_warehouse_schema_catalog(): + options = DbtTaskOptions( + task_type=TaskType.DBT, + warehouse_id="wh123", + schema="silver", + catalog="main", + ) + assert options.warehouse_id == "wh123" diff --git a/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_load_resources.py b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_load_resources.py new file mode 100644 index 00000000..2d31c6db --- /dev/null +++ b/contrib/templates/dbt-factory/template/{{.project_name}}/tests/test_load_resources.py @@ -0,0 +1,70 @@ +""" +Offline test for the PyDABs integration in resources/__init__.py. + +Exercises the whole glue path against the committed dbt manifest (no Databricks workspace +required): reading the manifest, wiring the factory, generating notebook tasks, and registering +the job on a Resources object (which runs Job.from_dict, validating the task dict schema). +""" + +import json +from pathlib import Path + +import pytest +from databricks.bundles.core import Bundle + +import resources + +PROJECT_ROOT = Path(__file__).resolve().parent.parent + +# The manifest is generated by `make manifest` (dbt parse). In a freshly initialized project it +# does not exist yet, so skip these tests rather than erroring until the user has run it. +pytestmark = pytest.mark.skipif( + not (PROJECT_ROOT / resources.MANIFEST_PATH).exists(), + reason="dbt manifest not found; run `make manifest` (dbt parse) first", +) + + +@pytest.fixture(autouse=True) +def _chdir_to_project_root(monkeypatch): + # resources.MANIFEST_PATH is relative to the bundle root, as it is at deploy time. + monkeypatch.chdir(PROJECT_ROOT) + + +def test_build_tasks_covers_every_buildable_node(): + # Every model/seed/snapshot node gets its own task regardless of the BUNDLE_TESTS mode + # (tests may be collapsed into per-resource `tests_*` tasks when bundling is on). + tasks = resources._build_tasks("dev") + task_keys = {task["task_key"] for task in tasks} + + manifest = json.loads((PROJECT_ROOT / resources.MANIFEST_PATH).read_text()) + buildable = { + name + for name, node in manifest.get("nodes", {}).items() + if node.get("resource_type") in {"model", "seed", "snapshot"} + } + expected_keys = {name.replace(".", "_") for name in buildable} + + assert buildable, "committed manifest has no model/seed/snapshot nodes" + assert expected_keys <= task_keys + + +def test_generated_tasks_are_serverless_notebook_tasks(): + tasks = resources._build_tasks("dev") + + for task in tasks: + assert "notebook_task" in task, task + assert task["environment_key"] == resources.ENVIRONMENT_KEY + assert "job_cluster_key" not in task # serverless only + notebook_task = task["notebook_task"] + assert notebook_task["notebook_path"] == resources.RUNNER_NOTEBOOK_PATH + # The dbt target is injected into every command. + assert "--target dev" in notebook_task["base_parameters"]["dbt_commands"] + + +def test_load_resources_registers_the_job(): + result = resources.load_resources(Bundle(target="dev")) + + assert resources.JOB_NAME in result.jobs + job = result.jobs[resources.JOB_NAME] + # Job.from_dict succeeded and the environment carries the dbt-databricks dependency. + assert job.environments[0].environment_key == resources.ENVIRONMENT_KEY