From 3ed77be7a870f4ef47b0becc22faa4c9f603fbef Mon Sep 17 00:00:00 2001 From: Samson Gebre Date: Tue, 5 May 2026 20:53:50 -0700 Subject: [PATCH 01/16] Add unit tests for records.list_pages() and migration script for v0 to v1 - Implemented unit tests for the `list_pages` method in `TestListPages` class, covering various scenarios including iterator return, page content validation, and parameter passing. - Added checks for deprecation warnings to ensure no warnings are raised during the usage of `list_pages`. - Introduced a new migration script `migrate_v0_to_v1.py` to automate the transition from beta (v0) to GA (v1) API calls, including method renaming and argument adjustments. - Created a new `tools` directory to house the migration script. --- .claude/skills/dataverse-sdk-use/SKILL.md | 64 +- README.md | 187 ++-- examples/advanced/alternate_keys_upsert.py | 51 +- examples/advanced/batch.py | 424 ++++----- examples/advanced/dataframe_operations.py | 19 +- .../advanced/datascience_risk_assessment.py | 46 +- examples/advanced/fetch_xml.py | 578 ++++++++++++ examples/advanced/prodev_quick_start.py | 47 +- examples/advanced/sql_examples.py | 200 ++-- examples/advanced/walkthrough.py | 68 +- examples/basic/functional_testing.py | 173 ++-- examples/basic/installation_example.py | 8 +- pyproject.toml | 1 + src/PowerPlatform/Dataverse/__init__.py | 6 +- .../claude_skill/dataverse-sdk-use/SKILL.md | 64 +- src/PowerPlatform/Dataverse/client.py | 578 +----------- src/PowerPlatform/Dataverse/data/_batch.py | 22 + src/PowerPlatform/Dataverse/data/_odata.py | 25 + .../Dataverse/models/__init__.py | 10 +- .../Dataverse/models/fetch_xml_query.py | 121 +++ src/PowerPlatform/Dataverse/models/filters.py | 532 ++++++++--- .../Dataverse/models/protocol.py | 69 ++ .../Dataverse/models/query_builder.py | 498 ++++------ src/PowerPlatform/Dataverse/models/record.py | 52 +- .../Dataverse/operations/batch.py | 96 +- .../Dataverse/operations/dataframe.py | 44 +- .../Dataverse/operations/query.py | 246 ++--- .../Dataverse/operations/records.py | 235 ++++- tests/unit/models/test_query_builder.py | 720 +++++++-------- tests/unit/test_client.py | 198 ++-- tests/unit/test_client_deprecations.py | 291 ++---- tests/unit/test_phase1_ga.py | 861 ++++++++++++++++++ tests/unit/test_phase2_ga.py | 415 +++++++++ tests/unit/test_phase3_ga.py | 424 +++++++++ tests/unit/test_phase4_ga.py | 416 +++++++++ tests/unit/test_query_operations.py | 392 +++----- tests/unit/test_records_operations.py | 60 ++ tools/__init__.py | 0 tools/migrate_v0_to_v1.py | 667 ++++++++++++++ 39 files changed, 6059 insertions(+), 2849 deletions(-) create mode 100644 examples/advanced/fetch_xml.py create mode 100644 src/PowerPlatform/Dataverse/models/fetch_xml_query.py create mode 100644 src/PowerPlatform/Dataverse/models/protocol.py create mode 100644 tests/unit/test_phase1_ga.py create mode 100644 tests/unit/test_phase2_ga.py create mode 100644 tests/unit/test_phase3_ga.py create mode 100644 tests/unit/test_phase4_ga.py create mode 100644 tools/__init__.py create mode 100644 tools/migrate_v0_to_v1.py diff --git a/.claude/skills/dataverse-sdk-use/SKILL.md b/.claude/skills/dataverse-sdk-use/SKILL.md index 72677468..bda1ed14 100644 --- a/.claude/skills/dataverse-sdk-use/SKILL.md +++ b/.claude/skills/dataverse-sdk-use/SKILL.md @@ -32,7 +32,7 @@ The SDK supports Dataverse's native bulk operations: Pass lists to `create()`, ` - Use `top` parameter to limit total records returned ### DataFrame Support -- DataFrame operations are accessed via the `client.dataframe` namespace: `client.dataframe.get()`, `client.dataframe.create()`, `client.dataframe.update()`, `client.dataframe.delete()` +- DataFrame operations are accessed via the `client.dataframe` namespace: `client.dataframe.create()`, `client.dataframe.update()`, `client.dataframe.delete()` — `client.dataframe.get()` is deprecated; use `client.query.builder(table).where(...).execute().to_dataframe()` instead ## Common Operations @@ -85,28 +85,40 @@ contact_ids = client.records.create("contact", contacts) #### Read Records ```python # Get single record by ID -account = client.records.get("account", account_id, select=["name", "telephone1"]) +account = client.records.retrieve("account", account_id, select=["name", "telephone1"]) -# Query with filter (paginated) -for page in client.records.get( +# Query with filter — follows @odata.nextLink automatically (multiple HTTP requests if needed), +# loads all matching records into memory, returns a single QueryResult. +# Page size is Dataverse's default (~5000/page); use top to bound total records and round-trips. +# For very large sets where memory is a concern, use query.builder().execute(by_page=True) instead. +result = client.records.list( "account", select=["accountid", "name"], # select is case-insensitive (automatically lowercased) filter="statecode eq 0", # filter must use lowercase logical names (not transformed) - top=100, -): + top=100, # bounds both total records returned and HTTP round-trips +) +for record in result: + print(record["name"]) + +# For large result sets that must be streamed page-by-page (caller controls memory, one page at a time) +for page in (client.query.builder("account") + .select("accountid", "name") + .where(col("statecode") == 0) + .page_size(500) # optional: override Dataverse default page size + .execute(by_page=True)): for record in page: print(record["name"]) -# Query with navigation property expansion (case-sensitive!) -for page in client.records.get( - "account", - select=["name"], - expand=["primarycontactid"], # Navigation properties are case-sensitive! - filter="statecode eq 0", # Column names must be lowercase logical names -): - for account in page: - contact = account.get("primarycontactid", {}) - print(f"{account['name']} - {contact.get('fullname', 'N/A')}") +# Query with navigation property expansion — use the query builder (records.list() has no expand) +from PowerPlatform.Dataverse.models.query_builder import ExpandOption +from PowerPlatform.Dataverse.models.filters import col +for record in (client.query.builder("account") + .select("name") + .expand(ExpandOption("primarycontactid").select("fullname")) + .where(col("statecode") == 0) + .execute()): + contact = record.get("primarycontactid", {}) + print(f"{record['name']} - {contact.get('fullname', 'N/A')}") ``` #### Create Records with Lookup Bindings (@odata.bind) @@ -179,18 +191,20 @@ client.records.delete("account", [id1, id2, id3], use_bulk_delete=True) The SDK provides DataFrame wrappers for all CRUD operations via the `client.dataframe` namespace, using pandas DataFrames and Series as input/output. +> **Note:** `client.dataframe.get()` is deprecated. Use `client.query.builder(table).select(...).filter(...).to_dataframe()` instead. + ```python import pandas as pd -# Query records -- returns a single DataFrame -df = client.dataframe.get("account", filter="statecode eq 0", select=["name"]) +# Query records -- returns a single DataFrame (GA builder pattern) +df = client.query.builder("account").filter("statecode eq 0").select("name").to_dataframe() print(f"Got {len(df)} rows") -# Limit results with top for large tables -df = client.dataframe.get("account", select=["name"], top=100) +# Limit results with top +df = client.query.builder("account").select("name").top(100).to_dataframe() # Fetch single record as one-row DataFrame -df = client.dataframe.get("account", record_id=account_id, select=["name"]) +df = client.records.retrieve("account", account_id, select=["name"]).to_dataframe() # Create records from a DataFrame (returns a Series of GUIDs) new_accounts = pd.DataFrame([ @@ -380,7 +394,8 @@ Use `client.batch` to send multiple operations in one HTTP request. All batch me batch = client.batch.new() batch.records.create("account", {"name": "Contoso"}) batch.records.update("account", account_id, {"telephone1": "555-0100"}) -batch.records.get("account", account_id, select=["name"]) +batch.records.retrieve("account", account_id, select=["name"]) # single record (GA) +batch.records.list("account", filter="statecode eq 0", top=50) # multi-record, single page batch.query.sql("SELECT TOP 5 name FROM account") result = batch.execute() @@ -412,7 +427,8 @@ print(f"Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") **Batch limitations:** - Maximum 1000 operations per batch -- Paginated `records.get()` (without `record_id`) is not supported in batch +- `batch.records.get()` is deprecated; use `batch.records.retrieve()` for single records +- `batch.records.list()` returns a single page (no pagination); use `top` to bound results - `flush_cache()` is not supported in batch ## Error Handling @@ -430,7 +446,7 @@ from PowerPlatform.Dataverse.core.errors import ( from PowerPlatform.Dataverse.client import DataverseClient try: - client.records.get("account", "invalid-id") + client.records.retrieve("account", "invalid-id") except HttpError as e: print(f"HTTP {e.status_code}: {e.message}") print(f"Error code: {e.code}") diff --git a/README.md b/README.md index eab08734..1f077b2d 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,7 @@ with DataverseClient("https://yourorg.crm.dynamics.com", credential) as client: contact_id = client.records.create("contact", {"firstname": "John", "lastname": "Doe"}) # Read the contact back - contact = client.records.get("contact", contact_id, select=["firstname", "lastname"]) + contact = client.records.retrieve("contact", contact_id, select=["firstname", "lastname"]) print(f"Created: {contact['firstname']} {contact['lastname']}") # Clean up @@ -159,7 +159,7 @@ with DataverseClient("https://yourorg.crm.dynamics.com", credential) as client: account_id = client.records.create("account", {"name": "Contoso Ltd"}) # Read a record -account = client.records.get("account", account_id) +account = client.records.retrieve("account", account_id) print(account["name"]) # Update a record @@ -242,18 +242,25 @@ client.records.upsert("account", [ The SDK provides pandas wrappers for all CRUD operations via the `client.dataframe` namespace, using DataFrames and Series for input and output. +> **Note:** `client.dataframe.get()` is deprecated. Use the GA patterns shown below instead. + ```python import pandas as pd +from PowerPlatform.Dataverse.models.filters import col -# Query records as a single DataFrame -df = client.dataframe.get("account", filter="statecode eq 0", select=["name", "telephone1"]) +# Query records as a single DataFrame (GA builder pattern) +df = (client.query.builder("account") + .select("name", "telephone1") + .where(col("statecode") == 0) + .execute() + .to_dataframe()) print(f"Found {len(df)} accounts") # Limit results with top for large tables -df = client.dataframe.get("account", select=["name"], top=100) +df = client.query.builder("account").select("name").top(100).execute().to_dataframe() # Fetch a single record as a one-row DataFrame -df = client.dataframe.get("account", record_id=account_id, select=["name"]) +df = client.records.retrieve("account", account_id, select=["name"]).to_dataframe() # Create records from a DataFrame (returns a Series of GUIDs) new_accounts = pd.DataFrame([ @@ -288,10 +295,12 @@ The **QueryBuilder** is the recommended way to query records. It provides a flue ```python # Fluent query builder (recommended) +from PowerPlatform.Dataverse.models.filters import col + for record in (client.query.builder("account") .select("name", "revenue") - .filter_eq("statecode", 0) - .filter_gt("revenue", 1000000) + .where(col("statecode") == 0) + .where(col("revenue") > 1000000) .order_by("revenue", descending=True) .top(100) .page_size(50) @@ -299,47 +308,48 @@ for record in (client.query.builder("account") print(f"{record['name']}: {record['revenue']}") ``` -The QueryBuilder handles value formatting, column name casing, and OData syntax automatically. All filter methods are discoverable via IDE autocomplete: +The QueryBuilder handles value formatting, column name casing, and OData syntax automatically. Filter expressions are built with `col()` and standard Python operators: ```python # Get results as a pandas DataFrame (consolidates all pages) df = (client.query.builder("account") .select("name", "telephone1") - .filter_eq("statecode", 0) + .where(col("statecode") == 0) .top(100) + .execute() .to_dataframe()) print(f"Got {len(df)} accounts") ``` ```python -# Comparison filters +# Comparison filters using col() expressions query = (client.query.builder("contact") - .filter_eq("statecode", 0) # statecode eq 0 - .filter_gt("revenue", 1000000) # revenue gt 1000000 - .filter_contains("name", "Corp") # contains(name, 'Corp') - .filter_in("statecode", [0, 1]) # Microsoft.Dynamics.CRM.In(...) - .filter_between("revenue", 100000, 500000) # (revenue ge 100000 and revenue le 500000) - .filter_null("telephone1") # telephone1 eq null + .where(col("statecode") == 0) # statecode eq 0 + .where(col("revenue") > 1000000) # revenue gt 1000000 + .where(col("name").contains("Corp")) # contains(name, 'Corp') + .where(col("statecode").in_([0, 1])) # Microsoft.Dynamics.CRM.In(...) + .where(col("revenue").between(100000, 500000)) # revenue ge 100000 and revenue le 500000 + .where(col("telephone1").is_null()) # telephone1 eq null ) ``` -For complex logic (OR, NOT, grouping), use the composable expression tree with `where()`: +For complex logic (OR, NOT, grouping), compose expressions with `&`, `|`, `~`: ```python -from PowerPlatform.Dataverse.models.filters import eq, gt, filter_in, between +from PowerPlatform.Dataverse.models.filters import col # OR conditions: (statecode = 0 OR statecode = 1) AND revenue > 100k for record in (client.query.builder("account") .select("name", "revenue") - .where((eq("statecode", 0) | eq("statecode", 1)) - & gt("revenue", 100000)) + .where(((col("statecode") == 0) | (col("statecode") == 1)) + & (col("revenue") > 100000)) .execute()): print(record["name"]) # NOT, between, and in operators for record in (client.query.builder("account") - .where(~eq("statecode", 2)) # NOT inactive - .where(between("revenue", 100000, 500000)) # revenue in range + .where(col("statecode") != 2) # NOT inactive + .where(col("revenue").between(100000, 500000)) # revenue in range .execute()): print(record["name"]) ``` @@ -373,16 +383,79 @@ for record in (client.query.builder("account") print(record["name"], record.get("Account_Tasks")) ``` +**Paging** -- `records.list()` follows `@odata.nextLink` automatically and loads all matching records into a single `QueryResult`. Use `top` to bound total records and round-trips. For very large result sets where memory matters, stream pages explicitly with `execute_pages()` or `records.list_pages()`: + +```python +# records.list() — automatic paging, all records in memory (good for small-to-medium sets) +result = client.records.list( + "account", + filter="statecode eq 0", + select=["name"], + top=500, # bounds total records returned and number of HTTP round-trips +) +for record in result: + print(record["name"]) + +# records.list_pages() — stream one page at a time without loading all records into memory +for page_num, page in enumerate( + client.records.list_pages("account", filter="statecode eq 0", select=["name"]) +): + print(f"Page {page_num + 1}: {len(page)} records") + for record in page: + print(record["name"]) + +# query.builder().execute_pages() — stream one page at a time, memory stays flat +# Use this for large result sets (exports, bulk processing) where loading all into memory is risky +for page_num, page in enumerate( + client.query.builder("account") + .select("accountid", "name", "revenue") + .where(col("statecode") == 0) + .order_by("name") + .page_size(500) # optional: override Dataverse default (~5000/page) + .execute_pages() +): + print(f"Page {page_num + 1}: {len(page)} records") + for record in page: + print(f" {record['name']}") +``` + +> **Deprecation note:** `execute(by_page=True)` and `execute(by_page=False)` are deprecated and emit a `UserWarning`. Replace with `execute_pages()` (streaming) or plain `execute()` (eager). The migration tool (`tools/migrate_v0_to_v1.py`) rewrites these automatically. + **Record count** -- include `$count=true` in the request: ```python # Request count alongside results results = (client.query.builder("account") - .filter_eq("statecode", 0) + .where(col("statecode") == 0) .count() .execute()) ``` +**FetchXML queries** -- `client.query.fetch_xml()` returns an inert `FetchXmlQuery` object; no HTTP request is made until you call `.execute()` or `.execute_pages()`: + +```python +xml = """ + + + + + + + +""" + +# .execute() — blocking, fetches all pages and returns a single QueryResult +result = client.query.fetch_xml(xml).execute() +df = result.to_dataframe() + +# .execute_pages() — streaming, yields one QueryResult per HTTP page +# Use count="N" in the FetchXML element to set page size +for page_num, page in enumerate(client.query.fetch_xml(xml).execute_pages()): + print(f"Page {page_num + 1}: {len(page)} records") + for record in page: + print(record["name"]) +``` + **SQL queries** provide an alternative read-only query syntax with support for JOINs, aggregates, GROUP BY, DISTINCT, and OFFSET FETCH pagination: @@ -405,52 +478,44 @@ df = client.dataframe.sql( "SELECT name, revenue FROM account ORDER BY revenue DESC" ) -# SQL helpers: discover columns and JOINs from metadata -cols = client.query.sql_select("account") # "accountid, name, revenue, ..." -join = client.query.sql_join("contact", "account", from_alias="c", to_alias="a") -# Returns: "JOIN account a ON c.parentcustomerid = a.accountid" +# Discover columns from metadata (schema-discovery helper, kept at GA) +cols_meta = client.query.sql_columns("account") +col_names = [c["LogicalName"] for c in cols_meta] -# Build queries using helpers -- no OData knowledge needed -sql = f"SELECT TOP 10 c.fullname, a.name FROM contact c {join}" +# Build queries using the discovered column names +sql = f"SELECT TOP 10 {', '.join(col_names[:5])} FROM account" df = client.dataframe.sql(sql) - -# Discover all possible JOINs from a table (including polymorphic) -joins = client.query.sql_joins("opportunity") -for j in joins: - print(f"{j['column']:30s} -> {j['target']}.{j['target_pk']}") ``` -**Raw OData queries** are available via `records.get()` for cases where you need direct control over the OData filter string. The SDK provides helpers to eliminate the most error-prone parts: +**Raw OData filter strings** can be passed directly to `records.list()` or the query builder for cases where you need direct control over the OData filter: ```python -# Discover columns for $select (returns list ready for select= parameter) -cols = client.query.odata_select("account") -for page in client.records.get("account", select=cols, top=10): - ... - -# Discover $expand navigation properties (auto-resolves PascalCase names) -nav = client.query.odata_expand("contact", "account") -# Returns: "parentcustomerid_account" -for page in client.records.get("contact", select=["fullname"], expand=[nav], top=5): - for r in page: - acct = r.get(nav) or {} - print(f"{r['fullname']} -> {acct.get('name')}") - -# Build @odata.bind for lookup fields (no manual name construction) -bind = client.query.odata_bind("contact", "account", account_id) -# Returns: {"parentcustomerid_account@odata.bind": "/accounts(guid)"} -client.records.create("contact", {"firstname": "Jane", **bind}) - -# Raw OData query with manual parameters -for page in client.records.get( +# Query with a raw OData filter string — follows @odata.nextLink automatically, +# all records loaded into memory. Use top to bound results and round-trips. +for record in client.records.list( "account", select=["name"], filter="statecode eq 0", # Raw OData: column names must be lowercase - expand=["primarycontactid"], # Navigation properties are case-sensitive top=100, ): - for record in page: - print(record["name"]) + print(record["name"]) + +# Discover navigation property names for $expand (metadata-discovery helper, kept at GA) +nav_props = client.query.odata_expands("account") # → list of navigation property metadata + +# Expand navigation properties using the query builder +from PowerPlatform.Dataverse.models.query_builder import ExpandOption +for record in (client.query.builder("contact") + .select("fullname") + .expand(ExpandOption("parentcustomerid_account").select("name")) + .execute()): + acct = record.get("parentcustomerid_account") or {} + print(f"{record['fullname']} -> {acct.get('name')}") + +# Build @odata.bind for lookup fields (deprecated helper, still functional with DeprecationWarning) +bind = client.query.odata_bind("contact", "account", account_id) +# Returns: {"parentcustomerid_account@odata.bind": "/accounts(guid)"} +client.records.create("contact", {"firstname": "Jane", **bind}) ``` ### Table management @@ -604,7 +669,7 @@ batch.records.create("account", {"name": "Contoso"}) batch.records.create("account", [{"name": "Fabrikam"}, {"name": "Woodgrove"}]) batch.records.update("account", account_id, {"telephone1": "555-0100"}) batch.records.delete("account", old_id) -batch.records.get("account", account_id, select=["name"]) +batch.records.retrieve("account", account_id, select=["name"]) # single record (GA) result = batch.execute() for item in result.responses: @@ -718,7 +783,7 @@ from PowerPlatform.Dataverse.client import DataverseClient from PowerPlatform.Dataverse.core.errors import HttpError, ValidationError try: - client.records.get("account", "invalid-id") + client.records.retrieve("account", "invalid-id") except HttpError as e: print(f"HTTP {e.status_code}: {e.message}") print(f"Error code: {e.code}") diff --git a/examples/advanced/alternate_keys_upsert.py b/examples/advanced/alternate_keys_upsert.py index 67e8a43e..3248282a 100644 --- a/examples/advanced/alternate_keys_upsert.py +++ b/examples/advanced/alternate_keys_upsert.py @@ -100,29 +100,37 @@ def main(): client = DataverseClient(base_url, credential) # ------------------------------------------------------------------ - # Step 1: Create table + # Step 1: Create table (skip if already exists) # ------------------------------------------------------------------ print("\n1. Creating table...") - table_info = backoff( - lambda: client.tables.create( - TABLE_NAME, - columns={ - KEY_COLUMN: "string", - "new_ProductName": "string", - "new_Price": "decimal", - }, + table_info = client.tables.get(TABLE_NAME) + if table_info: + print(f" Table already exists: {TABLE_NAME} (skipped)") + else: + table_info = backoff( + lambda: client.tables.create( + TABLE_NAME, + columns={ + KEY_COLUMN: "string", + "new_ProductName": "string", + "new_Price": "decimal", + }, + ) ) - ) - print(f" Created: {table_info.get('table_schema_name', TABLE_NAME)}") - - time.sleep(10) # Wait for metadata propagation + print(f" Created: {table_info.get('table_schema_name', TABLE_NAME)}") + time.sleep(10) # Wait for metadata propagation # ------------------------------------------------------------------ - # Step 2: Create alternate key + # Step 2: Create alternate key (skip if already exists) # ------------------------------------------------------------------ print("\n2. Creating alternate key...") - key_info = backoff(lambda: client.tables.create_alternate_key(TABLE_NAME, KEY_NAME, [KEY_COLUMN.lower()])) - print(f" Key created: {key_info.schema_name} (id={key_info.metadata_id})") + existing_keys = client.tables.get_alternate_keys(TABLE_NAME) + existing_key = next((k for k in existing_keys if k.schema_name == KEY_NAME), None) + if existing_key: + print(f" Alternate key already exists: {KEY_NAME} (skipped)") + else: + key_info = backoff(lambda: client.tables.create_alternate_key(TABLE_NAME, KEY_NAME, [KEY_COLUMN.lower()])) + print(f" Key created: {key_info.schema_name} (id={key_info.metadata_id})") # ------------------------------------------------------------------ # Step 3: Wait for key to become Active @@ -212,15 +220,14 @@ def main(): # Step 6: Verify # ------------------------------------------------------------------ print("\n6. Verifying records...") - for page in client.records.get( + for record in client.records.list( TABLE_NAME, select=["new_productname", "new_price", KEY_COLUMN.lower()], ): - for record in page: - ext_id = record.get(KEY_COLUMN.lower(), "?") - name = record.get("new_productname", "?") - price = record.get("new_price", "?") - print(f" {ext_id}: {name} @ ${price}") + ext_id = record.get(KEY_COLUMN.lower(), "?") + name = record.get("new_productname", "?") + price = record.get("new_price", "?") + print(f" {ext_id}: {name} @ ${price}") # ------------------------------------------------------------------ # Step 7: List alternate keys diff --git a/examples/advanced/batch.py b/examples/advanced/batch.py index a95aa303..bb180bfc 100644 --- a/examples/advanced/batch.py +++ b/examples/advanced/batch.py @@ -13,248 +13,258 @@ from __future__ import annotations -# --------------------------------------------------------------------------- -# Setup — replace with your environment URL and credential -# --------------------------------------------------------------------------- +import sys from azure.identity import InteractiveBrowserCredential from PowerPlatform.Dataverse.client import DataverseClient -credential = InteractiveBrowserCredential() -with DataverseClient("https://org.crm.dynamics.com", credential) as client: +def main(): + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") - # --------------------------------------------------------------------------- - # Example 1: Record CRUD in a single batch - # --------------------------------------------------------------------------- + credential = InteractiveBrowserCredential() - print("\n[INFO] Example 1: Record CRUD in a single batch") + with DataverseClient(base_url=base_url, credential=credential) as client: - batch = client.batch.new() + # --------------------------------------------------------------------------- + # Example 1: Record CRUD in a single batch + # --------------------------------------------------------------------------- - # Create a single record - batch.records.create("account", {"name": "Contoso Ltd", "telephone1": "555-0100"}) + print("\n[INFO] Example 1: Record CRUD in a single batch") - # Create multiple records via CreateMultiple (one batch item) - batch.records.create( - "contact", - [ - {"firstname": "Alice", "lastname": "Smith"}, - {"firstname": "Bob", "lastname": "Jones"}, - ], - ) + batch = client.batch.new() - # Assume we have an existing account_id from a prior operation - # batch.records.update("account", account_id, {"telephone1": "555-9999"}) - # batch.records.delete("account", old_id) + # Create a single record + batch.records.create("account", {"name": "Contoso Ltd", "telephone1": "555-0100"}) - result = batch.execute() + # Create multiple records via CreateMultiple (one batch item) + batch.records.create( + "contact", + [ + {"firstname": "Alice", "lastname": "Smith"}, + {"firstname": "Bob", "lastname": "Jones"}, + ], + ) - print(f"[OK] Total: {len(result.responses)}, Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") - for guid in result.entity_ids: - print(f"[OK] Created: {guid}") - for item in result.failed: - print(f"[ERR] {item.status_code}: {item.error_message}") + # Assume we have an existing account_id from a prior operation + # batch.records.update("account", account_id, {"telephone1": "555-9999"}) + # batch.records.delete("account", old_id) - # --------------------------------------------------------------------------- - # Example 2: Transactional changeset with content-ID chaining - # --------------------------------------------------------------------------- + result = batch.execute() - print("\n[INFO] Example 2: Transactional changeset") + print(f"[OK] Total: {len(result.responses)}, Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") + for guid in result.entity_ids: + print(f"[OK] Created: {guid}") + for item in result.failed: + print(f"[ERR] {item.status_code}: {item.error_message}") - batch = client.batch.new() + # --------------------------------------------------------------------------- + # Example 2: Transactional changeset with content-ID chaining + # --------------------------------------------------------------------------- - with batch.changeset() as cs: - # Each create() returns a "$n" reference usable in subsequent operations - lead_ref = cs.records.create( - "lead", - {"firstname": "Ada", "lastname": "Lovelace"}, - ) - contact_ref = cs.records.create("contact", {"firstname": "Ada"}) - - # Reference the newly created lead and contact in the account - cs.records.create( - "account", - { - "name": "Babbage & Co.", - "originatingleadid@odata.bind": lead_ref, - "primarycontactid@odata.bind": contact_ref, - }, + print("\n[INFO] Example 2: Transactional changeset") + + batch = client.batch.new() + + with batch.changeset() as cs: + # Each create() returns a "$n" reference usable in subsequent operations + lead_ref = cs.records.create( + "lead", + {"firstname": "Ada", "lastname": "Lovelace"}, + ) + contact_ref = cs.records.create("contact", {"firstname": "Ada"}) + + # Reference the newly created lead and contact in the account + cs.records.create( + "account", + { + "name": "Babbage & Co.", + "originatingleadid@odata.bind": lead_ref, + "primarycontactid@odata.bind": contact_ref, + }, + ) + + # Update using a content-ID reference as the record_id + cs.records.update("contact", contact_ref, {"lastname": "Lovelace"}) + + result = batch.execute() + + if result.has_errors: + print("[ERR] Changeset rolled back") + for item in result.failed: + print(f" {item.status_code}: {item.error_message}") + else: + print(f"[OK] {len(result.entity_ids)} records created atomically") + + # --------------------------------------------------------------------------- + # Example 3: Table metadata operations in a batch + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 3: Table metadata operations") + + batch = client.batch.new() + + # Create a new custom table + batch.tables.create( + "new_Product", + {"new_Price": "decimal", "new_InStock": "bool"}, + solution="MySolution", ) - # Update using a content-ID reference as the record_id - cs.records.update("contact", contact_ref, {"lastname": "Lovelace"}) + # Read table metadata + batch.tables.get("new_Product") - result = batch.execute() + # List all non-private tables + batch.tables.list() - if result.has_errors: - print("[ERR] Changeset rolled back") + result = batch.execute() + print(f"[OK] Table ops: {[(r.status_code, r.is_success) for r in result.responses]}") + + # --------------------------------------------------------------------------- + # Example 4: SQL query in a batch + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 4: SQL query in batch") + + batch = client.batch.new() + batch.query.sql("SELECT TOP 5 accountid, name FROM account ORDER BY name") + + result = batch.execute() + if result.responses and result.responses[0].is_success and result.responses[0].data: + rows = result.responses[0].data.get("value", []) + print(f"[OK] Retrieved {len(rows)} accounts") + for row in rows: + print(f" {row.get('name')}") + + # --------------------------------------------------------------------------- + # Example 5: Mixed batch — changeset writes + standalone GETs + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 5: Mixed batch") + + # NOTE: Commented out because it requires a pre-existing account_id. + # Uncomment and set account_id to run this example. + # batch = client.batch.new() + # + # with batch.changeset() as cs: + # cs.records.update("account", account_id, {"statecode": 0}) + # + # batch.records.get("account", account_id, select=["name", "statecode"]) + # + # result = batch.execute() + # update_response = result.responses[0] + # account_data = result.responses[1] + # if account_data.is_success and account_data.data: + # print(f"Account: {account_data.data.get('name')}") + + # --------------------------------------------------------------------------- + # Example 6: Continue on error + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 6: Continue on error") + + batch = client.batch.new() + batch.records.retrieve("account", "00000000-0000-0000-0000-000000000000") + batch.query.sql("SELECT TOP 1 name FROM account") + + result = batch.execute(continue_on_error=True) + print(f"[OK] Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") for item in result.failed: - print(f" {item.status_code}: {item.error_message}") - else: - print(f"[OK] {len(result.entity_ids)} records created atomically") - - # --------------------------------------------------------------------------- - # Example 3: Table metadata operations in a batch - # --------------------------------------------------------------------------- - - print("\n[INFO] Example 3: Table metadata operations") - - batch = client.batch.new() - - # Create a new custom table - batch.tables.create( - "new_Product", - {"new_Price": "decimal", "new_InStock": "bool"}, - solution="MySolution", - ) - - # Read table metadata - batch.tables.get("new_Product") - - # List all non-private tables - batch.tables.list() - - result = batch.execute() - print(f"[OK] Table ops: {[(r.status_code, r.is_success) for r in result.responses]}") - - # --------------------------------------------------------------------------- - # Example 4: SQL query in a batch - # --------------------------------------------------------------------------- - - print("\n[INFO] Example 4: SQL query in batch") - - batch = client.batch.new() - batch.query.sql("SELECT TOP 5 accountid, name FROM account ORDER BY name") - - result = batch.execute() - if result.responses and result.responses[0].is_success and result.responses[0].data: - rows = result.responses[0].data.get("value", []) - print(f"[OK] Retrieved {len(rows)} accounts") - for row in rows: - print(f" {row.get('name')}") - - # --------------------------------------------------------------------------- - # Example 5: Mixed batch — changeset writes + standalone GETs - # --------------------------------------------------------------------------- - - print("\n[INFO] Example 5: Mixed batch") - - # NOTE: Commented out because it requires a pre-existing account_id. - # Uncomment and set account_id to run this example. - # batch = client.batch.new() - # - # with batch.changeset() as cs: - # cs.records.update("account", account_id, {"statecode": 0}) - # - # batch.records.get("account", account_id, select=["name", "statecode"]) - # - # result = batch.execute() - # update_response = result.responses[0] - # account_data = result.responses[1] - # if account_data.is_success and account_data.data: - # print(f"Account: {account_data.data.get('name')}") - - # --------------------------------------------------------------------------- - # Example 6: Continue on error - # --------------------------------------------------------------------------- - - print("\n[INFO] Example 6: Continue on error") - - batch = client.batch.new() - batch.records.get("account", "00000000-0000-0000-0000-000000000000") - batch.query.sql("SELECT TOP 1 name FROM account") - - result = batch.execute(continue_on_error=True) - print(f"[OK] Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") - for item in result.failed: - print(f"[ERR] {item.status_code}: {item.error_message}") - - # --------------------------------------------------------------------------- - # Example 7: DataFrame integration - # --------------------------------------------------------------------------- - - print("\n[INFO] Example 7: DataFrame batch operations") - - import pandas as pd - - # Create records from a DataFrame - df = pd.DataFrame( - [ - {"name": "DF-Batch-A", "telephone1": "555-0100"}, - {"name": "DF-Batch-B", "telephone1": "555-0200"}, - ] - ) - batch = client.batch.new() - batch.dataframe.create("account", df) - result = batch.execute() - print(f"[OK] DataFrame create: {len(result.succeeded)} succeeded") - created_ids = list(result.entity_ids) - - # Update records from a DataFrame - if len(created_ids) >= 2: - update_df = pd.DataFrame( + print(f"[ERR] {item.status_code}: {item.error_message}") + + # --------------------------------------------------------------------------- + # Example 7: DataFrame integration + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 7: DataFrame batch operations") + + import pandas as pd + + # Create records from a DataFrame + df = pd.DataFrame( [ - {"accountid": created_ids[0], "telephone1": "555-9990"}, - {"accountid": created_ids[1], "telephone1": "555-9991"}, + {"name": "DF-Batch-A", "telephone1": "555-0100"}, + {"name": "DF-Batch-B", "telephone1": "555-0200"}, ] ) batch = client.batch.new() - batch.dataframe.update("account", update_df, id_column="accountid") + batch.dataframe.create("account", df) result = batch.execute() - print(f"[OK] DataFrame update: {len(result.succeeded)} succeeded") + print(f"[OK] DataFrame create: {len(result.succeeded)} succeeded") + created_ids = list(result.entity_ids) + + # Update records from a DataFrame + if len(created_ids) >= 2: + update_df = pd.DataFrame( + [ + {"accountid": created_ids[0], "telephone1": "555-9990"}, + {"accountid": created_ids[1], "telephone1": "555-9991"}, + ] + ) + batch = client.batch.new() + batch.dataframe.update("account", update_df, id_column="accountid") + result = batch.execute() + print(f"[OK] DataFrame update: {len(result.succeeded)} succeeded") + + # Delete records from a Series + if created_ids: + batch = client.batch.new() + batch.dataframe.delete("account", pd.Series(created_ids), use_bulk_delete=False) + result = batch.execute() + print(f"[OK] DataFrame delete: {len(result.succeeded)} succeeded") + + # --------------------------------------------------------------------------- + # Example 8: Understanding response data patterns + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 8: Response data patterns") + + # Every batch result maps 1:1 with the operations you added. + # Different operations return different response shapes: - # Delete records from a Series - if created_ids: batch = client.batch.new() - batch.dataframe.delete("account", pd.Series(created_ids), use_bulk_delete=False) - result = batch.execute() - print(f"[OK] DataFrame delete: {len(result.succeeded)} succeeded") - - # --------------------------------------------------------------------------- - # Example 8: Understanding response data patterns - # --------------------------------------------------------------------------- + # Op 0: single create -> 204 No Content, entity_id in OData-EntityId header + batch.records.create("account", {"name": "Pattern-Demo"}) + # Op 1: bulk create -> 200 OK, IDs in body as {"Ids": [...]} + batch.records.create("account", [{"name": "Bulk-A"}, {"name": "Bulk-B"}]) + # Op 2: SQL query -> 200 OK, rows in body as {"value": [...]} + batch.query.sql("SELECT TOP 3 name FROM account") - print("\n[INFO] Example 8: Response data patterns") + result = batch.execute() - # Every batch result maps 1:1 with the operations you added. - # Different operations return different response shapes: + for i, resp in enumerate(result.responses): + if not resp.is_success: + print(f" Op {i}: [FAIL] {resp.status_code}: {resp.error_message}") + continue - batch = client.batch.new() - # Op 0: single create -> 204 No Content, entity_id in OData-EntityId header - batch.records.create("account", {"name": "Pattern-Demo"}) - # Op 1: bulk create -> 200 OK, IDs in body as {"Ids": [...]} - batch.records.create("account", [{"name": "Bulk-A"}, {"name": "Bulk-B"}]) - # Op 2: SQL query -> 200 OK, rows in body as {"value": [...]} - batch.query.sql("SELECT TOP 3 name FROM account") + # Single create: entity_id from OData-EntityId header + if resp.entity_id: + print(f" Op {i}: [CREATE] entity_id={resp.entity_id}") - result = batch.execute() + # Bulk action (CreateMultiple/UpsertMultiple): IDs in body + elif resp.data and "Ids" in resp.data: + print(f" Op {i}: [BULK] {len(resp.data['Ids'])} IDs: {resp.data['Ids']}") - for i, resp in enumerate(result.responses): - if not resp.is_success: - print(f" Op {i}: [FAIL] {resp.status_code}: {resp.error_message}") - continue + # Query: rows in body + elif resp.data and "value" in resp.data: + print(f" Op {i}: [QUERY] {len(resp.data['value'])} rows") - # Single create: entity_id from OData-EntityId header - if resp.entity_id: - print(f" Op {i}: [CREATE] entity_id={resp.entity_id}") + # Delete or metadata operation: 204, no data + else: + print(f" Op {i}: [OK] {resp.status_code}") - # Bulk action (CreateMultiple/UpsertMultiple): IDs in body - elif resp.data and "Ids" in resp.data: - print(f" Op {i}: [BULK] {len(resp.data['Ids'])} IDs: {resp.data['Ids']}") + # Clean up demo records + for rid in result.entity_ids: + client.records.delete("account", rid) + for resp in result.succeeded: + if resp.data and "Ids" in resp.data: + for rid in resp.data["Ids"]: + client.records.delete("account", rid) - # Query: rows in body - elif resp.data and "value" in resp.data: - print(f" Op {i}: [QUERY] {len(resp.data['value'])} rows") - # Delete or metadata operation: 204, no data - else: - print(f" Op {i}: [OK] {resp.status_code}") - - # Clean up demo records - for rid in result.entity_ids: - client.records.delete("account", rid) - for resp in result.succeeded: - if resp.data and "Ids" in resp.data: - for rid in resp.data["Ids"]: - client.records.delete("account", rid) +if __name__ == "__main__": + main() diff --git a/examples/advanced/dataframe_operations.py b/examples/advanced/dataframe_operations.py index 7c0b6010..0a51b4c7 100644 --- a/examples/advanced/dataframe_operations.py +++ b/examples/advanced/dataframe_operations.py @@ -19,6 +19,7 @@ from azure.identity import InteractiveBrowserCredential from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.models.filters import col, raw def main(): @@ -81,7 +82,7 @@ def _run_walkthrough(client): print("2. Query records as a DataFrame") print("-" * 60) - df_all = client.dataframe.get(table, select=select_cols, filter=test_filter) + df_all = client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute().to_dataframe() print(f"[OK] Got {len(df_all)} records in one DataFrame") print(f" Columns: {list(df_all.columns)}") print(f"{df_all.to_string(index=False)}") @@ -91,7 +92,7 @@ def _run_walkthrough(client): print("3. Limit results with top") print("-" * 60) - df_top2 = client.dataframe.get(table, select=select_cols, filter=test_filter, top=2) + df_top2 = client.query.builder(table).select(*select_cols).where(raw(test_filter)).top(2).execute().to_dataframe() print(f"[OK] Got {len(df_top2)} records with top=2") print(f"{df_top2.to_string(index=False)}") @@ -102,7 +103,9 @@ def _run_walkthrough(client): first_id = new_accounts["accountid"].iloc[0] print(f" Fetching record {first_id}...") - single = client.dataframe.get(table, record_id=first_id, select=select_cols) + single = ( + client.query.builder(table).select(*select_cols).where(col("accountid") == first_id).execute().to_dataframe() + ) print(f"[OK] Single record DataFrame:\n{single.to_string(index=False)}") # -- 5. Update records from a DataFrame ------------------------ @@ -116,7 +119,7 @@ def _run_walkthrough(client): print("[OK] Updated 3 records") # Verify the updates - verified = client.dataframe.get(table, select=select_cols, filter=test_filter) + verified = client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute().to_dataframe() print(f" Verified:\n{verified.to_string(index=False)}") # -- 6. Broadcast update (same value to all records) ----------- @@ -131,7 +134,7 @@ def _run_walkthrough(client): print("[OK] Broadcast update complete") # Verify all records have the same websiteurl - verified = client.dataframe.get(table, select=select_cols, filter=test_filter) + verified = client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute().to_dataframe() print(f" Verified:\n{verified.to_string(index=False)}") # Default: NaN/None fields are skipped (not overridden on server) @@ -142,14 +145,14 @@ def _run_walkthrough(client): ] ) client.dataframe.update(table, sparse_df, id_column="accountid") - verified = client.dataframe.get(table, select=select_cols, filter=test_filter) + verified = client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute().to_dataframe() print(f" Verified (Contoso telephone1 updated, websiteurl unchanged):\n{verified.to_string(index=False)}") # Opt-in: clear_nulls=True sends None as null to clear the field print("\n Clearing websiteurl for Contoso with clear_nulls=True...") clear_df = pd.DataFrame([{"accountid": new_accounts["accountid"].iloc[0], "websiteurl": None}]) client.dataframe.update(table, clear_df, id_column="accountid", clear_nulls=True) - verified = client.dataframe.get(table, select=select_cols, filter=test_filter) + verified = client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute().to_dataframe() print(f" Verified (Contoso websiteurl should be empty):\n{verified.to_string(index=False)}") # -- 7. Delete records by passing a Series of GUIDs ------------ @@ -162,7 +165,7 @@ def _run_walkthrough(client): print(f"[OK] Deleted {len(new_accounts)} records") # Verify deletions - filter for our tagged records should return 0 - remaining = client.dataframe.get(table, select=select_cols, filter=test_filter) + remaining = client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute().to_dataframe() print(f" Verified: {len(remaining)} test records remaining (expected 0)") print("\n" + "=" * 60) diff --git a/examples/advanced/datascience_risk_assessment.py b/examples/advanced/datascience_risk_assessment.py index 338713e4..80dafdc4 100644 --- a/examples/advanced/datascience_risk_assessment.py +++ b/examples/advanced/datascience_risk_assessment.py @@ -50,6 +50,7 @@ from azure.identity import InteractiveBrowserCredential from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.models.filters import col, raw # -- Optional imports (graceful degradation if not installed) ------ @@ -272,43 +273,42 @@ def step1_extract(client): print("=" * 60) # Pull accounts - accounts = client.dataframe.get( - TABLE_ACCOUNTS, - select=["accountid", "name", "revenue", "numberofemployees", "industrycode"], - filter="statecode eq 0", - top=200, + accounts = ( + client.query.builder(TABLE_ACCOUNTS) + .select("accountid", "name", "revenue", "numberofemployees", "industrycode") + .where(col("statecode") == 0) + .top(200) + .execute() + .to_dataframe() ) print(f"[OK] Extracted {len(accounts)} active accounts") # Pull open cases (service incidents) - cases = client.dataframe.get( - TABLE_CASES, - select=[ - "incidentid", - "_customerid_value", - "title", - "severitycode", - "prioritycode", - "createdon", - ], - filter="statecode eq 0", - top=1000, + cases = ( + client.query.builder(TABLE_CASES) + .select("incidentid", "_customerid_value", "title", "severitycode", "prioritycode", "createdon") + .where(raw("statecode eq 0")) + .top(1000) + .execute() + .to_dataframe() ) print(f"[OK] Extracted {len(cases)} open cases") # Pull active opportunities - opportunities = client.dataframe.get( - TABLE_OPPORTUNITIES, - select=[ + opportunities = ( + client.query.builder(TABLE_OPPORTUNITIES) + .select( "opportunityid", "_parentaccountid_value", "name", "estimatedvalue", "closeprobability", "estimatedclosedate", - ], - filter="statecode eq 0", - top=1000, + ) + .where(col("statecode") == 0) + .top(1000) + .execute() + .to_dataframe() ) print(f"[OK] Extracted {len(opportunities)} active opportunities") diff --git a/examples/advanced/fetch_xml.py b/examples/advanced/fetch_xml.py new file mode 100644 index 00000000..b475df06 --- /dev/null +++ b/examples/advanced/fetch_xml.py @@ -0,0 +1,578 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +End-to-end FetchXML examples for Dataverse. + +Demonstrates ``client.query.fetch_xml()`` across the scenarios where FetchXML +is required or preferred over OData/SQL: + +- Basic attribute queries +- operators (eq, like, in, null, not-null, between) +- (inner and outer joins) +- Ordering +- Top N with automatic paging-cookie propagation +- Aggregate queries (count, sum, avg, min, max, group-by) +- Built-in system tables (account → contact join) + +FetchXML is the right tool when: +- You need a JOIN type OData $expand cannot express (many-to-many, outer link) +- You need server-side aggregates (count, sum, avg) without GROUP BY SQL +- You need ```` operators unavailable in OData ($filter) + +Prerequisites: +- pip install PowerPlatform-Dataverse-Client azure-identity +""" + +import sys +import time + +from azure.identity import InteractiveBrowserCredential +from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.core.errors import MetadataError +import requests + +# --------------------------------------------------------------------------- +# Helpers (same pattern as sql_examples.py) +# --------------------------------------------------------------------------- + + +def log_call(description): + print(f"\n-> {description}") + + +def heading(section_num, title): + print(f"\n{'=' * 80}") + print(f"{section_num}. {title}") + print("=" * 80) + + +def backoff(op, *, delays=(0, 2, 5, 10, 20, 20)): + """Retry an operation with exponential back-off.""" + last = None + total_delay = 0 + attempts = 0 + for d in delays: + if d: + time.sleep(d) + total_delay += d + attempts += 1 + try: + result = op() + if attempts > 1: + print(f" [INFO] Backoff succeeded after {attempts - 1} retry(s); waited {total_delay}s total.") + return result + except Exception as ex: + last = ex + continue + if last: + if attempts: + print( + f" [WARN] Backoff exhausted after {max(attempts - 1, 0)} retry(s); waited {total_delay}s total." + f"\n [ERROR] {last}" + ) + raise last + + +def main(): + print("=" * 80) + print("Dataverse SDK -- FetchXML End-to-End Examples") + print("=" * 80) + + heading(1, "Setup & Authentication") + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") + + log_call("InteractiveBrowserCredential()") + credential = InteractiveBrowserCredential() + + log_call(f"DataverseClient(base_url='{base_url}', credential=...)") + with DataverseClient(base_url=base_url, credential=credential) as client: + print(f"[OK] Connected to: {base_url}") + _run_examples(client) + + +def _run_examples(client): + project_table = "new_FXDemoProject" + task_table = "new_FXDemoTask" + + # =================================================================== + # 2. Create tables and seed data + # =================================================================== + heading(2, "Create Tables & Seed Data") + + log_call(f"client.tables.get('{project_table}')") + if client.tables.get(project_table): + print(f"[OK] Table already exists: {project_table}") + else: + log_call(f"client.tables.create('{project_table}', ...)") + try: + backoff( + lambda: client.tables.create( + project_table, + { + "new_Code": "string", + "new_Budget": "decimal", + "new_Active": "bool", + "new_Region": "int", + }, + ) + ) + print(f"[OK] Created table: {project_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {project_table} (skipped)") + else: + raise + + log_call(f"client.tables.get('{task_table}')") + if client.tables.get(task_table): + print(f"[OK] Table already exists: {task_table}") + else: + log_call(f"client.tables.create('{task_table}', ...)") + try: + backoff( + lambda: client.tables.create( + task_table, + { + "new_Title": "string", + "new_Hours": "int", + "new_Done": "bool", + "new_Priority": "int", + }, + ) + ) + print(f"[OK] Created table: {task_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {task_table} (skipped)") + else: + raise + + print("\n[INFO] Creating lookup field: tasks → projects ...") + try: + client.tables.create_lookup_field( + referencing_table=task_table, + lookup_field_name="new_ProjectId", + referenced_table=project_table, + display_name="Project", + ) + print("[OK] Created lookup: new_ProjectId on tasks → projects") + except Exception as e: + msg = str(e).lower() + if "already exists" in msg or "duplicate" in msg or "not unique" in msg: + print("[OK] Lookup already exists (skipped)") + else: + raise + + # Resolve entity set name for @odata.bind + project_set = f"{project_table.lower()}s" + try: + tinfo = client.tables.get(project_table) + if tinfo: + project_set = tinfo.get("entity_set_name", project_set) + except Exception: + pass + + log_call(f"client.records.create('{project_table}', [...])") + projects = [ + {"new_Code": "ALPHA", "new_Budget": 50000, "new_Active": True, "new_Region": 1}, + {"new_Code": "BRAVO", "new_Budget": 75000, "new_Active": True, "new_Region": 2}, + {"new_Code": "CHARLIE", "new_Budget": 30000, "new_Active": False, "new_Region": 3}, + {"new_Code": "DELTA", "new_Budget": 90000, "new_Active": True, "new_Region": 1}, + {"new_Code": "ECHO", "new_Budget": 42000, "new_Active": True, "new_Region": 2}, + ] + project_ids = backoff(lambda: client.records.create(project_table, projects)) + print(f"[OK] Seeded {len(project_ids)} projects") + + log_call(f"client.records.create('{task_table}', [...])") + tasks = [ + { + "new_Title": "Design mockups", + "new_Hours": 8, + "new_Done": True, + "new_Priority": 2, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[0]})", + }, + { + "new_Title": "Write unit tests", + "new_Hours": 12, + "new_Done": False, + "new_Priority": 3, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[0]})", + }, + { + "new_Title": "Code review", + "new_Hours": 3, + "new_Done": True, + "new_Priority": 1, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[1]})", + }, + { + "new_Title": "Deploy to staging", + "new_Hours": 5, + "new_Done": False, + "new_Priority": 3, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[1]})", + }, + { + "new_Title": "Update docs", + "new_Hours": 4, + "new_Done": True, + "new_Priority": 1, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[2]})", + }, + { + "new_Title": "Performance tuning", + "new_Hours": 10, + "new_Done": False, + "new_Priority": 2, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[3]})", + }, + { + "new_Title": "Security audit", + "new_Hours": 6, + "new_Done": False, + "new_Priority": 3, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[4]})", + }, + ] + task_ids = backoff(lambda: client.records.create(task_table, tasks)) + print(f"[OK] Seeded {len(task_ids)} tasks") + + project_logical = project_table.lower() # new_fxdemoproject + task_logical = task_table.lower() # new_fxdemotask + project_pk = f"{project_logical}id" # new_fxdemoprojectid + task_pk = f"{task_logical}id" # new_fxdemoproject id + lookup_attr = "new_projectid" # lookup logical name on task + + try: + # =============================================================== + # 3. Basic attribute query + # =============================================================== + heading(3, "Basic Attribute Query") + xml = f""" + + + + + + + + """ + log_call("client.query.fetch_xml(basic attribute query)") + result = backoff(lambda: client.query.fetch_xml(xml).execute()) + print(f"[OK] {len(result)} projects:") + for r in result: + print(f" {r.get('new_code', ''):<10s} Budget={r.get('new_budget')} Active={r.get('new_active')}") + + # =============================================================== + # 4. operators: eq, like, in, null, not-null, between + # =============================================================== + heading(4, " Operators") + + # eq + xml = f""" + + + + + + + + + """ + log_call('operator="eq" value="ALPHA"') + r = backoff(lambda: client.query.fetch_xml(xml).execute()) + print(f"[OK] eq: {[x.get('new_code') for x in r]}") + + # like + xml = f""" + + + + + + + + + """ + log_call('operator="like" value="%test%"') + r = backoff(lambda: client.query.fetch_xml(xml).execute()) + print(f"[OK] like: {len(r)} matches -> {[x.get('new_title') for x in r]}") + + # in + xml = f""" + + + + + + ALPHA + DELTA + + + + + """ + log_call('operator="in" values=[ALPHA, DELTA]') + r = backoff(lambda: client.query.fetch_xml(xml).execute()) + print(f"[OK] in: {[x.get('new_code') for x in r]}") + + # null / not-null + xml = f""" + + + + + + + + + """ + log_call('operator="not-null"') + r = backoff(lambda: client.query.fetch_xml(xml).execute()) + print(f"[OK] not-null: {len(r)} tasks have priority set") + + # between + xml = f""" + + + + + + + 40000 + 80000 + + + + + """ + log_call('operator="between" 40000 and 80000') + r = backoff(lambda: client.query.fetch_xml(xml).execute()) + print(f"[OK] between: {len(r)} projects -> {[(x.get('new_code'), x.get('new_budget')) for x in r]}") + + # =============================================================== + # 5. — inner join (tasks → projects) + # =============================================================== + heading(5, " Inner Join (Tasks → Projects)") + xml = f""" + + + + + + + + + + + """ + log_call("client.query.fetch_xml(link-entity inner join)") + try: + result = backoff(lambda: client.query.fetch_xml(xml).execute()) + print(f"[OK] {len(result)} rows:") + for r in result: + print( + f" Task={r.get('new_title', ''):<25s} " + f"Hours={r.get('new_hours')} " + f"Project={r.get('p.new_code', '')} " + f"Budget={r.get('p.new_budget')}" + ) + except Exception as e: + print(f"[WARN] link-entity join failed: {e}") + + # =============================================================== + # 6. — outer join (projects with or without tasks) + # =============================================================== + heading(6, " Outer Join (Projects With or Without Tasks)") + xml = f""" + + + + + + + + + """ + log_call("client.query.fetch_xml(link-entity outer join)") + try: + result = backoff(lambda: client.query.fetch_xml(xml).execute()) + print(f"[OK] {len(result)} rows (includes projects with no tasks):") + for r in result[:8]: + print(f" Project={r.get('new_code', ''):<10s} Task={r.get('t.new_title', '(none)')}") + except Exception as e: + print(f"[WARN] outer join failed: {e}") + + # =============================================================== + # 7. Ordering + # =============================================================== + heading(7, "Ordering ( element)") + + xml = f""" + + + + + + + + """ + log_call("client.query.fetch_xml(order by hours DESC)") + result = backoff(lambda: client.query.fetch_xml(xml).execute()) + print(f"[OK] Tasks by hours DESC:") + for r in result: + print(f" {r.get('new_title', ''):<25s} Hours={r.get('new_hours')}") + + # =============================================================== + # 8. Top N + paging-cookie propagation + # =============================================================== + heading(8, "Paging-Cookie Propagation") + print( + "[INFO] 'count' sets the page size in FetchXML (not 'top' — 'top' is a total-result limit).\n" + "With count='2' and 7 seeded tasks the server returns pages of 2, 2, 2, 1.\n" + ".execute() collects all pages eagerly; .execute_pages() yields one QueryResult per HTTP page." + ) + xml_paged = f""" + + + + + + + + """ + log_call("client.query.fetch_xml(xml).execute() — eager, all pages collected") + result = backoff(lambda: client.query.fetch_xml(xml_paged).execute()) + print(f"[OK] execute(): {len(result)} total tasks across all pages (seeded {len(task_ids)}):") + for r in result: + print(f" {r.get('new_title', ''):<25s} Hours={r.get('new_hours')}") + + log_call("client.query.fetch_xml(xml).execute_pages() — lazy, one QueryResult per HTTP page") + page_num = 0 + page_record_count = 0 + for page in backoff(lambda: client.query.fetch_xml(xml_paged).execute_pages()): + page_num += 1 + page_record_count += len(page) + print(f" Page {page_num}: {len(page)} record(s) — {[r.get('new_title') for r in page]}") + print(f"[OK] execute_pages(): {page_record_count} total tasks across {page_num} page(s)") + + # =============================================================== + # 9. Aggregates (count, sum, avg, min, max) + # =============================================================== + heading(9, "Aggregate Queries ()") + + # Global aggregates + xml = f""" + + + + + + + + + + """ + log_call("client.query.fetch_xml(aggregate: count, sum, avg, min, max)") + try: + result = backoff(lambda: client.query.fetch_xml(xml).execute()) + if result: + row = result[0] + print( + f"[OK] count={row.get('task_count')} sum={row.get('total_hours')} " + f"avg={row.get('avg_hours')} min={row.get('min_hours')} max={row.get('max_hours')}" + ) + except Exception as e: + print(f"[WARN] aggregate failed: {e}") + + # Group-by aggregate: total hours per project + xml = f""" + + + + + + + + + + """ + log_call("client.query.fetch_xml(aggregate group-by project)") + try: + result = backoff(lambda: client.query.fetch_xml(xml).execute()) + print(f"[OK] Hours per project ({len(result)} groups):") + for r in result: + print( + f" {r.get('project_code', ''):<10s} " + f"Tasks={r.get('task_count')} " + f"Hours={r.get('total_hours')}" + ) + except Exception as e: + print(f"[WARN] group-by aggregate failed: {e}") + + # =============================================================== + # 10. Built-in system tables (account → contact) + # =============================================================== + heading(10, "Built-In System Tables (account → contact Join)") + xml = """ + + + + + + + + + """ + log_call("client.query.fetch_xml(account → contact inner join)") + try: + result = backoff(lambda: client.query.fetch_xml(xml).execute()) + print(f"[OK] {len(result)} account-contact pairs:") + for r in result: + print(f" Account={r.get('name', ''):<25s} Contact={r.get('c.fullname', '')}") + except Exception as e: + print(f"[INFO] No account-contact data in this org: {e}") + + finally: + heading(11, "Cleanup") + for tbl in [task_table, project_table]: + log_call(f"client.tables.delete('{tbl}')") + try: + backoff(lambda tbl=tbl: client.tables.delete(tbl)) + print(f"[OK] Deleted table: {tbl}") + except Exception as ex: + code = getattr(getattr(ex, "response", None), "status_code", None) + if isinstance(ex, (requests.exceptions.HTTPError, MetadataError)) and code == 404: + print(f"[OK] Table already removed: {tbl}") + else: + print(f"[WARN] Could not delete {tbl}: {ex}") + + print("\n" + "=" * 80) + print("FetchXML Examples Complete!") + print("=" * 80) + + +if __name__ == "__main__": + main() diff --git a/examples/advanced/prodev_quick_start.py b/examples/advanced/prodev_quick_start.py index e28d1575..15448143 100644 --- a/examples/advanced/prodev_quick_start.py +++ b/examples/advanced/prodev_quick_start.py @@ -56,6 +56,7 @@ from azure.identity import InteractiveBrowserCredential from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.models.filters import col # -- Table schema names -- # Uses the standard 'new_' publisher prefix (default Dataverse publisher). @@ -298,10 +299,11 @@ def step3_populate_data(client, primary_name_col): print(f"[OK] Created {len(customers_df)} customers") # -- Projects (linked to customers via lookup) -- - # @odata.bind keys use the navigation property logical name (lowercase) - # and the entity set name (also lowercase) in the value. - customer_lookup = f"{TABLE_PROJECT}_CustomerId".lower() + "@odata.bind" - customer_set = TABLE_CUSTOMER.lower() + "s" + # @odata.bind keys use the lookup field schema name (case-sensitive) + # and the entity set name (from table metadata) in the value. + customer_lookup = f"{TABLE_PROJECT}_CustomerId@odata.bind" + customer_info = client.tables.get(TABLE_CUSTOMER) + customer_set = customer_info.get("entity_set_name") if customer_info else TABLE_CUSTOMER.lower() + "s" projects_df = pd.DataFrame( [ { @@ -352,8 +354,9 @@ def step3_populate_data(client, primary_name_col): for i, (task_name, priority, status, hours) in enumerate(task_names): proj_idx = project_assignment[i] - project_lookup = f"{TABLE_TASK}_ProjectId".lower() + "@odata.bind" - project_set = TABLE_PROJECT.lower() + "s" + project_lookup = f"{TABLE_TASK}_ProjectId@odata.bind" + project_info = client.tables.get(TABLE_PROJECT) + project_set = project_info.get("entity_set_name") if project_info else TABLE_PROJECT.lower() + "s" tasks_data.append( { name_col: task_name, @@ -392,26 +395,21 @@ def step4_query_and_analyze(client, customer_ids, primary_name_col): # Note: The SDK lowercases $select values automatically, so schema-name # casing (e.g., new_DemoProject_Budget) works -- it becomes the logical name. name_attr = primary_name_col - projects = client.dataframe.get( - TABLE_PROJECT, - select=[ - name_attr, - f"{TABLE_PROJECT}_Budget", - f"{TABLE_PROJECT}_Status", - ], + projects = ( + client.query.builder(TABLE_PROJECT) + .select(name_attr, f"{TABLE_PROJECT}_Budget", f"{TABLE_PROJECT}_Status") + .execute() + .to_dataframe() ) print(f"\n All projects ({len(projects)} rows):") print(f"{projects.to_string(index=False)}") # Query tasks and analyze - tasks = client.dataframe.get( - TABLE_TASK, - select=[ - name_attr, - f"{TABLE_TASK}_Priority", - f"{TABLE_TASK}_Status", - f"{TABLE_TASK}_EstimatedHours", - ], + tasks = ( + client.query.builder(TABLE_TASK) + .select(name_attr, f"{TABLE_TASK}_Priority", f"{TABLE_TASK}_Status", f"{TABLE_TASK}_EstimatedHours") + .execute() + .to_dataframe() ) print(f"\n All tasks ({len(tasks)} rows):") print(f"{tasks.to_string(index=False)}") @@ -440,7 +438,7 @@ def step4_query_and_analyze(client, customer_ids, primary_name_col): # Fetch single record by ID first_id = customer_ids.iloc[0] - single = client.dataframe.get(TABLE_CUSTOMER, record_id=first_id) + single = client.query.builder(TABLE_CUSTOMER).where(col(primary_id_col) == first_id).execute().to_dataframe() print(f"\n Single customer record (by ID):") print(f"{single.to_string(index=False)}") @@ -481,10 +479,7 @@ def step5_update_and_delete(client, task_ids, primary_name_col, primary_id_col): print(f"[OK] Deleted 1 task") # Verify - remaining = client.dataframe.get( - TABLE_TASK, - select=[primary_name_col, status_col], - ) + remaining = client.query.builder(TABLE_TASK).select(primary_name_col, status_col).execute().to_dataframe() print(f"\n Remaining tasks ({len(remaining)}):") print(f"{remaining.to_string(index=False)}") diff --git a/examples/advanced/sql_examples.py b/examples/advanced/sql_examples.py index d9fd9627..8c1dd55a 100644 --- a/examples/advanced/sql_examples.py +++ b/examples/advanced/sql_examples.py @@ -22,7 +22,7 @@ - SQL read -> DataFrame transform -> SDK write-back (full round-trip) - AND/OR, NOT IN, NOT LIKE boolean logic - Deep JOINs (5-8 tables) with no server depth limit -- SQL helper functions: sql_columns, sql_select, sql_joins, sql_join +- SQL helper functions: sql_columns (sql_select/sql_join/sql_joins removed at GA) - OData helper functions: odata_select, odata_expands, odata_expand, odata_bind - SQL vs OData side-by-side comparison @@ -133,58 +133,66 @@ def _run_examples(client): ) log_call(f"client.tables.get('{parent_table}')") - info = backoff(lambda: client.tables.get(parent_table)) - if info: + if client.tables.get(parent_table): print(f"[OK] Table already exists: {parent_table}") else: log_call(f"client.tables.create('{parent_table}', ...)") - info = backoff( - lambda: client.tables.create( - parent_table, - { - "new_Code": "string", - "new_Region": Region, - "new_Budget": "decimal", - "new_Active": "bool", - }, + try: + backoff( + lambda: client.tables.create( + parent_table, + { + "new_Code": "string", + "new_Region": Region, + "new_Budget": "decimal", + "new_Active": "bool", + }, + ) ) - ) - print(f"[OK] Created table: {parent_table}") + print(f"[OK] Created table: {parent_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {parent_table} (skipped)") + else: + raise log_call(f"client.tables.get('{child_table}')") - info2 = backoff(lambda: client.tables.get(child_table)) - if info2: + if client.tables.get(child_table): print(f"[OK] Table already exists: {child_table}") else: log_call(f"client.tables.create('{child_table}', ...)") - info2 = backoff( - lambda: client.tables.create( - child_table, - { - "new_Title": "string", - "new_Hours": "int", - "new_Done": "bool", - "new_Priority": "int", - }, + try: + backoff( + lambda: client.tables.create( + child_table, + { + "new_Title": "string", + "new_Hours": "int", + "new_Done": "bool", + "new_Priority": "int", + }, + ) ) - ) - print(f"[OK] Created table: {child_table}") + print(f"[OK] Created table: {child_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {child_table} (skipped)") + else: + raise # Create lookup so tasks reference teams via JOIN print("\n[INFO] Creating lookup field so tasks reference teams via JOIN...") try: - backoff( - lambda: client.tables.create_lookup_field( - referencing_table=child_table, - lookup_field_name="new_TeamId", - referenced_table=parent_table, - display_name="Team", - ) + client.tables.create_lookup_field( + referencing_table=child_table, + lookup_field_name="new_TeamId", + referenced_table=parent_table, + display_name="Team", ) print("[OK] Created lookup: new_TeamId on tasks -> teams") except Exception as e: msg = str(e).lower() - if "already exists" in msg or "duplicate" in msg: + if "already exists" in msg or "duplicate" in msg or "not unique" in msg: print("[OK] Lookup already exists (skipped)") else: raise @@ -419,19 +427,14 @@ def _run_examples(client): heading(13, "SQL -- INNER JOIN") print("Use the lookup attribute's logical name (e.g. new_teamid) for JOINs.") - # Use sql_join() to auto-discover the relationship and build - # the JOIN clause with proper aliases. + # Build the JOIN clause manually (sql_join() was removed at GA). + # Use the lookup attribute's logical name (not _..._value) as the join column. lookup_col = "new_teamid" # Lookup logical name, NOT _..._value - join_clause = client.query.sql_join( - from_table=child_table, - to_table=parent_table, - from_alias="tk", - to_alias="t", - ) + join_clause = f"JOIN {parent_table} t ON tk.{lookup_col} = t.{parent_logical}id" print(f"[INFO] Lookup column: {lookup_col}") - print(f"[INFO] Generated JOIN: {join_clause}") + print(f"[INFO] JOIN clause: {join_clause}") - sql = f"SELECT t.new_code, tk.new_title, tk.new_hours " f"FROM {child_table} tk " f"{join_clause}" + sql = f"SELECT t.new_code, tk.new_title, tk.new_hours FROM {child_table} tk {join_clause}" log_call('client.query.sql("...INNER JOIN...")') try: results = backoff(lambda: client.query.sql(sql)) @@ -912,71 +915,26 @@ def _run_examples(client): # ============================================================== heading(29, "SQL Helper Functions (query.sql_*)") print( - "The SDK provides helper functions that auto-discover column\n" - "names and JOIN clauses from metadata -- no guessing needed." + "At GA, sql_columns() is the only retained SQL schema-discovery helper.\n" + "sql_select(), sql_join(), and sql_joins() were removed — write JOIN\n" + "clauses directly or use client.query.fetch_xml() for complex queries." ) - # sql_columns + # sql_columns — still available at GA log_call(f"client.query.sql_columns('{parent_table}')") cols = client.query.sql_columns(parent_table) print(f"[OK] {len(cols)} columns:") for c in cols[:5]: print(f" {c['name']:30s} Type: {c['type']:15s} PK={c['is_pk']}") - # sql_select - log_call(f"client.query.sql_select('{parent_table}')") - select_str = client.query.sql_select(parent_table) - print(f"[OK] SELECT list: {select_str[:60]}...") - - # sql_joins - log_call(f"client.query.sql_joins('{child_table}')") - joins = client.query.sql_joins(child_table) - print(f"[OK] {len(joins)} possible JOINs:") - for j in joins[:5]: - print(f" {j['column']:25s} -> {j['target']}.{j['target_pk']}") - - # sql_joins -- alias uniqueness: multiple lookups to the same target - # table (e.g. ownerid + createdby + modifiedby all point to systemuser) - # must each get a distinct alias so the combined SQL is valid. - # Expected output: - # ownerid -> systemuser alias=s - # createdby -> systemuser alias=s2 - # modifiedby -> systemuser alias=s3 - log_call("client.query.sql_joins('contact') -- distinct aliases for same target table") - try: - contact_joins = client.query.sql_joins("contact") - systemuser_joins = [j for j in contact_joins if j["target"] == "systemuser"] - print(f"[OK] {len(systemuser_joins)} lookup(s) from contact -> systemuser:") - for j in systemuser_joins: - alias = j["join_clause"].split()[2] - print(f" {j['column']:30s} -> {j['target']} alias={alias}") - aliases = [j["join_clause"].split()[2] for j in contact_joins] - if len(aliases) != len(set(aliases)): - print("[WARN] Duplicate aliases detected") - else: - print(f"[OK] All {len(contact_joins)} aliases unique") - except Exception as e: - print(f"[INFO] Alias check skipped: {e}") - - # sql_join (auto-generate JOIN clause) - log_call(f"client.query.sql_join('{child_table}', '{parent_table}', ...)") - try: - join_clause = client.query.sql_join(child_table, parent_table, from_alias="tk", to_alias="t") - print(f"[OK] {join_clause}") - - sql = f"SELECT TOP 3 tk.new_title, t.new_code FROM {child_table} tk {join_clause}" - results = backoff(lambda: client.query.sql(sql)) - print(f"[OK] Live query with sql_join(): {len(results)} rows") - except Exception as e: - print(f"[WARN] {e}") - # ============================================================== # 30. OData Helper Functions # ============================================================== - heading(30, "OData Helper Functions (query.odata_*)") + heading(30, "OData Helper Functions (query.odata_* — deprecated at GA)") print( - "Parallel helpers for OData/records.get() users -- auto-discover\n" - "navigation properties and build @odata.bind payloads." + "odata_select(), odata_expand(), and odata_bind() still work at GA\n" + "but emit DeprecationWarning. Use the typed query builder instead.\n" + "odata_expands() is kept without deprecation." ) # odata_select @@ -998,7 +956,7 @@ def _run_examples(client): try: nav = client.query.odata_expand(child_table, parent_table) print(f"\n[OK] odata_expand('{child_table}', '{parent_table}') = '{nav}'") - print(" Usage: client.records.get('" + child_table + "', expand=['" + nav + "'])") + print(" Usage: client.query.builder('" + child_table + "').expand('" + nav + "').execute()") except Exception as e: print(f"[WARN] {e}") @@ -1030,8 +988,8 @@ def _run_examples(client): | DISTINCT | YES | Not directly | | Pagination | OFFSET FETCH | @odata.nextLink | | Max results | 5000 per query | 5000 per page | -| Column discovery | sql_columns/sql_select | odata_select | -| JOIN discovery | sql_joins/sql_join | odata_expands/expand | +| Column discovery | sql_columns | odata_expands (kept) | +| JOIN discovery | write manually/fetch_xml | odata_expand (deprecated)| | Lookup binding | N/A (read-only) | odata_bind | | SELECT * | YES (SDK auto-expands) | Not applicable | | Polymorphic lookups | Separate JOINs | $expand by nav prop | @@ -1077,21 +1035,20 @@ def _run_examples(client): # OData version (expand) t0 = _time.time() try: - odata_rows = [] - for page in backoff( - lambda: client.records.get( - "account", - select=["name"], - expand=["contact_customer_accounts"], - top=5, + odata_rows = list( + backoff( + lambda: client.records.list( + "account", + select=["name"], + top=5, + ) ) - ): - odata_rows.extend(page) + ) odata_time = _time.time() - t0 - print(f" OData $expand: {len(odata_rows)} rows in {odata_time:.2f}s") + print(f" OData records.list: {len(odata_rows)} rows in {odata_time:.2f}s") except Exception as e: odata_time = _time.time() - t0 - print(f" OData $expand: error ({odata_time:.2f}s): {e}") + print(f" OData records.list: error ({odata_time:.2f}s): {e}") # ============================================================== # 32. Anti-Patterns & Best Practices @@ -1144,8 +1101,8 @@ def _run_examples(client): -> ValidationError (blocked). - Pattern #2 (cartesian FROM a, b) -> UserWarning (advisory). - Server enforces 5000-row cap on all queries (#3, #5). - - Use sql_columns() or sql_select() to discover valid column names. - - Use sql_joins() or sql_join() to discover valid JOIN clauses. + - Use sql_columns() to discover valid column names. + - Write JOIN clauses manually or use fetch_xml() for complex queries. """) # ============================================================== @@ -1177,8 +1134,8 @@ def _run_examples(client): | Nested polymorphic chains | YES | e.g. opp -> acct -> contact -> owner | | Audit trail (createdby, etc.) | YES | JOIN to systemuser | | SQL read -> DF write-back | YES | dataframe.sql() + .update()/.create() | -| SQL column discovery | YES | query.sql_columns() / sql_select() | -| SQL JOIN discovery | YES | query.sql_joins() / sql_join() | +| SQL column discovery | YES | query.sql_columns() | +| SQL JOIN clause | manual | write directly or use fetch_xml() | | OData column discovery | YES | query.odata_select() | | OData expand discovery | YES | query.odata_expands() / odata_expand() | | OData bind builder | YES | query.odata_bind() | @@ -1195,12 +1152,11 @@ def _run_examples(client): SQL-First Workflow (no OData knowledge needed): 1. Discover schema: cols = client.query.sql_columns("account") - 2. Discover JOINs: joins = client.query.sql_joins("contact") - 3. Build JOIN: j = client.query.sql_join("contact", "account", from_alias="c", to_alias="a") - 4. Query with SQL: df = client.dataframe.sql(f"SELECT c.fullname, a.name FROM contact c {j}") - 5. Transform: df["col"] = df["col"] * 1.1 - 6. Write back: client.dataframe.update("account", df, id_column="accountid") - 7. Verify: df2 = client.dataframe.sql("SELECT ...") + 2. Write JOIN: j = "JOIN account a ON c.parentcustomerid = a.accountid" + 3. Query with SQL: df = client.dataframe.sql(f"SELECT c.fullname, a.name FROM contact c {j}") + 4. Transform: df["col"] = df["col"] * 1.1 + 5. Write back: client.dataframe.update("account", df, id_column="accountid") + 6. Verify: df2 = client.dataframe.sql("SELECT ...") """) finally: diff --git a/examples/advanced/walkthrough.py b/examples/advanced/walkthrough.py index 6c0a6184..d2cc4ff9 100644 --- a/examples/advanced/walkthrough.py +++ b/examples/advanced/walkthrough.py @@ -26,7 +26,7 @@ from azure.identity import InteractiveBrowserCredential from PowerPlatform.Dataverse.client import DataverseClient from PowerPlatform.Dataverse.core.errors import MetadataError -from PowerPlatform.Dataverse.models.filters import eq, gt, between +from PowerPlatform.Dataverse.models.filters import col from PowerPlatform.Dataverse.models.query_builder import ExpandOption import requests @@ -183,8 +183,8 @@ def _run_walkthrough(client): print("=" * 80) # Single read by ID - log_call(f"client.records.get('{table_name}', '{id1}')") - record = backoff(lambda: client.records.get(table_name, id1)) + log_call(f"client.records.retrieve('{table_name}', '{id1}')") + record = backoff(lambda: client.records.retrieve(table_name, id1)) print("[OK] Retrieved single record:") print( json.dumps( @@ -203,11 +203,8 @@ def _run_walkthrough(client): ) # Multiple read with filter - log_call(f"client.records.get('{table_name}', filter='new_quantity gt 5')") - all_records = [] - records_iterator = backoff(lambda: client.records.get(table_name, filter="new_quantity gt 5")) - for page in records_iterator: - all_records.extend(page) + log_call(f"client.records.list('{table_name}', filter='new_quantity gt 5')") + all_records = list(backoff(lambda: client.records.list(table_name, filter="new_quantity gt 5"))) print(f"[OK] Found {len(all_records)} records with new_quantity > 5") for rec in all_records: print(f" - new_Title='{rec.get('new_title')}', new_Quantity={rec.get('new_quantity')}") @@ -231,7 +228,7 @@ def _run_walkthrough(client): }, ) ) - updated = backoff(lambda: client.records.get(table_name, id1)) + updated = backoff(lambda: client.records.retrieve(table_name, id1)) print(f"[OK] Updated single record new_Quantity: {updated.get('new_quantity')}") print(f" new_Notes: {repr(updated.get('new_notes'))}") @@ -263,9 +260,11 @@ def _run_walkthrough(client): print(f"[OK] Created {len(paging_ids)} records for paging demo") # Query with paging - log_call(f"client.records.get('{table_name}', page_size=5)") + log_call(f"client.query.builder('{table_name}').order_by().page_size(5).execute_pages()") print("Fetching records with page_size=5...") - paging_iterator = backoff(lambda: client.records.get(table_name, orderby=["new_Quantity"], page_size=5)) + paging_iterator = backoff( + lambda: client.query.builder(table_name).order_by("new_Quantity").page_size(5).execute_pages() + ) for page_num, page in enumerate(paging_iterator, start=1): record_ids = [r.get("new_walkthroughdemoid")[:8] + "..." for r in page] print(f" Page {page_num}: {len(page)} records - IDs: {record_ids}") @@ -278,13 +277,13 @@ def _run_walkthrough(client): print("=" * 80) # Basic fluent query: active records sorted by amount (flat iteration) - log_call("client.query.builder(...).select().filter_eq().order_by().execute()") + log_call("client.query.builder(...).select().where(col(...)==...).order_by().execute()") print("Querying incomplete records ordered by amount (fluent builder)...") qb_records = list( backoff( lambda: client.query.builder(table_name) .select("new_Title", "new_Amount", "new_Priority") - .filter_eq("new_Completed", False) + .where(col("new_Completed") == False) .order_by("new_Amount", descending=True) .top(10) .execute() @@ -294,14 +293,14 @@ def _run_walkthrough(client): for rec in qb_records[:5]: print(f" - '{rec.get('new_title')}' Amount={rec.get('new_amount')}") - # filter_in: records with specific priorities - log_call("client.query.builder(...).filter_in('new_Priority', [HIGH, LOW]).execute()") - print("Querying records with HIGH or LOW priority (filter_in)...") + # col().in_(): records with specific priorities + log_call("client.query.builder(...).where(col('new_Priority').in_([HIGH, LOW])).execute()") + print("Querying records with HIGH or LOW priority (col().in_())...") priority_records = list( backoff( lambda: client.query.builder(table_name) .select("new_Title", "new_Priority") - .filter_in("new_Priority", [Priority.HIGH, Priority.LOW]) + .where(col("new_Priority").in_([Priority.HIGH, Priority.LOW])) .execute() ) ) @@ -309,14 +308,14 @@ def _run_walkthrough(client): for rec in priority_records[:5]: print(f" - '{rec.get('new_title')}' Priority={rec.get('new_priority')}") - # filter_between: amount in a range - log_call("client.query.builder(...).filter_between('new_Amount', 500, 1500).execute()") - print("Querying records with amount between 500 and 1500 (filter_between)...") + # col().between(): amount in a range + log_call("client.query.builder(...).where(col('new_Amount').between(500, 1500)).execute()") + print("Querying records with amount between 500 and 1500 (col().between())...") range_records = list( backoff( lambda: client.query.builder(table_name) .select("new_Title", "new_Amount") - .filter_between("new_Amount", 500, 1500) + .where(col("new_Amount").between(500, 1500)) .execute() ) ) @@ -325,13 +324,13 @@ def _run_walkthrough(client): print(f" - '{rec.get('new_title')}' Amount={rec.get('new_amount')}") # Composable expression tree with where() - log_call("client.query.builder(...).where((eq(...) | eq(...)) & gt(...)).execute()") + log_call("client.query.builder(...).where((col(...) == ...) & (col(...) > ...)).execute()") print("Querying with composable expression tree (where)...") expr_records = list( backoff( lambda: client.query.builder(table_name) .select("new_Title", "new_Amount", "new_Quantity") - .where((eq("new_Completed", False) & gt("new_Amount", 100))) + .where((col("new_Completed") == False) & (col("new_Amount") > 100)) .order_by("new_Amount", descending=True) .top(5) .execute() @@ -341,19 +340,19 @@ def _run_walkthrough(client): for rec in expr_records: print(f" - '{rec.get('new_title')}' Amount={rec.get('new_amount')} Qty={rec.get('new_quantity')}") - # Combined: fluent filters + expression tree + paging (by_page=True) - log_call("client.query.builder(...).filter_eq().where(between()).page_size().execute(by_page=True)") - print("Querying with combined fluent + expression filters and paging...") + # Multiple where() clauses + lazy paging (execute_pages) + log_call("client.query.builder(...).where(col(...)==...).where(col().between()).page_size().execute_pages()") + print("Querying with combined expression filters and paging...") combined_page_count = 0 combined_record_count = 0 for page in backoff( lambda: client.query.builder(table_name) .select("new_Title", "new_Quantity") - .filter_eq("new_Completed", False) - .where(between("new_Quantity", 1, 15)) + .where(col("new_Completed") == False) + .where(col("new_Quantity").between(1, 15)) .order_by("new_Quantity") .page_size(3) - .execute(by_page=True) + .execute_pages() ): combined_page_count += 1 combined_record_count += len(page) @@ -362,13 +361,14 @@ def _run_walkthrough(client): print(f"[OK] Combined query: {combined_record_count} records across {combined_page_count} page(s)") # to_dataframe: get results as a pandas DataFrame - log_call(f"client.query.builder('{table_name}').select(...).filter_eq(...).to_dataframe()") + log_call(f"client.query.builder('{table_name}').select(...).where(col(...)==...).execute().to_dataframe()") print("Querying completed records as a pandas DataFrame (to_dataframe)...") df = backoff( lambda: ( client.query.builder(table_name) .select("new_title", "new_quantity") - .filter_eq("new_completed", True) + .where(col("new_completed") == True) + .execute() .to_dataframe() ) ) @@ -453,7 +453,7 @@ def _run_walkthrough(client): "new_Priority": "High", # String label instead of int } label_id = backoff(lambda: client.records.create(table_name, label_record)) - retrieved = backoff(lambda: client.records.get(table_name, label_id)) + retrieved = backoff(lambda: client.records.retrieve(table_name, label_id)) print(f"[OK] Created record with string label 'High' for new_Priority") print(f" new_Priority stored as integer: {retrieved.get('new_priority')}") print(f" new_Priority@FormattedValue: {retrieved.get('new_priority@OData.Community.Display.V1.FormattedValue')}") @@ -461,7 +461,7 @@ def _run_walkthrough(client): # Update with a string label log_call(f"client.records.update('{table_name}', label_id, {{'new_Priority': 'Low'}})") backoff(lambda: client.records.update(table_name, label_id, {"new_Priority": "Low"})) - updated_label = backoff(lambda: client.records.get(table_name, label_id)) + updated_label = backoff(lambda: client.records.retrieve(table_name, label_id)) print(f"[OK] Updated record with string label 'Low' for new_Priority") print(f" new_Priority stored as integer: {updated_label.get('new_priority')}") print( @@ -615,7 +615,7 @@ def _run_walkthrough(client): print(" [OK] Reading records by ID and with filters") print(" [OK] Single and multiple record updates") print(" [OK] Paging through large result sets") - print(" [OK] QueryBuilder fluent queries (filter_eq, filter_in, filter_between, where, to_dataframe)") + print(" [OK] QueryBuilder fluent queries (where + col(), col().in_(), col().between(), to_dataframe)") print(" [OK] Expand navigation properties (simple + nested ExpandOption)") print(" [OK] SQL queries") print(" [OK] Picklist label-to-value conversion") diff --git a/examples/basic/functional_testing.py b/examples/basic/functional_testing.py index 1ea0d5f0..95603475 100644 --- a/examples/basic/functional_testing.py +++ b/examples/basic/functional_testing.py @@ -236,7 +236,7 @@ def test_read_record(client: DataverseClient, table_info: Dict[str, Any], record record = None for attempt in range(1, retries + 1): try: - record = client.records.get(table_schema_name, record_id) + record = client.records.retrieve(table_schema_name, record_id) if attempt > 1: print(f" [OK] Record read succeeded after {attempt} attempts.") break @@ -287,28 +287,30 @@ def test_query_records(client: DataverseClient, table_info: Dict[str, Any]) -> N retries = 5 delay_seconds = 3 + select_cols = [f"{attr_prefix}_name", f"{attr_prefix}_count", f"{attr_prefix}_amount"] + active_filter = f"{attr_prefix}_is_active eq true" + try: - print("Querying records from test table...") + # -- records.list() — eager, all pages collected into one QueryResult ---------- + print("Querying records with records.list()...") for attempt in range(1, retries + 1): try: - records_iterator = client.records.get( + result = client.records.list( table_schema_name, - select=[f"{attr_prefix}_name", f"{attr_prefix}_count", f"{attr_prefix}_amount"], - filter=f"{attr_prefix}_is_active eq true", + select=select_cols, + filter=active_filter, top=5, - orderby=[f"{attr_prefix}_name asc"], ) record_count = 0 - for batch in records_iterator: - for record in batch: - record_count += 1 - name = record.get(f"{attr_prefix}_name", "N/A") - count = record.get(f"{attr_prefix}_count", "N/A") - amount = record.get(f"{attr_prefix}_amount", "N/A") - print(f" Record {record_count}: {name} (Count: {count}, Amount: {amount})") - - print(f"[OK] Query completed! Found {record_count} active records.") + for record in result: + record_count += 1 + name = record.get(f"{attr_prefix}_name", "N/A") + count = record.get(f"{attr_prefix}_count", "N/A") + amount = record.get(f"{attr_prefix}_amount", "N/A") + print(f" Record {record_count}: {name} (Count: {count}, Amount: {amount})") + + print(f"[OK] records.list() completed! Found {record_count} active records.") break except HttpError as err: if getattr(err, "status_code", None) == 404 and attempt < retries: @@ -317,6 +319,21 @@ def test_query_records(client: DataverseClient, table_info: Dict[str, Any]) -> N continue raise + # -- records.list_pages() — lazy, one QueryResult per HTTP page --------------- + print("\nQuerying records with records.list_pages() (paged)...") + page_num = 0 + total_records = 0 + for page in client.records.list_pages( + table_schema_name, + select=select_cols, + filter=active_filter, + ): + page_num += 1 + total_records += len(page) + names = [r.get(f"{attr_prefix}_name", "N/A") for r in page] + print(f" Page {page_num}: {len(page)} record(s) — {names}") + print(f"[OK] records.list_pages() completed! {total_records} records across {page_num} page(s).") + except Exception as e: print(f"[WARN] Query test encountered an issue: {e}") print(" This might be expected if the table is very new.") @@ -954,32 +971,29 @@ def test_relationships(client: DataverseClient) -> None: # --- Create parent and child tables --- print("\nCreating relationship test tables...") - parent_info = backoff( - lambda: client.tables.create( - rel_parent_schema, - {"test_Code": "string"}, - ) - ) + def _get_or_create(schema, columns, label): + info = client.tables.get(schema) + if info: + print(f"[OK] Table already exists: {schema} (skipped)") + return info + try: + result = backoff(lambda: client.tables.create(schema, columns)) + print(f"[OK] Created {label}: {schema}") + return result + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {schema} (skipped)") + return client.tables.get(schema) + raise + + parent_info = _get_or_create(rel_parent_schema, {"test_Code": "string"}, "parent table") created_tables.append(rel_parent_schema) - print(f"[OK] Created parent table: {parent_info['table_schema_name']}") - child_info = backoff( - lambda: client.tables.create( - rel_child_schema, - {"test_Number": "string"}, - ) - ) + child_info = _get_or_create(rel_child_schema, {"test_Number": "string"}, "child table") created_tables.append(rel_child_schema) - print(f"[OK] Created child table: {child_info['table_schema_name']}") - proj_info = backoff( - lambda: client.tables.create( - rel_m2m_schema, - {"test_ProjectCode": "string"}, - ) - ) + proj_info = _get_or_create(rel_m2m_schema, {"test_ProjectCode": "string"}, "M:N table") created_tables.append(rel_m2m_schema) - print(f"[OK] Created M:N table: {proj_info['table_schema_name']}") # --- Wait for table metadata to propagate --- wait_for_table_metadata(client, rel_parent_schema) @@ -1008,42 +1022,52 @@ def test_relationships(client: DataverseClient) -> None: ), ) - result_1n = backoff( - lambda: client.tables.create_one_to_many_relationship( - lookup=lookup, - relationship=relationship, + existing_1n = client.tables.get_relationship("test_RelParent_RelChild") + if existing_1n: + result_1n = existing_1n + rel_id_1n = result_1n.relationship_id + print(f" [OK] Relationship already exists: {result_1n.relationship_schema_name} (skipped)") + else: + result_1n = backoff( + lambda: client.tables.create_one_to_many_relationship( + lookup=lookup, + relationship=relationship, + ) ) - ) - - assert result_1n.relationship_schema_name == "test_RelParent_RelChild" - assert result_1n.relationship_type == "one_to_many" - assert result_1n.lookup_schema_name is not None - rel_id_1n = result_1n.relationship_id - print(f" [OK] Created 1:N relationship: {result_1n.relationship_schema_name}") - print(f" Lookup: {result_1n.lookup_schema_name}") - print(f" ID: {rel_id_1n}") + assert result_1n.relationship_schema_name == "test_RelParent_RelChild" + assert result_1n.relationship_type == "one_to_many" + assert result_1n.lookup_schema_name is not None + rel_id_1n = result_1n.relationship_id + print(f" [OK] Created 1:N relationship: {result_1n.relationship_schema_name}") + print(f" Lookup: {result_1n.lookup_schema_name}") + print(f" ID: {rel_id_1n}") # --- Test 2: Create lookup field (convenience API) --- print("\n Test 2: Create lookup field (convenience API)") print(" " + "-" * 45) - result_lookup = backoff( - lambda: client.tables.create_lookup_field( - referencing_table=child_info["table_logical_name"], - lookup_field_name="test_ManagerId", - referenced_table="contact", - display_name="Manager", - description="The record's manager contact", - required=False, - cascade_delete=CASCADE_BEHAVIOR_REMOVE_LINK, + existing_lookup = client.tables.get_relationship("contact_test_relchild_test_ManagerId") + if existing_lookup: + result_lookup = existing_lookup + rel_id_lookup = result_lookup.relationship_id + print(f" [OK] Lookup already exists: {result_lookup.relationship_schema_name} (skipped)") + else: + result_lookup = backoff( + lambda: client.tables.create_lookup_field( + referencing_table=child_info["table_logical_name"], + lookup_field_name="test_ManagerId", + referenced_table="contact", + display_name="Manager", + description="The record's manager contact", + required=False, + cascade_delete=CASCADE_BEHAVIOR_REMOVE_LINK, + ) ) - ) - - assert result_lookup.relationship_type == "one_to_many" - assert result_lookup.lookup_schema_name is not None - rel_id_lookup = result_lookup.relationship_id - print(f" [OK] Created lookup: {result_lookup.lookup_schema_name}") - print(f" Relationship: {result_lookup.relationship_schema_name}") + assert result_lookup.relationship_type == "one_to_many" + assert result_lookup.lookup_schema_name is not None + rel_id_lookup = result_lookup.relationship_id + print(f" [OK] Created lookup: {result_lookup.lookup_schema_name}") + print(f" Relationship: {result_lookup.relationship_schema_name}") # --- Test 3: Create N:N relationship --- print("\n Test 3: Create N:N relationship") @@ -1055,13 +1079,18 @@ def test_relationships(client: DataverseClient) -> None: entity2_logical_name=proj_info["table_logical_name"], ) - result_nn = backoff(lambda: client.tables.create_many_to_many_relationship(relationship=m2m)) - - assert result_nn.relationship_schema_name == "test_relchild_relproject" - assert result_nn.relationship_type == "many_to_many" - rel_id_nn = result_nn.relationship_id - print(f" [OK] Created N:N relationship: {result_nn.relationship_schema_name}") - print(f" ID: {rel_id_nn}") + existing_nn = client.tables.get_relationship("test_relchild_relproject") + if existing_nn: + result_nn = existing_nn + rel_id_nn = result_nn.relationship_id + print(f" [OK] Relationship already exists: {result_nn.relationship_schema_name} (skipped)") + else: + result_nn = backoff(lambda: client.tables.create_many_to_many_relationship(relationship=m2m)) + assert result_nn.relationship_schema_name == "test_relchild_relproject" + assert result_nn.relationship_type == "many_to_many" + rel_id_nn = result_nn.relationship_id + print(f" [OK] Created N:N relationship: {result_nn.relationship_schema_name}") + print(f" ID: {rel_id_nn}") # --- Test 4: Get relationship metadata --- print("\n Test 4: Query relationship metadata") diff --git a/examples/basic/installation_example.py b/examples/basic/installation_example.py index 98189e58..f7821181 100644 --- a/examples/basic/installation_example.py +++ b/examples/basic/installation_example.py @@ -221,7 +221,7 @@ def show_usage_examples(): print(f"Created account: {account_id}") # Read a single record by ID -account = client.records.get("account", account_id) +account = client.records.retrieve("account", account_id) print(f"Account name: {account['name']}") # Update a record @@ -234,13 +234,13 @@ def show_usage_examples(): Querying Data: ```python # Query with OData filter -accounts = client.records.get("account", +accounts = client.records.list("account", filter="name eq 'Contoso Ltd'", select=["name", "telephone1"], top=10) -for batch in accounts: - for account in batch: +for page in accounts: + for account in page: print(f"Account: {account['name']}") # SQL queries (if enabled) diff --git a/pyproject.toml b/pyproject.toml index fd7d8d66..416e6680 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ dev = [ "mypy>=1.0.0", "ruff>=0.1.0", ] +migration = ["libcst>=1.0.0"] [tool.setuptools] package-dir = {"" = "src"} diff --git a/src/PowerPlatform/Dataverse/__init__.py b/src/PowerPlatform/Dataverse/__init__.py index 95b5171c..0d4ed6e2 100644 --- a/src/PowerPlatform/Dataverse/__init__.py +++ b/src/PowerPlatform/Dataverse/__init__.py @@ -3,6 +3,10 @@ from importlib.metadata import version +from .models.filters import col, raw +from .models.protocol import DataverseModel +from .models.record import QueryResult + __version__ = version("PowerPlatform-Dataverse-Client") -__all__ = ["__version__"] +__all__ = ["__version__", "col", "raw", "DataverseModel", "QueryResult"] diff --git a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md index 72677468..bda1ed14 100644 --- a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md +++ b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md @@ -32,7 +32,7 @@ The SDK supports Dataverse's native bulk operations: Pass lists to `create()`, ` - Use `top` parameter to limit total records returned ### DataFrame Support -- DataFrame operations are accessed via the `client.dataframe` namespace: `client.dataframe.get()`, `client.dataframe.create()`, `client.dataframe.update()`, `client.dataframe.delete()` +- DataFrame operations are accessed via the `client.dataframe` namespace: `client.dataframe.create()`, `client.dataframe.update()`, `client.dataframe.delete()` — `client.dataframe.get()` is deprecated; use `client.query.builder(table).where(...).execute().to_dataframe()` instead ## Common Operations @@ -85,28 +85,40 @@ contact_ids = client.records.create("contact", contacts) #### Read Records ```python # Get single record by ID -account = client.records.get("account", account_id, select=["name", "telephone1"]) +account = client.records.retrieve("account", account_id, select=["name", "telephone1"]) -# Query with filter (paginated) -for page in client.records.get( +# Query with filter — follows @odata.nextLink automatically (multiple HTTP requests if needed), +# loads all matching records into memory, returns a single QueryResult. +# Page size is Dataverse's default (~5000/page); use top to bound total records and round-trips. +# For very large sets where memory is a concern, use query.builder().execute(by_page=True) instead. +result = client.records.list( "account", select=["accountid", "name"], # select is case-insensitive (automatically lowercased) filter="statecode eq 0", # filter must use lowercase logical names (not transformed) - top=100, -): + top=100, # bounds both total records returned and HTTP round-trips +) +for record in result: + print(record["name"]) + +# For large result sets that must be streamed page-by-page (caller controls memory, one page at a time) +for page in (client.query.builder("account") + .select("accountid", "name") + .where(col("statecode") == 0) + .page_size(500) # optional: override Dataverse default page size + .execute(by_page=True)): for record in page: print(record["name"]) -# Query with navigation property expansion (case-sensitive!) -for page in client.records.get( - "account", - select=["name"], - expand=["primarycontactid"], # Navigation properties are case-sensitive! - filter="statecode eq 0", # Column names must be lowercase logical names -): - for account in page: - contact = account.get("primarycontactid", {}) - print(f"{account['name']} - {contact.get('fullname', 'N/A')}") +# Query with navigation property expansion — use the query builder (records.list() has no expand) +from PowerPlatform.Dataverse.models.query_builder import ExpandOption +from PowerPlatform.Dataverse.models.filters import col +for record in (client.query.builder("account") + .select("name") + .expand(ExpandOption("primarycontactid").select("fullname")) + .where(col("statecode") == 0) + .execute()): + contact = record.get("primarycontactid", {}) + print(f"{record['name']} - {contact.get('fullname', 'N/A')}") ``` #### Create Records with Lookup Bindings (@odata.bind) @@ -179,18 +191,20 @@ client.records.delete("account", [id1, id2, id3], use_bulk_delete=True) The SDK provides DataFrame wrappers for all CRUD operations via the `client.dataframe` namespace, using pandas DataFrames and Series as input/output. +> **Note:** `client.dataframe.get()` is deprecated. Use `client.query.builder(table).select(...).filter(...).to_dataframe()` instead. + ```python import pandas as pd -# Query records -- returns a single DataFrame -df = client.dataframe.get("account", filter="statecode eq 0", select=["name"]) +# Query records -- returns a single DataFrame (GA builder pattern) +df = client.query.builder("account").filter("statecode eq 0").select("name").to_dataframe() print(f"Got {len(df)} rows") -# Limit results with top for large tables -df = client.dataframe.get("account", select=["name"], top=100) +# Limit results with top +df = client.query.builder("account").select("name").top(100).to_dataframe() # Fetch single record as one-row DataFrame -df = client.dataframe.get("account", record_id=account_id, select=["name"]) +df = client.records.retrieve("account", account_id, select=["name"]).to_dataframe() # Create records from a DataFrame (returns a Series of GUIDs) new_accounts = pd.DataFrame([ @@ -380,7 +394,8 @@ Use `client.batch` to send multiple operations in one HTTP request. All batch me batch = client.batch.new() batch.records.create("account", {"name": "Contoso"}) batch.records.update("account", account_id, {"telephone1": "555-0100"}) -batch.records.get("account", account_id, select=["name"]) +batch.records.retrieve("account", account_id, select=["name"]) # single record (GA) +batch.records.list("account", filter="statecode eq 0", top=50) # multi-record, single page batch.query.sql("SELECT TOP 5 name FROM account") result = batch.execute() @@ -412,7 +427,8 @@ print(f"Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") **Batch limitations:** - Maximum 1000 operations per batch -- Paginated `records.get()` (without `record_id`) is not supported in batch +- `batch.records.get()` is deprecated; use `batch.records.retrieve()` for single records +- `batch.records.list()` returns a single page (no pagination); use `top` to bound results - `flush_cache()` is not supported in batch ## Error Handling @@ -430,7 +446,7 @@ from PowerPlatform.Dataverse.core.errors import ( from PowerPlatform.Dataverse.client import DataverseClient try: - client.records.get("account", "invalid-id") + client.records.retrieve("account", "invalid-id") except HttpError as e: print(f"HTTP {e.status_code}: {e.message}") print(f"Error code: {e.code}") diff --git a/src/PowerPlatform/Dataverse/client.py b/src/PowerPlatform/Dataverse/client.py index ea9dd6b8..bd7c1959 100644 --- a/src/PowerPlatform/Dataverse/client.py +++ b/src/PowerPlatform/Dataverse/client.py @@ -3,9 +3,8 @@ from __future__ import annotations -import warnings from contextlib import contextmanager -from typing import Any, Dict, Iterable, Iterator, List, Optional, Union +from typing import Iterator, Optional import requests @@ -199,581 +198,8 @@ def _check_closed(self) -> None: if self._closed: raise RuntimeError("DataverseClient is closed") - # ---------------- Unified CRUD: create/update/delete ---------------- - def create(self, table_schema_name: str, records: Union[Dict[str, Any], List[Dict[str, Any]]]) -> List[str]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.records.RecordOperations.create` instead. - - Create one or more records by table name. - - :param table_schema_name: Schema name of the table (e.g. ``"account"``, ``"contact"``, or ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param records: A single record dictionary or a list of record dictionaries. - Each dictionary should contain column schema names as keys. - :type records: dict or list[dict] - - :return: List of created record GUIDs. Returns a single-element list for a single input. - :rtype: list[str] - - :raises TypeError: If ``records`` is not a dict or list[dict], or if the internal - client returns an unexpected type. - - Example: - Create a single record:: - - client = DataverseClient(base_url, credential) - ids = client.create("account", {"name": "Contoso"}) - print(f"Created: {ids[0]}") - - Create multiple records:: - - records = [ - {"name": "Contoso"}, - {"name": "Fabrikam"} - ] - ids = client.create("account", records) - print(f"Created {len(ids)} accounts") - """ - warnings.warn( - "client.create() is deprecated. Use client.records.create() instead.", - DeprecationWarning, - stacklevel=2, - ) - # Old API always returned list[str]; new returns str for single - if isinstance(records, dict): - return [self.records.create(table_schema_name, records)] - return self.records.create(table_schema_name, records) - - def update( - self, table_schema_name: str, ids: Union[str, List[str]], changes: Union[Dict[str, Any], List[Dict[str, Any]]] - ) -> None: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.records.RecordOperations.update` instead. - - Update one or more records. - - This method supports three usage patterns: - - 1. Single record update: ``update("account", "guid", {"name": "New Name"})`` - 2. Broadcast update: ``update("account", [id1, id2], {"status": 1})`` - applies same changes to all IDs - 3. Paired updates: ``update("account", [id1, id2], [changes1, changes2])`` - one-to-one mapping - - :param table_schema_name: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param ids: Single GUID string or list of GUID strings to update. - :type ids: str or list[str] - :param changes: Dictionary of changes for single/broadcast mode, or list of dictionaries - for paired mode. When ``ids`` is a list and ``changes`` is a single dict, - the same changes are broadcast to all records. When both are lists, they must - have equal length for one-to-one mapping. - :type changes: dict or list[dict] - - :raises TypeError: If ``ids`` is not str or list[str], or if ``changes`` type doesn't match usage pattern. - - .. note:: - Single updates discard the response representation for better performance. For broadcast or paired updates, the method delegates to the internal client's batch update logic. - - Example: - Single record update:: - - client.update("account", account_id, {"telephone1": "555-0100"}) - - Broadcast same changes to multiple records:: - - client.update("account", [id1, id2, id3], {"statecode": 1}) - - Update multiple records with different values:: - - ids = [id1, id2] - changes = [ - {"name": "Updated Name 1"}, - {"name": "Updated Name 2"} - ] - client.update("account", ids, changes) - """ - warnings.warn( - "client.update() is deprecated. Use client.records.update() instead.", - DeprecationWarning, - stacklevel=2, - ) - self.records.update(table_schema_name, ids, changes) - - def delete( - self, - table_schema_name: str, - ids: Union[str, List[str]], - use_bulk_delete: bool = True, - ) -> Optional[str]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.records.RecordOperations.delete` instead. + # ---------------- Cache utilities ---------------- - Delete one or more records by GUID. - - :param table_schema_name: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param ids: Single GUID string or list of GUID strings to delete. - :type ids: str or list[str] - :param use_bulk_delete: When ``True`` (default) and ``ids`` is a list, execute the BulkDelete action and - return its async job identifier. When ``False`` each record is deleted sequentially. - :type use_bulk_delete: :class:`bool` - - :raises TypeError: If ``ids`` is not str or list[str]. - :raises HttpError: If the underlying Web API delete request fails. - - :return: BulkDelete job ID when deleting multiple records via BulkDelete; otherwise ``None``. - :rtype: :class:`str` or None - - Example: - Delete a single record:: - - client.delete("account", account_id) - - Delete multiple records:: - - job_id = client.delete("account", [id1, id2, id3]) - """ - warnings.warn( - "client.delete() is deprecated. Use client.records.delete() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.records.delete(table_schema_name, ids, use_bulk_delete=use_bulk_delete) - - def get( - self, - table_schema_name: str, - record_id: Optional[str] = None, - select: Optional[List[str]] = None, - filter: Optional[str] = None, - orderby: Optional[List[str]] = None, - top: Optional[int] = None, - expand: Optional[List[str]] = None, - page_size: Optional[int] = None, - ) -> Union[Dict[str, Any], Iterable[List[Dict[str, Any]]]]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.records.RecordOperations.get` instead. - - - **Single record by ID** -- ``client.records.get(table, record_id)`` - - **Query / filter multiple records** -- ``client.records.get(table, filter=..., select=...)`` - - Fetch a single record by ID or query multiple records. - - When ``record_id`` is provided, returns a single record dictionary. - When ``record_id`` is None, returns a generator yielding batches of records. - - :param table_schema_name: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param record_id: Optional GUID to fetch a specific record. If None, queries multiple records. - :type record_id: :class:`str` or None - :param select: Optional list of attribute logical names to retrieve. Column names are case-insensitive and automatically lowercased (e.g. ``["new_Title", "new_Amount"]`` becomes ``"new_title,new_amount"``). - :type select: list[str] or None - :param filter: Optional OData filter string, e.g. ``"name eq 'Contoso'"`` or ``"new_quantity gt 5"``. Column names in filter expressions must use exact lowercase logical names (e.g. ``"new_quantity"``, not ``"new_Quantity"``). The filter string is passed directly to the Dataverse Web API without transformation. - :type filter: :class:`str` or None - :param orderby: Optional list of attributes to sort by, e.g. ``["name asc", "createdon desc"]``. Column names are automatically lowercased. - :type orderby: list[str] or None - :param top: Optional maximum number of records to return. - :type top: :class:`int` or None - :param expand: Optional list of navigation properties to expand, e.g. ``["primarycontactid"]``. Navigation property names are case-sensitive and must match the server-defined names exactly. These are NOT automatically transformed. Consult entity metadata for correct casing. - :type expand: list[str] or None - :param page_size: Optional number of records per page for pagination. - :type page_size: :class:`int` or None - - :return: Single record dict if ``record_id`` is provided, otherwise a generator - yielding lists of record dictionaries (one list per page). - :rtype: dict or collections.abc.Iterable[list[dict]] - - :raises TypeError: If ``record_id`` is provided but not a string. - - Example: - Fetch a single record:: - - record = client.get("account", record_id=account_id, select=["name", "telephone1"]) - print(record["name"]) - - Query multiple records with filtering (note: exact logical names in filter):: - - for batch in client.get( - "account", - filter="statecode eq 0 and name eq 'Contoso'", # Must use exact logical names (lower-case) - select=["name", "telephone1"] - ): - for account in batch: - print(account["name"]) - - Query with navigation property expansion (note: case-sensitive property name):: - - for batch in client.get( - "account", - select=["name"], - expand=["primarycontactid"], # Case-sensitive! Check metadata for exact name - filter="statecode eq 0" - ): - for account in batch: - print(f"{account['name']} - Contact: {account.get('primarycontactid', {}).get('fullname')}") - - Query with sorting and pagination:: - - for batch in client.get( - "account", - orderby=["createdon desc"], - top=100, - page_size=50 - ): - print(f"Batch size: {len(batch)}") - """ - warnings.warn( - "client.get() is deprecated. Use client.records.get() instead.", - DeprecationWarning, - stacklevel=2, - ) - if record_id is not None: - return self.records.get(table_schema_name, record_id, select=select) - else: - return self.records.get( - table_schema_name, - select=select, - filter=filter, - orderby=orderby, - top=top, - expand=expand, - page_size=page_size, - ) - - # SQL via Web API sql parameter - def query_sql(self, sql: str) -> List[Dict[str, Any]]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.query.QueryOperations.sql` instead. - - Execute a read-only SQL query using the Dataverse Web API ``?sql`` capability. - - The SQL query must follow the supported subset: a single SELECT statement with - optional WHERE, TOP (integer literal), ORDER BY (column names only), and a simple - table alias after FROM. - - :param sql: Supported SQL SELECT statement. - :type sql: :class:`str` - - :return: List of result row dictionaries. Returns an empty list if no rows match. - :rtype: list[dict] - - :raises ~PowerPlatform.Dataverse.core.errors.SQLParseError: If the SQL query uses unsupported syntax. - :raises ~PowerPlatform.Dataverse.core.errors.HttpError: If the Web API returns an error. - - .. note:: - The SQL support is limited to read-only queries. Complex joins, subqueries, and certain SQL functions may not be supported. Consult the Dataverse documentation for the current feature set. - - Example: - Basic SQL query:: - - sql = "SELECT TOP 10 accountid, name FROM account WHERE name LIKE 'C%' ORDER BY name" - results = client.query_sql(sql) - for row in results: - print(row["name"]) - - Query with alias:: - - sql = "SELECT a.name, a.telephone1 FROM account AS a WHERE a.statecode = 0" - results = client.query_sql(sql) - """ - warnings.warn( - "client.query_sql() is deprecated. Use client.query.sql() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.query.sql(sql) - - # Table metadata helpers - def get_table_info(self, table_schema_name: str) -> Optional[Dict[str, Any]]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.tables.TableOperations.get` instead. - - Get basic metadata for a table if it exists. - - :param table_schema_name: Schema name of the table (e.g. ``"new_MyTestTable"`` or ``"account"``). - :type table_schema_name: :class:`str` - - :return: Dictionary containing table metadata with keys ``table_schema_name``, - ``table_logical_name``, ``entity_set_name``, and ``metadata_id``. - Returns None if the table is not found. - :rtype: :class:`dict` or None - - Example: - Retrieve table metadata:: - - info = client.get_table_info("new_MyTestTable") - if info: - print(f"Logical name: {info['table_logical_name']}") - print(f"Entity set: {info['entity_set_name']}") - """ - warnings.warn( - "client.get_table_info() is deprecated. Use client.tables.get() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.tables.get(table_schema_name) - - def create_table( - self, - table_schema_name: str, - columns: Dict[str, Any], - solution_unique_name: Optional[str] = None, - primary_column_schema_name: Optional[str] = None, - ) -> Dict[str, Any]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.tables.TableOperations.create` instead. - - Create a simple custom table with specified columns. - - :param table_schema_name: Schema name of the table with customization prefix value (e.g. ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param columns: Dictionary mapping column names (with customization prefix value) to their types. All custom column names must include the customization prefix value (e.g. ``"new_Title"``). - Supported types: - - - Primitive types: ``"string"`` (alias: ``"text"``), ``"int"`` (alias: ``"integer"``), ``"decimal"`` (alias: ``"money"``), ``"float"`` (alias: ``"double"``), ``"datetime"`` (alias: ``"date"``), ``"bool"`` (alias: ``"boolean"``), and ``"file"`` - - Enum subclass (IntEnum preferred): Creates a local option set. Optional multilingual - labels can be provided via ``__labels__`` class attribute, defined inside the Enum subclass:: - - class ItemStatus(IntEnum): - ACTIVE = 1 - INACTIVE = 2 - __labels__ = { - 1033: {"Active": "Active", "Inactive": "Inactive"}, - 1036: {"Active": "Actif", "Inactive": "Inactif"} - } - - :type columns: dict[str, typing.Any] - :param solution_unique_name: Optional solution unique name that should own the new table. When omitted the table is created in the default solution. - :type solution_unique_name: :class:`str` or None - :param primary_column_schema_name: Optional primary name column schema name with customization prefix value (e.g. ``"new_MyTestTable"``). If not provided, defaults to ``"{customization prefix value}_Name"``. - :type primary_column_schema_name: :class:`str` or None - - :return: Dictionary containing table metadata including ``table_schema_name``, - ``entity_set_name``, ``table_logical_name``, ``metadata_id``, and ``columns_created``. - :rtype: :class:`dict` - - :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: If table creation fails or the schema is invalid. - - Example: - Create a table with simple columns:: - - from enum import IntEnum - - class ItemStatus(IntEnum): - ACTIVE = 1 - INACTIVE = 2 - - columns = { - "new_Title": "string", # Note: includes 'new_' customization prefix value - "new_Quantity": "int", - "new_Price": "decimal", - "new_Available": "bool", - "new_Status": ItemStatus - } - - result = client.create_table("new_MyTestTable", columns) - print(f"Created table: {result['table_schema_name']}") - print(f"Columns: {result['columns_created']}") - - Create a table with a custom primary column name:: - - result = client.create_table( - "new_Product", - {"new_Price": "decimal"}, - primary_column_schema_name="new_ProductName" - ) - """ - warnings.warn( - "client.create_table() is deprecated. Use client.tables.create() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.tables.create( - table_schema_name, - columns, - solution=solution_unique_name, - primary_column=primary_column_schema_name, - ) - - def delete_table(self, table_schema_name: str) -> None: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.tables.TableOperations.delete` instead. - - Delete a custom table by name. - - :param table_schema_name: Schema name of the table (e.g. ``"new_MyTestTable"`` or ``"account"``). - :type table_schema_name: :class:`str` - - :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: If the table does not exist or deletion fails. - - .. warning:: - This operation is irreversible and will delete all records in the table along - with the table definition. Use with caution. - - Example: - Delete a custom table:: - - client.delete_table("new_MyTestTable") - """ - warnings.warn( - "client.delete_table() is deprecated. Use client.tables.delete() instead.", - DeprecationWarning, - stacklevel=2, - ) - self.tables.delete(table_schema_name) - - def list_tables(self) -> list[dict[str, Any]]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.tables.TableOperations.list` instead. - - List all non-private tables in the Dataverse environment. - - :return: List of EntityDefinition metadata dictionaries. - :rtype: list[dict] - - Example: - List all non-private tables and print their logical names:: - - tables = client.list_tables() - for table in tables: - print(table["LogicalName"]) - """ - warnings.warn( - "client.list_tables() is deprecated. Use client.tables.list() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.tables.list() - - def create_columns( - self, - table_schema_name: str, - columns: Dict[str, Any], - ) -> List[str]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.tables.TableOperations.add_columns` instead. - - Create one or more columns on an existing table using a schema-style mapping. - - :param table_schema_name: Schema name of the table (e.g. ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param columns: Mapping of column schema names (with customization prefix value) to supported types. All custom column names must include the customization prefix value** (e.g. ``"new_Notes"``). Primitive types include - ``"string"`` (alias: ``"text"``), ``"int"`` (alias: ``"integer"``), ``"decimal"`` (alias: ``"money"``), ``"float"`` (alias: ``"double"``), ``"datetime"`` (alias: ``"date"``), ``"bool"`` (alias: ``"boolean"``), and ``"file"``. Enum subclasses (IntEnum preferred) - generate a local option set and can specify localized labels via ``__labels__``. - :type columns: dict[str, typing.Any] - :returns: Schema names for the columns that were created. - :rtype: list[str] - Example: - Create multiple columns on the custom table:: - - created = client.create_columns( - "new_MyTestTable", - { - "new_Scratch": "string", - "new_Flags": "bool", - "new_Document": "file", - }, - ) - print(created) # ['new_Scratch', 'new_Flags', 'new_Document'] - """ - warnings.warn( - "client.create_columns() is deprecated. Use client.tables.add_columns() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.tables.add_columns(table_schema_name, columns) - - def delete_columns( - self, - table_schema_name: str, - columns: Union[str, List[str]], - ) -> List[str]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.tables.TableOperations.remove_columns` instead. - - Delete one or more columns from a table. - - :param table_schema_name: Schema name of the table (e.g. ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param columns: Column name or list of column names to remove. Must include customization prefix value (e.g. ``"new_TestColumn"``). - :type columns: str or list[str] - :returns: Schema names for the columns that were removed. - :rtype: list[str] - Example: - Remove two custom columns by schema name: - - removed = client.delete_columns( - "new_MyTestTable", - ["new_Scratch", "new_Flags"], - ) - print(removed) # ['new_Scratch', 'new_Flags'] - """ - warnings.warn( - "client.delete_columns() is deprecated. Use client.tables.remove_columns() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.tables.remove_columns(table_schema_name, columns) - - # File upload - def upload_file( - self, - table_schema_name: str, - record_id: str, - file_name_attribute: str, - path: str, - mode: Optional[str] = None, - mime_type: Optional[str] = None, - if_none_match: bool = True, - ) -> None: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.files.FileOperations.upload` instead. - - Upload a file to a Dataverse file column. - - :param table_schema_name: Schema name of the table. - :type table_schema_name: :class:`str` - :param record_id: GUID of the target record. - :type record_id: :class:`str` - :param file_name_attribute: Schema name of the file column attribute. - :type file_name_attribute: :class:`str` - :param path: Local filesystem path to the file. - :type path: :class:`str` - :param mode: Upload strategy: ``"auto"`` (default), ``"small"``, or ``"chunk"``. - :type mode: :class:`str` or None - :param mime_type: Explicit MIME type to store with the file. - :type mime_type: :class:`str` or None - :param if_none_match: When True (default), only succeed if the column is - currently empty. - :type if_none_match: :class:`bool` - """ - warnings.warn( - "client.upload_file() is deprecated. Use client.files.upload() instead.", - DeprecationWarning, - stacklevel=2, - ) - self.files.upload( - table_schema_name, - record_id, - file_name_attribute, - path, - mode=mode, - mime_type=mime_type, - if_none_match=if_none_match, - ) - - # Cache utilities def flush_cache(self, kind) -> int: """ Flush cached client metadata or state. diff --git a/src/PowerPlatform/Dataverse/data/_batch.py b/src/PowerPlatform/Dataverse/data/_batch.py index 2e880c01..1d5126e2 100644 --- a/src/PowerPlatform/Dataverse/data/_batch.py +++ b/src/PowerPlatform/Dataverse/data/_batch.py @@ -71,6 +71,15 @@ class _RecordGet: select: Optional[List[str]] = None +@dataclass +class _RecordList: + table: str + select: Optional[List[str]] = None + filter: Optional[str] = None + orderby: Optional[List[str]] = None + top: Optional[int] = None + + @dataclass class _RecordUpsert: table: str @@ -312,6 +321,8 @@ def _resolve_item(self, item: Any) -> List[_RawRequest]: return self._resolve_record_delete(item) if isinstance(item, _RecordGet): return self._resolve_record_get(item) + if isinstance(item, _RecordList): + return self._resolve_record_list(item) if isinstance(item, _RecordUpsert): return self._resolve_record_upsert(item) if isinstance(item, _TableCreate): @@ -384,6 +395,17 @@ def _resolve_record_delete(self, op: _RecordDelete) -> List[_RawRequest]: def _resolve_record_get(self, op: _RecordGet) -> List[_RawRequest]: return [self._od._build_get(op.table, op.record_id, select=op.select)] + def _resolve_record_list(self, op: _RecordList) -> List[_RawRequest]: + return [ + self._od._build_list( + op.table, + select=op.select, + filter=op.filter, + orderby=op.orderby, + top=op.top, + ) + ] + def _resolve_record_upsert(self, op: _RecordUpsert) -> List[_RawRequest]: entity_set = self._od._entity_set_from_schema_name(op.table) if len(op.items) == 1: diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index 4848e840..c5cc8fce 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -2317,6 +2317,31 @@ def _build_get( url += "?$select=" + ",".join(self._lowercase_list(select)) return _RawRequest(method="GET", url=url) + def _build_list( + self, + table: str, + *, + select: Optional[List[str]] = None, + filter: Optional[str] = None, + orderby: Optional[List[str]] = None, + top: Optional[int] = None, + ) -> _RawRequest: + """Build a multi-record GET request (single page, no pagination) without sending it.""" + entity_set = self._entity_set_from_schema_name(table) + params: List[str] = [] + if select: + params.append("$select=" + ",".join(self._lowercase_list(select))) + if filter: + params.append("$filter=" + filter) + if orderby: + params.append("$orderby=" + ",".join(orderby)) + if top is not None: + params.append(f"$top={top}") + url = f"{self.api}/{entity_set}" + if params: + url += "?" + "&".join(params) + return _RawRequest(method="GET", url=url) + def _build_create_entity( self, table: str, diff --git a/src/PowerPlatform/Dataverse/models/__init__.py b/src/PowerPlatform/Dataverse/models/__init__.py index dc10a4c0..50bd7326 100644 --- a/src/PowerPlatform/Dataverse/models/__init__.py +++ b/src/PowerPlatform/Dataverse/models/__init__.py @@ -8,12 +8,18 @@ - :class:`~PowerPlatform.Dataverse.models.query_builder.QueryBuilder`: Fluent query builder. - :mod:`~PowerPlatform.Dataverse.models.filters`: Composable OData filter expressions. +- :class:`~PowerPlatform.Dataverse.models.record.QueryResult`: Iterable result wrapper. - :class:`~PowerPlatform.Dataverse.models.upsert.UpsertItem`: Upsert operation item. Import directly from the specific module, e.g.:: from PowerPlatform.Dataverse.models.query_builder import QueryBuilder - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col, raw + from PowerPlatform.Dataverse.models.record import QueryResult """ -__all__ = [] +from .filters import col, raw +from .protocol import DataverseModel +from .record import QueryResult + +__all__ = ["col", "raw", "DataverseModel", "QueryResult"] diff --git a/src/PowerPlatform/Dataverse/models/fetch_xml_query.py b/src/PowerPlatform/Dataverse/models/fetch_xml_query.py new file mode 100644 index 00000000..cd08c0d1 --- /dev/null +++ b/src/PowerPlatform/Dataverse/models/fetch_xml_query.py @@ -0,0 +1,121 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""FetchXmlQuery — inert query object returned by QueryOperations.fetch_xml().""" + +from __future__ import annotations + +import xml.etree.ElementTree as _ET +from typing import Iterator, List, TYPE_CHECKING +from urllib.parse import unquote as _url_unquote + +from .record import QueryResult, Record + +if TYPE_CHECKING: + from ..client import DataverseClient + + +__all__ = ["FetchXmlQuery"] + +_PREFER_HEADER = ( + "odata.include-annotations=" '"Microsoft.Dynamics.CRM.fetchxmlpagingcookie,' 'Microsoft.Dynamics.CRM.morerecords"' +) + + +class FetchXmlQuery: + """Inert FetchXML query object. No HTTP request is made until + :meth:`execute` or :meth:`execute_pages` is called. + + Obtained via ``client.query.fetch_xml(xml)``. + + :param xml: Stripped, well-formed FetchXML string. + :param entity_name: Entity schema name from the ```` element. + :param client: Parent :class:`~PowerPlatform.Dataverse.client.DataverseClient`. + """ + + def __init__(self, xml: str, entity_name: str, client: "DataverseClient") -> None: + self._xml = xml + self._entity_name = entity_name + self._client = client + + def execute(self) -> QueryResult: + """Execute the FetchXML query and return all results as a :class:`QueryResult`. + + Blocking — fetches all pages upfront and holds every record in memory before + returning. Simple for small-to-medium result sets; use :meth:`execute_pages` + when the result set may be large or you want to process records as they arrive. + + :return: All matching records across all pages. + :rtype: :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + + Example:: + + rows = client.query.fetch_xml(xml).execute() + df = rows.to_dataframe() + """ + all_records: List[Record] = [] + for page in self.execute_pages(): + all_records.extend(page.records) + return QueryResult(all_records) + + def execute_pages(self) -> Iterator[QueryResult]: + """Lazily yield one :class:`QueryResult` per HTTP page. + + Streaming — each iteration fires one HTTP request and yields one page. + Prefer over :meth:`execute` when: + + - The result set may be large and you do not want all records in memory at once. + - You want early exit: stop iterating once you find what you need and the + remaining HTTP round-trips are skipped automatically. + - You need per-page progress reporting or batched downstream writes. + + One-shot — do not iterate more than once. + + :return: Iterator of per-page :class:`QueryResult` objects. + :rtype: Iterator[:class:`~PowerPlatform.Dataverse.models.record.QueryResult`] + + Example:: + + for page in client.query.fetch_xml(xml).execute_pages(): + process(page.to_dataframe()) + """ + current_xml = self._xml + page_num = 1 + + with self._client._scoped_odata() as od: + entity_set = od._entity_set_from_schema_name(self._entity_name) + base_url = f"{od.api}/{entity_set}" + + while True: + r = od._request( + "get", + base_url, + headers={"Prefer": _PREFER_HEADER}, + params={"fetchXml": current_xml}, + ) + data = r.json() if hasattr(r, "json") else {} + items = data.get("value") if isinstance(data, dict) else None + page_records: List[Record] = [] + if isinstance(items, list): + for item in items: + if isinstance(item, dict): + page_records.append(Record.from_api_response(self._entity_name, item)) + + yield QueryResult(page_records) + + more = bool(data.get("@Microsoft.Dynamics.CRM.morerecords", False)) if isinstance(data, dict) else False + if not more: + break + + raw_cookie = ( + data.get("@Microsoft.Dynamics.CRM.fetchxmlpagingcookie", "") if isinstance(data, dict) else "" + ) + if not raw_cookie: + break + + cookie = _url_unquote(_url_unquote(raw_cookie)) + page_num += 1 + fetch_el = _ET.fromstring(current_xml) + fetch_el.set("paging-cookie", cookie) + fetch_el.set("page", str(page_num)) + current_xml = _ET.tostring(fetch_el, encoding="unicode") diff --git a/src/PowerPlatform/Dataverse/models/filters.py b/src/PowerPlatform/Dataverse/models/filters.py index c5de258c..f0aca7d4 100644 --- a/src/PowerPlatform/Dataverse/models/filters.py +++ b/src/PowerPlatform/Dataverse/models/filters.py @@ -10,24 +10,26 @@ Example:: - from PowerPlatform.Dataverse.models.filters import eq, gt, filter_in + from PowerPlatform.Dataverse.models.filters import col, raw - # Simple comparison - expr = eq("statecode", 0) + # Preferred GA idiom — col() proxy + expr = col("statecode") == 0 print(expr.to_odata()) # statecode eq 0 # Complex composition with OR and AND - expr = (eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000) + expr = (col("statecode") == 0) | (col("statecode") == 1) & (col("revenue") > 100000) print(expr.to_odata()) - # ((statecode eq 0 or statecode eq 1) and revenue gt 100000) - # In operator (Dataverse function) - expr = filter_in("statecode", [0, 1, 2]) + # In / not-in + expr = col("statecode").in_([0, 1, 2]) print(expr.to_odata()) # Microsoft.Dynamics.CRM.In(PropertyName='statecode',PropertyValues=["0","1","2"]) + # Raw OData escape hatch (no deprecation warning) + expr = raw("Microsoft.Dynamics.CRM.Today(PropertyName='createdon')") + # Negation - expr = ~eq("statecode", 1) + expr = ~(col("statecode") == 1) print(expr.to_odata()) # not (statecode eq 1) """ @@ -35,11 +37,16 @@ import enum import uuid +import warnings from datetime import date, datetime, timezone -from typing import Any, Collection, Sequence +from typing import Any, Collection, List __all__ = [ "FilterExpression", + "ColumnProxy", + "col", + "raw", + # Deprecated factories — still functional, fire DeprecationWarning on call: "eq", "ne", "gt", @@ -55,7 +62,6 @@ "filter_in", "not_in", "not_between", - "raw", ] @@ -264,138 +270,458 @@ def to_odata(self) -> str: # --------------------------------------------------------------------------- -# Public factory functions +# Private implementation helpers (no warnings — used internally and by col()) # --------------------------------------------------------------------------- -def eq(column: str, value: Any) -> FilterExpression: - """Equality filter: ``column eq value``. +def _eq_impl(column: str, value: Any) -> FilterExpression: + return _ComparisonFilter(column, "eq", value) + + +def _ne_impl(column: str, value: Any) -> FilterExpression: + return _ComparisonFilter(column, "ne", value) + + +def _gt_impl(column: str, value: Any) -> FilterExpression: + return _ComparisonFilter(column, "gt", value) + + +def _ge_impl(column: str, value: Any) -> FilterExpression: + return _ComparisonFilter(column, "ge", value) + + +def _lt_impl(column: str, value: Any) -> FilterExpression: + return _ComparisonFilter(column, "lt", value) + + +def _le_impl(column: str, value: Any) -> FilterExpression: + return _ComparisonFilter(column, "le", value) + + +def _contains_impl(column: str, value: str) -> FilterExpression: + return _FunctionFilter("contains", column, value) + + +def _startswith_impl(column: str, value: str) -> FilterExpression: + return _FunctionFilter("startswith", column, value) + + +def _endswith_impl(column: str, value: str) -> FilterExpression: + return _FunctionFilter("endswith", column, value) + + +def _in_impl(column: str, values: Collection[Any]) -> FilterExpression: + return _InFilter(column, values) + + +def _not_in_impl(column: str, values: Collection[Any]) -> FilterExpression: + return _NotInFilter(column, values) + + +# --------------------------------------------------------------------------- +# ColumnProxy — GA idiom for building filter expressions +# --------------------------------------------------------------------------- - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: A filter expression. +_LIKE_WILDCARD = "%" + + +def _compile_like(column: str, pattern: str) -> FilterExpression: + """Compile a LIKE-style pattern to an OData FilterExpression. + + Pattern rules: + - ``val%`` → ``startswith(column, 'val')`` + - ``%val`` → ``endswith(column, 'val')`` + - ``%val%`` → ``contains(column, 'val')`` + - ``val`` (no wildcard) → ``column eq 'val'`` (equality) + - Anything else → :class:`ValueError` + + :param column: Lowercased column name. + :param pattern: The LIKE pattern string. + :raises ValueError: If the pattern contains wildcards in unsupported positions. + """ + has_start = pattern.startswith(_LIKE_WILDCARD) + has_end = pattern.endswith(_LIKE_WILDCARD) + inner = pattern.strip(_LIKE_WILDCARD) + + # Detect non-reducible interior wildcards: after stripping the leading/trailing + # % the inner value must contain no further % characters. + if _LIKE_WILDCARD in inner: + raise ValueError( + f"like() pattern {pattern!r} is not reducible to a single OData function. " + "Use raw(), fetch_xml(), or query.sql() for complex wildcard patterns." + ) + + if not has_start and has_end: + # "val%" — startswith + return _startswith_impl(column, inner) + if has_start and not has_end: + # "%val" — endswith + return _endswith_impl(column, inner) + if has_start and has_end: + # "%val%" — contains + return _contains_impl(column, inner) + # No wildcard at all — exact equality + return _eq_impl(column, pattern) + + +class ColumnProxy: + """Fluent proxy for building OData filter expressions from a column name. + + Returned by :func:`col`. Operator overloads and methods produce + :class:`FilterExpression` instances that can be passed to + ``QueryBuilder.where()``. Example:: - eq("statecode", 0).to_odata() # "statecode eq 0" + from PowerPlatform.Dataverse.models.filters import col + + expr = col("statecode") == 0 # equality + expr = col("revenue") > 1_000_000 # comparison + expr = col("name").like("Contoso%") # startswith + expr = col("name").is_null() # null check + expr = col("statecode").in_([0, 1]) # in """ - return _ComparisonFilter(column, "eq", value) + + __slots__ = ("_column",) + + def __init__(self, name: str) -> None: + if not name or not name.strip(): + raise ValueError("col() requires a non-empty column name") + self._column = name.strip().lower() + + # ---------------------------------------------------------------- comparisons + + def __eq__(self, other: Any) -> FilterExpression: # type: ignore[override] + return _eq_impl(self._column, other) + + def __ne__(self, other: Any) -> FilterExpression: # type: ignore[override] + return _ne_impl(self._column, other) + + def __gt__(self, other: Any) -> FilterExpression: + return _gt_impl(self._column, other) + + def __ge__(self, other: Any) -> FilterExpression: + return _ge_impl(self._column, other) + + def __lt__(self, other: Any) -> FilterExpression: + return _lt_impl(self._column, other) + + def __le__(self, other: Any) -> FilterExpression: + return _le_impl(self._column, other) + + # ---------------------------------------------------------------- null checks + + def is_null(self) -> FilterExpression: + """Column equals null: ``column eq null``.""" + return _eq_impl(self._column, None) + + def is_not_null(self) -> FilterExpression: + """Column not null: ``column ne null``.""" + return _ne_impl(self._column, None) + + # ---------------------------------------------------------------- in / not-in + + def in_(self, values: Collection[Any]) -> FilterExpression: + """In filter using ``Microsoft.Dynamics.CRM.In``. + + :param values: Non-empty collection of values. + :raises ValueError: If ``values`` is empty. + """ + return _in_impl(self._column, values) + + def not_in(self, values: Collection[Any]) -> FilterExpression: + """Not-in filter using ``Microsoft.Dynamics.CRM.NotIn``. + + :param values: Non-empty collection of values. + :raises ValueError: If ``values`` is empty. + """ + return _not_in_impl(self._column, values) + + # ---------------------------------------------------------------- range + + def between(self, lo: Any, hi: Any) -> FilterExpression: + """Between filter: ``(column ge lo and column le hi)``.""" + return _ge_impl(self._column, lo) & _le_impl(self._column, hi) + + def not_between(self, lo: Any, hi: Any) -> FilterExpression: + """Not-between filter: ``not (column ge lo and column le hi)``.""" + return ~(self.between(lo, hi)) + + # ---------------------------------------------------------------- string functions + + def contains(self, value: str) -> FilterExpression: + """Contains filter: ``contains(column, value)``.""" + return _contains_impl(self._column, value) + + def startswith(self, value: str) -> FilterExpression: + """Startswith filter: ``startswith(column, value)``.""" + return _startswith_impl(self._column, value) + + def endswith(self, value: str) -> FilterExpression: + """Endswith filter: ``endswith(column, value)``.""" + return _endswith_impl(self._column, value) + + # ---------------------------------------------------------------- like / not_like + + def like(self, pattern: str) -> FilterExpression: + """Pattern-match filter compiled to the closest OData equivalent. + + +-----------------+-----------------------------+-------------------------------------+ + | Pattern form | Example | Compiles to | + +=================+=============================+=====================================+ + | ``val%`` | ``like("Contoso%")`` | ``startswith(column,'Contoso')`` | + +-----------------+-----------------------------+-------------------------------------+ + | ``%val`` | ``like("%Ltd")`` | ``endswith(column,'Ltd')`` | + +-----------------+-----------------------------+-------------------------------------+ + | ``%val%`` | ``like("%Corp%")`` | ``contains(column,'Corp')`` | + +-----------------+-----------------------------+-------------------------------------+ + | No wildcard | ``like("Contoso")`` | ``column eq 'Contoso'`` | + +-----------------+-----------------------------+-------------------------------------+ + | Other | ``like("Con%oso")`` | :class:`ValueError` | + +-----------------+-----------------------------+-------------------------------------+ + + :param pattern: LIKE-style pattern string. + :raises ValueError: If the pattern cannot be reduced to a single OData function. + """ + return _compile_like(self._column, pattern) + + def not_like(self, pattern: str) -> FilterExpression: + """Negated pattern-match filter; mirrors :meth:`like` rules then negates. + + :param pattern: LIKE-style pattern string (same rules as :meth:`like`). + :raises ValueError: If the pattern cannot be reduced to a single OData function. + """ + return ~_compile_like(self._column, pattern) + + # ---------------------------------------------------------------- hash / repr + + def __hash__(self) -> int: + return hash(self._column) + + def __repr__(self) -> str: + return f"ColumnProxy({self._column!r})" + + +# --------------------------------------------------------------------------- +# Public factory: col() — no deprecation warning +# --------------------------------------------------------------------------- + + +def col(name: str) -> ColumnProxy: + """Return a :class:`ColumnProxy` for building filter expressions. + + This is the preferred GA idiom for constructing filter expressions:: + + from PowerPlatform.Dataverse.models.filters import col + + expr = col("statecode") == 0 + expr = col("revenue") > 1_000_000 + expr = col("name").like("Contoso%") + expr = col("statecode").in_([0, 1]) + expr = col("parentaccountid").is_null() + + :param name: Column logical name (case-insensitive, will be lowercased). + :return: A :class:`ColumnProxy` bound to the column. + :raises ValueError: If ``name`` is empty. + """ + return ColumnProxy(name) + + +# --------------------------------------------------------------------------- +# Public factory: raw() — no deprecation warning (OData escape hatch) +# --------------------------------------------------------------------------- + + +def raw(filter_string: str) -> FilterExpression: + """Verbatim OData filter expression (passed through unchanged). + + This function is **not** deprecated — it is the OData escape hatch with + no typed replacement. + + :param filter_string: Raw OData filter string. + :return: A :class:`FilterExpression`. + + Example:: + + raw("Microsoft.Dynamics.CRM.Today(PropertyName='createdon')") + """ + return _RawFilter(filter_string) + + +# --------------------------------------------------------------------------- +# Deprecated public factory functions — fire DeprecationWarning on CALL +# --------------------------------------------------------------------------- + +_DEP_MSG = "'{name}' is deprecated and will be removed in a future release. " "Use {replacement} instead." + + +def eq(column: str, value: Any) -> FilterExpression: + """Equality filter: ``column eq value``. + + .. deprecated:: + Use ``col(column) == value`` instead. + """ + warnings.warn( + _DEP_MSG.format(name="eq", replacement="col('column') == value"), + DeprecationWarning, + stacklevel=2, + ) + return _eq_impl(column, value) def ne(column: str, value: Any) -> FilterExpression: """Not-equal filter: ``column ne value``. - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: A filter expression. + .. deprecated:: + Use ``col(column) != value`` instead. """ - return _ComparisonFilter(column, "ne", value) + warnings.warn( + _DEP_MSG.format(name="ne", replacement="col('column') != value"), + DeprecationWarning, + stacklevel=2, + ) + return _ne_impl(column, value) def gt(column: str, value: Any) -> FilterExpression: """Greater-than filter: ``column gt value``. - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: A filter expression. + .. deprecated:: + Use ``col(column) > value`` instead. """ - return _ComparisonFilter(column, "gt", value) + warnings.warn( + _DEP_MSG.format(name="gt", replacement="col('column') > value"), + DeprecationWarning, + stacklevel=2, + ) + return _gt_impl(column, value) def ge(column: str, value: Any) -> FilterExpression: """Greater-than-or-equal filter: ``column ge value``. - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: A filter expression. + .. deprecated:: + Use ``col(column) >= value`` instead. """ - return _ComparisonFilter(column, "ge", value) + warnings.warn( + _DEP_MSG.format(name="ge", replacement="col('column') >= value"), + DeprecationWarning, + stacklevel=2, + ) + return _ge_impl(column, value) def lt(column: str, value: Any) -> FilterExpression: """Less-than filter: ``column lt value``. - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: A filter expression. + .. deprecated:: + Use ``col(column) < value`` instead. """ - return _ComparisonFilter(column, "lt", value) + warnings.warn( + _DEP_MSG.format(name="lt", replacement="col('column') < value"), + DeprecationWarning, + stacklevel=2, + ) + return _lt_impl(column, value) def le(column: str, value: Any) -> FilterExpression: """Less-than-or-equal filter: ``column le value``. - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: A filter expression. + .. deprecated:: + Use ``col(column) <= value`` instead. """ - return _ComparisonFilter(column, "le", value) + warnings.warn( + _DEP_MSG.format(name="le", replacement="col('column') <= value"), + DeprecationWarning, + stacklevel=2, + ) + return _le_impl(column, value) def contains(column: str, value: str) -> FilterExpression: """Contains filter: ``contains(column, value)``. - :param column: Column name (will be lowercased). - :param value: Substring to search for. - :return: A filter expression. + .. deprecated:: + Use ``col(column).contains(value)`` instead. """ - return _FunctionFilter("contains", column, value) + warnings.warn( + _DEP_MSG.format(name="contains", replacement="col('column').contains(value)"), + DeprecationWarning, + stacklevel=2, + ) + return _contains_impl(column, value) def startswith(column: str, value: str) -> FilterExpression: """Startswith filter: ``startswith(column, value)``. - :param column: Column name (will be lowercased). - :param value: Prefix to match. - :return: A filter expression. + .. deprecated:: + Use ``col(column).startswith(value)`` instead. """ - return _FunctionFilter("startswith", column, value) + warnings.warn( + _DEP_MSG.format(name="startswith", replacement="col('column').startswith(value)"), + DeprecationWarning, + stacklevel=2, + ) + return _startswith_impl(column, value) def endswith(column: str, value: str) -> FilterExpression: """Endswith filter: ``endswith(column, value)``. - :param column: Column name (will be lowercased). - :param value: Suffix to match. - :return: A filter expression. + .. deprecated:: + Use ``col(column).endswith(value)`` instead. """ - return _FunctionFilter("endswith", column, value) + warnings.warn( + _DEP_MSG.format(name="endswith", replacement="col('column').endswith(value)"), + DeprecationWarning, + stacklevel=2, + ) + return _endswith_impl(column, value) def between(column: str, low: Any, high: Any) -> FilterExpression: """Between filter: ``(column ge low and column le high)``. - Syntactic sugar that composes :func:`ge` and :func:`le` with ``&``. - - :param column: Column name (will be lowercased). - :param low: Lower bound (inclusive). - :param high: Upper bound (inclusive). - :return: A composed filter expression. - - Example:: - - between("revenue", 100000, 500000).to_odata() - # "(revenue ge 100000 and revenue le 500000)" + .. deprecated:: + Use ``col(column).between(low, high)`` instead. """ - return ge(column, low) & le(column, high) + warnings.warn( + _DEP_MSG.format(name="between", replacement="col('column').between(low, high)"), + DeprecationWarning, + stacklevel=2, + ) + # Use private helpers to avoid chaining through the deprecated ge/le wrappers + return _ge_impl(column, low) & _le_impl(column, high) def is_null(column: str) -> FilterExpression: """Null check: ``column eq null``. - :param column: Column name (will be lowercased). - :return: A filter expression. + .. deprecated:: + Use ``col(column).is_null()`` instead. """ - return _ComparisonFilter(column, "eq", None) + warnings.warn( + _DEP_MSG.format(name="is_null", replacement="col('column').is_null()"), + DeprecationWarning, + stacklevel=2, + ) + return _eq_impl(column, None) def is_not_null(column: str) -> FilterExpression: """Not-null check: ``column ne null``. - :param column: Column name (will be lowercased). - :return: A filter expression. + .. deprecated:: + Use ``col(column).is_not_null()`` instead. """ - return _ComparisonFilter(column, "ne", None) + warnings.warn( + _DEP_MSG.format(name="is_not_null", replacement="col('column').is_not_null()"), + DeprecationWarning, + stacklevel=2, + ) + return _ne_impl(column, None) def filter_in(column: str, values: Collection[Any]) -> FilterExpression: @@ -403,63 +729,45 @@ def filter_in(column: str, values: Collection[Any]) -> FilterExpression: Named ``filter_in`` because ``in`` is a Python keyword. - :param column: Column name (will be lowercased). - :param values: Non-empty sequence of values. - :return: A filter expression. - :raises ValueError: If ``values`` is empty. - - Example:: + .. deprecated:: + Use ``col(column).in_(values)`` instead. - filter_in("statecode", [0, 1, 2]).to_odata() - # "Microsoft.Dynamics.CRM.In(PropertyName='statecode',PropertyValues=["0","1","2"])" + :raises ValueError: If ``values`` is empty. """ - return _InFilter(column, values) + warnings.warn( + _DEP_MSG.format(name="filter_in", replacement="col('column').in_(values)"), + DeprecationWarning, + stacklevel=2, + ) + return _in_impl(column, values) def not_in(column: str, values: Collection[Any]) -> FilterExpression: """Not-in filter using ``Microsoft.Dynamics.CRM.NotIn``. - Named ``not_in`` to parallel :func:`filter_in`. + .. deprecated:: + Use ``col(column).not_in(values)`` instead. - :param column: Column name (will be lowercased). - :param values: Non-empty sequence of values. - :return: A filter expression. :raises ValueError: If ``values`` is empty. - - Example:: - - not_in("statecode", [0, 1]).to_odata() - # "Microsoft.Dynamics.CRM.NotIn(PropertyName='statecode',PropertyValues=[\"0\",\"1\"])" """ - return _NotInFilter(column, values) + warnings.warn( + _DEP_MSG.format(name="not_in", replacement="col('column').not_in(values)"), + DeprecationWarning, + stacklevel=2, + ) + return _not_in_impl(column, values) def not_between(column: str, low: Any, high: Any) -> FilterExpression: """Not-between filter: ``not (column ge low and column le high)``. - Syntactic sugar that negates :func:`between` with ``~``. - - :param column: Column name (will be lowercased). - :param low: Lower bound (inclusive, will be excluded). - :param high: Upper bound (inclusive, will be excluded). - :return: A composed filter expression. - - Example:: - - not_between("revenue", 100000, 500000).to_odata() - # "not ((revenue ge 100000 and revenue le 500000))" + .. deprecated:: + Use ``col(column).not_between(low, high)`` instead. """ - return ~between(column, low, high) - - -def raw(filter_string: str) -> FilterExpression: - """Verbatim OData filter expression (passed through unchanged). - - :param filter_string: Raw OData filter string. - :return: A filter expression. - - Example:: - - raw("Microsoft.Dynamics.CRM.Today(PropertyName='createdon')") - """ - return _RawFilter(filter_string) + warnings.warn( + _DEP_MSG.format(name="not_between", replacement="col('column').not_between(low, high)"), + DeprecationWarning, + stacklevel=2, + ) + # Use private helpers to avoid chaining through deprecated ge/le wrappers + return ~(_ge_impl(column, low) & _le_impl(column, high)) diff --git a/src/PowerPlatform/Dataverse/models/protocol.py b/src/PowerPlatform/Dataverse/models/protocol.py new file mode 100644 index 00000000..a3c48509 --- /dev/null +++ b/src/PowerPlatform/Dataverse/models/protocol.py @@ -0,0 +1,69 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""DataverseModel structural Protocol for typed entity integration.""" + +from __future__ import annotations + +from typing import Protocol, runtime_checkable + +__all__ = ["DataverseModel"] + + +@runtime_checkable +class DataverseModel(Protocol): + """Structural Protocol enabling typed entity instances to be passed to + ``records.create()`` and ``records.update()``. + + Implement this Protocol on any entity class (dataclass, Pydantic model, + hand-rolled) to enable it to be passed directly to CRUD operations without + specifying the table name or converting to dict manually. + + Required class variables: + + - ``__entity_logical_name__`` — Dataverse logical entity name (e.g. ``"account"``) + - ``__entity_set_name__`` — OData entity set name (e.g. ``"accounts"``) + + Required instance methods: + + - ``to_dict()`` — return record payload as ``dict`` + - ``from_dict(data)`` — classmethod to reconstruct from a response ``dict`` + + Example:: + + from dataclasses import dataclass + from PowerPlatform.Dataverse.models.protocol import DataverseModel + + @dataclass + class Account: + __entity_logical_name__ = "account" + __entity_set_name__ = "accounts" + name: str = "" + telephone1: str = "" + + def to_dict(self) -> dict: + return {"name": self.name, "telephone1": self.telephone1} + + @classmethod + def from_dict(cls, data: dict) -> "Account": + return cls( + name=data.get("name", ""), + telephone1=data.get("telephone1", ""), + ) + + # Use the entity directly with records operations: + guid = client.records.create(Account(name="Contoso")) + client.records.update(Account(name="Contoso Updated"), guid) + """ + + __entity_logical_name__: str + __entity_set_name__: str + + def to_dict(self) -> dict: + """Return the record payload as a plain dictionary.""" + ... + + @classmethod + def from_dict(cls, data: dict) -> DataverseModel: + """Reconstruct an instance from a response dictionary.""" + ... diff --git a/src/PowerPlatform/Dataverse/models/query_builder.py b/src/PowerPlatform/Dataverse/models/query_builder.py index dbd79b36..1192d16d 100644 --- a/src/PowerPlatform/Dataverse/models/query_builder.py +++ b/src/PowerPlatform/Dataverse/models/query_builder.py @@ -10,51 +10,58 @@ Example:: # Via client (recommended) -- flat iteration over records + from PowerPlatform.Dataverse.models.filters import col + for record in (client.query.builder("account") .select("name", "revenue") - .filter_eq("statecode", 0) - .filter_gt("revenue", 1000000) + .where(col("statecode") == 0) + .where(col("revenue") > 1_000_000) .order_by("revenue", descending=True) .top(100) .execute()): print(record["name"]) # With composable expression tree - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col, raw for record in (client.query.builder("account") .select("name", "revenue") - .where((eq("statecode", 0) | eq("statecode", 1)) - & gt("revenue", 100000)) + .where((col("statecode") == 0) | (col("statecode") == 1)) + .where(col("revenue") > 100000) .top(100) .execute()): print(record["name"]) - # Opt-in paged iteration (for batch processing) + # Lazy paged iteration (one QueryResult per HTTP page) for page in (client.query.builder("account") .select("name") - .execute(by_page=True)): + .execute_pages()): process_batch(page) # Get results as a pandas DataFrame df = (client.query.builder("account") .select("name", "telephone1") - .filter_eq("statecode", 0) + .where(col("statecode") == 0) .top(100) + .execute() .to_dataframe()) """ from __future__ import annotations -from typing import Any, Collection, Dict, Iterable, List, Optional, Sequence, TypedDict, Union +import warnings +from typing import Any, Dict, Iterable, Iterator, List, Optional, TypedDict, Union import pandas as pd from . import filters -from .record import Record +from .record import QueryResult, Record __all__ = ["QueryBuilder", "QueryParams", "ExpandOption"] +# Sentinel for detecting when by_page is explicitly passed to execute() +_BY_PAGE_UNSET = object() + class QueryParams(TypedDict, total=False): """Typed dictionary returned by :meth:`QueryBuilder.build`. @@ -168,7 +175,7 @@ class QueryBuilder: """Fluent interface for building OData queries. Provides method chaining for constructing complex queries with - type-safe filter operations. Can be used standalone (via :meth:`build`) + composable filter expressions. Can be used standalone (via :meth:`build`) or bound to a client (via :meth:`execute`). :param table: Table schema name to query. @@ -178,9 +185,11 @@ class QueryBuilder: Example: Standalone query construction:: + from PowerPlatform.Dataverse.models.filters import col + query = (QueryBuilder("account") .select("name") - .filter_eq("statecode", 0) + .where(col("statecode") == 0) .top(10)) params = query.build() # {"table": "account", "select": ["name"], @@ -220,222 +229,16 @@ def select(self, *columns: str) -> QueryBuilder: self._select.extend(columns) return self - # ----------------------------------------------------------- filter: comparison - - def filter_eq(self, column: str, value: Any) -> QueryBuilder: - """Add equality filter: ``column eq value``. - - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.eq(column, value)) - return self - - def filter_ne(self, column: str, value: Any) -> QueryBuilder: - """Add not-equal filter: ``column ne value``. - - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.ne(column, value)) - return self - - def filter_gt(self, column: str, value: Any) -> QueryBuilder: - """Add greater-than filter: ``column gt value``. - - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.gt(column, value)) - return self - - def filter_ge(self, column: str, value: Any) -> QueryBuilder: - """Add greater-than-or-equal filter: ``column ge value``. - - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.ge(column, value)) - return self - - def filter_lt(self, column: str, value: Any) -> QueryBuilder: - """Add less-than filter: ``column lt value``. - - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.lt(column, value)) - return self - - def filter_le(self, column: str, value: Any) -> QueryBuilder: - """Add less-than-or-equal filter: ``column le value``. - - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.le(column, value)) - return self - - # --------------------------------------------------------- filter: string functions - - def filter_contains(self, column: str, value: str) -> QueryBuilder: - """Add contains filter: ``contains(column, value)``. - - :param column: Column name (will be lowercased). - :param value: Substring to search for. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.contains(column, value)) - return self - - def filter_startswith(self, column: str, value: str) -> QueryBuilder: - """Add startswith filter: ``startswith(column, value)``. - - :param column: Column name (will be lowercased). - :param value: Prefix to match. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.startswith(column, value)) - return self - - def filter_endswith(self, column: str, value: str) -> QueryBuilder: - """Add endswith filter: ``endswith(column, value)``. - - :param column: Column name (will be lowercased). - :param value: Suffix to match. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.endswith(column, value)) - return self - - # --------------------------------------------------------- filter: null checks - - def filter_null(self, column: str) -> QueryBuilder: - """Add null check: ``column eq null``. - - :param column: Column name (will be lowercased). - :return: Self for method chaining. - """ - self._filter_parts.append(filters.is_null(column)) - return self - - def filter_not_null(self, column: str) -> QueryBuilder: - """Add not-null check: ``column ne null``. - - :param column: Column name (will be lowercased). - :return: Self for method chaining. - """ - self._filter_parts.append(filters.is_not_null(column)) - return self - - # --------------------------------------------------------- filter: special - - def filter_in(self, column: str, values: Collection[Any]) -> QueryBuilder: - """Add an ``in`` filter using ``Microsoft.Dynamics.CRM.In``. - - :param column: Column name (will be lowercased). - :param values: Non-empty list of values for the ``in`` clause. - :return: Self for method chaining. - :raises ValueError: If ``values`` is empty. - - Example:: - - query = QueryBuilder("account").filter_in("statecode", [0, 1, 2]) - # Produces: Microsoft.Dynamics.CRM.In( - # PropertyName='statecode',PropertyValues=["0","1","2"]) - """ - self._filter_parts.append(filters.filter_in(column, values)) - return self - - def filter_not_in(self, column: str, values: Collection[Any]) -> QueryBuilder: - """Add a ``not in`` filter using ``Microsoft.Dynamics.CRM.NotIn``. - - :param column: Column name (will be lowercased). - :param values: Non-empty list of values to exclude. - :return: Self for method chaining. - :raises ValueError: If ``values`` is empty. - - Example:: - - query = QueryBuilder("account").filter_not_in("statecode", [2, 3]) - # Produces: Microsoft.Dynamics.CRM.NotIn( - # PropertyName='statecode',PropertyValues=["2","3"]) - """ - self._filter_parts.append(filters.not_in(column, values)) - return self - - def filter_between(self, column: str, low: Any, high: Any) -> QueryBuilder: - """Add a between filter: ``(column ge low and column le high)``. - - :param column: Column name (will be lowercased). - :param low: Lower bound (inclusive). - :param high: Upper bound (inclusive). - :return: Self for method chaining. - - Example:: - - query = QueryBuilder("account").filter_between("revenue", 100000, 500000) - # Produces: (revenue ge 100000 and revenue le 500000) - """ - self._filter_parts.append(filters.between(column, low, high)) - return self - - def filter_not_between(self, column: str, low: Any, high: Any) -> QueryBuilder: - """Add a not-between filter: ``not (column ge low and column le high)``. - - :param column: Column name (will be lowercased). - :param low: Lower bound (inclusive, will be excluded). - :param high: Upper bound (inclusive, will be excluded). - :return: Self for method chaining. - - Example:: - - query = QueryBuilder("account").filter_not_between("revenue", 100000, 500000) - # Produces: not ((revenue ge 100000 and revenue le 500000)) - """ - self._filter_parts.append(filters.not_between(column, low, high)) - return self - - def filter_raw(self, filter_string: str) -> QueryBuilder: - """Add a raw OData filter string. - - Use this for complex filters not covered by other methods. - Column names in the filter string should be lowercase. - - .. warning:: - The filter string is passed directly to Dataverse without validation. - Ensure it follows OData filter syntax; a malformed expression will result - in a ``400 Bad Request`` error from the server. - - :param filter_string: Raw OData filter expression. - :return: Self for method chaining. - - Example:: - - query = QueryBuilder("account").filter_raw( - "(statecode eq 0 or statecode eq 1)" - ) - """ - self._filter_parts.append(filters.raw(filter_string)) - return self - # ------------------------------------------------------ filter: expression tree def where(self, expression: filters.FilterExpression) -> QueryBuilder: """Add a composable filter expression. Accepts a :class:`~PowerPlatform.Dataverse.models.filters.FilterExpression` - built using the convenience functions from - :mod:`~PowerPlatform.Dataverse.models.filters`. + built using :func:`~PowerPlatform.Dataverse.models.filters.col` or + :func:`~PowerPlatform.Dataverse.models.filters.raw`. - Multiple ``where()`` calls and ``filter_*()`` calls are all - AND-joined together in the order they were called. + Multiple ``where()`` calls are AND-joined together in call order. :param expression: A composable filter expression. :type expression: FilterExpression @@ -444,11 +247,11 @@ def where(self, expression: filters.FilterExpression) -> QueryBuilder: Example:: - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col query = (QueryBuilder("account") - .where((eq("statecode", 0) | eq("statecode", 1)) - & gt("revenue", 100000))) + .where((col("statecode") == 0) | (col("statecode") == 1)) + .where(col("revenue") > 100000)) """ if not isinstance(expression, filters.FilterExpression): raise TypeError(f"where() requires a FilterExpression, got {type(expression).__name__}") @@ -511,7 +314,7 @@ def count(self) -> QueryBuilder: Example:: results = (client.query.builder("account") - .filter_eq("statecode", 0) + .where(col("statecode") == 0) .count() .execute()) """ @@ -612,8 +415,8 @@ def build(self) -> QueryParams: """Build query parameters dictionary. Returns a :class:`QueryParams` dictionary suitable for passing to - the OData layer. All ``filter_*()`` and ``where()`` clauses are - AND-joined into a single ``filter`` string in call order. + the OData layer. All ``where()`` clauses are AND-joined into a + single ``filter`` string in call order. :return: Dictionary with ``table`` and optionally ``select``, ``filter``, ``orderby``, ``expand``, ``top``, ``page_size``, @@ -645,148 +448,215 @@ def build(self) -> QueryParams: params["include_annotations"] = self._include_annotations return params - # --------------------------------------------------------------- guards - - def _validate_constraints(self) -> None: - """Raise if the query has no limiting constraints. - - At least one of ``select``, ``filter``, or ``top`` must be set - before executing a query to prevent accidental full-table scans. - - :raises ValueError: If none of ``select()``, ``filter_*()``, - ``where()``, or ``top()`` has been called. - """ - if not (self._select or self._filter_parts or self._top is not None): - raise ValueError( - "Unbounded query: set at least one of select(), filter_*(), " - "where(), or top() before calling execute() or to_dataframe()." - ) - # --------------------------------------------------------------- execute - def execute(self, *, by_page: bool = False) -> Union[Iterable[Record], Iterable[List[Record]]]: + def execute(self, *, by_page=_BY_PAGE_UNSET) -> Union[QueryResult, Iterable[List[Record]]]: """Execute the query and return results. - By default, returns a flat iterator over individual records, - abstracting away OData paging. Pass ``by_page=True`` to get - page-level iteration instead (useful for batch processing). + Returns a :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + with all pages collected. Use :meth:`execute_pages` for lazy per-page + iteration. This method is only available when the QueryBuilder was created via ``client.query.builder(table)``. Standalone ``QueryBuilder`` instances should use :meth:`build` to get parameters and pass them to ``client.records.get()`` manually. - At least one of ``select()``, ``filter_*()``, ``where()``, or - ``top()`` must be called before ``execute()``; otherwise a - :class:`ValueError` is raised to prevent accidental full-table - scans. - - :param by_page: If ``True``, yield pages (lists of - :class:`~PowerPlatform.Dataverse.models.record.Record` objects) - instead of individual records. Defaults to ``False``. - :type by_page: bool - :return: Generator yielding individual - :class:`~PowerPlatform.Dataverse.models.record.Record` objects - (default) or pages of records (when ``by_page=True``). - :rtype: Iterable[Record] or Iterable[List[Record]] - :raises ValueError: If no ``select``, ``filter``, or ``top`` + At least one of ``select()``, ``where()``, or ``top()`` must be + called before ``execute()``; otherwise a :class:`ValueError` is + raised to prevent accidental full-table scans. + + .. deprecated:: + The ``by_page`` parameter is deprecated. Use :meth:`execute_pages` + for lazy per-page iteration, or plain ``execute()`` (no flag) for + the default eager result. + + :return: :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + with all pages collected (default), or page iterator (deprecated + ``by_page=True``). + :rtype: QueryResult or Iterable[List[Record]] + :raises ValueError: If no ``select``, ``where``, or ``top`` constraint has been set. :raises RuntimeError: If the query was not created via ``client.query.builder()``. - Example: - Flat iteration (default):: + Example:: - for record in (client.query.builder("account") - .select("name") - .filter_eq("statecode", 0) - .execute()): - print(record["name"]) + from PowerPlatform.Dataverse.models.filters import col - Paged iteration:: + for record in (client.query.builder("account") + .select("name") + .where(col("statecode") == 0) + .execute()): + print(record["name"]) + """ + use_by_page = False + if by_page is not _BY_PAGE_UNSET: + use_by_page = bool(by_page) + if use_by_page: + warnings.warn( + "'execute(by_page=True)' is deprecated; use 'execute_pages()' instead.", + UserWarning, + stacklevel=2, + ) + else: + warnings.warn( + "'execute(by_page=False)' is deprecated; " + "the by_page flag is redundant — use plain 'execute()' instead.", + UserWarning, + stacklevel=2, + ) - for page in (client.query.builder("account") - .select("name") - .execute(by_page=True)): - process_batch(page) - """ if self._query_ops is None: raise RuntimeError( "Cannot execute: query was not created via client.query.builder(). " "Use build() and pass parameters to client.records.get() instead." ) - self._validate_constraints() + + if not self._select and not self._filter_parts and self._top is None: + raise ValueError( + "At least one of select(), where(), or top() must be called before " + "execute() to prevent accidental full-table scans." + ) + params = self.build() client = self._query_ops._client - pages = client.records.get( - params["table"], - select=params.get("select"), - filter=params.get("filter"), - orderby=params.get("orderby"), - top=params.get("top"), - expand=params.get("expand"), - page_size=params.get("page_size"), - count=params.get("count", False), - include_annotations=params.get("include_annotations"), - ) + # Suppress DeprecationWarning from records.get() — execute() is GA; + # records.get() is deprecated but still used as the internal paging mechanism. + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + pages = client.records.get( + params["table"], + select=params.get("select"), + filter=params.get("filter"), + orderby=params.get("orderby"), + top=params.get("top"), + expand=params.get("expand"), + page_size=params.get("page_size"), + count=params.get("count", False), + include_annotations=params.get("include_annotations"), + ) - if by_page: + if use_by_page: return pages - def _flat() -> Iterable[Record]: - for page in pages: - yield from page + all_records: List[Record] = [] + for page in pages: + all_records.extend(page) + return QueryResult(all_records) + + # ---------------------------------------------------------- execute_pages + + def execute_pages(self) -> Iterator[QueryResult]: + """Lazily yield one :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + per HTTP page. + + Each iteration triggers a network request via ``@odata.nextLink``. + One-shot — do not iterate more than once. + + At least one of ``select()``, ``where()``, or ``top()`` must be + called before ``execute_pages()``; otherwise a :class:`ValueError` is + raised to prevent accidental full-table scans. - return _flat() + :return: Iterator of per-page :class:`~PowerPlatform.Dataverse.models.record.QueryResult`. + :rtype: Iterator[QueryResult] + :raises ValueError: If no ``select``, ``where``, or ``top`` + constraint has been set. + :raises RuntimeError: If the query was not created via + ``client.query.builder()``. + + Example:: + + from PowerPlatform.Dataverse.models.filters import col + + for page in (client.query.builder("account") + .select("name") + .where(col("statecode") == 0) + .execute_pages()): + process(page.to_dataframe()) + """ + if self._query_ops is None: + raise RuntimeError( + "Cannot execute: query was not created via client.query.builder(). " + "Use build() and pass parameters to client.records.get() instead." + ) + + if not self._select and not self._filter_parts and self._top is None: + raise ValueError( + "At least one of select(), where(), or top() must be called before " + "execute_pages() to prevent accidental full-table scans." + ) + + params = self.build() + client = self._query_ops._client + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + pages = client.records.get( + params["table"], + select=params.get("select"), + filter=params.get("filter"), + orderby=params.get("orderby"), + top=params.get("top"), + expand=params.get("expand"), + page_size=params.get("page_size"), + count=params.get("count", False), + include_annotations=params.get("include_annotations"), + ) + + for page in pages: + yield QueryResult(page) # ----------------------------------------------------------- to_dataframe def to_dataframe(self) -> pd.DataFrame: """Execute the query and return results as a pandas DataFrame. - All pages are consolidated into a single DataFrame, matching - the behavior of ``client.dataframe.get()``. + .. deprecated:: + Use ``QueryBuilder.execute().to_dataframe()`` instead. + ``QueryBuilder.to_dataframe()`` will be removed in a future release. + + All pages are consolidated into a single DataFrame. This method is only available when the QueryBuilder was created via ``client.query.builder(table)``. - At least one of ``select()``, ``filter_*()``, ``where()``, or - ``top()`` must be called before ``to_dataframe()``; otherwise a - :class:`ValueError` is raised to prevent accidental full-table - scans. + At least one of ``select()``, ``where()``, or ``top()`` must be + called before ``to_dataframe()``; otherwise a :class:`ValueError` + is raised to prevent accidental full-table scans. :return: DataFrame containing all matching records. Returns an empty DataFrame when no records match. :rtype: ~pandas.DataFrame - :raises ValueError: If no ``select``, ``filter``, or ``top`` + :raises ValueError: If no ``select``, ``where``, or ``top`` constraint has been set. :raises RuntimeError: If the query was not created via ``client.query.builder()``. Example:: + from PowerPlatform.Dataverse.models.filters import col + df = (client.query.builder("account") .select("name", "telephone1") - .filter_eq("statecode", 0) + .where(col("statecode") == 0) .top(100) + .execute() .to_dataframe()) """ + warnings.warn( + "'QueryBuilder.to_dataframe()' is deprecated; use " "'QueryBuilder.execute().to_dataframe()' instead.", + DeprecationWarning, + stacklevel=2, + ) if self._query_ops is None: raise RuntimeError( "Cannot execute: query was not created via client.query.builder(). " - "Use build() and pass parameters to client.dataframe.get() instead." + "Use build() and pass parameters to client.records.get() instead." ) - self._validate_constraints() - params = self.build() - return self._query_ops._client.dataframe.get( - params["table"], - select=params.get("select"), - filter=params.get("filter"), - orderby=params.get("orderby"), - top=params.get("top"), - expand=params.get("expand"), - page_size=params.get("page_size"), - count=params.get("count", False), - include_annotations=params.get("include_annotations"), - ) + + result = self.execute() + if not result: + return pd.DataFrame(columns=self._select) if self._select else pd.DataFrame() + return result.to_dataframe() diff --git a/src/PowerPlatform/Dataverse/models/record.py b/src/PowerPlatform/Dataverse/models/record.py index 43641032..f7a47941 100644 --- a/src/PowerPlatform/Dataverse/models/record.py +++ b/src/PowerPlatform/Dataverse/models/record.py @@ -6,9 +6,9 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any, Dict, Iterator, KeysView, Optional, ValuesView, ItemsView +from typing import Any, Dict, Iterator, KeysView, List, Optional, ValuesView, ItemsView -__all__ = ["Record"] +__all__ = ["Record", "QueryResult"] _ODATA_PREFIX = "@odata." @@ -112,3 +112,51 @@ def from_api_response( def to_dict(self) -> Dict[str, Any]: """Return a plain dict copy of the record data (excludes metadata).""" return dict(self.data) + + +class QueryResult: + """Iterable wrapper around a list of :class:`Record` objects. + + Returned by :meth:`~PowerPlatform.Dataverse.models.query_builder.QueryBuilder.execute` + (flat mode) and :meth:`~PowerPlatform.Dataverse.operations.records.RecordOperations.list`. + + Backward-compatible: ``for r in result`` continues to work without change. + + :param records: Collected records from all pages. + :type records: list[:class:`Record`] + """ + + def __init__(self, records: List[Record]) -> None: + self.records: List[Record] = records + + def __iter__(self) -> Iterator[Record]: + return iter(self.records) + + def __len__(self) -> int: + return len(self.records) + + def __bool__(self) -> bool: + return bool(self.records) + + def __repr__(self) -> str: + return f"QueryResult({len(self.records)} records)" + + def first(self) -> Optional[Record]: + """Return the first record, or ``None`` if the result is empty.""" + return self.records[0] if self.records else None + + def to_dataframe(self) -> Any: + """Return all records as a pandas DataFrame. + + :raises ImportError: If pandas is not installed. + :rtype: ~pandas.DataFrame + """ + try: + import pandas as pd + except ImportError as exc: + raise ImportError("pandas is required for to_dataframe(). " "Install it with: pip install pandas") from exc + + if not self.records: + return pd.DataFrame() + rows = [r.data if hasattr(r, "data") else dict(r) for r in self.records] + return pd.DataFrame.from_records(rows) diff --git a/src/PowerPlatform/Dataverse/operations/batch.py b/src/PowerPlatform/Dataverse/operations/batch.py index b2751d93..c3f040e8 100644 --- a/src/PowerPlatform/Dataverse/operations/batch.py +++ b/src/PowerPlatform/Dataverse/operations/batch.py @@ -5,6 +5,7 @@ from __future__ import annotations +import warnings from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union import pandas as pd @@ -18,6 +19,7 @@ _RecordUpdate, _RecordDelete, _RecordGet, + _RecordList, _RecordUpsert, _TableCreate, _TableDelete, @@ -43,6 +45,7 @@ if TYPE_CHECKING: from ..client import DataverseClient + from ..models.filters import FilterExpression __all__ = [ "BatchRecordOperations", @@ -166,11 +169,14 @@ class BatchRecordOperations: """ Record operations on a :class:`BatchRequest`. - Mirrors ``client.records`` exactly: same method names, same signatures. + Mirrors ``client.records``: same method names, same signatures. All methods return ``None``; results are available via :class:`~PowerPlatform.Dataverse.models.batch.BatchResult` after :meth:`BatchRequest.execute`. + GA methods: :meth:`retrieve` (single record) and :meth:`list` (multi-record, + single page). :meth:`get` is deprecated — use :meth:`retrieve` instead. + Do not instantiate directly; use ``batch.records``. """ @@ -253,16 +259,9 @@ def get( """ Add a single-record get operation to the batch. - Only the single-record overload (``record_id`` provided) is supported. - The paginated/multi-record overload of ``client.records.get()`` - (``filter``, ``orderby``, etc., without ``record_id``) is **not** - supported in batch — pagination requires following - ``@odata.nextLink`` across multiple round-trips, which is - incompatible with a single batch request. - - The response body will be available in - :attr:`~PowerPlatform.Dataverse.models.batch.BatchItemResponse.data` - after :meth:`BatchRequest.execute`. + .. deprecated:: + Use :meth:`retrieve` instead. ``batch.records.get()`` is deprecated + and will be removed in a future release. :param table: Table schema name. :type table: :class:`str` @@ -271,6 +270,11 @@ def get( :param select: Optional list of column names to include. :type select: list[str] or None """ + warnings.warn( + "'batch.records.get()' is deprecated; use 'batch.records.retrieve(table, record_id)' instead.", + DeprecationWarning, + stacklevel=2, + ) self._batch._items.append(_RecordGet(table=table, record_id=record_id, select=select)) def upsert( @@ -325,6 +329,76 @@ def upsert( raise TypeError("Each item must be an UpsertItem or a dict with 'alternate_key' and 'record' keys") self._batch._items.append(_RecordUpsert(table=table, items=normalized)) + def retrieve( + self, + table: str, + record_id: str, + *, + select: Optional[List[str]] = None, + ) -> None: + """ + Add a single-record retrieve operation to the batch. + + GA replacement for the deprecated :meth:`get`. Enqueues a GET request + for one record by its GUID. The response body will be available in + :attr:`~PowerPlatform.Dataverse.models.batch.BatchItemResponse.data` + after :meth:`BatchRequest.execute`. + + :param table: Table schema name (e.g. ``"account"``). + :type table: :class:`str` + :param record_id: GUID of the record to retrieve. + :type record_id: :class:`str` + :param select: Optional list of column logical names to include. + :type select: list[str] or None + + Example:: + + batch = client.batch.new() + batch.records.retrieve("account", account_id, select=["name", "telephone1"]) + result = batch.execute() + record = result.responses[0].data + """ + self._batch._items.append(_RecordGet(table=table, record_id=record_id, select=select)) + + def list( + self, + table: str, + *, + filter: "Optional[Union[str, FilterExpression]]" = None, + select: Optional[List[str]] = None, + top: Optional[int] = None, + ) -> None: + """ + Add a multi-record list operation to the batch (single page, no pagination). + + Enqueues a GET request for multiple records. Because batch requests are + a single HTTP round-trip, pagination (``@odata.nextLink``) is **not** + supported — use ``top`` to bound the result size, or rely on the + server's default page limit. + + The response body (``{"value": [...]}`` JSON) will be available in + :attr:`~PowerPlatform.Dataverse.models.batch.BatchItemResponse.data` + after :meth:`BatchRequest.execute`. + + :param table: Table schema name (e.g. ``"account"``). + :type table: :class:`str` + :param filter: Optional OData ``$filter`` expression or :class:`FilterExpression`. + :type filter: str or FilterExpression or None + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :param top: Maximum number of records to return. + :type top: int or None + + Example:: + + batch = client.batch.new() + batch.records.list("account", filter="statecode eq 0", select=["name"], top=50) + result = batch.execute() + records = result.responses[0].data.get("value", []) + """ + filter_str: Optional[str] = str(filter) if filter is not None else None + self._batch._items.append(_RecordList(table=table, select=select, filter=filter_str, top=top)) + class BatchTableOperations: """ diff --git a/src/PowerPlatform/Dataverse/operations/dataframe.py b/src/PowerPlatform/Dataverse/operations/dataframe.py index e6ec2033..28647c58 100644 --- a/src/PowerPlatform/Dataverse/operations/dataframe.py +++ b/src/PowerPlatform/Dataverse/operations/dataframe.py @@ -5,6 +5,7 @@ from __future__ import annotations +import warnings from typing import TYPE_CHECKING, Any, Dict, List, Optional import pandas as pd @@ -164,6 +165,12 @@ def get( df = client.dataframe.get("account", select=["name"], top=100) """ + warnings.warn( + "'dataframe.get()' is deprecated; use " + "client.query.builder(table).where(...).execute().to_dataframe() instead.", + DeprecationWarning, + stacklevel=2, + ) if record_id is not None: if not isinstance(record_id, str) or not record_id.strip(): raise ValueError("record_id must be a non-empty string") @@ -173,25 +180,30 @@ def get( "Cannot specify query parameters (filter, orderby, top, " "expand, page_size) when fetching a single record by ID" ) - result = self._client.records.get( - table, - record_id, - select=select, - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = self._client.records.get( + table, + record_id, + select=select, + ) return pd.DataFrame([result.data]) rows: List[dict] = [] - for batch in self._client.records.get( - table, - select=select, - filter=filter, - orderby=orderby, - top=top, - expand=expand, - page_size=page_size, - count=count, - include_annotations=include_annotations, - ): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + pages = self._client.records.get( + table, + select=select, + filter=filter, + orderby=orderby, + top=top, + expand=expand, + page_size=page_size, + count=count, + include_annotations=include_annotations, + ) + for batch in pages: rows.extend(row.data for row in batch) if not rows: diff --git a/src/PowerPlatform/Dataverse/operations/query.py b/src/PowerPlatform/Dataverse/operations/query.py index 83e82677..d6ab9283 100644 --- a/src/PowerPlatform/Dataverse/operations/query.py +++ b/src/PowerPlatform/Dataverse/operations/query.py @@ -5,9 +5,12 @@ from __future__ import annotations +import warnings +import xml.etree.ElementTree as _ET from typing import Any, Dict, List, Optional, TYPE_CHECKING from ..core.errors import MetadataError +from ..models.fetch_xml_query import FetchXmlQuery from ..models.record import Record from ..models.query_builder import QueryBuilder @@ -146,6 +149,60 @@ def sql(self, sql: str) -> List[Record]: rows = od._query_sql(sql) return [Record.from_api_response("", row) for row in rows] + # --------------------------------------------------------------- fetch_xml + + def fetch_xml(self, xml: str) -> FetchXmlQuery: + """Return an inert :class:`~PowerPlatform.Dataverse.models.fetch_xml_query.FetchXmlQuery` object. + + No HTTP request is made until + :meth:`~PowerPlatform.Dataverse.models.fetch_xml_query.FetchXmlQuery.execute` + or + :meth:`~PowerPlatform.Dataverse.models.fetch_xml_query.FetchXmlQuery.execute_pages` + is called on the returned object. + + Use for SQL-JOIN scenarios, aggregate queries, or other operations that + the OData builder endpoint cannot express. + + :param xml: Well-formed FetchXML query string. The root ```` + element determines the entity set endpoint. + :type xml: :class:`str` + :return: Inert query object with ``.execute()`` and ``.execute_pages()`` methods. + :rtype: :class:`~PowerPlatform.Dataverse.models.fetch_xml_query.FetchXmlQuery` + :raises ValueError: If the FetchXML is missing a root ```` element + or the entity ``name`` attribute. + + Example:: + + query = client.query.fetch_xml(\"\"\" + + + + + + + + + \"\"\") + + # Eager — collect all pages: + result = query.execute() + df = result.to_dataframe() + + # Lazy — process one page at a time: + for page in query.execute_pages(): + process(page.to_dataframe()) + """ + root_el = _ET.fromstring(xml.strip()) + entity_el = root_el.find("entity") + if entity_el is None: + raise ValueError("FetchXML must contain an child element") + entity_name = entity_el.get("name", "") + if not entity_name: + raise ValueError("FetchXML element must have a 'name' attribute") + + return FetchXmlQuery(xml.strip(), entity_name, self._client) + # --------------------------------------------------------------- sql_columns def sql_columns( @@ -230,178 +287,6 @@ def sql_columns( result.sort(key=lambda x: (not x["is_pk"], not x["is_name"], x["name"])) return result - # --------------------------------------------------------------- sql_select - - def sql_select( - self, - table: str, - *, - include_system: bool = False, - ) -> str: - """Return a comma-separated column list for use in SQL SELECT. - - Excludes virtual columns and optionally system columns. The result - can be embedded directly in a SQL query string. - - :param table: Schema name of the table (e.g. ``"account"``). - :type table: :class:`str` - :param include_system: Include system columns (default ``False``). - :type include_system: :class:`bool` - - :return: Comma-separated column names. - :rtype: :class:`str` - - Example:: - - cols = client.query.sql_select("account") - sql = f"SELECT TOP 10 {cols} FROM account" - df = client.dataframe.sql(sql) - """ - columns = self.sql_columns(table, include_system=include_system) - return ", ".join(c["name"] for c in columns) - - # --------------------------------------------------------------- sql_joins - - def sql_joins( - self, - table: str, - ) -> List[Dict[str, Any]]: - """Discover all possible SQL JOINs from a table. - - Returns one entry per outgoing lookup relationship, with the - exact column names needed for SQL ``JOIN ... ON`` clauses. - - For **polymorphic** lookups (e.g. ``customerid`` targeting both - ``account`` and ``contact``), multiple entries are returned with - the same ``column`` but different ``target`` values. - - :param table: Schema name of the table (e.g. ``"contact"``). - :type table: :class:`str` - - :return: List of JOIN metadata dicts, each containing: - - - ``column`` -- the lookup attribute on this table (use in ON clause) - - ``target`` -- the referenced entity name - - ``target_pk`` -- the referenced entity's primary key column - - ``relationship`` -- the schema name of the relationship - - ``join_clause`` -- a ready-to-use ``JOIN ... ON ...`` fragment - - :rtype: list[dict[str, typing.Any]] - - .. note:: - - The ``join_clause`` value references the source table by its - **full name** (e.g. ``ON contact.col = ...``), so the FROM - clause must also use the unaliased table name. For queries - that need aliases, use :meth:`sql_join` instead. - - Example:: - - joins = client.query.sql_joins("contact") - for j in joins: - print(f"{j['column']:30s} -> {j['target']}.{j['target_pk']}") - print(f" {j['join_clause']}") - - # Use in a query (no alias on the FROM table) - j = next(j for j in joins if j['target'] == 'account') - sql = f"SELECT TOP 10 contact.fullname, a.name FROM contact {j['join_clause']}" - """ - table_lower = table.lower() - rels = self._client.tables.list_table_relationships(table) - - used_aliases: set = set() - result: List[Dict[str, Any]] = [] - for r in rels: - ref_entity = (r.get("ReferencingEntity") or "").lower() - if ref_entity != table_lower: - continue - col = r.get("ReferencingAttribute", "") - target = r.get("ReferencedEntity", "") - target_pk = r.get("ReferencedAttribute", "") - schema = r.get("SchemaName", "") - if not all([col, target, target_pk]): - continue - - # Generate a unique alias — add a numeric suffix on collision so - # two lookups to tables starting with the same letter (e.g. - # "account" and "annotation") or two lookups to the same table - # (e.g. "ownerid" and "createdby" both to "systemuser") produce - # distinct aliases and valid SQL. - base = target[0] if target else "j" - alias = base - counter = 2 - while alias in used_aliases: - alias = f"{base}{counter}" - counter += 1 - used_aliases.add(alias) - join_clause = f"JOIN {target} {alias} ON {table_lower}.{col} = {alias}.{target_pk}" - - result.append( - { - "column": col, - "target": target, - "target_pk": target_pk, - "relationship": schema, - "join_clause": join_clause, - } - ) - - result.sort(key=lambda x: (x["target"], x["column"])) - return result - - # --------------------------------------------------------------- sql_join - - def sql_join( - self, - from_table: str, - to_table: str, - *, - from_alias: Optional[str] = None, - to_alias: Optional[str] = None, - ) -> str: - """Generate a SQL JOIN clause between two tables. - - Discovers the relationship automatically via metadata. If multiple - relationships exist (e.g. polymorphic lookups), picks the first - match. Use :meth:`sql_joins` to see all options. - - :param from_table: Schema name of the FROM table (e.g. ``"contact"``). - :type from_table: :class:`str` - :param to_table: Schema name of the target table (e.g. ``"account"``). - :type to_table: :class:`str` - :param from_alias: Optional alias for the FROM table in the JOIN - clause. If ``None``, uses the full table name. - :type from_alias: :class:`str` or None - :param to_alias: Optional alias for the target table. If ``None``, - uses the first letter of the target table name. - :type to_alias: :class:`str` or None - - :return: A ready-to-use ``JOIN ... ON ...`` clause. - :rtype: :class:`str` - - :raises ValueError: If no relationship is found between the tables. - - Example:: - - j = client.query.sql_join("contact", "account", from_alias="c", to_alias="a") - # Returns: "JOIN account a ON c.parentcustomerid = a.accountid" - sql = f"SELECT TOP 10 c.fullname, a.name FROM contact c {j}" - df = client.dataframe.sql(sql) - """ - to_lower = to_table.lower() - joins = self.sql_joins(from_table) - match = [j for j in joins if j["target"].lower() == to_lower] - if not match: - raise ValueError( - f"No relationship found from '{from_table}' to '{to_table}'. " - f"Use client.query.sql_joins('{from_table}') to see available targets." - ) - - j = match[0] - src = from_alias or from_table.lower() - tgt = to_alias or to_lower[0] - return f"JOIN {to_lower} {tgt} " f"ON {src}.{j['column']} = {tgt}.{j['target_pk']}" - # =========================================================== # OData helpers -- eliminate friction for records.get() users # =========================================================== @@ -433,6 +318,12 @@ def odata_select( for r in page: print(r) """ + warnings.warn( + "'odata_select' is deprecated; use the typed builder (1.x) " + "or client.query.sql_columns() to discover columns.", + DeprecationWarning, + stacklevel=2, + ) columns = self.sql_columns(table, include_system=include_system) return [c["name"] for c in columns] @@ -545,6 +436,12 @@ def odata_expand( acct = r.get(nav) or {} print(f"{r['fullname']} -> {acct.get('name', 'N/A')}") """ + warnings.warn( + "'odata_expand' is deprecated; use the typed builder (1.x) " + "with .expand() or client.query.odata_expands() to discover navigation properties.", + DeprecationWarning, + stacklevel=2, + ) to_lower = to_table.lower() expands = self.odata_expands(from_table) match = [e for e in expands if e["target_table"].lower() == to_lower] @@ -594,6 +491,11 @@ def odata_bind( **bind, }) """ + warnings.warn( + "'odata_bind' is deprecated; use the typed builder (1.x) " "or pass the @odata.bind dict manually.", + DeprecationWarning, + stacklevel=2, + ) to_lower = to_table.lower() expands = self.odata_expands(from_table) match = [e for e in expands if e["target_table"].lower() == to_lower and e["target_entity_set"]] diff --git a/src/PowerPlatform/Dataverse/operations/records.py b/src/PowerPlatform/Dataverse/operations/records.py index ef867f52..600f9d87 100644 --- a/src/PowerPlatform/Dataverse/operations/records.py +++ b/src/PowerPlatform/Dataverse/operations/records.py @@ -5,12 +5,15 @@ from __future__ import annotations -from typing import Any, Dict, Iterable, List, Optional, Union, overload, TYPE_CHECKING +import warnings +from typing import Any, Dict, Iterable, Iterator, List, Optional, Union, overload, TYPE_CHECKING -from ..models.record import Record +from ..models.protocol import DataverseModel +from ..models.record import QueryResult, Record from ..models.upsert import UpsertItem if TYPE_CHECKING: + from ..models.filters import FilterExpression from ..client import DataverseClient @@ -54,10 +57,16 @@ def create(self, table: str, data: Dict[str, Any]) -> str: ... @overload def create(self, table: str, data: List[Dict[str, Any]]) -> List[str]: ... + @overload + def create(self, table_or_entity: DataverseModel, data: None = None) -> str: ... + + @overload + def create(self, table_or_entity: List[DataverseModel], data: None = None) -> List[str]: ... + def create( self, - table: str, - data: Union[Dict[str, Any], List[Dict[str, Any]]], + table_or_entity: Union[str, DataverseModel, List[DataverseModel]], + data: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None, ) -> Union[str, List[str]]: """Create one or more records in a Dataverse table. @@ -78,19 +87,49 @@ def create( :raises TypeError: If ``data`` is not a dict or list[dict]. Example: - Create a single record:: + Create a single record (dict form):: guid = client.records.create("account", {"name": "Contoso"}) print(f"Created: {guid}") - Create multiple records:: + Create multiple records (dict form):: guids = client.records.create("account", [ {"name": "Contoso"}, {"name": "Fabrikam"}, ]) print(f"Created {len(guids)} accounts") + + Create from a DataverseModel instance:: + + guid = client.records.create(Account(name="Contoso")) """ + # DataverseModel dispatch: list of entities + if isinstance(table_or_entity, list) and table_or_entity and isinstance(table_or_entity[0], DataverseModel): + entities = table_or_entity + table = entities[0].__entity_logical_name__ + data_list = [e.to_dict() for e in entities] + with self._client._scoped_odata() as od: + entity_set = od._entity_set_from_schema_name(table) + ids = od._create_multiple(entity_set, table, data_list) + if not isinstance(ids, list) or not all(isinstance(x, str) for x in ids): + raise TypeError("_create (multi) did not return list[str]") + return ids + + # DataverseModel dispatch: single entity + if isinstance(table_or_entity, DataverseModel): + entity = table_or_entity + table = entity.__entity_logical_name__ + record_data = entity.to_dict() + with self._client._scoped_odata() as od: + entity_set = od._entity_set_from_schema_name(table) + rid = od._create(entity_set, table, record_data) + if not isinstance(rid, str): + raise TypeError("_create (single) did not return GUID string") + return rid + + # Existing str/dict path + table = table_or_entity # type: ignore[assignment] with self._client._scoped_odata() as od: entity_set = od._entity_set_from_schema_name(table) if isinstance(data, dict): @@ -109,9 +148,9 @@ def create( def update( self, - table: str, - ids: Union[str, List[str]], - changes: Union[Dict[str, Any], List[Dict[str, Any]]], + table_or_entity: Union[str, DataverseModel], + ids: Optional[Union[str, List[str]]] = None, + changes: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None, ) -> None: """Update one or more records in a Dataverse table. @@ -151,7 +190,29 @@ def update( [id1, id2], [{"name": "Name A"}, {"name": "Name B"}], ) + + Update from a DataverseModel instance:: + + client.records.update(Account(name="Contoso Updated"), account_id) """ + # DataverseModel dispatch: entity provides table + changes + if isinstance(table_or_entity, DataverseModel): + entity = table_or_entity + table = entity.__entity_logical_name__ + record_data = entity.to_dict() + if ids is None: + raise TypeError("record_id must be provided when updating from a DataverseModel") + with self._client._scoped_odata() as od: + if isinstance(ids, str): + od._update(table, ids, record_data) + return None + if isinstance(ids, list): + od._update_by_ids(table, ids, record_data) + return None + return None + + # Existing str/dict path + table: str = table_or_entity # type: ignore[assignment] with self._client._scoped_odata() as od: if isinstance(ids, str): if not isinstance(changes, dict): @@ -424,6 +485,12 @@ def get( print(record["name"]) """ if record_id is not None: + warnings.warn( + "'records.get()' with a record_id is deprecated; " + "use 'client.records.retrieve(table, record_id)' instead.", + DeprecationWarning, + stacklevel=2, + ) if not isinstance(record_id, str): raise TypeError("record_id must be str") if ( @@ -444,6 +511,12 @@ def get( raw = od._get(table, record_id, select=select) return Record.from_api_response(table, raw, record_id=record_id) + warnings.warn( + "'records.get()' is deprecated; " "use 'client.records.list(table, filter=...)' instead.", + DeprecationWarning, + stacklevel=2, + ) + def _paged() -> Iterable[List[Record]]: with self._client._scoped_odata() as od: for page in od._get_multiple( @@ -461,6 +534,150 @@ def _paged() -> Iterable[List[Record]]: return _paged() + # --------------------------------------------------------------- retrieve + + def retrieve( + self, + table: str, + record_id: str, + *, + select: Optional[List[str]] = None, + ) -> Optional[Record]: + """Fetch a single record by its GUID, returning ``None`` if not found. + + GA replacement for ``records.get(table, record_id)``. Returns ``None`` + instead of raising when the record does not exist (HTTP 404). + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param record_id: GUID of the record to retrieve. + :type record_id: :class:`str` + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :return: Typed record, or ``None`` if not found. + :rtype: :class:`~PowerPlatform.Dataverse.models.record.Record` or None + + Example:: + + record = client.records.retrieve("account", account_id, select=["name"]) + if record is not None: + print(record["name"]) + """ + with self._client._scoped_odata() as od: + try: + raw = od._get(table, record_id, select=select) + except Exception as exc: + resp = getattr(exc, "response", None) + if resp is not None and getattr(resp, "status_code", None) == 404: + return None + raise + return Record.from_api_response(table, raw, record_id=record_id) + + # ------------------------------------------------------------------ list + + def list( + self, + table: str, + *, + filter: Optional[Union[str, "FilterExpression"]] = None, + select: Optional[List[str]] = None, + top: Optional[int] = None, + ) -> QueryResult: + """Fetch multiple records and return them as a :class:`QueryResult`. + + GA replacement for ``records.get(table, filter=...)``. All pages are + collected eagerly and returned as a single :class:`QueryResult`. + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param filter: Optional OData filter string or :class:`FilterExpression`. + :type filter: str or FilterExpression or None + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :param top: Maximum total number of records to return. + :type top: int or None + :return: All matching records collected into a :class:`QueryResult`. + :rtype: :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + + Example:: + + from PowerPlatform.Dataverse import col + + result = client.records.list( + "account", + filter=col("statecode") == 0, + select=["name"], + top=100, + ) + for record in result: + print(record["name"]) + + df = result.to_dataframe() + """ + filter_str: Optional[str] = str(filter) if filter is not None else None + all_records: List[Record] = [] + with self._client._scoped_odata() as od: + for page in od._get_multiple( + table, + select=select, + filter=filter_str, + orderby=None, + top=top, + expand=None, + page_size=None, + count=False, + include_annotations=None, + ): + all_records.extend(Record.from_api_response(table, row) for row in page) + return QueryResult(all_records) + + # --------------------------------------------------------------- list_pages + + def list_pages( + self, + table: str, + *, + filter: Optional[Union[str, "FilterExpression"]] = None, + select: Optional[List[str]] = None, + top: Optional[int] = None, + ) -> Iterator[QueryResult]: + """Lazily yield one :class:`QueryResult` per HTTP page. + + Symmetric with :meth:`~PowerPlatform.Dataverse.models.query_builder.QueryBuilder.execute_pages` + on the builder. Each iteration triggers a network request via + ``@odata.nextLink``. One-shot — do not iterate more than once. + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param filter: Optional OData filter string or :class:`FilterExpression`. + :type filter: str or FilterExpression or None + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :param top: Maximum total number of records to return. + :type top: int or None + :return: Iterator of per-page :class:`QueryResult` objects. + :rtype: Iterator[:class:`~PowerPlatform.Dataverse.models.record.QueryResult`] + + Example:: + + for page in client.records.list_pages("account", filter="statecode eq 0"): + process(page.to_dataframe()) + """ + filter_str: Optional[str] = str(filter) if filter is not None else None + with self._client._scoped_odata() as od: + for page in od._get_multiple( + table, + select=select, + filter=filter_str, + orderby=None, + top=top, + expand=None, + page_size=None, + count=False, + include_annotations=None, + ): + yield QueryResult([Record.from_api_response(table, row) for row in page]) + # ------------------------------------------------------------------ upsert def upsert(self, table: str, items: List[Union[UpsertItem, Dict[str, Any]]]) -> None: diff --git a/tests/unit/models/test_query_builder.py b/tests/unit/models/test_query_builder.py index 9f094912..160e5072 100644 --- a/tests/unit/models/test_query_builder.py +++ b/tests/unit/models/test_query_builder.py @@ -51,349 +51,120 @@ def test_select_returns_self(self): self.assertIs(qb.select("name"), qb) -class TestComparisonFilters(unittest.TestCase): - """Tests for comparison filter methods.""" +class TestRemovedFilterMethods(unittest.TestCase): + """Verify all 16 filter_* builder methods were removed in 1.0 GA.""" - def test_filter_eq_string(self): - qb = QueryBuilder("account").filter_eq("name", "Contoso") - self.assertEqual(qb.build()["filter"], "name eq 'Contoso'") - - def test_filter_eq_integer(self): - qb = QueryBuilder("account").filter_eq("statecode", 0) - self.assertEqual(qb.build()["filter"], "statecode eq 0") + def setUp(self): + self.qb = QueryBuilder("account") - def test_filter_eq_boolean_true(self): - qb = QueryBuilder("account").filter_eq("active", True) - self.assertEqual(qb.build()["filter"], "active eq true") + def test_filter_eq_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_eq("name", "Contoso") - def test_filter_eq_boolean_false(self): - qb = QueryBuilder("account").filter_eq("active", False) - self.assertEqual(qb.build()["filter"], "active eq false") + def test_filter_ne_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_ne("statecode", 1) - def test_filter_eq_none(self): - qb = QueryBuilder("account").filter_eq("telephone1", None) - self.assertEqual(qb.build()["filter"], "telephone1 eq null") + def test_filter_gt_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_gt("revenue", 0) - def test_filter_eq_float(self): - qb = QueryBuilder("account").filter_eq("revenue", 1000000.5) - self.assertEqual(qb.build()["filter"], "revenue eq 1000000.5") + def test_filter_ge_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_ge("revenue", 0) - def test_filter_eq_datetime(self): - from datetime import datetime, timezone + def test_filter_lt_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_lt("revenue", 0) - dt = datetime(2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc) - qb = QueryBuilder("account").filter_eq("createdon", dt) - self.assertEqual(qb.build()["filter"], "createdon eq 2024-01-15T10:30:00Z") + def test_filter_le_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_le("revenue", 0) + def test_filter_contains_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_contains("name", "Corp") -class TestFilterIn(unittest.TestCase): - """Tests for the filter_in() method.""" + def test_filter_startswith_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_startswith("name", "Con") - def test_filter_in_integers(self): - qb = QueryBuilder("account").filter_in("statecode", [0, 1, 2]) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1","2"])', - ) - - def test_filter_in_strings(self): - qb = QueryBuilder("account").filter_in("name", ["Contoso", "Fabrikam"]) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.In(PropertyName=\'name\',PropertyValues=["Contoso","Fabrikam"])', - ) + def test_filter_endswith_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_endswith("name", "Ltd") - def test_filter_in_single_value(self): - qb = QueryBuilder("account").filter_in("statecode", [0]) - self.assertEqual( - qb.build()["filter"], - "Microsoft.Dynamics.CRM.In(PropertyName='statecode',PropertyValues=[\"0\"])", - ) + def test_filter_null_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_null("telephone1") - def test_filter_in_column_lowercased(self): - qb = QueryBuilder("account").filter_in("StateCode", [0, 1]) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1"])', - ) + def test_filter_not_null_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_not_null("telephone1") - def test_filter_in_empty_raises(self): - with self.assertRaises(ValueError): - QueryBuilder("account").filter_in("statecode", []) + def test_filter_in_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_in("statecode", [0, 1]) - def test_filter_in_returns_self(self): - qb = QueryBuilder("account") - self.assertIs(qb.filter_in("statecode", [0, 1]), qb) - - def test_filter_in_with_set(self): - qb = QueryBuilder("account").filter_in("statecode", {0, 1}) - result = qb.build()["filter"] - self.assertIn("Microsoft.Dynamics.CRM.In", result) - self.assertIn("statecode", result) - - def test_filter_in_with_tuple(self): - qb = QueryBuilder("account").filter_in("statecode", (0, 1, 2)) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1","2"])', - ) - - def test_filter_in_int_enum(self): - from enum import IntEnum - - class Priority(IntEnum): - LOW = 1 - HIGH = 3 - - qb = QueryBuilder("account").filter_in("priority", [Priority.LOW, Priority.HIGH]) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.In(PropertyName=\'priority\',PropertyValues=["1","3"])', - ) - - def test_filter_in_combined_with_other_filters(self): - qb = QueryBuilder("account").filter_eq("statecode", 0).filter_in("priority", [1, 2, 3]) - self.assertEqual( - qb.build()["filter"], - 'statecode eq 0 and Microsoft.Dynamics.CRM.In(PropertyName=\'priority\',PropertyValues=["1","2","3"])', - ) + def test_filter_not_in_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_not_in("statecode", [0, 1]) - def test_filter_ne(self): - qb = QueryBuilder("account").filter_ne("statecode", 1) - self.assertEqual(qb.build()["filter"], "statecode ne 1") + def test_filter_between_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_between("revenue", 100, 500) - def test_filter_gt(self): - qb = QueryBuilder("account").filter_gt("revenue", 1000000) - self.assertEqual(qb.build()["filter"], "revenue gt 1000000") + def test_filter_not_between_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_not_between("revenue", 100, 500) - def test_filter_ge(self): - qb = QueryBuilder("account").filter_ge("revenue", 1000000) - self.assertEqual(qb.build()["filter"], "revenue ge 1000000") - - def test_filter_lt(self): - qb = QueryBuilder("account").filter_lt("revenue", 500000) - self.assertEqual(qb.build()["filter"], "revenue lt 500000") - - def test_filter_le(self): - qb = QueryBuilder("account").filter_le("revenue", 500000) - self.assertEqual(qb.build()["filter"], "revenue le 500000") - - def test_column_names_lowercased(self): - qb = QueryBuilder("account").filter_eq("StateCode", 0).order_by("Revenue") - params = qb.build() - self.assertEqual(params["filter"], "statecode eq 0") - self.assertEqual(params["orderby"], ["revenue"]) - - def test_string_with_quotes_escaped(self): - qb = QueryBuilder("account").filter_eq("name", "O'Brien's Corp") - self.assertEqual(qb.build()["filter"], "name eq 'O''Brien''s Corp'") - - def test_multiple_filters_and_joined(self): - qb = QueryBuilder("account").filter_eq("statecode", 0).filter_gt("revenue", 1000000) - self.assertEqual(qb.build()["filter"], "statecode eq 0 and revenue gt 1000000") - - -class TestStringFunctionFilters(unittest.TestCase): - """Tests for string function filter methods.""" - - def test_filter_contains(self): - qb = QueryBuilder("account").filter_contains("name", "Corp") - self.assertEqual(qb.build()["filter"], "contains(name, 'Corp')") - - def test_filter_startswith(self): - qb = QueryBuilder("account").filter_startswith("name", "Con") - self.assertEqual(qb.build()["filter"], "startswith(name, 'Con')") - - def test_filter_endswith(self): - qb = QueryBuilder("account").filter_endswith("name", "Ltd") - self.assertEqual(qb.build()["filter"], "endswith(name, 'Ltd')") - - def test_filter_contains_single_quotes(self): - qb = QueryBuilder("account").filter_contains("name", "O'Brien") - self.assertEqual(qb.build()["filter"], "contains(name, 'O''Brien')") - - -class TestNullFilters(unittest.TestCase): - """Tests for null/not-null filter methods.""" - - def test_filter_null(self): - qb = QueryBuilder("account").filter_null("telephone1") - self.assertEqual(qb.build()["filter"], "telephone1 eq null") - - def test_filter_not_null(self): - qb = QueryBuilder("account").filter_not_null("telephone1") - self.assertEqual(qb.build()["filter"], "telephone1 ne null") - - -class TestFilterBetween(unittest.TestCase): - """Tests for the filter_between() method.""" - - def test_filter_between_parenthesized(self): - qb = QueryBuilder("account").filter_between("revenue", 100000, 500000) - self.assertEqual( - qb.build()["filter"], - "(revenue ge 100000 and revenue le 500000)", - ) - - def test_filter_between_column_lowercased(self): - qb = QueryBuilder("account").filter_between("Revenue", 100, 500) - self.assertEqual( - qb.build()["filter"], - "(revenue ge 100 and revenue le 500)", - ) - - def test_filter_between_returns_self(self): - qb = QueryBuilder("account") - self.assertIs(qb.filter_between("revenue", 100, 500), qb) - - def test_filter_between_combined_with_other_filters(self): - qb = QueryBuilder("account").filter_eq("statecode", 0).filter_between("revenue", 100000, 500000) - self.assertEqual( - qb.build()["filter"], - "statecode eq 0 and (revenue ge 100000 and revenue le 500000)", - ) - - def test_filter_between_datetimes(self): - from datetime import datetime, timezone - - start = datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc) - end = datetime(2024, 12, 31, 23, 59, 59, tzinfo=timezone.utc) - qb = QueryBuilder("account").filter_between("createdon", start, end) - self.assertEqual( - qb.build()["filter"], - "(createdon ge 2024-01-01T00:00:00Z and createdon le 2024-12-31T23:59:59Z)", - ) - - -class TestFilterNotIn(unittest.TestCase): - """Tests for the filter_not_in() method.""" - - def test_filter_not_in_ints(self): - qb = QueryBuilder("account").filter_not_in("statecode", [2, 3]) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["2","3"])', - ) - - def test_filter_not_in_strings(self): - qb = QueryBuilder("account").filter_not_in("name", ["Contoso", "Fabrikam"]) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'name\',PropertyValues=["Contoso","Fabrikam"])', - ) - - def test_filter_not_in_empty_raises(self): - with self.assertRaises(ValueError): - QueryBuilder("account").filter_not_in("statecode", []) - - def test_filter_not_in_returns_self(self): - qb = QueryBuilder("account") - self.assertIs(qb.filter_not_in("statecode", [0, 1]), qb) - - def test_filter_not_in_combined_with_other_filters(self): - qb = QueryBuilder("account").filter_eq("statecode", 0).filter_not_in("priority", [1, 2]) - self.assertEqual( - qb.build()["filter"], - 'statecode eq 0 and Microsoft.Dynamics.CRM.NotIn(PropertyName=\'priority\',PropertyValues=["1","2"])', - ) - - def test_filter_not_in_with_set(self): - qb = QueryBuilder("account").filter_not_in("statecode", {2, 3}) - result = qb.build()["filter"] - self.assertIn("Microsoft.Dynamics.CRM.NotIn", result) - self.assertIn("statecode", result) - - def test_filter_not_in_with_tuple(self): - qb = QueryBuilder("account").filter_not_in("statecode", (2, 3)) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["2","3"])', - ) - - -class TestFilterNotBetween(unittest.TestCase): - """Tests for the filter_not_between() method.""" - - def test_filter_not_between_ints(self): - qb = QueryBuilder("account").filter_not_between("revenue", 100000, 500000) - self.assertEqual( - qb.build()["filter"], - "not ((revenue ge 100000 and revenue le 500000))", - ) - - def test_filter_not_between_returns_self(self): - qb = QueryBuilder("account") - self.assertIs(qb.filter_not_between("revenue", 100, 500), qb) - - def test_filter_not_between_combined_with_other_filters(self): - qb = QueryBuilder("account").filter_eq("statecode", 0).filter_not_between("revenue", 100000, 500000) - self.assertEqual( - qb.build()["filter"], - "statecode eq 0 and not ((revenue ge 100000 and revenue le 500000))", - ) - - -class TestFilterRaw(unittest.TestCase): - """Tests for the filter_raw() method.""" - - def test_filter_raw(self): - qb = QueryBuilder("account").filter_raw("(statecode eq 0 or statecode eq 1)") - self.assertEqual(qb.build()["filter"], "(statecode eq 0 or statecode eq 1)") - - def test_filter_raw_returns_self(self): - qb = QueryBuilder("account") - self.assertIs(qb.filter_raw("a eq 1"), qb) - - def test_build_with_plain_string_filter_part(self): - """build() handles plain string entries in _filter_parts (internal path).""" - qb = QueryBuilder("account") - qb._filter_parts.append("name eq 'Contoso'") - self.assertEqual(qb.build()["filter"], "name eq 'Contoso'") + def test_filter_raw_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_raw("statecode eq 0") class TestWhere(unittest.TestCase): """Tests for the where() method with composable expressions.""" def test_where_simple(self): - from PowerPlatform.Dataverse.models.filters import eq + from PowerPlatform.Dataverse.models.filters import col - qb = QueryBuilder("account").where(eq("statecode", 0)) + qb = QueryBuilder("account").where(col("statecode") == 0) self.assertEqual(qb.build()["filter"], "statecode eq 0") def test_where_complex(self): - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col - expr = (eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000) + expr = ((col("statecode") == 0) | (col("statecode") == 1)) & (col("revenue") > 100000) qb = QueryBuilder("account").where(expr) self.assertEqual( qb.build()["filter"], "((statecode eq 0 or statecode eq 1) and revenue gt 100000)", ) - def test_where_combined_with_filter_methods(self): - from PowerPlatform.Dataverse.models.filters import gt + def test_where_combined_with_raw(self): + from PowerPlatform.Dataverse.models.filters import col, raw - qb = QueryBuilder("account").filter_eq("statecode", 0).where(gt("revenue", 100000)) + qb = QueryBuilder("account").where(raw("statecode eq 0")).where(col("revenue") > 100000) self.assertEqual(qb.build()["filter"], "statecode eq 0 and revenue gt 100000") def test_where_multiple_calls(self): - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col - qb = QueryBuilder("account").where(eq("statecode", 0)).where(gt("revenue", 100000)) + qb = QueryBuilder("account").where(col("statecode") == 0).where(col("revenue") > 100000) self.assertEqual(qb.build()["filter"], "statecode eq 0 and revenue gt 100000") def test_where_preserves_call_order(self): - """Interleaved filter_*() and where() should preserve call order.""" - from PowerPlatform.Dataverse.models.filters import eq, gt + """Multiple where() calls preserve insertion order.""" + from PowerPlatform.Dataverse.models.filters import col - qb = QueryBuilder("account").where(eq("a", 1)).filter_eq("b", 2).where(gt("c", 3)) + qb = QueryBuilder("account").where(col("a") == 1).where(col("b") == 2).where(col("c") > 3) self.assertEqual(qb.build()["filter"], "a eq 1 and b eq 2 and c gt 3") def test_where_returns_self(self): - from PowerPlatform.Dataverse.models.filters import eq + from PowerPlatform.Dataverse.models.filters import col qb = QueryBuilder("account") - self.assertIs(qb.where(eq("statecode", 0)), qb) + self.assertIs(qb.where(col("statecode") == 0), qb) def test_where_non_expression_raises(self): qb = QueryBuilder("account") @@ -401,21 +172,54 @@ def test_where_non_expression_raises(self): qb.where("statecode eq 0") # type: ignore def test_where_with_not(self): - from PowerPlatform.Dataverse.models.filters import eq + from PowerPlatform.Dataverse.models.filters import col - qb = QueryBuilder("account").where(~eq("statecode", 1)) + qb = QueryBuilder("account").where(~(col("statecode") == 1)) self.assertEqual(qb.build()["filter"], "not (statecode eq 1)") def test_where_with_filter_in(self): - from PowerPlatform.Dataverse.models.filters import filter_in, gt + from PowerPlatform.Dataverse.models.filters import col - expr = filter_in("statecode", [0, 1]) & gt("revenue", 100000) + expr = col("statecode").in_([0, 1]) & (col("revenue") > 100000) qb = QueryBuilder("account").where(expr) self.assertEqual( qb.build()["filter"], '(Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1"]) and revenue gt 100000)', ) + def test_where_with_raw_preserves_string(self): + from PowerPlatform.Dataverse.models.filters import raw + + qb = QueryBuilder("account").where(raw("(statecode eq 0 or statecode eq 1)")) + self.assertEqual(qb.build()["filter"], "(statecode eq 0 or statecode eq 1)") + + def test_where_negation_of_in(self): + from PowerPlatform.Dataverse.models.filters import col + + qb = QueryBuilder("account").where(col("statecode").not_in([2, 3])) + self.assertEqual( + qb.build()["filter"], + 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["2","3"])', + ) + + def test_where_between(self): + from PowerPlatform.Dataverse.models.filters import col + + qb = QueryBuilder("account").where(col("revenue").between(100000, 500000)) + self.assertEqual( + qb.build()["filter"], + "(revenue ge 100000 and revenue le 500000)", + ) + + def test_where_not_between(self): + from PowerPlatform.Dataverse.models.filters import col + + qb = QueryBuilder("account").where(col("revenue").not_between(100000, 500000)) + self.assertEqual( + qb.build()["filter"], + "not ((revenue ge 100000 and revenue le 500000))", + ) + class TestOrderBy(unittest.TestCase): """Tests for the order_by() method.""" @@ -635,11 +439,13 @@ def test_empty_builder_only_has_table(self): self.assertNotIn("page_size", params) def test_full_query_build(self): + from PowerPlatform.Dataverse.models.filters import col, raw + qb = ( QueryBuilder("account") .select("name", "revenue", "telephone1") - .filter_eq("statecode", 0) - .filter_gt("revenue", 1000000) + .where(raw("statecode eq 0")) + .where(col("revenue") > 1000000) .order_by("revenue", descending=True) .order_by("name") .expand("primarycontactid") @@ -663,33 +469,34 @@ def test_build_returns_fresh_lists(self): self.assertEqual(params1["select"], params2["select"]) self.assertIsNot(params1["select"], params2["select"]) + def test_build_with_plain_string_filter_part(self): + """build() handles plain string entries in _filter_parts (internal path).""" + qb = QueryBuilder("account") + qb._filter_parts.append("name eq 'Contoso'") + self.assertEqual(qb.build()["filter"], "name eq 'Contoso'") + + def test_build_mixed_string_and_expression_filter_parts(self): + """build() AND-joins raw strings and FilterExpression objects in order.""" + from PowerPlatform.Dataverse.models.filters import col + + qb = QueryBuilder("account") + qb._filter_parts.append("statecode eq 0") + qb.where(col("revenue") > 100000) + self.assertEqual(qb.build()["filter"], "statecode eq 0 and revenue gt 100000") + class TestMethodChainingReturnsSelf(unittest.TestCase): - """Verify all methods return self for chaining.""" + """Verify all public methods return self for chaining.""" def test_all_methods_return_self(self): - from PowerPlatform.Dataverse.models.filters import eq + from PowerPlatform.Dataverse.models.filters import col qb = QueryBuilder("account") self.assertIs(qb.select("name"), qb) - self.assertIs(qb.filter_eq("a", 1), qb) - self.assertIs(qb.filter_ne("b", 2), qb) - self.assertIs(qb.filter_gt("c", 3), qb) - self.assertIs(qb.filter_ge("d", 4), qb) - self.assertIs(qb.filter_lt("e", 5), qb) - self.assertIs(qb.filter_le("f", 6), qb) - self.assertIs(qb.filter_contains("g", "x"), qb) - self.assertIs(qb.filter_startswith("h", "y"), qb) - self.assertIs(qb.filter_endswith("i", "z"), qb) - self.assertIs(qb.filter_null("j"), qb) - self.assertIs(qb.filter_not_null("k"), qb) - self.assertIs(qb.filter_raw("l eq 1"), qb) - self.assertIs(qb.filter_in("m", [1, 2]), qb) - self.assertIs(qb.filter_between("n", 1, 10), qb) - self.assertIs(qb.where(eq("o", 1)), qb) - self.assertIs(qb.order_by("p"), qb) - self.assertIs(qb.expand("q"), qb) + self.assertIs(qb.where(col("statecode") == 0), qb) + self.assertIs(qb.order_by("name"), qb) + self.assertIs(qb.expand("primarycontactid"), qb) self.assertIs(qb.top(10), qb) self.assertIs(qb.page_size(5), qb) self.assertIs(qb.count(), qb) @@ -701,22 +508,31 @@ class TestExecute(unittest.TestCase): """Tests for the execute() terminal method.""" def test_execute_without_query_ops_raises(self): - qb = QueryBuilder("account").filter_eq("statecode", 0) + from PowerPlatform.Dataverse.models.filters import raw + + qb = QueryBuilder("account").where(raw("statecode eq 0")) with self.assertRaises(RuntimeError) as ctx: qb.execute() self.assertIn("client.query.builder()", str(ctx.exception)) def test_execute_calls_records_get(self): """execute() should delegate to client.records.get() with built params.""" + from PowerPlatform.Dataverse.models.filters import raw + mock_query_ops = MagicMock() mock_client = mock_query_ops._client mock_client.records.get.return_value = iter([[{"name": "Test"}]]) qb = QueryBuilder("account") qb._query_ops = mock_query_ops - qb.select("name", "revenue").filter_eq("statecode", 0).order_by("revenue", descending=True).top(100).page_size( - 50 - ).expand("primarycontactid") + ( + qb.select("name", "revenue") + .where(raw("statecode eq 0")) + .order_by("revenue", descending=True) + .top(100) + .page_size(50) + .expand("primarycontactid") + ) list(qb.execute()) @@ -764,15 +580,6 @@ def test_execute_by_page_returns_pages(self): self.assertEqual(pages[0], page1) self.assertEqual(pages[1], page2) - def test_execute_unbounded_raises(self): - """execute() with no select/filter/top should raise ValueError.""" - mock_query_ops = MagicMock() - qb = QueryBuilder("account") - qb._query_ops = mock_query_ops - with self.assertRaises(ValueError) as ctx: - qb.execute() - self.assertIn("Unbounded query", str(ctx.exception)) - def test_execute_with_only_select_succeeds(self): """execute() with select only should not raise.""" mock_query_ops = MagicMock() @@ -787,13 +594,15 @@ def test_execute_with_only_select_succeeds(self): def test_execute_with_only_filter_succeeds(self): """execute() with filter only should not raise.""" + from PowerPlatform.Dataverse.models.filters import raw + mock_query_ops = MagicMock() mock_client = mock_query_ops._client mock_client.records.get.return_value = iter([]) qb = QueryBuilder("account") qb._query_ops = mock_query_ops - qb.filter_eq("statecode", 0) + qb.where(raw("statecode eq 0")) list(qb.execute()) # should not raise mock_client.records.get.assert_called_once() @@ -828,7 +637,7 @@ def test_execute_with_only_count_raises(self): qb.execute() def test_execute_with_where_expressions(self): - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col mock_query_ops = MagicMock() mock_client = mock_query_ops._client @@ -836,7 +645,7 @@ def test_execute_with_where_expressions(self): qb = QueryBuilder("account") qb._query_ops = mock_query_ops - qb.where((eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000)) + qb.where(((col("statecode") == 0) | (col("statecode") == 1)) & (col("revenue") > 100000)) list(qb.execute()) call_args = mock_client.records.get.call_args @@ -846,13 +655,15 @@ def test_execute_with_where_expressions(self): ) def test_execute_with_filter_in(self): + from PowerPlatform.Dataverse.models.filters import col + mock_query_ops = MagicMock() mock_client = mock_query_ops._client mock_client.records.get.return_value = iter([]) qb = QueryBuilder("account") qb._query_ops = mock_query_ops - qb.filter_in("statecode", [0, 1, 2]) + qb.where(col("statecode").in_([0, 1, 2])) list(qb.execute()) call_args = mock_client.records.get.call_args @@ -885,88 +696,209 @@ def test_execute_passes_count_and_annotations(self): ) +class TestExecutePages(unittest.TestCase): + """Tests for execute_pages() — lazy per-page QueryResult iterator.""" + + def _make_qb(self): + mock_query_ops = MagicMock() + mock_query_ops._client.records.get.return_value = iter([[{"name": "A"}], [{"name": "B"}]]) + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name") + return qb, mock_query_ops + + def test_execute_pages_returns_iterator(self): + qb, _ = self._make_qb() + result = qb.execute_pages() + import types + + self.assertIsInstance(result, types.GeneratorType) + + def test_execute_pages_yields_query_result_per_page(self): + from PowerPlatform.Dataverse.models.record import QueryResult + + qb, _ = self._make_qb() + pages = list(qb.execute_pages()) + self.assertEqual(len(pages), 2) + for page in pages: + self.assertIsInstance(page, QueryResult) + + def test_execute_pages_page_contents(self): + qb, _ = self._make_qb() + pages = list(qb.execute_pages()) + self.assertEqual(pages[0].first()["name"], "A") + self.assertEqual(pages[1].first()["name"], "B") + + def test_execute_pages_without_query_ops_raises(self): + from PowerPlatform.Dataverse.models.filters import raw + + qb = QueryBuilder("account").where(raw("statecode eq 0")) + with self.assertRaises(RuntimeError): + list(qb.execute_pages()) + + def test_execute_pages_without_constraints_raises(self): + mock_query_ops = MagicMock() + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + with self.assertRaises(ValueError): + list(qb.execute_pages()) + + +class TestByPageWarning(unittest.TestCase): + """execute(by_page=...) fires UserWarning; plain execute() does not.""" + + def _make_qb(self): + mock_query_ops = MagicMock() + mock_query_ops._client.records.get.return_value = iter([[{"name": "A"}]]) + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name") + return qb + + def test_execute_no_flag_no_warning(self): + """execute() with no by_page argument fires no warning.""" + import warnings + + qb = self._make_qb() + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + qb.execute() + user_warnings = [w for w in caught if issubclass(w.category, UserWarning)] + self.assertEqual(len(user_warnings), 0) + + def test_execute_by_page_true_fires_user_warning(self): + """execute(by_page=True) fires UserWarning pointing to execute_pages().""" + qb = self._make_qb() + with self.assertWarns(UserWarning) as ctx: + list(qb.execute(by_page=True)) + self.assertIn("execute_pages()", str(ctx.warning)) + + def test_execute_by_page_false_fires_user_warning(self): + """execute(by_page=False) fires UserWarning — redundant flag.""" + qb = self._make_qb() + with self.assertWarns(UserWarning) as ctx: + qb.execute(by_page=False) + self.assertIn("redundant", str(ctx.warning)) + + def test_execute_by_page_true_still_functional(self): + """execute(by_page=True) still returns the raw page iterator.""" + import warnings + + qb = self._make_qb() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + result = qb.execute(by_page=True) + pages = list(result) + self.assertEqual(len(pages), 1) + + def test_execute_by_page_false_still_returns_query_result(self): + """execute(by_page=False) still returns QueryResult.""" + import warnings + + from PowerPlatform.Dataverse.models.record import QueryResult + + qb = self._make_qb() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + result = qb.execute(by_page=False) + self.assertIsInstance(result, QueryResult) + + class TestToDataframe(unittest.TestCase): """Tests for the to_dataframe() terminal method.""" def test_to_dataframe_without_query_ops_raises(self): - qb = QueryBuilder("account").filter_eq("statecode", 0) + from PowerPlatform.Dataverse.models.filters import raw + + qb = QueryBuilder("account").where(raw("statecode eq 0")) with self.assertRaises(RuntimeError) as ctx: qb.to_dataframe() self.assertIn("client.query.builder()", str(ctx.exception)) - def test_to_dataframe_delegates_to_dataframe_get(self): - """to_dataframe() should delegate to client.dataframe.get() with built params.""" + def test_to_dataframe_returns_dataframe(self): + """to_dataframe() collects execute() results into a DataFrame.""" import pandas as pd mock_query_ops = MagicMock() mock_client = mock_query_ops._client - expected_df = pd.DataFrame([{"name": "Contoso", "revenue": 1000}]) - mock_client.dataframe.get.return_value = expected_df + mock_client.records.get.return_value = iter([[{"name": "A", "revenue": 100}, {"name": "B", "revenue": 200}]]) qb = QueryBuilder("account") qb._query_ops = mock_query_ops - qb.select("name", "revenue").filter_eq("statecode", 0).order_by("revenue", descending=True).top(100).page_size( - 50 - ).expand("primarycontactid") + qb.select("name", "revenue") result = qb.to_dataframe() - mock_client.dataframe.get.assert_called_once_with( - "account", - select=["name", "revenue"], - filter="statecode eq 0", - orderby=["revenue desc"], - top=100, - expand=["primarycontactid"], - page_size=50, - count=False, - include_annotations=None, - ) - pd.testing.assert_frame_equal(result, expected_df) + self.assertIsInstance(result, pd.DataFrame) + self.assertEqual(len(result), 2) + self.assertListEqual(list(result.columns), ["name", "revenue"]) + + def test_to_dataframe_empty_result_returns_empty_dataframe(self): + """to_dataframe() with no matching records returns empty DataFrame.""" + import pandas as pd - def test_to_dataframe_unbounded_raises(self): - """to_dataframe() with no select/filter/top should raise ValueError.""" mock_query_ops = MagicMock() + mock_client = mock_query_ops._client + mock_client.records.get.return_value = iter([]) + qb = QueryBuilder("account") qb._query_ops = mock_query_ops - with self.assertRaises(ValueError) as ctx: - qb.to_dataframe() - self.assertIn("Unbounded query", str(ctx.exception)) + qb.select("name", "revenue") - def test_to_dataframe_returns_dataframe(self): - """to_dataframe() should return a pandas DataFrame.""" + result = qb.to_dataframe() + + self.assertIsInstance(result, pd.DataFrame) + self.assertEqual(len(result), 0) + + def test_to_dataframe_calls_records_get_with_params(self): + """to_dataframe() should call records.get() with the built query params.""" import pandas as pd + from PowerPlatform.Dataverse.models.filters import raw mock_query_ops = MagicMock() mock_client = mock_query_ops._client - mock_client.dataframe.get.return_value = pd.DataFrame( - [{"name": "A", "revenue": 100}, {"name": "B", "revenue": 200}] - ) + mock_client.records.get.return_value = iter([[{"name": "Contoso", "revenue": 1000}]]) qb = QueryBuilder("account") qb._query_ops = mock_query_ops - qb.select("name", "revenue") + ( + qb.select("name", "revenue") + .where(raw("statecode eq 0")) + .order_by("revenue", descending=True) + .top(100) + .page_size(50) + .expand("primarycontactid") + ) result = qb.to_dataframe() + mock_client.records.get.assert_called_once_with( + "account", + select=["name", "revenue"], + filter="statecode eq 0", + orderby=["revenue desc"], + top=100, + expand=["primarycontactid"], + page_size=50, + count=False, + include_annotations=None, + ) self.assertIsInstance(result, pd.DataFrame) - self.assertEqual(len(result), 2) - self.assertListEqual(list(result.columns), ["name", "revenue"]) + self.assertEqual(len(result), 1) + self.assertEqual(result.iloc[0]["name"], "Contoso") def test_to_dataframe_forwards_count_and_annotations(self): """to_dataframe() should forward count and include_annotations when set.""" - import pandas as pd - mock_query_ops = MagicMock() mock_client = mock_query_ops._client - mock_client.dataframe.get.return_value = pd.DataFrame() + mock_client.records.get.return_value = iter([]) qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name").count().include_formatted_values() qb.to_dataframe() - mock_client.dataframe.get.assert_called_once_with( + mock_client.records.get.assert_called_once_with( "account", select=["name"], filter=None, @@ -978,6 +910,46 @@ def test_to_dataframe_forwards_count_and_annotations(self): include_annotations="OData.Community.Display.V1.FormattedValue", ) + def test_to_dataframe_with_record_objects(self): + """to_dataframe() handles Record objects (with .data attribute).""" + import pandas as pd + from PowerPlatform.Dataverse.models.record import Record + + mock_query_ops = MagicMock() + mock_client = mock_query_ops._client + records = [ + Record(id="id-1", table="account", data={"name": "Contoso", "revenue": 1000}), + Record(id="id-2", table="account", data={"name": "Fabrikam", "revenue": 2000}), + ] + mock_client.records.get.return_value = iter([records]) + + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name", "revenue") + + result = qb.to_dataframe() + + self.assertIsInstance(result, pd.DataFrame) + self.assertEqual(len(result), 2) + self.assertEqual(result.iloc[0]["name"], "Contoso") + self.assertEqual(result.iloc[1]["revenue"], 2000) + + def test_to_dataframe_emits_deprecation_warning(self): + """QueryBuilder.to_dataframe() fires DeprecationWarning; use execute().to_dataframe() instead.""" + mock_query_ops = MagicMock() + mock_client = mock_query_ops._client + mock_client.records.get.return_value = iter([]) + + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name") + + with self.assertWarns(DeprecationWarning) as ctx: + qb.to_dataframe() + + self.assertIn("QueryBuilder.to_dataframe()", str(ctx.warning)) + self.assertIn("execute().to_dataframe()", str(ctx.warning)) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 52d5a716..2ed03ddd 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -9,135 +9,87 @@ from PowerPlatform.Dataverse.client import DataverseClient -class TestDataverseClient(unittest.TestCase): +class TestDataverseClientConstruction(unittest.TestCase): + """Tests for DataverseClient construction and lifecycle.""" + + def test_empty_base_url_raises(self): + """DataverseClient raises ValueError when base_url is empty.""" + mock_credential = MagicMock(spec=TokenCredential) + with self.assertRaises(ValueError): + DataverseClient("", mock_credential) + + def test_trailing_slash_stripped(self): + """DataverseClient strips trailing slash from base_url.""" + mock_credential = MagicMock(spec=TokenCredential) + client = DataverseClient("https://example.crm.dynamics.com/", mock_credential) + self.assertEqual(client._base_url, "https://example.crm.dynamics.com") + + def test_namespace_attributes_present(self): + """Client exposes records, query, tables, files, dataframe, batch namespaces.""" + mock_credential = MagicMock(spec=TokenCredential) + client = DataverseClient("https://example.crm.dynamics.com", mock_credential) + for attr in ("records", "query", "tables", "files", "dataframe", "batch"): + self.assertTrue(hasattr(client, attr), f"Missing namespace: {attr}") + + +class TestRemovedClientMethods(unittest.TestCase): + """Verify all 12 deprecated flat methods were removed from DataverseClient in 1.0 GA. + + These methods previously delegated to namespace equivalents (records.*, query.*, + tables.*, files.*). They were fully removed; callers must use the namespaces directly. + """ + def setUp(self): - """Set up test fixtures before each test method.""" - # Create mock credential self.mock_credential = MagicMock(spec=TokenCredential) - self.base_url = "https://example.crm.dynamics.com" + self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) - # Initialize the client under test - self.client = DataverseClient(self.base_url, self.mock_credential) + def test_create_removed(self): + with self.assertRaises(AttributeError): + self.client.create("account", {"name": "Test"}) - # Mock the internal _odata client - # This ensures we verify logic without making actual HTTP calls - self.client._odata = MagicMock() - - def test_create_single(self): - """Test create method with a single record.""" - # Setup mock return values - # _create must return a GUID string - self.client._odata._create.return_value = "00000000-0000-0000-0000-000000000000" - # _entity_set_from_schema_name should return the plural entity set name - self.client._odata._entity_set_from_schema_name.return_value = "accounts" - - # Execute test - self.client.create("account", {"name": "Contoso Ltd"}) - - # Verify - # Ensure _entity_set_from_schema_name was called and its result ("accounts") was passed to _create - self.client._odata._create.assert_called_once_with("accounts", "account", {"name": "Contoso Ltd"}) - - def test_create_multiple(self): - """Test create method with multiple records.""" - payloads = [{"name": "Company A"}, {"name": "Company B"}, {"name": "Company C"}] - - # Setup mock return values - # _create_multiple must return a list of GUID strings - self.client._odata._create_multiple.return_value = [ - "00000000-0000-0000-0000-000000000001", - "00000000-0000-0000-0000-000000000002", - "00000000-0000-0000-0000-000000000003", - ] - self.client._odata._entity_set_from_schema_name.return_value = "accounts" - - # Execute test - self.client.create("account", payloads) - - # Verify - self.client._odata._create_multiple.assert_called_once_with("accounts", "account", payloads) - - def test_update_single(self): - """Test update method with a single record.""" - self.client.update("account", "00000000-0000-0000-0000-000000000000", {"telephone1": "555-0199"}) - self.client._odata._update.assert_called_once_with( - "account", "00000000-0000-0000-0000-000000000000", {"telephone1": "555-0199"} - ) + def test_update_removed(self): + with self.assertRaises(AttributeError): + self.client.update("account", "guid-1", {"name": "Test"}) - def test_update_multiple(self): - """Test update method with multiple records (broadcast).""" - ids = [ - "00000000-0000-0000-0000-000000000001", - "00000000-0000-0000-0000-000000000002", - ] - changes = {"statecode": 1} - - self.client.update("account", ids, changes) - self.client._odata._update_by_ids.assert_called_once_with("account", ids, changes) - - def test_delete_single(self): - """Test delete method with a single record.""" - self.client.delete("account", "00000000-0000-0000-0000-000000000000") - self.client._odata._delete.assert_called_once_with("account", "00000000-0000-0000-0000-000000000000") - - def test_delete_multiple(self): - """Test delete method with multiple records.""" - ids = [ - "00000000-0000-0000-0000-000000000001", - "00000000-0000-0000-0000-000000000002", - ] - # Mock return value for bulk delete job ID - self.client._odata._delete_multiple.return_value = "job-guid-123" - - job_id = self.client.delete("account", ids) - - self.client._odata._delete_multiple.assert_called_once_with("account", ids) - self.assertEqual(job_id, "job-guid-123") - - def test_get_single(self): - """Test get method with a single record ID.""" - # Setup mock return value - expected_record = {"accountid": "00000000-0000-0000-0000-000000000000", "name": "Contoso"} - self.client._odata._get.return_value = expected_record - - result = self.client.get("account", "00000000-0000-0000-0000-000000000000") - - self.client._odata._get.assert_called_once_with("account", "00000000-0000-0000-0000-000000000000", select=None) - self.assertEqual(result["accountid"], "00000000-0000-0000-0000-000000000000") - self.assertEqual(result["name"], "Contoso") - - def test_get_multiple(self): - """Test get method for querying multiple records.""" - # Setup mock return value (iterator) - expected_batch = [{"accountid": "1", "name": "A"}, {"accountid": "2", "name": "B"}] - self.client._odata._get_multiple.return_value = iter([expected_batch]) - - # Execute query - result_iterator = self.client.get("account", filter="statecode eq 0", top=10) - - # Consume iterator to verify content - results = list(result_iterator) - - self.client._odata._get_multiple.assert_called_once_with( - "account", - select=None, - filter="statecode eq 0", - orderby=None, - top=10, - expand=None, - page_size=None, - count=False, - include_annotations=None, - ) - self.assertEqual(len(results), 1) - self.assertEqual(len(results[0]), 2) - self.assertEqual(results[0][0]["name"], "A") - self.assertEqual(results[0][1]["name"], "B") + def test_delete_removed(self): + with self.assertRaises(AttributeError): + self.client.delete("account", "guid-1") - def test_empty_base_url_raises(self): - """DataverseClient raises ValueError when base_url is empty.""" - with self.assertRaises(ValueError): - DataverseClient("", self.mock_credential) + def test_get_removed(self): + with self.assertRaises(AttributeError): + self.client.get("account", "guid-1") + + def test_query_sql_removed(self): + with self.assertRaises(AttributeError): + self.client.query_sql("SELECT name FROM account") + + def test_get_table_info_removed(self): + with self.assertRaises(AttributeError): + self.client.get_table_info("account") + + def test_create_table_removed(self): + with self.assertRaises(AttributeError): + self.client.create_table("new_Test", {}) + + def test_delete_table_removed(self): + with self.assertRaises(AttributeError): + self.client.delete_table("new_Test") + + def test_list_tables_removed(self): + with self.assertRaises(AttributeError): + self.client.list_tables() + + def test_create_columns_removed(self): + with self.assertRaises(AttributeError): + self.client.create_columns("account", {}) + + def test_delete_columns_removed(self): + with self.assertRaises(AttributeError): + self.client.delete_columns("account", []) + + def test_upload_file_removed(self): + with self.assertRaises(AttributeError): + self.client.upload_file("account", "guid-1", "file_col", "/path/file.pdf") class TestCreateLookupField(unittest.TestCase): diff --git a/tests/unit/test_client_deprecations.py b/tests/unit/test_client_deprecations.py index fbf1382d..b11486be 100644 --- a/tests/unit/test_client_deprecations.py +++ b/tests/unit/test_client_deprecations.py @@ -1,12 +1,11 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Tests for deprecated flat methods on DataverseClient. +"""Tests confirming all 12 deprecated flat methods were removed from DataverseClient. -Each deprecated method on the client should: -1. Emit a DeprecationWarning. -2. Delegate to the correct namespace method (records / query / tables / files). -3. Return the expected value, including any backward-compatibility shims. +These methods previously delegated to namespace equivalents with a DeprecationWarning. +In 1.0 GA they are fully removed; each call now raises AttributeError. +Callers must use the operation namespaces directly (records.*, query.*, tables.*, files.*). """ import unittest @@ -18,281 +17,95 @@ class TestClientDeprecations(unittest.TestCase): - """Verify every deprecated flat method warns and delegates correctly.""" + """All formerly-deprecated flat methods are now removed and raise AttributeError.""" def setUp(self): - """Set up test fixtures before each test method.""" self.mock_credential = MagicMock(spec=TokenCredential) self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) - # Mock the internal OData client so namespace methods resolve without - # making real HTTP calls. - self.client._odata = MagicMock() - # ---------------------------------------------------------------- create + # ---------------------------------------------------------------- records - def test_create_warns(self): - """client.create() emits a DeprecationWarning.""" - self.client._odata._entity_set_from_schema_name.return_value = "accounts" - self.client._odata._create.return_value = "guid-123" - - with self.assertWarns(DeprecationWarning): + def test_create_removed(self): + """client.create() → use client.records.create()""" + with self.assertRaises(AttributeError): self.client.create("account", {"name": "Test"}) def test_create_single_returns_list(self): - """client.create() wraps a single GUID in a list for backward compat. - - records.create() returns a bare string for a single dict, but the - deprecated client.create() always returned list[str]. - """ - self.client._odata._entity_set_from_schema_name.return_value = "accounts" - self.client._odata._create.return_value = "guid-123" - - with self.assertWarns(DeprecationWarning): - result = self.client.create("account", {"name": "A"}) - - self.assertIsInstance(result, list) - self.assertEqual(result, ["guid-123"]) + """client.create() single-dict shim is gone; client.records.create() returns str.""" + with self.assertRaises(AttributeError): + self.client.create("account", {"name": "A"}) def test_create_bulk_returns_list(self): - """client.create() with a list payload returns list[str] directly.""" - self.client._odata._entity_set_from_schema_name.return_value = "accounts" - self.client._odata._create_multiple.return_value = ["guid-1", "guid-2"] - - with self.assertWarns(DeprecationWarning): - result = self.client.create("account", [{"name": "A"}, {"name": "B"}]) - - self.assertIsInstance(result, list) - self.assertEqual(result, ["guid-1", "guid-2"]) - - # ---------------------------------------------------------------- update + """client.create() list-payload shim is gone; client.records.create() returns list[str].""" + with self.assertRaises(AttributeError): + self.client.create("account", [{"name": "A"}, {"name": "B"}]) def test_update_warns_and_delegates(self): - """client.update() emits a DeprecationWarning and delegates to records.update.""" - with self.assertWarns(DeprecationWarning): - self.client.update( - "account", - "00000000-0000-0000-0000-000000000001", - {"telephone1": "555-0199"}, - ) - - self.client._odata._update.assert_called_once_with( - "account", - "00000000-0000-0000-0000-000000000001", - {"telephone1": "555-0199"}, - ) - - # ---------------------------------------------------------------- delete + """client.update() → use client.records.update()""" + with self.assertRaises(AttributeError): + self.client.update("account", "guid-1", {"telephone1": "555-0199"}) def test_delete_warns_and_delegates(self): - """client.delete() emits a DeprecationWarning and delegates to records.delete.""" - with self.assertWarns(DeprecationWarning): - self.client.delete("account", "00000000-0000-0000-0000-000000000001") - - self.client._odata._delete.assert_called_once_with("account", "00000000-0000-0000-0000-000000000001") - - # ------------------------------------------------------------------- get + """client.delete() → use client.records.delete()""" + with self.assertRaises(AttributeError): + self.client.delete("account", "guid-1") def test_get_single_warns(self): - """client.get() with record_id emits a DeprecationWarning and delegates - to records.get. - """ - expected = {"accountid": "guid-1", "name": "Contoso"} - self.client._odata._get.return_value = expected - - with self.assertWarns(DeprecationWarning): - result = self.client.get("account", record_id="guid-1") - - self.client._odata._get.assert_called_once_with("account", "guid-1", select=None) - self.assertEqual(result["accountid"], "guid-1") - self.assertEqual(result["name"], "Contoso") + """client.get(record_id=...) → use client.records.get()""" + with self.assertRaises(AttributeError): + self.client.get("account", record_id="guid-1") def test_get_multiple_warns(self): - """client.get() without record_id emits a DeprecationWarning and delegates - to records.get. - """ - page = [{"accountid": "1", "name": "A"}, {"accountid": "2", "name": "B"}] - self.client._odata._get_multiple.return_value = iter([page]) - - with self.assertWarns(DeprecationWarning): - result = self.client.get("account", filter="statecode eq 0", top=10) + """client.get(filter=...) → use client.records.get()""" + with self.assertRaises(AttributeError): + self.client.get("account", filter="statecode eq 0", top=10) - # The result is a generator; consume it. - pages = list(result) - self.assertEqual(len(pages), 1) - self.assertEqual(pages[0][0]["name"], "A") - self.assertEqual(pages[0][1]["name"], "B") - - self.client._odata._get_multiple.assert_called_once_with( - "account", - select=None, - filter="statecode eq 0", - orderby=None, - top=10, - expand=None, - page_size=None, - count=False, - include_annotations=None, - ) - - # ------------------------------------------------------------- query_sql + # ----------------------------------------------------------------- query def test_query_sql_warns(self): - """client.query_sql() emits a DeprecationWarning and delegates to - query.sql. - """ - expected_rows = [{"name": "Contoso"}, {"name": "Fabrikam"}] - self.client._odata._query_sql.return_value = expected_rows - - with self.assertWarns(DeprecationWarning): - result = self.client.query_sql("SELECT name FROM account") - - self.client._odata._query_sql.assert_called_once_with("SELECT name FROM account") - self.assertEqual(len(result), 2) - self.assertEqual(result[0]["name"], "Contoso") - self.assertEqual(result[1]["name"], "Fabrikam") + """client.query_sql() → use client.query.sql()""" + with self.assertRaises(AttributeError): + self.client.query_sql("SELECT name FROM account") - # -------------------------------------------------------- get_table_info + # --------------------------------------------------------------- tables def test_get_table_info_warns(self): - """client.get_table_info() emits a DeprecationWarning and delegates to - tables.get. - """ - expected_info = { - "table_schema_name": "new_MyTable", - "table_logical_name": "new_mytable", - "entity_set_name": "new_mytables", - "metadata_id": "meta-guid", - } - self.client._odata._get_table_info.return_value = expected_info - - with self.assertWarns(DeprecationWarning): - result = self.client.get_table_info("new_MyTable") - - self.client._odata._get_table_info.assert_called_once_with("new_MyTable") - self.assertEqual(result["table_schema_name"], "new_MyTable") - self.assertEqual(result["entity_set_name"], "new_mytables") - - # --------------------------------------------------------- create_table + """client.get_table_info() → use client.tables.get()""" + with self.assertRaises(AttributeError): + self.client.get_table_info("new_MyTable") def test_create_table_warns(self): - """client.create_table() emits a DeprecationWarning and maps legacy - parameter names (solution_unique_name -> solution, - primary_column_schema_name -> primary_column) when delegating to - tables.create. - """ - expected = { - "table_schema_name": "new_Product", - "entity_set_name": "new_products", - "table_logical_name": "new_product", - "metadata_id": "meta-guid", - "columns_created": ["new_Price"], - } - self.client._odata._create_table.return_value = expected - - with self.assertWarns(DeprecationWarning): - result = self.client.create_table( - "new_Product", - {"new_Price": "decimal"}, - solution_unique_name="MySolution", - primary_column_schema_name="new_ProductName", - ) - - # Verify that the internal _create_table received the mapped params. - self.client._odata._create_table.assert_called_once_with( - "new_Product", - {"new_Price": "decimal"}, - "MySolution", - "new_ProductName", - None, - ) - self.assertEqual(result["table_schema_name"], "new_Product") - self.assertEqual(result["columns_created"], ["new_Price"]) - - # --------------------------------------------------------- delete_table + """client.create_table() → use client.tables.create()""" + with self.assertRaises(AttributeError): + self.client.create_table("new_Product", {"new_Price": "decimal"}) def test_delete_table_warns(self): - """client.delete_table() emits a DeprecationWarning and delegates to - tables.delete. - """ - with self.assertWarns(DeprecationWarning): + """client.delete_table() → use client.tables.delete()""" + with self.assertRaises(AttributeError): self.client.delete_table("new_MyTestTable") - self.client._odata._delete_table.assert_called_once_with("new_MyTestTable") - - # ---------------------------------------------------------- list_tables - def test_list_tables_warns(self): - """client.list_tables() emits a DeprecationWarning and delegates to - tables.list. - """ - expected = [{"LogicalName": "account"}, {"LogicalName": "contact"}] - self.client._odata._list_tables.return_value = expected - - with self.assertWarns(DeprecationWarning): - result = self.client.list_tables() - - self.client._odata._list_tables.assert_called_once() - self.assertEqual(result, expected) - - # ------------------------------------------------------- create_columns + """client.list_tables() → use client.tables.list()""" + with self.assertRaises(AttributeError): + self.client.list_tables() def test_create_columns_warns(self): - """client.create_columns() emits a DeprecationWarning and delegates to - tables.add_columns. - """ - self.client._odata._create_columns.return_value = ["new_Notes", "new_Active"] - - with self.assertWarns(DeprecationWarning): - result = self.client.create_columns( - "new_MyTestTable", - {"new_Notes": "string", "new_Active": "bool"}, - ) - - self.client._odata._create_columns.assert_called_once_with( - "new_MyTestTable", - {"new_Notes": "string", "new_Active": "bool"}, - ) - self.assertEqual(result, ["new_Notes", "new_Active"]) - - # ------------------------------------------------------- delete_columns + """client.create_columns() → use client.tables.add_columns()""" + with self.assertRaises(AttributeError): + self.client.create_columns("new_MyTestTable", {"new_Notes": "string"}) def test_delete_columns_warns(self): - """client.delete_columns() emits a DeprecationWarning and delegates to - tables.remove_columns. - """ - self.client._odata._delete_columns.return_value = ["new_Notes", "new_Active"] - - with self.assertWarns(DeprecationWarning): - result = self.client.delete_columns( - "new_MyTestTable", - ["new_Notes", "new_Active"], - ) + """client.delete_columns() → use client.tables.remove_columns()""" + with self.assertRaises(AttributeError): + self.client.delete_columns("new_MyTestTable", ["new_Notes"]) - self.client._odata._delete_columns.assert_called_once_with( - "new_MyTestTable", - ["new_Notes", "new_Active"], - ) - self.assertEqual(result, ["new_Notes", "new_Active"]) - - # ----------------------------------------------------------- upload_file + # ----------------------------------------------------------------- files def test_upload_file_warns(self): - """client.upload_file() emits a DeprecationWarning and delegates - to files.upload. - """ - with self.assertWarns(DeprecationWarning): + """client.upload_file() → use client.files.upload()""" + with self.assertRaises(AttributeError): self.client.upload_file("account", "guid-1", "new_Document", "/path/to/file.pdf") - self.client._odata._upload_file.assert_called_once_with( - "account", - "guid-1", - "new_Document", - "/path/to/file.pdf", - mode=None, - mime_type=None, - if_none_match=True, - ) - if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_phase1_ga.py b/tests/unit/test_phase1_ga.py new file mode 100644 index 00000000..ee832029 --- /dev/null +++ b/tests/unit/test_phase1_ga.py @@ -0,0 +1,861 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Comprehensive Phase 1 GA regression tests. + +Verifies all Phase 1 breaking changes and deprecations in one place: + +1. All 12 deprecated client flat methods raise AttributeError (removed). +2. All 15 deprecated filter factory functions emit DeprecationWarning on CALL + (not on import). +3. GA filter functions col() and raw() emit NO deprecation warning. +4. dataframe.get() emits DeprecationWarning on call. +5. All deprecated factories remain functional (correct OData output). +6. All 16 filter_* builder methods raise AttributeError on QueryBuilder (removed). +7. No DeprecationWarning is emitted at module import time. +""" + +import unittest +import warnings +from unittest.mock import MagicMock + +from azure.core.credentials import TokenCredential + +from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.models.query_builder import QueryBuilder + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_client(): + """Create a DataverseClient with a mock credential and mock _odata.""" + credential = MagicMock(spec=TokenCredential) + client = DataverseClient("https://example.crm.dynamics.com", credential) + client._odata = MagicMock() + return client + + +def _catch_warnings(*categories): + """Context manager: catch all warnings, return the list.""" + return warnings.catch_warnings(record=True) + + +# --------------------------------------------------------------------------- +# 1. Removed client flat methods raise AttributeError +# --------------------------------------------------------------------------- + + +class TestRemovedClientFlatMethods(unittest.TestCase): + """All 12 formerly-deprecated client flat methods are removed in GA.""" + + def setUp(self): + self.client = _make_client() + + def _assert_removed(self, method_name, *args, **kwargs): + with self.assertRaises(AttributeError, msg=f"client.{method_name} should not exist"): + getattr(self.client, method_name)(*args, **kwargs) + + def test_create_removed(self): + self._assert_removed("create", "account", {"name": "Test"}) + + def test_update_removed(self): + self._assert_removed("update", "account", "guid-1", {"name": "Test"}) + + def test_delete_removed(self): + self._assert_removed("delete", "account", "guid-1") + + def test_get_removed(self): + self._assert_removed("get", "account", "guid-1") + + def test_query_sql_removed(self): + self._assert_removed("query_sql", "SELECT name FROM account") + + def test_get_table_info_removed(self): + self._assert_removed("get_table_info", "account") + + def test_create_table_removed(self): + self._assert_removed("create_table", "new_Test", {}) + + def test_delete_table_removed(self): + self._assert_removed("delete_table", "new_Test") + + def test_list_tables_removed(self): + self._assert_removed("list_tables") + + def test_create_columns_removed(self): + self._assert_removed("create_columns", "account", {}) + + def test_delete_columns_removed(self): + self._assert_removed("delete_columns", "account", []) + + def test_upload_file_removed(self): + self._assert_removed("upload_file", "account", "guid-1", "col", "/path") + + +# --------------------------------------------------------------------------- +# 2. Deprecated filter factories emit DeprecationWarning on CALL (not import) +# --------------------------------------------------------------------------- + + +class TestDeprecatedFilterFactoriesWarnOnCall(unittest.TestCase): + """All 15 deprecated filter factories emit DeprecationWarning when called. + + The warning must fire on CALL, not on import — importing the module must + be warning-free. + """ + + def _assert_warns_on_call(self, func, *args, **kwargs): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = func(*args, **kwargs) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertGreater(len(dep_warnings), 0, f"{func.__name__}() did not emit DeprecationWarning") + return result, dep_warnings + + def _assert_single_warning(self, func, *args, **kwargs): + """Verify exactly one DeprecationWarning is emitted (no chained warnings).""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + func(*args, **kwargs) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual( + len(dep_warnings), 1, f"{func.__name__}() emitted {len(dep_warnings)} warnings (expected exactly 1)" + ) + + def test_no_warning_on_import(self): + """Importing the filters module emits no DeprecationWarning.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + import importlib + import PowerPlatform.Dataverse.models.filters as _f + + importlib.reload(_f) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0, "Import must not emit DeprecationWarning") + + def test_eq_warns(self): + from PowerPlatform.Dataverse.models.filters import eq + + self._assert_warns_on_call(eq, "name", "Contoso") + + def test_ne_warns(self): + from PowerPlatform.Dataverse.models.filters import ne + + self._assert_warns_on_call(ne, "statecode", 1) + + def test_gt_warns(self): + from PowerPlatform.Dataverse.models.filters import gt + + self._assert_warns_on_call(gt, "revenue", 1000000) + + def test_ge_warns(self): + from PowerPlatform.Dataverse.models.filters import ge + + self._assert_warns_on_call(ge, "revenue", 1000000) + + def test_lt_warns(self): + from PowerPlatform.Dataverse.models.filters import lt + + self._assert_warns_on_call(lt, "revenue", 500000) + + def test_le_warns(self): + from PowerPlatform.Dataverse.models.filters import le + + self._assert_warns_on_call(le, "revenue", 500000) + + def test_contains_warns(self): + from PowerPlatform.Dataverse.models.filters import contains + + self._assert_warns_on_call(contains, "name", "Corp") + + def test_startswith_warns(self): + from PowerPlatform.Dataverse.models.filters import startswith + + self._assert_warns_on_call(startswith, "name", "Con") + + def test_endswith_warns(self): + from PowerPlatform.Dataverse.models.filters import endswith + + self._assert_warns_on_call(endswith, "name", "Ltd") + + def test_between_warns(self): + from PowerPlatform.Dataverse.models.filters import between + + self._assert_warns_on_call(between, "revenue", 100000, 500000) + + def test_is_null_warns(self): + from PowerPlatform.Dataverse.models.filters import is_null + + self._assert_warns_on_call(is_null, "telephone1") + + def test_is_not_null_warns(self): + from PowerPlatform.Dataverse.models.filters import is_not_null + + self._assert_warns_on_call(is_not_null, "telephone1") + + def test_filter_in_warns(self): + from PowerPlatform.Dataverse.models.filters import filter_in + + self._assert_warns_on_call(filter_in, "statecode", [0, 1]) + + def test_not_in_warns(self): + from PowerPlatform.Dataverse.models.filters import not_in + + self._assert_warns_on_call(not_in, "statecode", [2, 3]) + + def test_not_between_warns(self): + from PowerPlatform.Dataverse.models.filters import not_between + + self._assert_warns_on_call(not_between, "revenue", 100000, 500000) + + def test_between_emits_exactly_one_warning(self): + """between() must not chain through deprecated ge/le (would emit 3 warnings).""" + from PowerPlatform.Dataverse.models.filters import between + + self._assert_single_warning(between, "revenue", 100000, 500000) + + def test_not_between_emits_exactly_one_warning(self): + """not_between() must not chain through deprecated ge/le.""" + from PowerPlatform.Dataverse.models.filters import not_between + + self._assert_single_warning(not_between, "revenue", 100000, 500000) + + def test_warning_is_deprecation_warning_class(self): + """Each factory's warning category must be DeprecationWarning, not its subclass.""" + from PowerPlatform.Dataverse.models.filters import eq + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + eq("name", "Test") + w = caught[0] + self.assertIs(w.category, DeprecationWarning) + + def test_warning_message_names_replacement(self): + """Each warning message should name the col()-based replacement.""" + from PowerPlatform.Dataverse.models.filters import eq + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + eq("name", "Test") + self.assertIn("col(", str(caught[0].message)) + + +# --------------------------------------------------------------------------- +# 3. GA functions col() and raw() emit NO deprecation warning +# --------------------------------------------------------------------------- + + +class TestGAFunctionsNoWarning(unittest.TestCase): + """col() and raw() are GA — must never emit DeprecationWarning.""" + + def _assert_no_dep_warning(self, func, *args, **kwargs): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + func(*args, **kwargs) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual( + len(dep_warnings), 0, f"{func.__name__}() emitted unexpected DeprecationWarning: {dep_warnings}" + ) + + def test_col_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + self._assert_no_dep_warning(col, "statecode") + + def test_raw_no_warning(self): + from PowerPlatform.Dataverse.models.filters import raw + + self._assert_no_dep_warning(raw, "statecode eq 0") + + def test_col_eq_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("statecode") == 0 + _ = expr.to_odata() + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_comparison_chain_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = (col("statecode") == 0) & (col("revenue") > 100000) + _ = expr.to_odata() + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_between_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("revenue").between(100000, 500000) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_not_between_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("revenue").not_between(100000, 500000) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_in_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("statecode").in_([0, 1]) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_not_in_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("statecode").not_in([2, 3]) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_like_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("name").like("Contoso%") + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_not_like_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("name").not_like("%Corp%") + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_where_with_col_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + QueryBuilder("account").where(col("statecode") == 0) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + +# --------------------------------------------------------------------------- +# 4. dataframe.get() emits DeprecationWarning on call +# --------------------------------------------------------------------------- + + +class TestDataframeGetDeprecation(unittest.TestCase): + """dataframe.get() must emit DeprecationWarning on call (not on import).""" + + def setUp(self): + self.client = _make_client() + # Set up _odata so records.get() can be called + self.client._odata._get_multiple.return_value = iter([]) + self.client._odata._get.return_value = MagicMock(data={}) + + def test_dataframe_get_warns_on_call(self): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + try: + self.client.dataframe.get("account", select=["name"], top=10) + except Exception: + pass + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertGreater(len(dep_warnings), 0, "dataframe.get() did not emit DeprecationWarning") + + def test_dataframe_get_warning_message(self): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + try: + self.client.dataframe.get("account", select=["name"], top=10) + except Exception: + pass + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + msg = str(dep_warnings[0].message) + self.assertIn("dataframe.get()", msg) + self.assertIn("builder", msg) + + def test_dataframe_other_methods_no_warning(self): + """dataframe.sql(), dataframe.create(), etc. must NOT warn.""" + import pandas as pd + from PowerPlatform.Dataverse.models.record import Record + + self.client._odata._query_sql.return_value = [] + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.dataframe.sql("SELECT name FROM account") + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0, "dataframe.sql() must not warn") + + +# --------------------------------------------------------------------------- +# 5. Deprecated factories remain functional (correct OData output) +# --------------------------------------------------------------------------- + + +class TestDeprecatedFactoriesStillFunctional(unittest.TestCase): + """Despite the warning, deprecated factories produce correct OData strings.""" + + def _odata(self, func, *args): + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + return func(*args).to_odata() + + def test_eq_functional(self): + from PowerPlatform.Dataverse.models.filters import eq + + self.assertEqual(self._odata(eq, "name", "Contoso"), "name eq 'Contoso'") + + def test_ne_functional(self): + from PowerPlatform.Dataverse.models.filters import ne + + self.assertEqual(self._odata(ne, "statecode", 1), "statecode ne 1") + + def test_gt_functional(self): + from PowerPlatform.Dataverse.models.filters import gt + + self.assertEqual(self._odata(gt, "revenue", 1000000), "revenue gt 1000000") + + def test_ge_functional(self): + from PowerPlatform.Dataverse.models.filters import ge + + self.assertEqual(self._odata(ge, "revenue", 1000000), "revenue ge 1000000") + + def test_lt_functional(self): + from PowerPlatform.Dataverse.models.filters import lt + + self.assertEqual(self._odata(lt, "revenue", 500000), "revenue lt 500000") + + def test_le_functional(self): + from PowerPlatform.Dataverse.models.filters import le + + self.assertEqual(self._odata(le, "revenue", 500000), "revenue le 500000") + + def test_contains_functional(self): + from PowerPlatform.Dataverse.models.filters import contains + + self.assertEqual(self._odata(contains, "name", "Corp"), "contains(name, 'Corp')") + + def test_startswith_functional(self): + from PowerPlatform.Dataverse.models.filters import startswith + + self.assertEqual(self._odata(startswith, "name", "Con"), "startswith(name, 'Con')") + + def test_endswith_functional(self): + from PowerPlatform.Dataverse.models.filters import endswith + + self.assertEqual(self._odata(endswith, "name", "Ltd"), "endswith(name, 'Ltd')") + + def test_between_functional(self): + from PowerPlatform.Dataverse.models.filters import between + + self.assertEqual( + self._odata(between, "revenue", 100000, 500000), + "(revenue ge 100000 and revenue le 500000)", + ) + + def test_is_null_functional(self): + from PowerPlatform.Dataverse.models.filters import is_null + + self.assertEqual(self._odata(is_null, "telephone1"), "telephone1 eq null") + + def test_is_not_null_functional(self): + from PowerPlatform.Dataverse.models.filters import is_not_null + + self.assertEqual(self._odata(is_not_null, "telephone1"), "telephone1 ne null") + + def test_filter_in_functional(self): + from PowerPlatform.Dataverse.models.filters import filter_in + + self.assertEqual( + self._odata(filter_in, "statecode", [0, 1]), + 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1"])', + ) + + def test_not_in_functional(self): + from PowerPlatform.Dataverse.models.filters import not_in + + self.assertEqual( + self._odata(not_in, "statecode", [2, 3]), + 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["2","3"])', + ) + + def test_not_between_functional(self): + from PowerPlatform.Dataverse.models.filters import not_between + + self.assertEqual( + self._odata(not_between, "revenue", 100000, 500000), + "not ((revenue ge 100000 and revenue le 500000))", + ) + + def test_deprecated_factory_usable_in_where(self): + """Deprecated factories still produce valid FilterExpression for where().""" + from PowerPlatform.Dataverse.models.filters import eq + + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + expr = eq("statecode", 0) + qb = QueryBuilder("account").where(expr) + self.assertEqual(qb.build()["filter"], "statecode eq 0") + + def test_deprecated_factories_composable(self): + """Deprecated factories still compose correctly with & and |.""" + from PowerPlatform.Dataverse.models.filters import eq, gt + + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + expr = (eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000) + self.assertEqual( + expr.to_odata(), + "((statecode eq 0 or statecode eq 1) and revenue gt 100000)", + ) + + +# --------------------------------------------------------------------------- +# 6. All 16 filter_* builder methods raise AttributeError on QueryBuilder +# --------------------------------------------------------------------------- + + +class TestRemovedBuilderFilterMethods(unittest.TestCase): + """All 16 filter_* methods were removed from QueryBuilder in GA.""" + + def setUp(self): + self.qb = QueryBuilder("account") + + def _assert_removed(self, method_name, *args): + with self.assertRaises(AttributeError, msg=f"QueryBuilder.{method_name} should not exist"): + getattr(self.qb, method_name)(*args) + + def test_filter_eq_removed(self): + self._assert_removed("filter_eq", "name", "Contoso") + + def test_filter_ne_removed(self): + self._assert_removed("filter_ne", "statecode", 1) + + def test_filter_gt_removed(self): + self._assert_removed("filter_gt", "revenue", 0) + + def test_filter_ge_removed(self): + self._assert_removed("filter_ge", "revenue", 0) + + def test_filter_lt_removed(self): + self._assert_removed("filter_lt", "revenue", 0) + + def test_filter_le_removed(self): + self._assert_removed("filter_le", "revenue", 0) + + def test_filter_contains_removed(self): + self._assert_removed("filter_contains", "name", "Corp") + + def test_filter_startswith_removed(self): + self._assert_removed("filter_startswith", "name", "Con") + + def test_filter_endswith_removed(self): + self._assert_removed("filter_endswith", "name", "Ltd") + + def test_filter_null_removed(self): + self._assert_removed("filter_null", "telephone1") + + def test_filter_not_null_removed(self): + self._assert_removed("filter_not_null", "telephone1") + + def test_filter_in_removed(self): + self._assert_removed("filter_in", "statecode", [0, 1]) + + def test_filter_not_in_removed(self): + self._assert_removed("filter_not_in", "statecode", [0, 1]) + + def test_filter_between_removed(self): + self._assert_removed("filter_between", "revenue", 100, 500) + + def test_filter_not_between_removed(self): + self._assert_removed("filter_not_between", "revenue", 100, 500) + + def test_filter_raw_removed(self): + self._assert_removed("filter_raw", "statecode eq 0") + + +# --------------------------------------------------------------------------- +# 7. ColumnProxy (col()) correctness — all operators and methods +# --------------------------------------------------------------------------- + + +class TestColumnProxyOperators(unittest.TestCase): + """col() proxy covers all operators and methods correctly.""" + + def test_eq(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("name") == "Contoso").to_odata(), "name eq 'Contoso'") + + def test_ne(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("statecode") != 1).to_odata(), "statecode ne 1") + + def test_gt(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("revenue") > 1000000).to_odata(), "revenue gt 1000000") + + def test_ge(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("revenue") >= 1000000).to_odata(), "revenue ge 1000000") + + def test_lt(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("revenue") < 500000).to_odata(), "revenue lt 500000") + + def test_le(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("revenue") <= 500000).to_odata(), "revenue le 500000") + + def test_is_null(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("telephone1").is_null().to_odata(), "telephone1 eq null") + + def test_is_not_null(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("telephone1").is_not_null().to_odata(), "telephone1 ne null") + + def test_in_(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("statecode").in_([0, 1]).to_odata(), + 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1"])', + ) + + def test_not_in(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("statecode").not_in([2, 3]).to_odata(), + 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["2","3"])', + ) + + def test_between(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("revenue").between(100000, 500000).to_odata(), + "(revenue ge 100000 and revenue le 500000)", + ) + + def test_not_between(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("revenue").not_between(100000, 500000).to_odata(), + "not ((revenue ge 100000 and revenue le 500000))", + ) + + def test_contains(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").contains("Corp").to_odata(), "contains(name, 'Corp')") + + def test_startswith(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").startswith("Con").to_odata(), "startswith(name, 'Con')") + + def test_endswith(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").endswith("Ltd").to_odata(), "endswith(name, 'Ltd')") + + def test_like_startswith_pattern(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").like("Contoso%").to_odata(), "startswith(name, 'Contoso')") + + def test_like_endswith_pattern(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").like("%Ltd").to_odata(), "endswith(name, 'Ltd')") + + def test_like_contains_pattern(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").like("%Corp%").to_odata(), "contains(name, 'Corp')") + + def test_like_no_wildcard_equality(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").like("Contoso").to_odata(), "name eq 'Contoso'") + + def test_like_interior_wildcard_raises(self): + from PowerPlatform.Dataverse.models.filters import col + + with self.assertRaises(ValueError): + col("name").like("Con%oso") + + def test_not_like_startswith_pattern(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("name").not_like("Contoso%").to_odata(), + "not (startswith(name, 'Contoso'))", + ) + + def test_not_like_endswith_pattern(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("name").not_like("%Ltd").to_odata(), + "not (endswith(name, 'Ltd'))", + ) + + def test_not_like_contains_pattern(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("name").not_like("%Corp%").to_odata(), + "not (contains(name, 'Corp'))", + ) + + def test_not_like_no_wildcard_negated_equality(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("name").not_like("Contoso").to_odata(), + "not (name eq 'Contoso')", + ) + + def test_not_like_interior_wildcard_raises(self): + from PowerPlatform.Dataverse.models.filters import col + + with self.assertRaises(ValueError): + col("name").not_like("Con%oso") + + def test_column_name_lowercased(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("StateCode") == 0).to_odata(), "statecode eq 0") + + def test_empty_column_name_raises(self): + from PowerPlatform.Dataverse.models.filters import col + + with self.assertRaises(ValueError): + col("") + + def test_whitespace_column_name_raises(self): + from PowerPlatform.Dataverse.models.filters import col + + with self.assertRaises(ValueError): + col(" ") + + def test_boolean_eq(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("active") == True).to_odata(), "active eq true") # noqa: E712 + self.assertEqual((col("active") == False).to_odata(), "active eq false") # noqa: E712 + + def test_none_eq(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("telephone1") == None).to_odata(), "telephone1 eq null") # noqa: E711 + + def test_and_composition(self): + from PowerPlatform.Dataverse.models.filters import col + + expr = (col("statecode") == 0) & (col("revenue") > 100000) + self.assertEqual(expr.to_odata(), "(statecode eq 0 and revenue gt 100000)") + + def test_or_composition(self): + from PowerPlatform.Dataverse.models.filters import col + + expr = (col("statecode") == 0) | (col("statecode") == 1) + self.assertEqual(expr.to_odata(), "(statecode eq 0 or statecode eq 1)") + + def test_not_composition(self): + from PowerPlatform.Dataverse.models.filters import col + + expr = ~(col("statecode") == 1) + self.assertEqual(expr.to_odata(), "not (statecode eq 1)") + + def test_in_empty_raises(self): + from PowerPlatform.Dataverse.models.filters import col + + with self.assertRaises(ValueError): + col("statecode").in_([]) + + def test_not_in_empty_raises(self): + from PowerPlatform.Dataverse.models.filters import col + + with self.assertRaises(ValueError): + col("statecode").not_in([]) + + +# --------------------------------------------------------------------------- +# 8. GA namespace-level fluent builder (where + col) end-to-end +# --------------------------------------------------------------------------- + + +class TestGABuilderEndToEnd(unittest.TestCase): + """End-to-end verification of the GA query builder without any deprecated APIs.""" + + def setUp(self): + self.client = _make_client() + self.client._odata._get_multiple.return_value = iter([]) + + def test_builder_with_col_exprs_no_warnings(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + qb = ( + self.client.query.builder("account") + .select("name", "revenue") + .where((col("statecode") == 0) | (col("statecode") == 1)) + .where(col("revenue") > 100000) + .order_by("revenue", descending=True) + .top(50) + ) + params = qb.build() + + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0, f"GA API emitted warnings: {dep_warnings}") + self.assertEqual( + params["filter"], + "(statecode eq 0 or statecode eq 1) and revenue gt 100000", + ) + self.assertEqual(params["select"], ["name", "revenue"]) + self.assertEqual(params["top"], 50) + + def test_builder_with_raw_no_warnings(self): + from PowerPlatform.Dataverse.models.filters import raw + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + qb = self.client.query.builder("account").select("name").where(raw("statecode eq 0")).top(10) + params = qb.build() + + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + self.assertEqual(params["filter"], "statecode eq 0") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_phase2_ga.py b/tests/unit/test_phase2_ga.py new file mode 100644 index 00000000..b8a0aa5d --- /dev/null +++ b/tests/unit/test_phase2_ga.py @@ -0,0 +1,415 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Phase 2 GA regression tests. + +Covers: +- QueryResult class (models/record.py) +- execute() returns QueryResult in flat mode +- execute(by_page=True) still returns Iterable[list[Record]] +- col, raw, QueryResult re-exports from models/__init__ and package root +- pyproject.toml migration optional dep +""" + +import unittest +import warnings +from unittest.mock import MagicMock + +from azure.core.credentials import TokenCredential + +from PowerPlatform.Dataverse.models.record import QueryResult, Record +from PowerPlatform.Dataverse.models.query_builder import QueryBuilder + + +def _make_client(): + cred = MagicMock(spec=TokenCredential) + from PowerPlatform.Dataverse.client import DataverseClient + + client = DataverseClient("https://example.crm.dynamics.com", cred) + client._odata = MagicMock() + client._odata._get_multiple = MagicMock() + client._odata._get_single = MagicMock() + return client + + +class TestQueryResultClass(unittest.TestCase): + """Unit tests for QueryResult.""" + + def _records(self, n=3): + return [Record(id=f"id-{i}", table="account", data={"name": f"R{i}"}) for i in range(n)] + + # ----- construction / dunder + + def test_init_stores_records(self): + recs = self._records(2) + qr = QueryResult(recs) + self.assertIs(qr.records, recs) + + def test_empty_result(self): + qr = QueryResult([]) + self.assertEqual(len(qr), 0) + self.assertFalse(bool(qr)) + self.assertIsNone(qr.first()) + + def test_len(self): + self.assertEqual(len(QueryResult(self._records(5))), 5) + + def test_bool_true_when_nonempty(self): + self.assertTrue(bool(QueryResult(self._records(1)))) + + def test_bool_false_when_empty(self): + self.assertFalse(bool(QueryResult([]))) + + def test_iter_yields_records(self): + recs = self._records(3) + qr = QueryResult(recs) + self.assertEqual(list(qr), recs) + + def test_iter_multiple_times(self): + recs = self._records(2) + qr = QueryResult(recs) + self.assertEqual(list(qr), list(qr)) + + def test_repr_contains_count(self): + qr = QueryResult(self._records(7)) + self.assertIn("7", repr(qr)) + + # ----- first() + + def test_first_returns_first_record(self): + recs = self._records(3) + qr = QueryResult(recs) + self.assertIs(qr.first(), recs[0]) + + def test_first_returns_none_when_empty(self): + self.assertIsNone(QueryResult([]).first()) + + # ----- to_dataframe() + + def test_to_dataframe_nonempty(self): + import pandas as pd + + recs = [ + Record(id="id-1", table="account", data={"name": "Contoso", "revenue": 1000}), + Record(id="id-2", table="account", data={"name": "Fabrikam", "revenue": 2000}), + ] + qr = QueryResult(recs) + df = qr.to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 2) + self.assertIn("name", df.columns) + self.assertIn("revenue", df.columns) + self.assertEqual(df.iloc[0]["name"], "Contoso") + self.assertEqual(df.iloc[1]["revenue"], 2000) + + def test_to_dataframe_empty_returns_empty_df(self): + import pandas as pd + + df = QueryResult([]).to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 0) + + def test_to_dataframe_handles_plain_dicts(self): + """QueryResult.to_dataframe() handles plain dicts (no .data attribute).""" + import pandas as pd + + qr = QueryResult([{"name": "A"}, {"name": "B"}]) # type: ignore[arg-type] + df = qr.to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 2) + + # ----- for r in result (backward compat) + + def test_backward_compat_for_loop(self): + recs = self._records(3) + qr = QueryResult(recs) + collected = [] + for r in qr: + collected.append(r) + self.assertEqual(collected, recs) + + def test_list_conversion(self): + recs = self._records(4) + qr = QueryResult(recs) + self.assertEqual(list(qr), recs) + + +class TestExecuteReturnsQueryResult(unittest.TestCase): + """execute() flat mode returns QueryResult.""" + + def setUp(self): + self.client = _make_client() + + def test_execute_flat_returns_query_result(self): + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}], + [{"name": "B", "accountid": "2"}], + ] + ) + result = self.client.query.builder("account").select("name").execute() + self.assertIsInstance(result, QueryResult) + + def test_execute_flat_collects_all_pages(self): + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}], + [{"name": "B", "accountid": "2"}, {"name": "C", "accountid": "3"}], + ] + ) + result = self.client.query.builder("account").select("name").execute() + self.assertEqual(len(result), 3) + + def test_execute_flat_records_accessible(self): + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "Contoso", "accountid": "abc"}], + ] + ) + result = self.client.query.builder("account").select("name").execute() + first = result.first() + self.assertIsNotNone(first) + self.assertEqual(first["name"], "Contoso") + + def test_execute_flat_for_loop_backward_compat(self): + """for r in execute() still works — backward-compatible iteration.""" + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}, {"name": "B", "accountid": "2"}], + ] + ) + records = [] + for r in self.client.query.builder("account").select("name").execute(): + records.append(r) + self.assertEqual(len(records), 2) + + def test_execute_flat_list_backward_compat(self): + """list(execute()) still works.""" + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "X", "accountid": "x"}], + ] + ) + records = list(self.client.query.builder("account").select("name").execute()) + self.assertEqual(len(records), 1) + self.assertEqual(records[0]["name"], "X") + + def test_execute_by_page_not_query_result(self): + """execute(by_page=True) still returns page iterator, not QueryResult.""" + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}], + [{"name": "B", "accountid": "2"}], + ] + ) + result = self.client.query.builder("account").select("name").execute(by_page=True) + self.assertNotIsInstance(result, QueryResult) + pages = list(result) + self.assertEqual(len(pages), 2) + + def test_execute_empty_returns_empty_query_result(self): + self.client._odata._get_multiple.return_value = iter([]) + result = self.client.query.builder("account").select("name").execute() + self.assertIsInstance(result, QueryResult) + self.assertEqual(len(result), 0) + self.assertFalse(bool(result)) + self.assertIsNone(result.first()) + + def test_execute_result_to_dataframe(self): + """execute().to_dataframe() works end-to-end.""" + import pandas as pd + + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "Contoso", "accountid": "1"}, {"name": "Fabrikam", "accountid": "2"}], + ] + ) + df = self.client.query.builder("account").select("name").execute().to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 2) + + def test_execute_no_deprecation_warnings(self): + """execute() flat mode emits no DeprecationWarnings.""" + from PowerPlatform.Dataverse.models.filters import col + + self.client._odata._get_multiple.return_value = iter([]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + (self.client.query.builder("account").select("name").where(col("statecode") == 0).execute()) + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"Unexpected warnings: {dep}") + + +class TestQueryBuilderToDataframe(unittest.TestCase): + """to_dataframe() delegates to QueryResult.to_dataframe() after execute().""" + + def setUp(self): + self.client = _make_client() + + def test_to_dataframe_returns_dataframe(self): + import pandas as pd + + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}], + ] + ) + df = self.client.query.builder("account").select("name").to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 1) + + def test_to_dataframe_empty_preserves_select_columns(self): + """to_dataframe() on empty result keeps column names from select().""" + import pandas as pd + + self.client._odata._get_multiple.return_value = iter([]) + df = self.client.query.builder("account").select("name", "revenue").to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 0) + self.assertListEqual(list(df.columns), ["name", "revenue"]) + + def test_to_dataframe_empty_no_select(self): + """to_dataframe() on empty result with no select() returns bare empty DataFrame.""" + import pandas as pd + + self.client._odata._get_multiple.return_value = iter([]) + df = self.client.query.builder("account").top(10).to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 0) + + +class TestExports(unittest.TestCase): + """col, raw, QueryResult are importable from models and package root.""" + + def test_col_importable_from_models(self): + from PowerPlatform.Dataverse.models import col + + self.assertIsNotNone(col) + + def test_raw_importable_from_models(self): + from PowerPlatform.Dataverse.models import raw + + self.assertIsNotNone(raw) + + def test_query_result_importable_from_models(self): + from PowerPlatform.Dataverse.models import QueryResult + + self.assertIsNotNone(QueryResult) + + def test_col_importable_from_package_root(self): + from PowerPlatform.Dataverse import col + + self.assertIsNotNone(col) + + def test_raw_importable_from_package_root(self): + from PowerPlatform.Dataverse import raw + + self.assertIsNotNone(raw) + + def test_query_result_importable_from_package_root(self): + from PowerPlatform.Dataverse import QueryResult + + self.assertIsNotNone(QueryResult) + + def test_col_from_root_produces_filter_expression(self): + from PowerPlatform.Dataverse import col as root_col + from PowerPlatform.Dataverse.models.filters import FilterExpression + + expr = root_col("statecode") == 0 + self.assertIsInstance(expr, FilterExpression) + + def test_raw_from_root_produces_filter_expression(self): + from PowerPlatform.Dataverse import raw as root_raw + from PowerPlatform.Dataverse.models.filters import FilterExpression + + expr = root_raw("statecode eq 0") + self.assertIsInstance(expr, FilterExpression) + + def test_query_result_from_root_is_correct_class(self): + from PowerPlatform.Dataverse import QueryResult as root_qr + + qr = root_qr([]) + self.assertIsInstance(qr, root_qr) + self.assertEqual(len(qr), 0) + + def test_col_from_package_root_no_warning(self): + """Importing col from package root fires no DeprecationWarning.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + from PowerPlatform.Dataverse import col # noqa: F401 + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"Unexpected warnings: {dep}") + + def test_col_call_no_warning(self): + """col() emits no DeprecationWarning.""" + from PowerPlatform.Dataverse import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = col("statecode") == 0 + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"Unexpected warnings: {dep}") + self.assertIsNotNone(result) + + def test_raw_call_no_warning(self): + """raw() emits no DeprecationWarning.""" + from PowerPlatform.Dataverse import raw + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = raw("statecode eq 0") + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"Unexpected warnings: {dep}") + self.assertIsNotNone(result) + + +class TestQueryResultAcceptanceCriteria(unittest.TestCase): + """Verify all QueryResult acceptance criteria from the spec.""" + + def setUp(self): + self.client = _make_client() + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "Contoso", "accountid": "id-1"}], + ] + ) + + def _execute(self): + return self.client.query.builder("account").select("name").execute() + + def test_result_is_query_result(self): + result = self._execute() + self.assertIsInstance(result, QueryResult) + + def test_for_loop_still_works(self): + result = self._execute() + records = [r for r in result] + self.assertEqual(len(records), 1) + + def test_first_returns_record_or_none(self): + result = self._execute() + r = result.first() + self.assertIsNotNone(r) + self.assertIsInstance(r, Record) + + def test_to_dataframe_returns_dataframe(self): + import pandas as pd + + result = self._execute() + df = result.to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + + def test_builder_to_dataframe_returns_dataframe(self): + import pandas as pd + + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "Contoso", "accountid": "id-1"}], + ] + ) + df = self.client.query.builder("account").select("name").to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_phase3_ga.py b/tests/unit/test_phase3_ga.py new file mode 100644 index 00000000..34a3cb06 --- /dev/null +++ b/tests/unit/test_phase3_ga.py @@ -0,0 +1,424 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Phase 3 GA regression tests. + +Covers: +- records.get() deprecation (DeprecationWarning, still functional) +- records.retrieve() — single record, None on 404 +- records.list() — QueryResult, accepts str/FilterExpression filter +- DataverseModel Protocol and isinstance() check +- records.create(DataverseModel) dispatch +- records.update(DataverseModel, record_id) dispatch +- DataverseModel exported from models and package root +- execute() emits zero DeprecationWarning (internal records.get() suppressed) +""" + +import unittest +import warnings +from dataclasses import dataclass +from unittest.mock import MagicMock, patch + +from azure.core.credentials import TokenCredential + +from PowerPlatform.Dataverse.models.record import QueryResult, Record +from PowerPlatform.Dataverse.models.protocol import DataverseModel + + +def _make_client(): + cred = MagicMock(spec=TokenCredential) + from PowerPlatform.Dataverse.client import DataverseClient + + client = DataverseClient("https://example.crm.dynamics.com", cred) + client._odata = MagicMock() + client._odata._get_multiple = MagicMock() + client._odata._get_single = MagicMock() + client._odata._get = MagicMock() + client._odata._create = MagicMock() + client._odata._create_multiple = MagicMock() + client._odata._update = MagicMock() + client._odata._update_by_ids = MagicMock() + client._odata._entity_set_from_schema_name = MagicMock(side_effect=lambda t: t + "s") + return client + + +# --------------------------------------------------------------------------- +# Sample DataverseModel implementation for tests + + +@dataclass +class _Account: + __entity_logical_name__ = "account" + __entity_set_name__ = "accounts" + name: str = "" + telephone1: str = "" + + def to_dict(self) -> dict: + return {"name": self.name, "telephone1": self.telephone1} + + @classmethod + def from_dict(cls, data: dict) -> "_Account": + return cls(name=data.get("name", ""), telephone1=data.get("telephone1", "")) + + +# --------------------------------------------------------------------------- + + +class TestDataverseModelProtocol(unittest.TestCase): + """DataverseModel Protocol structural checks.""" + + def test_account_satisfies_protocol(self): + self.assertIsInstance(_Account(), DataverseModel) + + def test_plain_dict_does_not_satisfy_protocol(self): + self.assertNotIsInstance({"name": "X"}, DataverseModel) + + def test_missing_entity_logical_name_fails(self): + class _Bad: + __entity_set_name__ = "bads" + + def to_dict(self): + return {} + + @classmethod + def from_dict(cls, d): + return cls() + + self.assertNotIsInstance(_Bad(), DataverseModel) + + def test_missing_to_dict_fails(self): + class _Bad: + __entity_logical_name__ = "bad" + __entity_set_name__ = "bads" + + @classmethod + def from_dict(cls, d): + return cls() + + self.assertNotIsInstance(_Bad(), DataverseModel) + + def test_importable_from_models(self): + from PowerPlatform.Dataverse.models import DataverseModel as dm + + self.assertIsNotNone(dm) + + def test_importable_from_package_root(self): + from PowerPlatform.Dataverse import DataverseModel as dm + + self.assertIsNotNone(dm) + + def test_import_no_deprecation_warning(self): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + from PowerPlatform.Dataverse import DataverseModel # noqa: F401 + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, []) + + +class TestRecordsGetDeprecation(unittest.TestCase): + """records.get() fires DeprecationWarning, still functional.""" + + def setUp(self): + self.client = _make_client() + + def test_get_single_warns(self): + self.client._odata._get.return_value = {"accountid": "1", "name": "Contoso"} + with self.assertWarns(DeprecationWarning) as ctx: + self.client.records.get("account", "guid-1") + self.assertIn("retrieve", str(ctx.warning)) + + def test_get_multiple_warns(self): + self.client._odata._get_multiple.return_value = iter([]) + with self.assertWarns(DeprecationWarning) as ctx: + list(self.client.records.get("account", filter="statecode eq 0")) + self.assertIn("list", str(ctx.warning)) + + def test_get_single_still_returns_record(self): + self.client._odata._get.return_value = {"accountid": "1", "name": "Contoso"} + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + record = self.client.records.get("account", "guid-1") + self.assertIsInstance(record, Record) + self.assertEqual(record["name"], "Contoso") + + def test_get_multiple_still_returns_pages(self): + self.client._odata._get_multiple.return_value = iter([[{"name": "A", "accountid": "1"}]]) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + pages = list(self.client.records.get("account", filter="statecode eq 0")) + self.assertEqual(len(pages), 1) + + def test_get_warning_message_single_id(self): + self.client._odata._get.return_value = {"accountid": "1"} + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.records.get("account", "guid-1") + msgs = [str(w.message) for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertTrue(any("retrieve" in m for m in msgs)) + + def test_get_warning_message_filter_form(self): + self.client._odata._get_multiple.return_value = iter([]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + list(self.client.records.get("account", filter="statecode eq 0")) + msgs = [str(w.message) for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertTrue(any("list" in m for m in msgs)) + + +class TestRecordsRetrieve(unittest.TestCase): + """records.retrieve() returns Record or None, no warning.""" + + def setUp(self): + self.client = _make_client() + + def test_retrieve_returns_record(self): + self.client._odata._get.return_value = {"accountid": "abc", "name": "Contoso"} + record = self.client.records.retrieve("account", "abc") + self.assertIsInstance(record, Record) + self.assertEqual(record["name"], "Contoso") + + def test_retrieve_passes_select(self): + self.client._odata._get.return_value = {"accountid": "abc", "name": "Contoso"} + self.client.records.retrieve("account", "abc", select=["name"]) + self.client._odata._get.assert_called_once_with("account", "abc", select=["name"]) + + def test_retrieve_no_deprecation_warning(self): + self.client._odata._get.return_value = {"accountid": "abc", "name": "Contoso"} + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.records.retrieve("account", "abc") + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"retrieve() must not emit DeprecationWarning: {dep}") + + def test_retrieve_returns_none_on_404(self): + exc = Exception("Not Found") + resp_mock = MagicMock() + resp_mock.status_code = 404 + exc.response = resp_mock + self.client._odata._get.side_effect = exc + result = self.client.records.retrieve("account", "nonexistent-guid") + self.assertIsNone(result) + + def test_retrieve_reraises_non_404(self): + exc = Exception("Server Error") + resp_mock = MagicMock() + resp_mock.status_code = 500 + exc.response = resp_mock + self.client._odata._get.side_effect = exc + with self.assertRaises(Exception): + self.client.records.retrieve("account", "some-guid") + + def test_retrieve_reraises_when_no_response_attr(self): + self.client._odata._get.side_effect = ValueError("Bad input") + with self.assertRaises(ValueError): + self.client.records.retrieve("account", "some-guid") + + def test_retrieve_record_id_set(self): + self.client._odata._get.return_value = {"name": "Contoso"} + record = self.client.records.retrieve("account", "my-id") + self.assertEqual(record.id, "my-id") + + def test_retrieve_table_set(self): + self.client._odata._get.return_value = {"name": "Contoso"} + record = self.client.records.retrieve("account", "my-id") + self.assertEqual(record.table, "account") + + +class TestRecordsList(unittest.TestCase): + """records.list() returns QueryResult, no warning.""" + + def setUp(self): + self.client = _make_client() + + def test_list_returns_query_result(self): + self.client._odata._get_multiple.return_value = iter([]) + result = self.client.records.list("account") + self.assertIsInstance(result, QueryResult) + + def test_list_collects_all_pages(self): + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}], + [{"name": "B", "accountid": "2"}, {"name": "C", "accountid": "3"}], + ] + ) + result = self.client.records.list("account") + self.assertEqual(len(result), 3) + + def test_list_no_deprecation_warning(self): + self.client._odata._get_multiple.return_value = iter([]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.records.list("account", filter="statecode eq 0") + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"list() must not emit DeprecationWarning: {dep}") + + def test_list_passes_string_filter(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", filter="statecode eq 0") + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["filter"], "statecode eq 0") + + def test_list_passes_filter_expression(self): + from PowerPlatform.Dataverse.models.filters import col + + self.client._odata._get_multiple.return_value = iter([]) + expr = col("statecode") == 0 + self.client.records.list("account", filter=expr) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["filter"], "statecode eq 0") + + def test_list_passes_select(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", select=["name", "revenue"]) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["select"], ["name", "revenue"]) + + def test_list_passes_top(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", top=50) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["top"], 50) + + def test_list_none_filter_passes_none(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account") + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertIsNone(call_kwargs["filter"]) + + def test_list_result_iterable(self): + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "X", "accountid": "1"}], + ] + ) + result = self.client.records.list("account") + records = list(result) + self.assertEqual(len(records), 1) + self.assertEqual(records[0]["name"], "X") + + def test_list_result_to_dataframe(self): + import pandas as pd + + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}, {"name": "B", "accountid": "2"}], + ] + ) + df = self.client.records.list("account", select=["name"]).to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 2) + + +class TestCreateWithDataverseModel(unittest.TestCase): + """records.create() accepts DataverseModel.""" + + def setUp(self): + self.client = _make_client() + self.client._odata._create.return_value = "new-guid-123" + self.client._odata._create_multiple.return_value = ["guid-1", "guid-2"] + + def test_create_single_entity(self): + account = _Account(name="Contoso", telephone1="555-0100") + result = self.client.records.create(account) + self.assertEqual(result, "new-guid-123") + + def test_create_single_entity_uses_logical_name(self): + account = _Account(name="Contoso") + self.client.records.create(account) + self.client._odata._entity_set_from_schema_name.assert_called_with("account") + + def test_create_single_entity_calls_to_dict(self): + account = _Account(name="Contoso", telephone1="555-0100") + self.client.records.create(account) + self.client._odata._create.assert_called_once() + call_args = self.client._odata._create.call_args + self.assertEqual(call_args[0][2]["name"], "Contoso") + + def test_create_list_of_entities(self): + entities = [_Account(name="A"), _Account(name="B")] + result = self.client.records.create(entities) + self.assertEqual(result, ["guid-1", "guid-2"]) + + def test_create_list_uses_first_entity_logical_name(self): + entities = [_Account(name="A"), _Account(name="B")] + self.client.records.create(entities) + self.client._odata._entity_set_from_schema_name.assert_called_with("account") + + def test_create_entity_no_deprecation_warning(self): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.records.create(_Account(name="Contoso")) + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, []) + + def test_create_dict_path_still_works(self): + result = self.client.records.create("account", {"name": "Contoso"}) + self.assertEqual(result, "new-guid-123") + + def test_create_list_dict_path_still_works(self): + result = self.client.records.create("account", [{"name": "A"}, {"name": "B"}]) + self.assertEqual(result, ["guid-1", "guid-2"]) + + +class TestUpdateWithDataverseModel(unittest.TestCase): + """records.update() accepts DataverseModel as first arg.""" + + def setUp(self): + self.client = _make_client() + + def test_update_single_entity_with_id(self): + account = _Account(name="Updated Name") + self.client.records.update(account, "guid-abc") + self.client._odata._update.assert_called_once_with( + "account", "guid-abc", {"name": "Updated Name", "telephone1": ""} + ) + + def test_update_entity_no_id_raises(self): + account = _Account(name="Updated") + with self.assertRaises(TypeError): + self.client.records.update(account) + + def test_update_entity_no_deprecation_warning(self): + account = _Account(name="Updated") + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.records.update(account, "guid-abc") + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, []) + + def test_update_dict_path_still_works(self): + self.client.records.update("account", "guid-1", {"name": "New Name"}) + self.client._odata._update.assert_called_with("account", "guid-1", {"name": "New Name"}) + + +class TestExecuteNoDeprecationFromRecordsGet(unittest.TestCase): + """execute() suppresses DeprecationWarning from the internal records.get() call.""" + + def setUp(self): + self.client = _make_client() + self.client._odata._get_multiple.return_value = iter([]) + + def test_execute_flat_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.query.builder("account").select("name").where(col("statecode") == 0).execute() + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"execute() leaked DeprecationWarning: {dep}") + + def test_to_dataframe_no_records_get_warning(self): + """to_dataframe() emits its own deprecation but must not leak records.get()'s warning.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.query.builder("account").select("name").to_dataframe() + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + # Exactly one DeprecationWarning: from QueryBuilder.to_dataframe() itself. + # The internal records.get() deprecation must remain suppressed. + self.assertEqual(len(dep), 1, f"Unexpected warnings: {dep}") + self.assertIn("QueryBuilder.to_dataframe()", str(dep[0].message)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_phase4_ga.py b/tests/unit/test_phase4_ga.py new file mode 100644 index 00000000..6ea41b52 --- /dev/null +++ b/tests/unit/test_phase4_ga.py @@ -0,0 +1,416 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Phase 4 GA regression tests. + +Covers: +- fetch_xml(): basic, pagination, missing-entity-element error +- sql_select / sql_join / sql_joins raise AttributeError (removed at GA) +- odata_select / odata_expand / odata_bind emit DeprecationWarning (deprecated at GA) +- sql_columns / odata_expands / sql() emit zero DeprecationWarning (still GA-clean) +""" + +import unittest +import warnings +import xml.etree.ElementTree as ET +from unittest.mock import MagicMock, patch + +from azure.core.credentials import TokenCredential + +from PowerPlatform.Dataverse.models.record import QueryResult, Record + + +def _make_client(): + cred = MagicMock(spec=TokenCredential) + from PowerPlatform.Dataverse.client import DataverseClient + + client = DataverseClient("https://example.crm.dynamics.com", cred) + client._odata = MagicMock() + client._odata._entity_set_from_schema_name = MagicMock(side_effect=lambda t: t + "s") + client._odata.api = "https://example.crm.dynamics.com/api/data/v9.2" + return client + + +# --------------------------------------------------------------------------- +# fetch_xml() +# --------------------------------------------------------------------------- + + +class TestFetchXml(unittest.TestCase): + + def setUp(self): + self.client = _make_client() + + def _fetch_xml(self, entity="account"): + return f"""""" + + def _mock_response(self, records, more=False, cookie=""): + resp = MagicMock() + payload = {"value": records} + if more: + payload["@Microsoft.Dynamics.CRM.morerecords"] = True + payload["@Microsoft.Dynamics.CRM.fetchxmlpagingcookie"] = cookie + else: + payload["@Microsoft.Dynamics.CRM.morerecords"] = False + resp.json.return_value = payload + return resp + + def test_fetch_xml_inert_no_http_request(self): + """fetch_xml() alone must not fire any HTTP request.""" + from PowerPlatform.Dataverse.models.fetch_xml_query import FetchXmlQuery + + query = self.client.query.fetch_xml(self._fetch_xml()) + self.assertIsInstance(query, FetchXmlQuery) + self.client._odata._request.assert_not_called() + + def test_basic_returns_query_result(self): + self.client._odata._request.return_value = self._mock_response([{"name": "Contoso", "accountid": "1"}]) + result = self.client.query.fetch_xml(self._fetch_xml()).execute() + self.assertIsInstance(result, QueryResult) + + def test_basic_record_count(self): + self.client._odata._request.return_value = self._mock_response([{"name": "A"}, {"name": "B"}]) + result = self.client.query.fetch_xml(self._fetch_xml()).execute() + self.assertEqual(len(result), 2) + + def test_record_values_accessible(self): + self.client._odata._request.return_value = self._mock_response([{"name": "Contoso", "accountid": "abc-123"}]) + result = self.client.query.fetch_xml(self._fetch_xml()).execute() + self.assertEqual(result.first()["name"], "Contoso") + + def test_empty_result_returns_empty_query_result(self): + self.client._odata._request.return_value = self._mock_response([]) + result = self.client.query.fetch_xml(self._fetch_xml()).execute() + self.assertIsInstance(result, QueryResult) + self.assertEqual(len(result), 0) + self.assertFalse(result) + + def test_pagination_fetches_all_pages(self): + """execute_pages() drives the HTTP loop; each page yields one QueryResult.""" + cookie_raw = "%25253Cpagingcookie%252520pagingcookie%25253D%252522" + page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + pages = list(self.client.query.fetch_xml(self._fetch_xml()).execute_pages()) + self.assertEqual(len(pages), 2) + self.assertEqual(self.client._odata._request.call_count, 2) + + def test_pagination_second_request_includes_page_and_cookie(self): + """execute_pages() injects the decoded paging cookie into the second request.""" + cookie_raw = "%25253Cpagingcookie%252520test%25253D%252522" + page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + list(self.client.query.fetch_xml(self._fetch_xml()).execute_pages()) + + second_call_kwargs = self.client._odata._request.call_args_list[1] + params = ( + second_call_kwargs.kwargs.get("params") or second_call_kwargs.args[2] + if len(second_call_kwargs.args) > 2 + else {} + ) + if not params: + params = second_call_kwargs[1].get("params", {}) + xml_sent = params.get("fetchXml", "") + fetch_el = ET.fromstring(xml_sent) + self.assertEqual(fetch_el.get("page"), "2") + self.assertIsNotNone(fetch_el.get("paging-cookie")) + + def test_missing_entity_element_raises_value_error(self): + with self.assertRaises(ValueError) as ctx: + self.client.query.fetch_xml("") + self.assertIn("entity", str(ctx.exception).lower()) + + def test_entity_missing_name_attr_raises_value_error(self): + with self.assertRaises(ValueError) as ctx: + self.client.query.fetch_xml("") + self.assertIn("name", str(ctx.exception).lower()) + + def test_entity_set_resolved_from_entity_name(self): + self.client._odata._request.return_value = self._mock_response([]) + self.client.query.fetch_xml(self._fetch_xml("account")).execute() + self.client._odata._entity_set_from_schema_name.assert_called_with("account") + + def test_request_uses_prefer_header(self): + self.client._odata._request.return_value = self._mock_response([]) + self.client.query.fetch_xml(self._fetch_xml()).execute() + call_kwargs = self.client._odata._request.call_args + headers = call_kwargs.kwargs.get("headers", {}) + self.assertIn("Prefer", headers) + self.assertIn("fetchxmlpagingcookie", headers["Prefer"]) + + def test_result_iterable(self): + self.client._odata._request.return_value = self._mock_response([{"name": "A"}, {"name": "B"}]) + result = self.client.query.fetch_xml(self._fetch_xml()).execute() + names = [r["name"] for r in result] + self.assertEqual(names, ["A", "B"]) + + def test_result_to_dataframe(self): + try: + import pandas as pd + except ImportError: + self.skipTest("pandas not installed") + self.client._odata._request.return_value = self._mock_response([{"name": "Contoso"}, {"name": "Fabrikam"}]) + result = self.client.query.fetch_xml(self._fetch_xml()).execute() + df = result.to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 2) + + def test_no_deprecation_warning_emitted(self): + self.client._odata._request.return_value = self._mock_response([]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.query.fetch_xml(self._fetch_xml()).execute() + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(deprecations), 0, "fetch_xml().execute() should not emit DeprecationWarning") + + def test_execute_pages_returns_iterator_of_query_result(self): + """execute_pages() yields QueryResult objects, one per HTTP page.""" + cookie_raw = "%25253Cpagingcookie%252520pagingcookie%25253D%252522" + page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + pages = list(self.client.query.fetch_xml(self._fetch_xml()).execute_pages()) + self.assertEqual(len(pages), 2) + for page in pages: + self.assertIsInstance(page, QueryResult) + + def test_execute_pages_one_http_call_per_page(self): + """Each execute_pages() iteration fires exactly one HTTP request.""" + cookie_raw = "%25253Cpagingcookie%252520pagingcookie%25253D%252522" + page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + count = 0 + for _page in self.client.query.fetch_xml(self._fetch_xml()).execute_pages(): + count += 1 + self.assertEqual(self.client._odata._request.call_count, 2) + self.assertEqual(count, 2) + + def test_execute_pages_per_page_records(self): + """Each page yielded by execute_pages() contains only its own records.""" + cookie_raw = "%25253Cpagingcookie%252520pagingcookie%25253D%252522" + page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) + page2 = self._mock_response([{"name": "B"}, {"name": "C"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + pages = list(self.client.query.fetch_xml(self._fetch_xml()).execute_pages()) + self.assertEqual(len(pages[0]), 1) + self.assertEqual(len(pages[1]), 2) + self.assertEqual(pages[0].first()["name"], "A") + self.assertEqual(pages[1].first()["name"], "B") + + +# --------------------------------------------------------------------------- +# Removed SQL helpers — raise AttributeError +# --------------------------------------------------------------------------- + + +class TestRemovedSqlHelpers(unittest.TestCase): + + def setUp(self): + self.client = _make_client() + + def test_sql_select_raises_attribute_error(self): + with self.assertRaises(AttributeError): + self.client.query.sql_select("account") + + def test_sql_joins_raises_attribute_error(self): + with self.assertRaises(AttributeError): + self.client.query.sql_joins("contact") + + def test_sql_join_raises_attribute_error(self): + with self.assertRaises(AttributeError): + self.client.query.sql_join("contact", "account") + + +# --------------------------------------------------------------------------- +# Deprecated OData helpers — emit DeprecationWarning, still functional +# --------------------------------------------------------------------------- + + +class TestDeprecatedOdataHelpers(unittest.TestCase): + + def setUp(self): + self.client = _make_client() + + # --- odata_select --- + + def test_odata_select_emits_deprecation_warning(self): + self.client._odata._list_columns.return_value = [] + with self.assertWarns(DeprecationWarning): + self.client.query.odata_select("account") + + def test_odata_select_still_returns_list(self): + self.client._odata._list_columns.return_value = [ + { + "LogicalName": "name", + "AttributeType": "String", + "IsPrimaryId": False, + "IsPrimaryName": True, + "DisplayName": {}, + } + ] + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + cols = self.client.query.odata_select("account") + self.assertIsInstance(cols, list) + self.assertIn("name", cols) + + # --- odata_expand --- + + def _contact_to_account_rel(self): + return [ + { + "ReferencingEntity": "contact", + "ReferencingAttribute": "parentcustomerid", + "ReferencedEntity": "account", + "ReferencedAttribute": "accountid", + "ReferencingEntityNavigationPropertyName": "parentcustomerid_account", + "SchemaName": "contact_customer_accounts", + } + ] + + def test_odata_expand_emits_deprecation_warning(self): + self.client._odata._list_table_relationships.return_value = self._contact_to_account_rel() + with self.assertWarns(DeprecationWarning): + self.client.query.odata_expand("contact", "account") + + def test_odata_expand_still_returns_nav_property(self): + self.client._odata._list_table_relationships.return_value = self._contact_to_account_rel() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + nav = self.client.query.odata_expand("contact", "account") + self.assertEqual(nav, "parentcustomerid_account") + + def test_odata_expand_no_match_raises_value_error(self): + self.client._odata._list_table_relationships.return_value = [] + with self.assertRaises(ValueError): + self.client.query.odata_expand("contact", "nonexistent") + + # --- odata_bind --- + + def test_odata_bind_emits_deprecation_warning(self): + self.client._odata._list_table_relationships.return_value = self._contact_to_account_rel() + with self.assertWarns(DeprecationWarning): + self.client.query.odata_bind("contact", "account", "some-guid") + + def test_odata_bind_still_returns_bind_dict(self): + self.client._odata._list_table_relationships.return_value = self._contact_to_account_rel() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = self.client.query.odata_bind("contact", "account", "guid-123") + self.assertIsInstance(result, dict) + key = list(result.keys())[0] + self.assertEqual(key, "parentcustomerid_account@odata.bind") + self.assertIn("guid-123", result[key]) + + def test_odata_bind_no_match_raises_value_error(self): + self.client._odata._list_table_relationships.return_value = [] + with self.assertRaises(ValueError): + self.client.query.odata_bind("contact", "nonexistent", "guid") + + +# --------------------------------------------------------------------------- +# GA-clean methods: no DeprecationWarning +# --------------------------------------------------------------------------- + + +class TestGaCleanMethods(unittest.TestCase): + + def setUp(self): + self.client = _make_client() + + def test_sql_columns_no_warning(self): + self.client._odata._list_columns.return_value = [] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.query.sql_columns("account") + deps = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(deps), 0) + + def test_odata_expands_no_warning(self): + self.client._odata._list_table_relationships.return_value = [] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.query.odata_expands("contact") + deps = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(deps), 0) + + def test_sql_no_warning(self): + self.client._odata._query_sql.return_value = [] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.query.sql("SELECT name FROM account") + deps = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(deps), 0) + + def test_builder_no_warning(self): + self.client._odata._get_multiple.return_value = iter([[]]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + list(self.client.query.builder("account").select("name").execute()) + deps = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(deps), 0) + + +# --------------------------------------------------------------------------- +# Codemod — execute(by_page=...) transforms +# --------------------------------------------------------------------------- + + +class TestCodemodByPage(unittest.TestCase): + """migrate_source() rewrites literal by_page arguments.""" + + @classmethod + def setUpClass(cls): + try: + from tools.migrate_v0_to_v1 import migrate_source + + cls.migrate = staticmethod(migrate_source) + except ImportError: + cls.migrate = None + + def setUp(self): + if self.migrate is None: + self.skipTest("libcst not installed or tools package not on path") + + def test_execute_by_page_true_becomes_execute_pages(self): + src = "result = builder.execute(by_page=True)\n" + out = self.migrate(src) + self.assertIn("execute_pages()", out) + self.assertNotIn("by_page", out) + self.assertNotIn("execute(", out) + + def test_execute_by_page_false_removes_flag(self): + src = "result = builder.execute(by_page=False)\n" + out = self.migrate(src) + self.assertIn("execute()", out) + self.assertNotIn("by_page", out) + + def test_execute_by_page_variable_not_rewritten(self): + """Variable by_page argument must not be rewritten — requires manual review.""" + src = "result = builder.execute(by_page=flag)\n" + out = self.migrate(src) + self.assertIn("by_page=flag", out) + + def test_idempotent_execute_pages(self): + """Codemod is idempotent — running again changes nothing.""" + src = "result = builder.execute(by_page=True)\n" + once = self.migrate(src) + twice = self.migrate(once) + self.assertEqual(once, twice) + + def test_idempotent_execute_no_flag(self): + src = "result = builder.execute(by_page=False)\n" + once = self.migrate(src) + twice = self.migrate(once) + self.assertEqual(once, twice) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_query_operations.py b/tests/unit/test_query_operations.py index 1add6288..c6520e74 100644 --- a/tests/unit/test_query_operations.py +++ b/tests/unit/test_query_operations.py @@ -179,9 +179,13 @@ def test_builder_returns_query_builder(self): def test_builder_execute_flat_default(self): """builder().execute() should return flat records by default.""" + from PowerPlatform.Dataverse.models.filters import col + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1", "name": "Test"}]]) - records = list(self.client.query.builder("account").select("name").filter_eq("statecode", 0).top(10).execute()) + records = list( + self.client.query.builder("account").select("name").where(col("statecode") == 0).top(10).execute() + ) self.client._odata._get_multiple.assert_called_once_with( "account", @@ -220,13 +224,15 @@ def test_builder_execute_by_page(self): def test_builder_execute_all_params(self): """builder().execute() should forward all parameters.""" + from PowerPlatform.Dataverse.models.filters import col + self.client._odata._get_multiple.return_value = iter([[{"name": "Test"}]]) list( self.client.query.builder("account") .select("name", "revenue") - .filter_eq("statecode", 0) - .filter_gt("revenue", 1000000) + .where(col("statecode") == 0) + .where(col("revenue") > 1000000) .order_by("revenue", descending=True) .expand("primarycontactid") .top(50) @@ -248,13 +254,13 @@ def test_builder_execute_all_params(self): def test_builder_execute_with_where(self): """builder().where().execute() should compile expression to filter.""" - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col self.client._odata._get_multiple.return_value = iter([[{"name": "Test"}]]) list( self.client.query.builder("account") - .where((eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000)) + .where(((col("statecode") == 0) | (col("statecode") == 1)) & (col("revenue") > 100000)) .execute() ) @@ -265,10 +271,12 @@ def test_builder_execute_with_where(self): ) def test_builder_execute_with_filter_in(self): - """builder().filter_in().execute() should forward CRM.In filter to _get_multiple.""" + """builder().where(col().in_()).execute() should forward CRM.In filter to _get_multiple.""" + from PowerPlatform.Dataverse.models.filters import col + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) - list(self.client.query.builder("account").select("name").filter_in("statecode", [0, 1, 2]).execute()) + list(self.client.query.builder("account").select("name").where(col("statecode").in_([0, 1, 2])).execute()) call_kwargs = self.client._odata._get_multiple.call_args self.assertEqual( @@ -277,13 +285,15 @@ def test_builder_execute_with_filter_in(self): ) def test_builder_execute_with_where_filter_in(self): - """builder().where(filter_in(...) & ...).execute() should compile composed expression.""" - from PowerPlatform.Dataverse.models.filters import filter_in, gt + """builder().where(col().in_() & ...).execute() should compile composed expression.""" + from PowerPlatform.Dataverse.models.filters import col self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) list( - self.client.query.builder("account").where(filter_in("statecode", [0, 1]) & gt("revenue", 100000)).execute() + self.client.query.builder("account") + .where(col("statecode").in_([0, 1]) & (col("revenue") > 100000)) + .execute() ) call_kwargs = self.client._odata._get_multiple.call_args @@ -293,14 +303,15 @@ def test_builder_execute_with_where_filter_in(self): ) def test_builder_execute_with_filter_between_datetimes(self): - """builder().filter_between() with datetimes should forward correct OData.""" + """builder().where(col().between()).execute() should forward correct OData.""" from datetime import datetime, timezone + from PowerPlatform.Dataverse.models.filters import col self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) start = datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc) end = datetime(2024, 12, 31, 23, 59, 59, tzinfo=timezone.utc) - list(self.client.query.builder("account").filter_between("createdon", start, end).execute()) + list(self.client.query.builder("account").where(col("createdon").between(start, end)).execute()) call_kwargs = self.client._odata._get_multiple.call_args self.assertEqual( @@ -309,10 +320,12 @@ def test_builder_execute_with_filter_between_datetimes(self): ) def test_builder_execute_with_filter_not_in(self): - """builder().filter_not_in().execute() should forward CRM.NotIn filter.""" + """builder().where(col().not_in()).execute() should forward CRM.NotIn filter.""" + from PowerPlatform.Dataverse.models.filters import col + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) - list(self.client.query.builder("account").select("name").filter_not_in("statecode", [2, 3]).execute()) + list(self.client.query.builder("account").select("name").where(col("statecode").not_in([2, 3])).execute()) call_kwargs = self.client._odata._get_multiple.call_args self.assertEqual( @@ -321,10 +334,12 @@ def test_builder_execute_with_filter_not_in(self): ) def test_builder_execute_with_filter_not_between(self): - """builder().filter_not_between().execute() should forward negated between filter.""" + """builder().where(col().not_between()).execute() should forward negated between filter.""" + from PowerPlatform.Dataverse.models.filters import col + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) - list(self.client.query.builder("account").filter_not_between("revenue", 100000, 500000).execute()) + list(self.client.query.builder("account").where(col("revenue").not_between(100000, 500000)).execute()) call_kwargs = self.client._odata._get_multiple.call_args self.assertEqual( @@ -334,6 +349,8 @@ def test_builder_execute_with_filter_not_between(self): def test_builder_full_fluent_workflow(self): """End-to-end test of the fluent query workflow.""" + from PowerPlatform.Dataverse.models.filters import col + expected_records = [ {"accountid": "1", "name": "Big Corp", "revenue": 5000000}, {"accountid": "2", "name": "Mega Inc", "revenue": 4000000}, @@ -343,8 +360,8 @@ def test_builder_full_fluent_workflow(self): records = list( self.client.query.builder("account") .select("name", "revenue") - .filter_eq("statecode", 0) - .filter_gt("revenue", 1000000) + .where(col("statecode") == 0) + .where(col("revenue") > 1000000) .order_by("revenue", descending=True) .expand("primarycontactid") .top(10) @@ -357,23 +374,24 @@ def test_builder_full_fluent_workflow(self): self.assertEqual(records[1]["name"], "Mega Inc") def test_builder_to_dataframe(self): - """builder().to_dataframe() should delegate to client.dataframe.get().""" + """builder().to_dataframe() should collect records into a DataFrame.""" import pandas as pd + from PowerPlatform.Dataverse.models.filters import raw - expected_df = pd.DataFrame([{"name": "Contoso", "revenue": 1000}]) - self.client.dataframe = MagicMock() - self.client.dataframe.get.return_value = expected_df + expected_records = [{"name": "Contoso", "revenue": 1000}] + self.client._odata._get_multiple.return_value = iter([expected_records]) result = ( self.client.query.builder("account") .select("name", "revenue") - .filter_eq("statecode", 0) + .where(raw("statecode eq 0")) .order_by("name") .top(50) + .execute() .to_dataframe() ) - self.client.dataframe.get.assert_called_once_with( + self.client._odata._get_multiple.assert_called_once_with( "account", select=["name", "revenue"], filter="statecode eq 0", @@ -384,7 +402,9 @@ def test_builder_to_dataframe(self): count=False, include_annotations=None, ) - pd.testing.assert_frame_equal(result, expected_df) + self.assertIsInstance(result, pd.DataFrame) + self.assertEqual(len(result), 1) + self.assertEqual(result.iloc[0]["name"], "Contoso") # =================================================================== @@ -526,239 +546,25 @@ def test_excludes_attribute_of_columns(self): self.assertNotIn("createdbyname", names) -class TestSqlSelect(unittest.TestCase): - """Tests for client.query.sql_select().""" - - def setUp(self): - self.mock_credential = MagicMock(spec=TokenCredential) - self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) - self.client._odata = MagicMock() - - def test_returns_comma_separated(self): - self.client._odata._list_columns.return_value = [ - { - "LogicalName": "accountid", - "AttributeType": "Uniqueidentifier", - "IsPrimaryId": True, - "IsPrimaryName": False, - "DisplayName": {}, - }, - { - "LogicalName": "name", - "AttributeType": "String", - "IsPrimaryId": False, - "IsPrimaryName": True, - "DisplayName": {}, - }, - { - "LogicalName": "revenue", - "AttributeType": "Money", - "IsPrimaryId": False, - "IsPrimaryName": False, - "DisplayName": {}, - }, - ] - result = self.client.query.sql_select("account") - self.assertIn("accountid", result) - self.assertIn("name", result) - self.assertIn("revenue", result) - self.assertEqual(result.count(","), 2) # 3 cols = 2 commas - - -class TestSqlJoins(unittest.TestCase): - """Tests for client.query.sql_joins().""" - - def setUp(self): - self.mock_credential = MagicMock(spec=TokenCredential) - self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) - self.client._odata = MagicMock() - - def _mock_rels(self, rels): - self.client._odata._list_table_relationships.return_value = rels - - def test_outgoing_lookups(self): - self._mock_rels( - [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "parentcustomerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "SchemaName": "contact_customer_accounts", - }, - ] - ) - joins = self.client.query.sql_joins("contact") - self.assertEqual(len(joins), 1) - j = joins[0] - self.assertEqual(j["column"], "parentcustomerid") - self.assertEqual(j["target"], "account") - self.assertEqual(j["target_pk"], "accountid") - self.assertIn("JOIN account", j["join_clause"]) - self.assertIn("parentcustomerid", j["join_clause"]) - - def test_ignores_incoming_rels(self): - self._mock_rels( - [ - # This is an incoming relationship (account is referenced, not referencing) - { - "ReferencingEntity": "opportunity", - "ReferencingAttribute": "customerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "SchemaName": "opp_customer_accounts", - }, - ] - ) - joins = self.client.query.sql_joins("account") - self.assertEqual(len(joins), 0) - - def test_polymorphic_returns_multiple(self): - self._mock_rels( - [ - { - "ReferencingEntity": "opportunity", - "ReferencingAttribute": "customerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "SchemaName": "opp_customer_accounts", - }, - { - "ReferencingEntity": "opportunity", - "ReferencingAttribute": "customerid", - "ReferencedEntity": "contact", - "ReferencedAttribute": "contactid", - "SchemaName": "opp_customer_contacts", - }, - ] - ) - joins = self.client.query.sql_joins("opportunity") - self.assertEqual(len(joins), 2) - targets = {j["target"] for j in joins} - self.assertEqual(targets, {"account", "contact"}) - # Both use the same source column - self.assertTrue(all(j["column"] == "customerid" for j in joins)) - - def test_empty_relationships(self): - self._mock_rels([]) - joins = self.client.query.sql_joins("account") - self.assertEqual(joins, []) - - def test_alias_collision_same_first_letter(self): - """Two targets starting with the same letter get distinct aliases.""" - self._mock_rels( - [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "parentcustomerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "SchemaName": "contact_customer_accounts", - }, - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "regardingobjectid", - "ReferencedEntity": "annotation", - "ReferencedAttribute": "annotationid", - "SchemaName": "contact_annotation", - }, - ] - ) - joins = self.client.query.sql_joins("contact") - self.assertEqual(len(joins), 2) - aliases = [j["join_clause"].split()[2] for j in joins] - self.assertEqual(len(set(aliases)), 2, "aliases must be unique") - self.assertNotEqual(aliases[0], aliases[1]) - - def test_alias_collision_same_target_table(self): - """Two lookups to the same table (e.g. ownerid + createdby -> systemuser) get distinct aliases.""" - self._mock_rels( - [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "ownerid", - "ReferencedEntity": "systemuser", - "ReferencedAttribute": "systemuserid", - "SchemaName": "contact_ownerid_systemuser", - }, - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "createdby", - "ReferencedEntity": "systemuser", - "ReferencedAttribute": "systemuserid", - "SchemaName": "contact_createdby_systemuser", - }, - ] - ) - joins = self.client.query.sql_joins("contact") - self.assertEqual(len(joins), 2) - aliases = [j["join_clause"].split()[2] for j in joins] - self.assertEqual(len(set(aliases)), 2, "aliases must be unique") - self.assertNotEqual(aliases[0], aliases[1]) - - -class TestSqlJoin(unittest.TestCase): - """Tests for client.query.sql_join().""" +class TestRemovedSqlHelpers(unittest.TestCase): + """sql_select(), sql_join(), sql_joins() are removed at GA — raise AttributeError.""" def setUp(self): self.mock_credential = MagicMock(spec=TokenCredential) self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) self.client._odata = MagicMock() - def _mock_rels(self, rels): - self.client._odata._list_table_relationships.return_value = rels - - def test_generates_join_clause(self): - self._mock_rels( - [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "parentcustomerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "SchemaName": "contact_customer_accounts", - }, - ] - ) - result = self.client.query.sql_join("contact", "account", from_alias="c", to_alias="a") - self.assertEqual(result, "JOIN account a ON c.parentcustomerid = a.accountid") - - def test_default_aliases(self): - self._mock_rels( - [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "parentcustomerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "SchemaName": "contact_customer_accounts", - }, - ] - ) - result = self.client.query.sql_join("contact", "account") - self.assertIn("JOIN account a ON contact.parentcustomerid = a.accountid", result) + def test_sql_select_removed(self): + with self.assertRaises(AttributeError): + self.client.query.sql_select("account") - def test_no_relationship_raises(self): - self._mock_rels([]) - with self.assertRaises(ValueError) as ctx: - self.client.query.sql_join("contact", "nonexistent") - self.assertIn("No relationship found", str(ctx.exception)) + def test_sql_joins_removed(self): + with self.assertRaises(AttributeError): + self.client.query.sql_joins("contact") - def test_case_insensitive_target(self): - self._mock_rels( - [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "ownerid", - "ReferencedEntity": "systemuser", - "ReferencedAttribute": "systemuserid", - "SchemaName": "contact_owner", - }, - ] - ) - result = self.client.query.sql_join("contact", "SystemUser", from_alias="c", to_alias="su") - self.assertIn("JOIN systemuser su", result) - self.assertIn("c.ownerid = su.systemuserid", result) + def test_sql_join_removed(self): + with self.assertRaises(AttributeError): + self.client.query.sql_join("contact", "account") # =================================================================== @@ -767,13 +573,18 @@ def test_case_insensitive_target(self): class TestOdataSelect(unittest.TestCase): - """Tests for client.query.odata_select().""" + """Tests for client.query.odata_select() — deprecated at GA, still functional.""" def setUp(self): self.mock_credential = MagicMock(spec=TokenCredential) self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) self.client._odata = MagicMock() + def test_emits_deprecation_warning(self): + self.client._odata._list_columns.return_value = [] + with self.assertWarns(DeprecationWarning): + self.client.query.odata_select("account") + def test_returns_list_of_strings(self): self.client._odata._list_columns.return_value = [ { @@ -791,7 +602,11 @@ def test_returns_list_of_strings(self): "DisplayName": {}, }, ] - result = self.client.query.odata_select("account") + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = self.client.query.odata_select("account") self.assertIsInstance(result, list) self.assertIn("accountid", result) self.assertIn("name", result) @@ -807,7 +622,11 @@ def test_result_usable_in_records_get(self): "DisplayName": {}, }, ] - cols = self.client.query.odata_select("account") + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + cols = self.client.query.odata_select("account") self.assertEqual(cols, ["name"]) @@ -914,13 +733,28 @@ def test_metadata_error_on_entity_set_resolution_is_swallowed(self): class TestOdataExpand(unittest.TestCase): - """Tests for client.query.odata_expand().""" + """Tests for client.query.odata_expand() — deprecated at GA, still functional.""" def setUp(self): self.mock_credential = MagicMock(spec=TokenCredential) self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) self.client._odata = MagicMock() + def test_emits_deprecation_warning(self): + self.client._odata._list_table_relationships.return_value = [ + { + "ReferencingEntity": "contact", + "ReferencingAttribute": "parentcustomerid", + "ReferencedEntity": "account", + "ReferencedAttribute": "accountid", + "ReferencingEntityNavigationPropertyName": "parentcustomerid_account", + "SchemaName": "contact_customer_accounts", + }, + ] + self.client._odata._entity_set_from_schema_name.return_value = "accounts" + with self.assertWarns(DeprecationWarning): + self.client.query.odata_expand("contact", "account") + def test_returns_nav_property(self): self.client._odata._list_table_relationships.return_value = [ { @@ -933,7 +767,11 @@ def test_returns_nav_property(self): }, ] self.client._odata._entity_set_from_schema_name.return_value = "accounts" - result = self.client.query.odata_expand("contact", "account") + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = self.client.query.odata_expand("contact", "account") self.assertEqual(result, "parentcustomerid_account") def test_no_relationship_raises(self): @@ -954,20 +792,24 @@ def test_case_insensitive_target(self): }, ] self.client._odata._entity_set_from_schema_name.return_value = "systemusers" - result = self.client.query.odata_expand("contact", "SystemUser") + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = self.client.query.odata_expand("contact", "SystemUser") self.assertEqual(result, "ownerid_systemuser") class TestOdataBind(unittest.TestCase): - """Tests for client.query.odata_bind().""" + """Tests for client.query.odata_bind() — deprecated at GA, still functional.""" def setUp(self): self.mock_credential = MagicMock(spec=TokenCredential) self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) self.client._odata = MagicMock() - def test_returns_bind_dict(self): - self.client._odata._list_table_relationships.return_value = [ + def _rel(self): + return [ { "ReferencingEntity": "contact", "ReferencingAttribute": "parentcustomerid", @@ -977,10 +819,23 @@ def test_returns_bind_dict(self): "SchemaName": "contact_customer_accounts", }, ] + + def test_emits_deprecation_warning(self): + self.client._odata._list_table_relationships.return_value = self._rel() + self.client._odata._entity_set_from_schema_name.return_value = "accounts" + with self.assertWarns(DeprecationWarning): + self.client.query.odata_bind("contact", "account", "some-guid") + + def test_returns_bind_dict(self): + self.client._odata._list_table_relationships.return_value = self._rel() self.client._odata._entity_set_from_schema_name.return_value = "accounts" guid = "12345678-1234-1234-1234-123456789abc" - result = self.client.query.odata_bind("contact", "account", guid) + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = self.client.query.odata_bind("contact", "account", guid) self.assertIsInstance(result, dict) self.assertEqual(len(result), 1) key = list(result.keys())[0] @@ -994,19 +849,14 @@ def test_no_relationship_raises(self): def test_usable_in_create_payload(self): """Result can be merged into a create payload via **spread.""" - self.client._odata._list_table_relationships.return_value = [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "parentcustomerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "ReferencingEntityNavigationPropertyName": "parentcustomerid_account", - "SchemaName": "contact_customer_accounts", - }, - ] + self.client._odata._list_table_relationships.return_value = self._rel() self.client._odata._entity_set_from_schema_name.return_value = "accounts" - bind = self.client.query.odata_bind("contact", "account", "some-guid") + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + bind = self.client.query.odata_bind("contact", "account", "some-guid") payload = {"firstname": "Jane", "lastname": "Doe", **bind} self.assertIn("parentcustomerid_account@odata.bind", payload) self.assertEqual(payload["firstname"], "Jane") diff --git a/tests/unit/test_records_operations.py b/tests/unit/test_records_operations.py index 09df6869..104521e2 100644 --- a/tests/unit/test_records_operations.py +++ b/tests/unit/test_records_operations.py @@ -2,6 +2,7 @@ # Licensed under the MIT license. import unittest +import warnings from unittest.mock import MagicMock from azure.core.credentials import TokenCredential @@ -389,5 +390,64 @@ def test_upsert_composite_alternate_key(self): self.client._odata._upsert_multiple.assert_not_called() +class TestListPages(unittest.TestCase): + """Unit tests for records.list_pages().""" + + def setUp(self): + self.mock_credential = MagicMock(spec=TokenCredential) + self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) + self.client._odata = MagicMock() + + def test_list_pages_returns_iterator(self): + self.client._odata._get_multiple.return_value = iter([[{"name": "A"}], [{"name": "B"}]]) + result = self.client.records.list_pages("account") + import types + + self.assertIsInstance(result, types.GeneratorType) + + def test_list_pages_yields_query_result_per_page(self): + from PowerPlatform.Dataverse.models.record import QueryResult + + self.client._odata._get_multiple.return_value = iter([[{"name": "A"}], [{"name": "B"}]]) + pages = list(self.client.records.list_pages("account")) + self.assertEqual(len(pages), 2) + for page in pages: + self.assertIsInstance(page, QueryResult) + + def test_list_pages_page_contents(self): + self.client._odata._get_multiple.return_value = iter([[{"name": "A"}], [{"name": "B"}, {"name": "C"}]]) + pages = list(self.client.records.list_pages("account")) + self.assertEqual(len(pages[0]), 1) + self.assertEqual(len(pages[1]), 2) + + def test_list_pages_passes_filter(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", filter="statecode eq 0")) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("filter") or call_kwargs[1].get("filter"), "statecode eq 0") + + def test_list_pages_passes_select(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", select=["name"])) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("select") or call_kwargs[1].get("select"), ["name"]) + + def test_list_pages_passes_top(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", top=50)) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("top") or call_kwargs[1].get("top"), 50) + + def test_list_pages_no_deprecation_warning(self): + import warnings + + self.client._odata._get_multiple.return_value = iter([]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + list(self.client.records.list_pages("account", filter="statecode eq 0")) + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(deprecations), 0) + + if __name__ == "__main__": unittest.main() diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tools/migrate_v0_to_v1.py b/tools/migrate_v0_to_v1.py new file mode 100644 index 00000000..553bc026 --- /dev/null +++ b/tools/migrate_v0_to_v1.py @@ -0,0 +1,667 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +DV-Python-SDK v0 → v1 GA migration codemod. + +Mechanically rewrites beta (0.1.0b*) call sites to their GA (1.0) equivalents +using LibCST (concrete syntax tree — preserves all whitespace and comments). + +Usage:: + + pip install PowerPlatform-Dataverse-Client[migration] + python -m tools.migrate_v0_to_v1 path/to/scripts/ + python -m tools.migrate_v0_to_v1 examples/ # _codemon.py files only + +Transformations applied +----------------------- +Builder methods (.filter_* → .where(col(...)...)):: + + .filter_eq("col", v) → .where(col("col") == v) + .filter_ne("col", v) → .where(col("col") != v) + .filter_gt("col", v) → .where(col("col") > v) + .filter_ge("col", v) → .where(col("col") >= v) + .filter_lt("col", v) → .where(col("col") < v) + .filter_le("col", v) → .where(col("col") <= v) + .filter_contains("col", v) → .where(col("col").contains(v)) + .filter_startswith("col", v) → .where(col("col").startswith(v)) + .filter_endswith("col", v) → .where(col("col").endswith(v)) + .filter_in("col", vals) → .where(col("col").in_(vals)) + .filter_not_in("col", vals) → .where(col("col").not_in(vals)) + .filter_null("col") → .where(col("col").is_null()) + .filter_not_null("col") → .where(col("col").is_not_null()) + .filter_between("col", lo, hi) → .where(col("col").between(lo, hi)) + .filter_not_between("col", lo, hi) → .where(col("col").not_between(lo, hi)) + .filter_raw("expr") → .where(raw("expr")) + .filter("expr") → .where(raw("expr")) + .execute(by_page=True) → .execute_pages() + .execute(by_page=False) → .execute() (flag removed) + +Record namespace:: + + batch.records.get(t, id) → batch.records.retrieve(t, id) + +Top-level shortcuts (removed at GA):: + + client.create(t, d) → client.records.create(t, d) + client.update(t, id, d) → client.records.update(t, id, d) + client.delete(t, id) → client.records.delete(t, id) + client.get(t, id) → client.records.retrieve(t, id) + client.query_sql(sql) → client.query.sql(sql) + client.get_table_info(t) → client.tables.get(t) + client.create_table(t, …) → client.tables.create(t, …) + client.delete_table(t) → client.tables.delete(t) + client.list_tables() → client.tables.list() + client.create_columns(t, …) → client.tables.add_columns(t, …) + client.delete_columns(t, …) → client.tables.remove_columns(t, …) + client.upload_file(…) → client.files.upload(…) + +Import management: + Adds ``from PowerPlatform.Dataverse.models.filters import col`` when a + .filter_* method is rewritten (if col is not already imported). + Adds ``raw`` to the same import when .filter_raw or .filter is rewritten. + +NOT handled by this codemod (manual migration required): + execute(by_page=variable) → manual review required (variable argument, not literal) + client.records.get(t, id) → client.records.retrieve(t, id) + Return type changes: beta returns Record (raises on 404); GA retrieve() returns + Record | None. Callers that do not guard against None will fail silently. + client.records.get(t, kw=…) → client.records.list(t, kw=…) + Return type changes: beta returns Iterable[List[Record]] (pages); GA list() + returns QueryResult (flat iterable over Records). Any ``for page in result: + for rec in page:`` iteration pattern breaks after a mechanical rename. + client.dataframe.get() → client.query.builder(…).execute().to_dataframe() + Expression reconstruction requires understanding caller intent. + client.query.sql_select()/sql_join()/sql_joins() → removed (no mechanical replacement) +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import List, Optional, Sequence, Set + +try: + import libcst as cst +except ImportError: + print( + "ERROR: libcst is required. Install with:\n" + " pip install PowerPlatform-Dataverse-Client[migration]\n" + " # or: pip install 'libcst>=1.0.0'", + file=sys.stderr, + ) + sys.exit(1) + + +# --------------------------------------------------------------------------- +# Filter-method → .where(col(...)) mapping +# --------------------------------------------------------------------------- + +_UNARY_FILTER_MAP = { + "filter_null": "is_null", + "filter_not_null": "is_not_null", +} + +_BINARY_OP_MAP = { + "filter_eq": cst.Equal(), + "filter_ne": cst.NotEqual(), + "filter_gt": cst.GreaterThan(), + "filter_ge": cst.GreaterThanEqual(), + "filter_lt": cst.LessThan(), + "filter_le": cst.LessThanEqual(), +} + +_METHOD_FILTER_MAP = { + "filter_contains": "contains", + "filter_startswith": "startswith", + "filter_endswith": "endswith", + "filter_in": "in_", + "filter_not_in": "not_in", + "filter_between": "between", + "filter_not_between": "not_between", +} + +_ALL_FILTER_METHODS: Set[str] = set(_UNARY_FILTER_MAP) | set(_BINARY_OP_MAP) | set(_METHOD_FILTER_MAP) | {"filter_raw"} + +# Standalone filter functions from filters module (beta API) → col() equivalents +# eq("f", v) → col("f") == v, between("f", lo, hi) → col("f").between(lo, hi), etc. +_FUNC_BINARY_OP_MAP = { + "eq": cst.Equal(), + "ne": cst.NotEqual(), + "gt": cst.GreaterThan(), + "ge": cst.GreaterThanEqual(), + "lt": cst.LessThan(), + "le": cst.LessThanEqual(), +} +_FUNC_METHOD_MAP = { + "contains": "contains", + "startswith": "startswith", + "endswith": "endswith", + "filter_in": "in_", + "not_in": "not_in", + "between": "between", + "not_between": "not_between", +} +_FUNC_UNARY_MAP = { + "is_null": "is_null", + "is_not_null": "is_not_null", +} +_ALL_FILTER_FUNCS: Set[str] = set(_FUNC_BINARY_OP_MAP) | set(_FUNC_METHOD_MAP) | set(_FUNC_UNARY_MAP) + +# Top-level client shortcut → (new_namespace, new_method) +_CLIENT_SHORTCUTS = { + "create": ("records", "create"), + "update": ("records", "update"), + "delete": ("records", "delete"), + "get": ("records", "retrieve"), + "query_sql": ("query", "sql"), + "get_table_info": ("tables", "get"), + "create_table": ("tables", "create"), + "delete_table": ("tables", "delete"), + "list_tables": ("tables", "list"), + "create_columns": ("tables", "add_columns"), + "delete_columns": ("tables", "remove_columns"), + "upload_file": ("files", "upload"), +} + +_FILTERS_MODULE = "PowerPlatform.Dataverse.models.filters" + + +# --------------------------------------------------------------------------- +# Node helpers +# --------------------------------------------------------------------------- + + +def _name(s: str) -> cst.Name: + return cst.Name(s) + + +def _attr(obj: cst.BaseExpression, attr: str) -> cst.Attribute: + return cst.Attribute(value=obj, attr=cst.Name(attr)) + + +def _call(func: cst.BaseExpression, *args: cst.BaseExpression) -> cst.Call: + cst_args = [] + for i, a in enumerate(args): + comma = ( + cst.MaybeSentinel.DEFAULT if i == len(args) - 1 else cst.Comma(whitespace_after=cst.SimpleWhitespace(" ")) + ) + cst_args.append(cst.Arg(value=a, comma=comma)) + return cst.Call(func=func, args=cst_args) + + +def _col_call(col_name_node: cst.BaseExpression) -> cst.Call: + """col("field_name") call node.""" + return _call(_name("col"), col_name_node) + + +def _filters_module_attr() -> cst.Attribute: + """Build the Attribute chain for PowerPlatform.Dataverse.models.filters.""" + return _attr( + _attr( + _attr(_name("PowerPlatform"), "Dataverse"), + "models", + ), + "filters", + ) + + +# --------------------------------------------------------------------------- +# Positional argument helpers +# --------------------------------------------------------------------------- + + +def _pos_arg(args: Sequence[cst.Arg], n: int) -> Optional[cst.BaseExpression]: + """Return the n-th (0-indexed) positional argument value, or None.""" + count = 0 + for a in args: + if a.keyword is None: + if count == n: + return a.value + count += 1 + return None + + +def _positional_count(args: Sequence[cst.Arg]) -> int: + return sum(1 for a in args if a.keyword is None) + + +# --------------------------------------------------------------------------- +# Main transformer +# --------------------------------------------------------------------------- + + +class _V1Migrator(cst.CSTTransformer): + """LibCST transformer rewriting DV-Python-SDK beta → v1 GA.""" + + def __init__(self, client_var: str = "client") -> None: + self._client_var = client_var + self._needs_col = False + self._needs_raw = False + self._has_col = False + self._has_raw = False + # Names imported from filters module in this file (e.g. eq, gt, between) + self._imported_filter_funcs: Set[str] = set() + + # ------------------------------------------------------------------ + # Track existing col / raw imports + # ------------------------------------------------------------------ + + def visit_ImportFrom(self, node: cst.ImportFrom) -> None: + if isinstance(node.names, cst.ImportStar): + return + module_str = _dotted_name(node.module) + if module_str != _FILTERS_MODULE: + return + for alias in node.names: + name = alias.name.value if isinstance(alias.name, cst.Name) else "" + if name == "col": + self._has_col = True + elif name == "raw": + self._has_raw = True + elif name in _ALL_FILTER_FUNCS: + self._imported_filter_funcs.add(name) + + # ------------------------------------------------------------------ + # Rewrite call nodes + # ------------------------------------------------------------------ + + def leave_Call(self, original_node: cst.Call, updated_node: cst.Call) -> cst.BaseExpression: + func = updated_node.func + + # ---------------------------------------------------------------- + # Standalone filter functions: eq("f", v) → col("f") == v, etc. + # Only transform names that were actually imported from filters module. + # Wrap Comparison nodes in explicit parentheses so that combining with + # & / | doesn't hit Python precedence bugs (& binds tighter than ==/>). + # ---------------------------------------------------------------- + if isinstance(func, cst.Name) and func.value in self._imported_filter_funcs: + result = self._build_filter_func_arg(func.value, updated_node.args) + if result is not None: + if isinstance(result, cst.Comparison): + result = result.with_changes(lpar=[cst.LeftParen()], rpar=[cst.RightParen()]) + return result + + if not isinstance(func, cst.Attribute): + return updated_node + + method_name = func.attr.value if isinstance(func.attr, cst.Name) else "" + + # ---------------------------------------------------------------- + # .filter_*(...) → .where(col(...) ...) + # ---------------------------------------------------------------- + if method_name in _ALL_FILTER_METHODS: + where_arg = self._build_filter_arg(method_name, updated_node.args) + if where_arg is not None: + return updated_node.with_changes( + func=func.with_changes(attr=_name("where")), + args=[cst.Arg(value=where_arg)], + ) + + # ---------------------------------------------------------------- + # .filter("expr") → .where(raw("expr")) + # QueryBuilder.filter() was removed at GA (not deprecated). Wrapping + # in raw() preserves the OData string exactly for string-literal callers. + # ---------------------------------------------------------------- + if method_name == "filter": + expr_node = _pos_arg(updated_node.args, 0) + if expr_node is not None and _positional_count(updated_node.args) == 1: + self._needs_raw = True + return updated_node.with_changes( + func=func.with_changes(attr=_name("where")), + args=[cst.Arg(value=_call(_name("raw"), expr_node))], + ) + + # ---------------------------------------------------------------- + # .execute(by_page=True) → .execute_pages() + # .execute(by_page=False) → .execute() (flag removed) + # Only literal True/False are codemod-able; variable by_page requires + # manual review per section 8.5 of the GA spec. + # ---------------------------------------------------------------- + if method_name == "execute": + by_page_val = self._kwarg_bool_literal(updated_node.args, "by_page") + if by_page_val is True: + return updated_node.with_changes( + func=func.with_changes(attr=_name("execute_pages")), + args=[], + ) + if by_page_val is False: + other_args = [ + a + for a in updated_node.args + if not (isinstance(a.keyword, cst.Name) and a.keyword.value == "by_page") + ] + return updated_node.with_changes(args=other_args) + + # ---------------------------------------------------------------- + # batch.records.get(table, id) → batch.records.retrieve(table, id) + # NOTE: client.records.get() is NOT codemodded — the return type changes + # between beta and GA (Record | None vs Record for single-id; QueryResult vs + # Iterable[List[Record]] for multi-record). Surrounding iteration patterns + # would silently break after a mechanical rename. + # ---------------------------------------------------------------- + if method_name == "get" and isinstance(func.value, cst.Attribute): + inner = func.value + if isinstance(inner.attr, cst.Name) and inner.attr.value == "records": + if isinstance(inner.value, cst.Name) and inner.value.value == "batch": + # batch.records.get() returns None in both versions — safe to rename + return updated_node.with_changes(func=func.with_changes(attr=_name("retrieve"))) + + # ---------------------------------------------------------------- + # client.(...) top-level shortcuts removed at GA + # Only match when receiver is the known client variable name to avoid + # false positives on record.get("field"), table_info.get("field"), etc. + # ---------------------------------------------------------------- + if ( + isinstance(func.value, cst.Name) + and func.value.value == self._client_var + and method_name in _CLIENT_SHORTCUTS + ): + new_ns, new_method = _CLIENT_SHORTCUTS[method_name] + new_func = _attr(_attr(func.value, new_ns), new_method) + return updated_node.with_changes(func=new_func) + + return updated_node + + # ------------------------------------------------------------------ + # Keyword-argument helpers + # ------------------------------------------------------------------ + + @staticmethod + def _kwarg_bool_literal(args: Sequence[cst.Arg], keyword: str) -> Optional[bool]: + """Return True/False if *keyword* is a literal bool kwarg, else None.""" + for a in args: + if isinstance(a.keyword, cst.Name) and a.keyword.value == keyword: + if isinstance(a.value, cst.Name): + if a.value.value == "True": + return True + if a.value.value == "False": + return False + return None + + # ------------------------------------------------------------------ + # Build the argument for .where() from .filter_*() args + # ------------------------------------------------------------------ + + def _build_filter_arg( + self, + method_name: str, + args: Sequence[cst.Arg], + ) -> Optional[cst.BaseExpression]: + + field_node = _pos_arg(args, 0) + if field_node is None: + return None + + # .filter_raw(expr) → raw(expr) + if method_name == "filter_raw": + self._needs_raw = True + return _call(_name("raw"), field_node) + + # .filter_null / .filter_not_null → col("f").is_null() / .is_not_null() + if method_name in _UNARY_FILTER_MAP: + self._needs_col = True + proxy = _UNARY_FILTER_MAP[method_name] + return _call(_attr(_col_call(field_node), proxy)) + + # .filter_eq / .filter_ne / ... → col("f") OP val + if method_name in _BINARY_OP_MAP: + val_node = _pos_arg(args, 1) + if val_node is None: + return None + self._needs_col = True + return cst.Comparison( + left=_col_call(field_node), + comparisons=[ + cst.ComparisonTarget( + operator=_BINARY_OP_MAP[method_name], + comparator=val_node, + ) + ], + ) + + # .filter_between / .filter_not_between → col("f").between(lo, hi) + if method_name in ("filter_between", "filter_not_between"): + lo = _pos_arg(args, 1) + hi = _pos_arg(args, 2) + if lo is None or hi is None: + return None + self._needs_col = True + proxy = _METHOD_FILTER_MAP[method_name] + return _call(_attr(_col_call(field_node), proxy), lo, hi) + + # .filter_in / .filter_not_in / .filter_contains / etc. + if method_name in _METHOD_FILTER_MAP: + val_node = _pos_arg(args, 1) + if val_node is None: + return None + self._needs_col = True + proxy = _METHOD_FILTER_MAP[method_name] + return _call(_attr(_col_call(field_node), proxy), val_node) + + return None + + # ------------------------------------------------------------------ + # Standalone filter function: eq("f", v) → col("f") == v, etc. + # ------------------------------------------------------------------ + + def _build_filter_func_arg( + self, + func_name: str, + args: Sequence[cst.Arg], + ) -> Optional[cst.BaseExpression]: + """Return the replacement expression node for a standalone filter call.""" + field_node = _pos_arg(args, 0) + if field_node is None: + return None + + if func_name in _FUNC_UNARY_MAP: + self._needs_col = True + proxy = _FUNC_UNARY_MAP[func_name] + return _call(_attr(_col_call(field_node), proxy)) + + if func_name in _FUNC_BINARY_OP_MAP: + val_node = _pos_arg(args, 1) + if val_node is None: + return None + self._needs_col = True + return cst.Comparison( + left=_col_call(field_node), + comparisons=[ + cst.ComparisonTarget( + operator=_FUNC_BINARY_OP_MAP[func_name], + comparator=val_node, + ) + ], + ) + + if func_name in ("between", "not_between"): + lo = _pos_arg(args, 1) + hi = _pos_arg(args, 2) + if lo is None or hi is None: + return None + self._needs_col = True + proxy = _FUNC_METHOD_MAP[func_name] + return _call(_attr(_col_call(field_node), proxy), lo, hi) + + if func_name in _FUNC_METHOD_MAP: + val_node = _pos_arg(args, 1) + if val_node is None: + return None + self._needs_col = True + proxy = _FUNC_METHOD_MAP[func_name] + return _call(_attr(_col_call(field_node), proxy), val_node) + + return None + + # ------------------------------------------------------------------ + # Inject missing col / raw imports at module level + # ------------------------------------------------------------------ + + def leave_Module(self, original_node: cst.Module, updated_node: cst.Module) -> cst.Module: + to_add: List[str] = [] + if self._needs_col and not self._has_col: + to_add.append("col") + if self._needs_raw and not self._has_raw: + to_add.append("raw") + if not to_add: + return updated_node + + new_body = list(updated_node.body) + + # Try to augment an existing filters import line + for i, stmt in enumerate(new_body): + if not ( + isinstance(stmt, cst.SimpleStatementLine) + and len(stmt.body) == 1 + and isinstance(stmt.body[0], cst.ImportFrom) + ): + continue + imp = stmt.body[0] + if isinstance(imp.names, cst.ImportStar): + continue + if _dotted_name(imp.module) != _FILTERS_MODULE: + continue + existing_names = {alias.name.value for alias in imp.names if isinstance(alias.name, cst.Name)} + need = [n for n in to_add if n not in existing_names] + if not need: + return updated_node # already present + all_aliases = list(imp.names) + [cst.ImportAlias(name=_name(n)) for n in need] + # Re-apply commas + fixed = _comma_separated(all_aliases) + new_imp = imp.with_changes(names=fixed) + new_body[i] = stmt.with_changes(body=[new_imp]) + return updated_node.with_changes(body=new_body) + + # No existing filters import — insert a new one after the last import block + new_import_stmt = cst.SimpleStatementLine( + body=[ + cst.ImportFrom( + module=_filters_module_attr(), + names=_comma_separated([cst.ImportAlias(name=_name(n)) for n in to_add]), + ) + ] + ) + last_import_idx = 0 + for i, stmt in enumerate(new_body): + if isinstance(stmt, cst.SimpleStatementLine) and any( + isinstance(s, (cst.Import, cst.ImportFrom)) for s in stmt.body + ): + last_import_idx = i + new_body.insert(last_import_idx + 1, new_import_stmt) + return updated_node.with_changes(body=new_body) + + +def _comma_separated( + aliases: List[cst.ImportAlias], +) -> List[cst.ImportAlias]: + """Return aliases with commas between each, last one without.""" + result = [] + for i, alias in enumerate(aliases): + if i < len(aliases) - 1: + result.append(alias.with_changes(comma=cst.Comma(whitespace_after=cst.SimpleWhitespace(" ")))) + else: + result.append(alias.with_changes(comma=cst.MaybeSentinel.DEFAULT)) + return result + + +# --------------------------------------------------------------------------- +# Utility: dotted-name string from libcst Attribute / Name tree +# --------------------------------------------------------------------------- + + +def _dotted_name(node: Optional[cst.BaseExpression]) -> str: + if node is None: + return "" + if isinstance(node, cst.Name): + return node.value + if isinstance(node, cst.Attribute): + return f"{_dotted_name(node.value)}.{node.attr.value}" + return "" + + +# --------------------------------------------------------------------------- +# File-level migration +# --------------------------------------------------------------------------- + + +def migrate_source(source: str, *, client_var: str = "client") -> str: + """Parse *source*, apply transformations, return migrated source.""" + try: + tree = cst.parse_module(source) + except cst.ParserSyntaxError as exc: + raise ValueError(f"Parse error: {exc}") from exc + new_tree = tree.visit(_V1Migrator(client_var=client_var)) + return new_tree.code + + +def migrate_file(path: Path, *, dry_run: bool = False) -> bool: + """Migrate *path* in place. Returns True if the file was changed.""" + original = path.read_text(encoding="utf-8") + try: + migrated = migrate_source(original) + except ValueError as exc: + print(f" [SKIP] {path}: {exc}", file=sys.stderr) + return False + if migrated == original: + return False + if not dry_run: + path.write_text(migrated, encoding="utf-8") + return True + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def _collect_targets(paths: List[str]) -> List[Path]: + targets: List[Path] = [] + for p_str in paths: + p = Path(p_str) + if p.is_dir(): + root = p.resolve() + for candidate in sorted(p.rglob("*.py")): + resolved = candidate.resolve() + if root == resolved or root in resolved.parents: + targets.append(candidate) + else: + print(f"[WARN] Skipping symlink outside target directory: {candidate}", file=sys.stderr) + elif p.is_file() and p.suffix == ".py": + targets.append(p) + else: + print(f"[WARN] Not a file or directory: {p}", file=sys.stderr) + return targets + + +def main(argv: Optional[List[str]] = None) -> int: + args = sys.argv[1:] if argv is None else list(argv) + dry_run = "--dry-run" in args + remaining = [a for a in args if a != "--dry-run"] + + if not remaining: + print(__doc__) + print("\nUsage: python -m tools.migrate_v0_to_v1 [--dry-run] [ ...]") + return 1 + + targets = _collect_targets(remaining) + if not targets: + print("[ERROR] No Python files found.", file=sys.stderr) + return 1 + + changed = skipped = 0 + for path in targets: + if migrate_file(path, dry_run=dry_run): + changed += 1 + tag = "[DRY-RUN]" if dry_run else "[MIGRATED]" + print(f"{tag} {path}") + else: + skipped += 1 + + print(f"\nDone: {changed} file(s) {'would be ' if dry_run else ''}modified, " f"{skipped} unchanged.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From e35f2973f7a3580630c5ce77085e553e9eb13e0f Mon Sep 17 00:00:00 2001 From: Samson Gebre Date: Tue, 5 May 2026 21:45:22 -0700 Subject: [PATCH 02/16] Enhance SKILL.md and QueryBuilder to support advanced streaming and update deprecated methods - Added simple and advanced streaming options in SKILL.md for records.list_pages() and execute_pages(). - Updated QueryBuilder to replace records.get() with records.list() in documentation and method calls. - Improved unit tests to validate new streaming functionality and ensure correct method delegation. --- .claude/skills/dataverse-sdk-use/SKILL.md | 59 ++++++- .../claude_skill/dataverse-sdk-use/SKILL.md | 59 ++++++- .../Dataverse/models/query_builder.py | 48 +++--- .../Dataverse/operations/records.py | 12 +- tests/unit/models/test_query_builder.py | 159 ++++++++---------- 5 files changed, 203 insertions(+), 134 deletions(-) diff --git a/.claude/skills/dataverse-sdk-use/SKILL.md b/.claude/skills/dataverse-sdk-use/SKILL.md index bda1ed14..32c2b275 100644 --- a/.claude/skills/dataverse-sdk-use/SKILL.md +++ b/.claude/skills/dataverse-sdk-use/SKILL.md @@ -30,6 +30,16 @@ The SDK supports Dataverse's native bulk operations: Pass lists to `create()`, ` ### Paging - Control page size with `page_size` parameter - Use `top` parameter to limit total records returned +- Simple streaming: `records.list_pages(table, filter, select)` — yields one `QueryResult` per HTTP page (3 params only; use builder for advanced options) +- Advanced streaming: `client.query.builder(table)....execute_pages()` — full builder options, one `QueryResult` per page +- `execute(by_page=True/False)` is **deprecated** and emits `UserWarning`; use `execute_pages()` instead + +### QueryResult +- Returned by `records.list()`, `records.retrieve()`, `execute()`, and each page from `list_pages()` / `execute_pages()` +- Iterable: `for record in result` — each item is a `dict`-like `Record` +- `.to_dataframe()` — convert to pandas DataFrame +- `.first()` — return the first record or `None` +- `len(result)` — number of records in this result/page ### DataFrame Support - DataFrame operations are accessed via the `client.dataframe` namespace: `client.dataframe.create()`, `client.dataframe.update()`, `client.dataframe.delete()` — `client.dataframe.get()` is deprecated; use `client.query.builder(table).where(...).execute().to_dataframe()` instead @@ -90,7 +100,7 @@ account = client.records.retrieve("account", account_id, select=["name", "teleph # Query with filter — follows @odata.nextLink automatically (multiple HTTP requests if needed), # loads all matching records into memory, returns a single QueryResult. # Page size is Dataverse's default (~5000/page); use top to bound total records and round-trips. -# For very large sets where memory is a concern, use query.builder().execute(by_page=True) instead. +# For very large sets where memory is a concern, use records.list_pages() or execute_pages() instead. result = client.records.list( "account", select=["accountid", "name"], # select is case-insensitive (automatically lowercased) @@ -100,12 +110,22 @@ result = client.records.list( for record in result: print(record["name"]) -# For large result sets that must be streamed page-by-page (caller controls memory, one page at a time) +# Simple streaming — page-by-page (3 params only; use builder for ordering/expand/count) +for page in client.records.list_pages( + "account", + select=["accountid", "name"], + filter="statecode eq 0", +): + for record in page: + print(record["name"]) + +# Advanced streaming — full builder options, one QueryResult per HTTP page +from PowerPlatform.Dataverse.models.filters import col for page in (client.query.builder("account") .select("accountid", "name") .where(col("statecode") == 0) .page_size(500) # optional: override Dataverse default page size - .execute(by_page=True)): + .execute_pages()): for record in page: print(record["name"]) @@ -191,13 +211,14 @@ client.records.delete("account", [id1, id2, id3], use_bulk_delete=True) The SDK provides DataFrame wrappers for all CRUD operations via the `client.dataframe` namespace, using pandas DataFrames and Series as input/output. -> **Note:** `client.dataframe.get()` is deprecated. Use `client.query.builder(table).select(...).filter(...).to_dataframe()` instead. +> **Note:** `client.dataframe.get()` is deprecated. Use `client.query.builder(table).select(...).where(...).to_dataframe()` instead. ```python import pandas as pd # Query records -- returns a single DataFrame (GA builder pattern) -df = client.query.builder("account").filter("statecode eq 0").select("name").to_dataframe() +from PowerPlatform.Dataverse.models.filters import col +df = client.query.builder("account").where(col("statecode") == 0).select("name").to_dataframe() print(f"Got {len(df)} rows") # Limit results with top @@ -237,6 +258,34 @@ for record in results: print(record["name"]) ``` +### FetchXML Queries + +`client.query.fetch_xml(xml)` returns an inert `FetchXmlQuery` object — **no HTTP request is made** until `.execute()` or `.execute_pages()` is called. + +```python +xml = """ + + + + + + + + + +""" + +# Load all results into memory (simple, small-to-medium sets) +query = client.query.fetch_xml(xml) +result = query.execute() # returns QueryResult — all pages fetched upfront +for record in result: + print(record["name"]) + +# Stream page-by-page (large sets or early exit) +for page in query.execute_pages(): # yields one QueryResult per HTTP page + process(page.to_dataframe()) +``` + ### Table Management #### Create Custom Tables diff --git a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md index bda1ed14..32c2b275 100644 --- a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md +++ b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md @@ -30,6 +30,16 @@ The SDK supports Dataverse's native bulk operations: Pass lists to `create()`, ` ### Paging - Control page size with `page_size` parameter - Use `top` parameter to limit total records returned +- Simple streaming: `records.list_pages(table, filter, select)` — yields one `QueryResult` per HTTP page (3 params only; use builder for advanced options) +- Advanced streaming: `client.query.builder(table)....execute_pages()` — full builder options, one `QueryResult` per page +- `execute(by_page=True/False)` is **deprecated** and emits `UserWarning`; use `execute_pages()` instead + +### QueryResult +- Returned by `records.list()`, `records.retrieve()`, `execute()`, and each page from `list_pages()` / `execute_pages()` +- Iterable: `for record in result` — each item is a `dict`-like `Record` +- `.to_dataframe()` — convert to pandas DataFrame +- `.first()` — return the first record or `None` +- `len(result)` — number of records in this result/page ### DataFrame Support - DataFrame operations are accessed via the `client.dataframe` namespace: `client.dataframe.create()`, `client.dataframe.update()`, `client.dataframe.delete()` — `client.dataframe.get()` is deprecated; use `client.query.builder(table).where(...).execute().to_dataframe()` instead @@ -90,7 +100,7 @@ account = client.records.retrieve("account", account_id, select=["name", "teleph # Query with filter — follows @odata.nextLink automatically (multiple HTTP requests if needed), # loads all matching records into memory, returns a single QueryResult. # Page size is Dataverse's default (~5000/page); use top to bound total records and round-trips. -# For very large sets where memory is a concern, use query.builder().execute(by_page=True) instead. +# For very large sets where memory is a concern, use records.list_pages() or execute_pages() instead. result = client.records.list( "account", select=["accountid", "name"], # select is case-insensitive (automatically lowercased) @@ -100,12 +110,22 @@ result = client.records.list( for record in result: print(record["name"]) -# For large result sets that must be streamed page-by-page (caller controls memory, one page at a time) +# Simple streaming — page-by-page (3 params only; use builder for ordering/expand/count) +for page in client.records.list_pages( + "account", + select=["accountid", "name"], + filter="statecode eq 0", +): + for record in page: + print(record["name"]) + +# Advanced streaming — full builder options, one QueryResult per HTTP page +from PowerPlatform.Dataverse.models.filters import col for page in (client.query.builder("account") .select("accountid", "name") .where(col("statecode") == 0) .page_size(500) # optional: override Dataverse default page size - .execute(by_page=True)): + .execute_pages()): for record in page: print(record["name"]) @@ -191,13 +211,14 @@ client.records.delete("account", [id1, id2, id3], use_bulk_delete=True) The SDK provides DataFrame wrappers for all CRUD operations via the `client.dataframe` namespace, using pandas DataFrames and Series as input/output. -> **Note:** `client.dataframe.get()` is deprecated. Use `client.query.builder(table).select(...).filter(...).to_dataframe()` instead. +> **Note:** `client.dataframe.get()` is deprecated. Use `client.query.builder(table).select(...).where(...).to_dataframe()` instead. ```python import pandas as pd # Query records -- returns a single DataFrame (GA builder pattern) -df = client.query.builder("account").filter("statecode eq 0").select("name").to_dataframe() +from PowerPlatform.Dataverse.models.filters import col +df = client.query.builder("account").where(col("statecode") == 0).select("name").to_dataframe() print(f"Got {len(df)} rows") # Limit results with top @@ -237,6 +258,34 @@ for record in results: print(record["name"]) ``` +### FetchXML Queries + +`client.query.fetch_xml(xml)` returns an inert `FetchXmlQuery` object — **no HTTP request is made** until `.execute()` or `.execute_pages()` is called. + +```python +xml = """ + + + + + + + + + +""" + +# Load all results into memory (simple, small-to-medium sets) +query = client.query.fetch_xml(xml) +result = query.execute() # returns QueryResult — all pages fetched upfront +for record in result: + print(record["name"]) + +# Stream page-by-page (large sets or early exit) +for page in query.execute_pages(): # yields one QueryResult per HTTP page + process(page.to_dataframe()) +``` + ### Table Management #### Create Custom Tables diff --git a/src/PowerPlatform/Dataverse/models/query_builder.py b/src/PowerPlatform/Dataverse/models/query_builder.py index 1192d16d..52cd8155 100644 --- a/src/PowerPlatform/Dataverse/models/query_builder.py +++ b/src/PowerPlatform/Dataverse/models/query_builder.py @@ -50,7 +50,7 @@ from __future__ import annotations import warnings -from typing import Any, Dict, Iterable, Iterator, List, Optional, TypedDict, Union +from typing import Any, Dict, Iterator, List, Optional, TypedDict, Union import pandas as pd @@ -67,7 +67,7 @@ class QueryParams(TypedDict, total=False): """Typed dictionary returned by :meth:`QueryBuilder.build`. Provides IDE autocomplete when passing build results to - ``client.records.get()`` manually. + ``client.records.list()`` manually. """ table: str @@ -450,7 +450,7 @@ def build(self) -> QueryParams: # --------------------------------------------------------------- execute - def execute(self, *, by_page=_BY_PAGE_UNSET) -> Union[QueryResult, Iterable[List[Record]]]: + def execute(self, *, by_page=_BY_PAGE_UNSET) -> Union[QueryResult, Iterator[QueryResult]]: """Execute the query and return results. Returns a :class:`~PowerPlatform.Dataverse.models.record.QueryResult` @@ -460,7 +460,7 @@ def execute(self, *, by_page=_BY_PAGE_UNSET) -> Union[QueryResult, Iterable[List This method is only available when the QueryBuilder was created via ``client.query.builder(table)``. Standalone ``QueryBuilder`` instances should use :meth:`build` to get parameters and pass them - to ``client.records.get()`` manually. + to ``client.records.list()`` manually. At least one of ``select()``, ``where()``, or ``top()`` must be called before ``execute()``; otherwise a :class:`ValueError` is @@ -474,7 +474,7 @@ def execute(self, *, by_page=_BY_PAGE_UNSET) -> Union[QueryResult, Iterable[List :return: :class:`~PowerPlatform.Dataverse.models.record.QueryResult` with all pages collected (default), or page iterator (deprecated ``by_page=True``). - :rtype: QueryResult or Iterable[List[Record]] + :rtype: QueryResult or Iterator[QueryResult] :raises ValueError: If no ``select``, ``where``, or ``top`` constraint has been set. :raises RuntimeError: If the query was not created via @@ -510,7 +510,7 @@ def execute(self, *, by_page=_BY_PAGE_UNSET) -> Union[QueryResult, Iterable[List if self._query_ops is None: raise RuntimeError( "Cannot execute: query was not created via client.query.builder(). " - "Use build() and pass parameters to client.records.get() instead." + "Use build() and pass parameters to client.records.list() instead." ) if not self._select and not self._filter_parts and self._top is None: @@ -522,11 +522,12 @@ def execute(self, *, by_page=_BY_PAGE_UNSET) -> Union[QueryResult, Iterable[List params = self.build() client = self._query_ops._client - # Suppress DeprecationWarning from records.get() — execute() is GA; - # records.get() is deprecated but still used as the internal paging mechanism. - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - pages = client.records.get( + if use_by_page: + return self.execute_pages() + + all_records: List[Record] = [] + with client._scoped_odata() as od: + for page in od._get_multiple( params["table"], select=params.get("select"), filter=params.get("filter"), @@ -536,14 +537,8 @@ def execute(self, *, by_page=_BY_PAGE_UNSET) -> Union[QueryResult, Iterable[List page_size=params.get("page_size"), count=params.get("count", False), include_annotations=params.get("include_annotations"), - ) - - if use_by_page: - return pages - - all_records: List[Record] = [] - for page in pages: - all_records.extend(page) + ): + all_records.extend(Record.from_api_response(params["table"], row) for row in page) return QueryResult(all_records) # ---------------------------------------------------------- execute_pages @@ -579,7 +574,7 @@ def execute_pages(self) -> Iterator[QueryResult]: if self._query_ops is None: raise RuntimeError( "Cannot execute: query was not created via client.query.builder(). " - "Use build() and pass parameters to client.records.get() instead." + "Use build() and pass parameters to client.records.list() instead." ) if not self._select and not self._filter_parts and self._top is None: @@ -591,9 +586,8 @@ def execute_pages(self) -> Iterator[QueryResult]: params = self.build() client = self._query_ops._client - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - pages = client.records.get( + with client._scoped_odata() as od: + for page in od._get_multiple( params["table"], select=params.get("select"), filter=params.get("filter"), @@ -603,10 +597,8 @@ def execute_pages(self) -> Iterator[QueryResult]: page_size=params.get("page_size"), count=params.get("count", False), include_annotations=params.get("include_annotations"), - ) - - for page in pages: - yield QueryResult(page) + ): + yield QueryResult([Record.from_api_response(params["table"], row) for row in page]) # ----------------------------------------------------------- to_dataframe @@ -653,7 +645,7 @@ def to_dataframe(self) -> pd.DataFrame: if self._query_ops is None: raise RuntimeError( "Cannot execute: query was not created via client.query.builder(). " - "Use build() and pass parameters to client.records.get() instead." + "Use build() and pass parameters to client.records.list() instead." ) result = self.execute() diff --git a/src/PowerPlatform/Dataverse/operations/records.py b/src/PowerPlatform/Dataverse/operations/records.py index 600f9d87..dc8a5129 100644 --- a/src/PowerPlatform/Dataverse/operations/records.py +++ b/src/PowerPlatform/Dataverse/operations/records.py @@ -588,6 +588,9 @@ def list( GA replacement for ``records.get(table, filter=...)``. All pages are collected eagerly and returned as a single :class:`QueryResult`. + For advanced query options (ordering, expand, count, page size, or + OData annotations) use ``client.query.builder()`` instead. + :param table: Schema name of the table (e.g. ``"account"``). :type table: :class:`str` :param filter: Optional OData filter string or :class:`FilterExpression`. @@ -643,9 +646,12 @@ def list_pages( ) -> Iterator[QueryResult]: """Lazily yield one :class:`QueryResult` per HTTP page. - Symmetric with :meth:`~PowerPlatform.Dataverse.models.query_builder.QueryBuilder.execute_pages` - on the builder. Each iteration triggers a network request via - ``@odata.nextLink``. One-shot — do not iterate more than once. + Streaming counterpart to :meth:`list`. Each iteration triggers one + network request via ``@odata.nextLink``. One-shot — do not iterate + more than once. + + For advanced query options (ordering, expand, count, page size, or + OData annotations) use ``client.query.builder().execute_pages()`` instead. :param table: Schema name of the table (e.g. ``"account"``). :type table: :class:`str` diff --git a/tests/unit/models/test_query_builder.py b/tests/unit/models/test_query_builder.py index 160e5072..7ffc09f8 100644 --- a/tests/unit/models/test_query_builder.py +++ b/tests/unit/models/test_query_builder.py @@ -515,14 +515,19 @@ def test_execute_without_query_ops_raises(self): qb.execute() self.assertIn("client.query.builder()", str(ctx.exception)) - def test_execute_calls_records_get(self): - """execute() should delegate to client.records.get() with built params.""" - from PowerPlatform.Dataverse.models.filters import raw - + def _make_od(self, pages=None): + """Return (mock_query_ops, mock_od) with _get_multiple pre-wired.""" mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([[{"name": "Test"}]]) + mock_od = MagicMock() + mock_od._get_multiple.side_effect = lambda *a, **kw: iter(pages or []) + mock_query_ops._client._scoped_odata.return_value.__enter__.return_value = mock_od + return mock_query_ops, mock_od + def test_execute_calls_get_multiple(self): + """execute() calls _get_multiple() via _scoped_odata with all built params.""" + from PowerPlatform.Dataverse.models.filters import raw + + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops ( @@ -534,9 +539,9 @@ def test_execute_calls_records_get(self): .expand("primarycontactid") ) - list(qb.execute()) + qb.execute() - mock_client.records.get.assert_called_once_with( + mock_od._get_multiple.assert_called_once_with( "account", select=["name", "revenue"], filter="statecode eq 0", @@ -549,10 +554,7 @@ def test_execute_calls_records_get(self): ) def test_execute_returns_flat_records_by_default(self): - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([[{"name": "A"}, {"name": "B"}], [{"name": "C"}]]) - + mock_query_ops, _ = self._make_od([[{"name": "A"}, {"name": "B"}], [{"name": "C"}]]) qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name") @@ -564,59 +566,47 @@ def test_execute_returns_flat_records_by_default(self): self.assertEqual(records[2]["name"], "C") def test_execute_by_page_returns_pages(self): - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - - page1 = [{"name": "A"}, {"name": "B"}] - page2 = [{"name": "C"}] - mock_client.records.get.return_value = iter([page1, page2]) + from PowerPlatform.Dataverse.models.record import QueryResult + mock_query_ops, _ = self._make_od([[{"name": "A"}, {"name": "B"}], [{"name": "C"}]]) qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name") pages = list(qb.execute(by_page=True)) self.assertEqual(len(pages), 2) - self.assertEqual(pages[0], page1) - self.assertEqual(pages[1], page2) + self.assertIsInstance(pages[0], QueryResult) + self.assertEqual(len(pages[0]), 2) + self.assertEqual(len(pages[1]), 1) def test_execute_with_only_select_succeeds(self): """execute() with select only should not raise.""" - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) - + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name") - list(qb.execute()) # should not raise - mock_client.records.get.assert_called_once() + qb.execute() # should not raise + mock_od._get_multiple.assert_called_once() def test_execute_with_only_filter_succeeds(self): """execute() with filter only should not raise.""" from PowerPlatform.Dataverse.models.filters import raw - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) - + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.where(raw("statecode eq 0")) - list(qb.execute()) # should not raise - mock_client.records.get.assert_called_once() + qb.execute() # should not raise + mock_od._get_multiple.assert_called_once() def test_execute_with_only_top_succeeds(self): """execute() with top only should not raise.""" - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) - + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.top(10) - list(qb.execute()) # should not raise - mock_client.records.get.assert_called_once() + qb.execute() # should not raise + mock_od._get_multiple.assert_called_once() def test_execute_with_only_expand_raises(self): """expand alone is not a sufficient constraint.""" @@ -639,51 +629,40 @@ def test_execute_with_only_count_raises(self): def test_execute_with_where_expressions(self): from PowerPlatform.Dataverse.models.filters import col - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) - + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.where(((col("statecode") == 0) | (col("statecode") == 1)) & (col("revenue") > 100000)) - list(qb.execute()) + qb.execute() - call_args = mock_client.records.get.call_args self.assertEqual( - call_args.kwargs["filter"], + mock_od._get_multiple.call_args.kwargs["filter"], "((statecode eq 0 or statecode eq 1) and revenue gt 100000)", ) def test_execute_with_filter_in(self): from PowerPlatform.Dataverse.models.filters import col - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) - + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.where(col("statecode").in_([0, 1, 2])) - list(qb.execute()) + qb.execute() - call_args = mock_client.records.get.call_args self.assertEqual( - call_args.kwargs["filter"], + mock_od._get_multiple.call_args.kwargs["filter"], 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1","2"])', ) def test_execute_passes_count_and_annotations(self): - """execute() should forward count and include_annotations when set.""" - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) - + """execute() should forward count and include_annotations to _get_multiple.""" + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name").count().include_formatted_values() - list(qb.execute()) + qb.execute() - mock_client.records.get.assert_called_once_with( + mock_od._get_multiple.assert_called_once_with( "account", select=["name"], filter=None, @@ -701,7 +680,9 @@ class TestExecutePages(unittest.TestCase): def _make_qb(self): mock_query_ops = MagicMock() - mock_query_ops._client.records.get.return_value = iter([[{"name": "A"}], [{"name": "B"}]]) + mock_od = MagicMock() + mock_od._get_multiple.side_effect = lambda *a, **kw: iter([[{"name": "A"}], [{"name": "B"}]]) + mock_query_ops._client._scoped_odata.return_value.__enter__.return_value = mock_od qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name") @@ -749,7 +730,9 @@ class TestByPageWarning(unittest.TestCase): def _make_qb(self): mock_query_ops = MagicMock() - mock_query_ops._client.records.get.return_value = iter([[{"name": "A"}]]) + mock_od = MagicMock() + mock_od._get_multiple.side_effect = lambda *a, **kw: iter([[{"name": "A"}]]) + mock_query_ops._client._scoped_odata.return_value.__enter__.return_value = mock_od qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name") @@ -807,6 +790,13 @@ def test_execute_by_page_false_still_returns_query_result(self): class TestToDataframe(unittest.TestCase): """Tests for the to_dataframe() terminal method.""" + def _make_od(self, pages=None): + mock_query_ops = MagicMock() + mock_od = MagicMock() + mock_od._get_multiple.side_effect = lambda *a, **kw: iter(pages or []) + mock_query_ops._client._scoped_odata.return_value.__enter__.return_value = mock_od + return mock_query_ops, mock_od + def test_to_dataframe_without_query_ops_raises(self): from PowerPlatform.Dataverse.models.filters import raw @@ -819,10 +809,7 @@ def test_to_dataframe_returns_dataframe(self): """to_dataframe() collects execute() results into a DataFrame.""" import pandas as pd - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([[{"name": "A", "revenue": 100}, {"name": "B", "revenue": 200}]]) - + mock_query_ops, _ = self._make_od([[{"name": "A", "revenue": 100}, {"name": "B", "revenue": 200}]]) qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name", "revenue") @@ -837,10 +824,7 @@ def test_to_dataframe_empty_result_returns_empty_dataframe(self): """to_dataframe() with no matching records returns empty DataFrame.""" import pandas as pd - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) - + mock_query_ops, _ = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name", "revenue") @@ -850,15 +834,12 @@ def test_to_dataframe_empty_result_returns_empty_dataframe(self): self.assertIsInstance(result, pd.DataFrame) self.assertEqual(len(result), 0) - def test_to_dataframe_calls_records_get_with_params(self): - """to_dataframe() should call records.get() with the built query params.""" + def test_to_dataframe_calls_get_multiple_with_params(self): + """to_dataframe() passes all built query params to _get_multiple.""" import pandas as pd from PowerPlatform.Dataverse.models.filters import raw - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([[{"name": "Contoso", "revenue": 1000}]]) - + mock_query_ops, mock_od = self._make_od([[{"name": "Contoso", "revenue": 1000}]]) qb = QueryBuilder("account") qb._query_ops = mock_query_ops ( @@ -872,7 +853,7 @@ def test_to_dataframe_calls_records_get_with_params(self): result = qb.to_dataframe() - mock_client.records.get.assert_called_once_with( + mock_od._get_multiple.assert_called_once_with( "account", select=["name", "revenue"], filter="statecode eq 0", @@ -889,16 +870,13 @@ def test_to_dataframe_calls_records_get_with_params(self): def test_to_dataframe_forwards_count_and_annotations(self): """to_dataframe() should forward count and include_annotations when set.""" - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) - + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name").count().include_formatted_values() qb.to_dataframe() - mock_client.records.get.assert_called_once_with( + mock_od._get_multiple.assert_called_once_with( "account", select=["name"], filter=None, @@ -915,14 +893,12 @@ def test_to_dataframe_with_record_objects(self): import pandas as pd from PowerPlatform.Dataverse.models.record import Record - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - records = [ - Record(id="id-1", table="account", data={"name": "Contoso", "revenue": 1000}), - Record(id="id-2", table="account", data={"name": "Fabrikam", "revenue": 2000}), - ] - mock_client.records.get.return_value = iter([records]) - + mock_query_ops, _ = self._make_od([ + [ + {"name": "Contoso", "revenue": 1000}, + {"name": "Fabrikam", "revenue": 2000}, + ] + ]) qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name", "revenue") @@ -936,10 +912,7 @@ def test_to_dataframe_with_record_objects(self): def test_to_dataframe_emits_deprecation_warning(self): """QueryBuilder.to_dataframe() fires DeprecationWarning; use execute().to_dataframe() instead.""" - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) - + mock_query_ops, _ = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name") From e6a3b1688a95053b8b747c8860b0aa0f842086a7 Mon Sep 17 00:00:00 2001 From: Samson Gebre Date: Wed, 6 May 2026 09:59:56 -0700 Subject: [PATCH 03/16] Refactor FetchXML method names and update documentation - Changed method name from `fetch_xml` to `fetchxml` across the codebase for consistency. - Updated relevant documentation to reflect the new method name. - Added a new example script for FetchXML usage demonstrating various scenarios. - Adjusted unit tests to accommodate the method name change and ensure proper functionality. --- .claude/skills/dataverse-sdk-use/SKILL.md | 4 +- README.md | 6 +-- .../advanced/{fetch_xml.py => fetchxml.py} | 50 +++++++++---------- examples/advanced/sql_examples.py | 8 +-- .../claude_skill/dataverse-sdk-use/SKILL.md | 4 +- .../{fetch_xml_query.py => fetchxml_query.py} | 22 +++++--- src/PowerPlatform/Dataverse/models/filters.py | 2 +- .../Dataverse/operations/query.py | 16 +++--- tests/unit/models/test_query_builder.py | 10 ++-- tests/unit/test_phase4_ga.py | 46 ++++++++--------- 10 files changed, 90 insertions(+), 78 deletions(-) rename examples/advanced/{fetch_xml.py => fetchxml.py} (91%) rename src/PowerPlatform/Dataverse/models/{fetch_xml_query.py => fetchxml_query.py} (83%) diff --git a/.claude/skills/dataverse-sdk-use/SKILL.md b/.claude/skills/dataverse-sdk-use/SKILL.md index 32c2b275..00f5a7fa 100644 --- a/.claude/skills/dataverse-sdk-use/SKILL.md +++ b/.claude/skills/dataverse-sdk-use/SKILL.md @@ -260,7 +260,7 @@ for record in results: ### FetchXML Queries -`client.query.fetch_xml(xml)` returns an inert `FetchXmlQuery` object — **no HTTP request is made** until `.execute()` or `.execute_pages()` is called. +`client.query.fetchxml(xml)` returns an inert `FetchXmlQuery` object — **no HTTP request is made** until `.execute()` or `.execute_pages()` is called. ```python xml = """ @@ -276,7 +276,7 @@ xml = """ """ # Load all results into memory (simple, small-to-medium sets) -query = client.query.fetch_xml(xml) +query = client.query.fetchxml(xml) result = query.execute() # returns QueryResult — all pages fetched upfront for record in result: print(record["name"]) diff --git a/README.md b/README.md index 1f077b2d..4e72fa5b 100644 --- a/README.md +++ b/README.md @@ -431,7 +431,7 @@ results = (client.query.builder("account") .execute()) ``` -**FetchXML queries** -- `client.query.fetch_xml()` returns an inert `FetchXmlQuery` object; no HTTP request is made until you call `.execute()` or `.execute_pages()`: +**FetchXML queries** -- `client.query.fetchxml()` returns an inert `FetchXmlQuery` object; no HTTP request is made until you call `.execute()` or `.execute_pages()`: ```python xml = """ @@ -445,12 +445,12 @@ xml = """ """ # .execute() — blocking, fetches all pages and returns a single QueryResult -result = client.query.fetch_xml(xml).execute() +result = client.query.fetchxml(xml).execute() df = result.to_dataframe() # .execute_pages() — streaming, yields one QueryResult per HTTP page # Use count="N" in the FetchXML element to set page size -for page_num, page in enumerate(client.query.fetch_xml(xml).execute_pages()): +for page_num, page in enumerate(client.query.fetchxml(xml).execute_pages()): print(f"Page {page_num + 1}: {len(page)} records") for record in page: print(record["name"]) diff --git a/examples/advanced/fetch_xml.py b/examples/advanced/fetchxml.py similarity index 91% rename from examples/advanced/fetch_xml.py rename to examples/advanced/fetchxml.py index b475df06..9bfecc33 100644 --- a/examples/advanced/fetch_xml.py +++ b/examples/advanced/fetchxml.py @@ -4,7 +4,7 @@ """ End-to-end FetchXML examples for Dataverse. -Demonstrates ``client.query.fetch_xml()`` across the scenarios where FetchXML +Demonstrates ``client.query.fetchxml()`` across the scenarios where FetchXML is required or preferred over OData/SQL: - Basic attribute queries @@ -263,8 +263,8 @@ def _run_examples(client): """ - log_call("client.query.fetch_xml(basic attribute query)") - result = backoff(lambda: client.query.fetch_xml(xml).execute()) + log_call("client.query.fetchxml(basic attribute query)") + result = backoff(lambda: client.query.fetchxml(xml).execute()) print(f"[OK] {len(result)} projects:") for r in result: print(f" {r.get('new_code', ''):<10s} Budget={r.get('new_budget')} Active={r.get('new_active')}") @@ -286,7 +286,7 @@ def _run_examples(client): """ log_call('operator="eq" value="ALPHA"') - r = backoff(lambda: client.query.fetch_xml(xml).execute()) + r = backoff(lambda: client.query.fetchxml(xml).execute()) print(f"[OK] eq: {[x.get('new_code') for x in r]}") # like @@ -301,7 +301,7 @@ def _run_examples(client): """ log_call('operator="like" value="%test%"') - r = backoff(lambda: client.query.fetch_xml(xml).execute()) + r = backoff(lambda: client.query.fetchxml(xml).execute()) print(f"[OK] like: {len(r)} matches -> {[x.get('new_title') for x in r]}") # in @@ -319,7 +319,7 @@ def _run_examples(client): """ log_call('operator="in" values=[ALPHA, DELTA]') - r = backoff(lambda: client.query.fetch_xml(xml).execute()) + r = backoff(lambda: client.query.fetchxml(xml).execute()) print(f"[OK] in: {[x.get('new_code') for x in r]}") # null / not-null @@ -334,7 +334,7 @@ def _run_examples(client): """ log_call('operator="not-null"') - r = backoff(lambda: client.query.fetch_xml(xml).execute()) + r = backoff(lambda: client.query.fetchxml(xml).execute()) print(f"[OK] not-null: {len(r)} tasks have priority set") # between @@ -353,7 +353,7 @@ def _run_examples(client): """ log_call('operator="between" 40000 and 80000') - r = backoff(lambda: client.query.fetch_xml(xml).execute()) + r = backoff(lambda: client.query.fetchxml(xml).execute()) print(f"[OK] between: {len(r)} projects -> {[(x.get('new_code'), x.get('new_budget')) for x in r]}") # =============================================================== @@ -376,9 +376,9 @@ def _run_examples(client): """ - log_call("client.query.fetch_xml(link-entity inner join)") + log_call("client.query.fetchxml(link-entity inner join)") try: - result = backoff(lambda: client.query.fetch_xml(xml).execute()) + result = backoff(lambda: client.query.fetchxml(xml).execute()) print(f"[OK] {len(result)} rows:") for r in result: print( @@ -408,9 +408,9 @@ def _run_examples(client): """ - log_call("client.query.fetch_xml(link-entity outer join)") + log_call("client.query.fetchxml(link-entity outer join)") try: - result = backoff(lambda: client.query.fetch_xml(xml).execute()) + result = backoff(lambda: client.query.fetchxml(xml).execute()) print(f"[OK] {len(result)} rows (includes projects with no tasks):") for r in result[:8]: print(f" Project={r.get('new_code', ''):<10s} Task={r.get('t.new_title', '(none)')}") @@ -431,8 +431,8 @@ def _run_examples(client): """ - log_call("client.query.fetch_xml(order by hours DESC)") - result = backoff(lambda: client.query.fetch_xml(xml).execute()) + log_call("client.query.fetchxml(order by hours DESC)") + result = backoff(lambda: client.query.fetchxml(xml).execute()) print(f"[OK] Tasks by hours DESC:") for r in result: print(f" {r.get('new_title', ''):<25s} Hours={r.get('new_hours')}") @@ -455,16 +455,16 @@ def _run_examples(client): """ - log_call("client.query.fetch_xml(xml).execute() — eager, all pages collected") - result = backoff(lambda: client.query.fetch_xml(xml_paged).execute()) + log_call("client.query.fetchxml(xml).execute() — eager, all pages collected") + result = backoff(lambda: client.query.fetchxml(xml_paged).execute()) print(f"[OK] execute(): {len(result)} total tasks across all pages (seeded {len(task_ids)}):") for r in result: print(f" {r.get('new_title', ''):<25s} Hours={r.get('new_hours')}") - log_call("client.query.fetch_xml(xml).execute_pages() — lazy, one QueryResult per HTTP page") + log_call("client.query.fetchxml(xml).execute_pages() — lazy, one QueryResult per HTTP page") page_num = 0 page_record_count = 0 - for page in backoff(lambda: client.query.fetch_xml(xml_paged).execute_pages()): + for page in backoff(lambda: client.query.fetchxml(xml_paged).execute_pages()): page_num += 1 page_record_count += len(page) print(f" Page {page_num}: {len(page)} record(s) — {[r.get('new_title') for r in page]}") @@ -487,11 +487,11 @@ def _run_examples(client): """ - log_call("client.query.fetch_xml(aggregate: count, sum, avg, min, max)") + log_call("client.query.fetchxml(aggregate: count, sum, avg, min, max)") try: - result = backoff(lambda: client.query.fetch_xml(xml).execute()) + result = backoff(lambda: client.query.fetchxml(xml).execute()) if result: - row = result[0] + row = result.first() print( f"[OK] count={row.get('task_count')} sum={row.get('total_hours')} " f"avg={row.get('avg_hours')} min={row.get('min_hours')} max={row.get('max_hours')}" @@ -515,9 +515,9 @@ def _run_examples(client): """ - log_call("client.query.fetch_xml(aggregate group-by project)") + log_call("client.query.fetchxml(aggregate group-by project)") try: - result = backoff(lambda: client.query.fetch_xml(xml).execute()) + result = backoff(lambda: client.query.fetchxml(xml).execute()) print(f"[OK] Hours per project ({len(result)} groups):") for r in result: print( @@ -546,9 +546,9 @@ def _run_examples(client): """ - log_call("client.query.fetch_xml(account → contact inner join)") + log_call("client.query.fetchxml(account → contact inner join)") try: - result = backoff(lambda: client.query.fetch_xml(xml).execute()) + result = backoff(lambda: client.query.fetchxml(xml).execute()) print(f"[OK] {len(result)} account-contact pairs:") for r in result: print(f" Account={r.get('name', ''):<25s} Contact={r.get('c.fullname', '')}") diff --git a/examples/advanced/sql_examples.py b/examples/advanced/sql_examples.py index 8c1dd55a..372a3567 100644 --- a/examples/advanced/sql_examples.py +++ b/examples/advanced/sql_examples.py @@ -917,7 +917,7 @@ def _run_examples(client): print( "At GA, sql_columns() is the only retained SQL schema-discovery helper.\n" "sql_select(), sql_join(), and sql_joins() were removed — write JOIN\n" - "clauses directly or use client.query.fetch_xml() for complex queries." + "clauses directly or use client.query.fetchxml() for complex queries." ) # sql_columns — still available at GA @@ -989,7 +989,7 @@ def _run_examples(client): | Pagination | OFFSET FETCH | @odata.nextLink | | Max results | 5000 per query | 5000 per page | | Column discovery | sql_columns | odata_expands (kept) | -| JOIN discovery | write manually/fetch_xml | odata_expand (deprecated)| +| JOIN discovery | write manually/fetchxml | odata_expand (deprecated)| | Lookup binding | N/A (read-only) | odata_bind | | SELECT * | YES (SDK auto-expands) | Not applicable | | Polymorphic lookups | Separate JOINs | $expand by nav prop | @@ -1102,7 +1102,7 @@ def _run_examples(client): - Pattern #2 (cartesian FROM a, b) -> UserWarning (advisory). - Server enforces 5000-row cap on all queries (#3, #5). - Use sql_columns() to discover valid column names. - - Write JOIN clauses manually or use fetch_xml() for complex queries. + - Write JOIN clauses manually or use fetchxml() for complex queries. """) # ============================================================== @@ -1135,7 +1135,7 @@ def _run_examples(client): | Audit trail (createdby, etc.) | YES | JOIN to systemuser | | SQL read -> DF write-back | YES | dataframe.sql() + .update()/.create() | | SQL column discovery | YES | query.sql_columns() | -| SQL JOIN clause | manual | write directly or use fetch_xml() | +| SQL JOIN clause | manual | write directly or use fetchxml() | | OData column discovery | YES | query.odata_select() | | OData expand discovery | YES | query.odata_expands() / odata_expand() | | OData bind builder | YES | query.odata_bind() | diff --git a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md index 32c2b275..00f5a7fa 100644 --- a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md +++ b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md @@ -260,7 +260,7 @@ for record in results: ### FetchXML Queries -`client.query.fetch_xml(xml)` returns an inert `FetchXmlQuery` object — **no HTTP request is made** until `.execute()` or `.execute_pages()` is called. +`client.query.fetchxml(xml)` returns an inert `FetchXmlQuery` object — **no HTTP request is made** until `.execute()` or `.execute_pages()` is called. ```python xml = """ @@ -276,7 +276,7 @@ xml = """ """ # Load all results into memory (simple, small-to-medium sets) -query = client.query.fetch_xml(xml) +query = client.query.fetchxml(xml) result = query.execute() # returns QueryResult — all pages fetched upfront for record in result: print(record["name"]) diff --git a/src/PowerPlatform/Dataverse/models/fetch_xml_query.py b/src/PowerPlatform/Dataverse/models/fetchxml_query.py similarity index 83% rename from src/PowerPlatform/Dataverse/models/fetch_xml_query.py rename to src/PowerPlatform/Dataverse/models/fetchxml_query.py index cd08c0d1..dbd05502 100644 --- a/src/PowerPlatform/Dataverse/models/fetch_xml_query.py +++ b/src/PowerPlatform/Dataverse/models/fetchxml_query.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""FetchXmlQuery — inert query object returned by QueryOperations.fetch_xml().""" +"""FetchXmlQuery — inert query object returned by QueryOperations.fetchxml().""" from __future__ import annotations @@ -26,7 +26,7 @@ class FetchXmlQuery: """Inert FetchXML query object. No HTTP request is made until :meth:`execute` or :meth:`execute_pages` is called. - Obtained via ``client.query.fetch_xml(xml)``. + Obtained via ``client.query.fetchxml(xml)``. :param xml: Stripped, well-formed FetchXML string. :param entity_name: Entity schema name from the ```` element. @@ -50,7 +50,7 @@ def execute(self) -> QueryResult: Example:: - rows = client.query.fetch_xml(xml).execute() + rows = client.query.fetchxml(xml).execute() df = rows.to_dataframe() """ all_records: List[Record] = [] @@ -76,7 +76,7 @@ def execute_pages(self) -> Iterator[QueryResult]: Example:: - for page in client.query.fetch_xml(xml).execute_pages(): + for page in client.query.fetchxml(xml).execute_pages(): process(page.to_dataframe()) """ current_xml = self._xml @@ -113,8 +113,18 @@ def execute_pages(self) -> Iterator[QueryResult]: if not raw_cookie: break - cookie = _url_unquote(_url_unquote(raw_cookie)) - page_num += 1 + # The annotation is outer XML: + # The pagingcookie attribute is double URL-encoded; decode twice to get raw cookie XML. + try: + cookie_el = _ET.fromstring(raw_cookie) + except _ET.ParseError: + break + inner_encoded = cookie_el.get("pagingcookie", "") + if not inner_encoded: + break + cookie = _url_unquote(_url_unquote(inner_encoded)) + page_num = int(cookie_el.get("pagenumber", str(page_num + 1))) + fetch_el = _ET.fromstring(current_xml) fetch_el.set("paging-cookie", cookie) fetch_el.set("page", str(page_num)) diff --git a/src/PowerPlatform/Dataverse/models/filters.py b/src/PowerPlatform/Dataverse/models/filters.py index f0aca7d4..2a7929cc 100644 --- a/src/PowerPlatform/Dataverse/models/filters.py +++ b/src/PowerPlatform/Dataverse/models/filters.py @@ -348,7 +348,7 @@ def _compile_like(column: str, pattern: str) -> FilterExpression: if _LIKE_WILDCARD in inner: raise ValueError( f"like() pattern {pattern!r} is not reducible to a single OData function. " - "Use raw(), fetch_xml(), or query.sql() for complex wildcard patterns." + "Use raw(), fetchxml(), or query.sql() for complex wildcard patterns." ) if not has_start and has_end: diff --git a/src/PowerPlatform/Dataverse/operations/query.py b/src/PowerPlatform/Dataverse/operations/query.py index d6ab9283..84b1ec52 100644 --- a/src/PowerPlatform/Dataverse/operations/query.py +++ b/src/PowerPlatform/Dataverse/operations/query.py @@ -10,7 +10,7 @@ from typing import Any, Dict, List, Optional, TYPE_CHECKING from ..core.errors import MetadataError -from ..models.fetch_xml_query import FetchXmlQuery +from ..models.fetchxml_query import FetchXmlQuery from ..models.record import Record from ..models.query_builder import QueryBuilder @@ -149,15 +149,15 @@ def sql(self, sql: str) -> List[Record]: rows = od._query_sql(sql) return [Record.from_api_response("", row) for row in rows] - # --------------------------------------------------------------- fetch_xml + # --------------------------------------------------------------- fetchxml - def fetch_xml(self, xml: str) -> FetchXmlQuery: - """Return an inert :class:`~PowerPlatform.Dataverse.models.fetch_xml_query.FetchXmlQuery` object. + def fetchxml(self, xml: str) -> FetchXmlQuery: + """Return an inert :class:`~PowerPlatform.Dataverse.models.fetchxml_query.FetchXmlQuery` object. No HTTP request is made until - :meth:`~PowerPlatform.Dataverse.models.fetch_xml_query.FetchXmlQuery.execute` + :meth:`~PowerPlatform.Dataverse.models.fetchxml_query.FetchXmlQuery.execute` or - :meth:`~PowerPlatform.Dataverse.models.fetch_xml_query.FetchXmlQuery.execute_pages` + :meth:`~PowerPlatform.Dataverse.models.fetchxml_query.FetchXmlQuery.execute_pages` is called on the returned object. Use for SQL-JOIN scenarios, aggregate queries, or other operations that @@ -167,13 +167,13 @@ def fetch_xml(self, xml: str) -> FetchXmlQuery: element determines the entity set endpoint. :type xml: :class:`str` :return: Inert query object with ``.execute()`` and ``.execute_pages()`` methods. - :rtype: :class:`~PowerPlatform.Dataverse.models.fetch_xml_query.FetchXmlQuery` + :rtype: :class:`~PowerPlatform.Dataverse.models.fetchxml_query.FetchXmlQuery` :raises ValueError: If the FetchXML is missing a root ```` element or the entity ``name`` attribute. Example:: - query = client.query.fetch_xml(\"\"\" + query = client.query.fetchxml(\"\"\" diff --git a/tests/unit/models/test_query_builder.py b/tests/unit/models/test_query_builder.py index 7ffc09f8..f40d1e5d 100644 --- a/tests/unit/models/test_query_builder.py +++ b/tests/unit/models/test_query_builder.py @@ -893,12 +893,14 @@ def test_to_dataframe_with_record_objects(self): import pandas as pd from PowerPlatform.Dataverse.models.record import Record - mock_query_ops, _ = self._make_od([ + mock_query_ops, _ = self._make_od( [ - {"name": "Contoso", "revenue": 1000}, - {"name": "Fabrikam", "revenue": 2000}, + [ + {"name": "Contoso", "revenue": 1000}, + {"name": "Fabrikam", "revenue": 2000}, + ] ] - ]) + ) qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name", "revenue") diff --git a/tests/unit/test_phase4_ga.py b/tests/unit/test_phase4_ga.py index 6ea41b52..ed28cbeb 100644 --- a/tests/unit/test_phase4_ga.py +++ b/tests/unit/test_phase4_ga.py @@ -4,7 +4,7 @@ """Phase 4 GA regression tests. Covers: -- fetch_xml(): basic, pagination, missing-entity-element error +- fetchxml(): basic, pagination, missing-entity-element error - sql_select / sql_join / sql_joins raise AttributeError (removed at GA) - odata_select / odata_expand / odata_bind emit DeprecationWarning (deprecated at GA) - sql_columns / odata_expands / sql() emit zero DeprecationWarning (still GA-clean) @@ -32,7 +32,7 @@ def _make_client(): # --------------------------------------------------------------------------- -# fetch_xml() +# fetchxml() # --------------------------------------------------------------------------- @@ -55,32 +55,32 @@ def _mock_response(self, records, more=False, cookie=""): resp.json.return_value = payload return resp - def test_fetch_xml_inert_no_http_request(self): - """fetch_xml() alone must not fire any HTTP request.""" - from PowerPlatform.Dataverse.models.fetch_xml_query import FetchXmlQuery + def test_fetchxml_inert_no_http_request(self): + """fetchxml() alone must not fire any HTTP request.""" + from PowerPlatform.Dataverse.models.fetchxml_query import FetchXmlQuery - query = self.client.query.fetch_xml(self._fetch_xml()) + query = self.client.query.fetchxml(self._fetch_xml()) self.assertIsInstance(query, FetchXmlQuery) self.client._odata._request.assert_not_called() def test_basic_returns_query_result(self): self.client._odata._request.return_value = self._mock_response([{"name": "Contoso", "accountid": "1"}]) - result = self.client.query.fetch_xml(self._fetch_xml()).execute() + result = self.client.query.fetchxml(self._fetch_xml()).execute() self.assertIsInstance(result, QueryResult) def test_basic_record_count(self): self.client._odata._request.return_value = self._mock_response([{"name": "A"}, {"name": "B"}]) - result = self.client.query.fetch_xml(self._fetch_xml()).execute() + result = self.client.query.fetchxml(self._fetch_xml()).execute() self.assertEqual(len(result), 2) def test_record_values_accessible(self): self.client._odata._request.return_value = self._mock_response([{"name": "Contoso", "accountid": "abc-123"}]) - result = self.client.query.fetch_xml(self._fetch_xml()).execute() + result = self.client.query.fetchxml(self._fetch_xml()).execute() self.assertEqual(result.first()["name"], "Contoso") def test_empty_result_returns_empty_query_result(self): self.client._odata._request.return_value = self._mock_response([]) - result = self.client.query.fetch_xml(self._fetch_xml()).execute() + result = self.client.query.fetchxml(self._fetch_xml()).execute() self.assertIsInstance(result, QueryResult) self.assertEqual(len(result), 0) self.assertFalse(result) @@ -92,7 +92,7 @@ def test_pagination_fetches_all_pages(self): page2 = self._mock_response([{"name": "B"}], more=False) self.client._odata._request.side_effect = [page1, page2] - pages = list(self.client.query.fetch_xml(self._fetch_xml()).execute_pages()) + pages = list(self.client.query.fetchxml(self._fetch_xml()).execute_pages()) self.assertEqual(len(pages), 2) self.assertEqual(self.client._odata._request.call_count, 2) @@ -103,7 +103,7 @@ def test_pagination_second_request_includes_page_and_cookie(self): page2 = self._mock_response([{"name": "B"}], more=False) self.client._odata._request.side_effect = [page1, page2] - list(self.client.query.fetch_xml(self._fetch_xml()).execute_pages()) + list(self.client.query.fetchxml(self._fetch_xml()).execute_pages()) second_call_kwargs = self.client._odata._request.call_args_list[1] params = ( @@ -120,22 +120,22 @@ def test_pagination_second_request_includes_page_and_cookie(self): def test_missing_entity_element_raises_value_error(self): with self.assertRaises(ValueError) as ctx: - self.client.query.fetch_xml("") + self.client.query.fetchxml("") self.assertIn("entity", str(ctx.exception).lower()) def test_entity_missing_name_attr_raises_value_error(self): with self.assertRaises(ValueError) as ctx: - self.client.query.fetch_xml("") + self.client.query.fetchxml("") self.assertIn("name", str(ctx.exception).lower()) def test_entity_set_resolved_from_entity_name(self): self.client._odata._request.return_value = self._mock_response([]) - self.client.query.fetch_xml(self._fetch_xml("account")).execute() + self.client.query.fetchxml(self._fetch_xml("account")).execute() self.client._odata._entity_set_from_schema_name.assert_called_with("account") def test_request_uses_prefer_header(self): self.client._odata._request.return_value = self._mock_response([]) - self.client.query.fetch_xml(self._fetch_xml()).execute() + self.client.query.fetchxml(self._fetch_xml()).execute() call_kwargs = self.client._odata._request.call_args headers = call_kwargs.kwargs.get("headers", {}) self.assertIn("Prefer", headers) @@ -143,7 +143,7 @@ def test_request_uses_prefer_header(self): def test_result_iterable(self): self.client._odata._request.return_value = self._mock_response([{"name": "A"}, {"name": "B"}]) - result = self.client.query.fetch_xml(self._fetch_xml()).execute() + result = self.client.query.fetchxml(self._fetch_xml()).execute() names = [r["name"] for r in result] self.assertEqual(names, ["A", "B"]) @@ -153,7 +153,7 @@ def test_result_to_dataframe(self): except ImportError: self.skipTest("pandas not installed") self.client._odata._request.return_value = self._mock_response([{"name": "Contoso"}, {"name": "Fabrikam"}]) - result = self.client.query.fetch_xml(self._fetch_xml()).execute() + result = self.client.query.fetchxml(self._fetch_xml()).execute() df = result.to_dataframe() self.assertIsInstance(df, pd.DataFrame) self.assertEqual(len(df), 2) @@ -162,9 +162,9 @@ def test_no_deprecation_warning_emitted(self): self.client._odata._request.return_value = self._mock_response([]) with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") - self.client.query.fetch_xml(self._fetch_xml()).execute() + self.client.query.fetchxml(self._fetch_xml()).execute() deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] - self.assertEqual(len(deprecations), 0, "fetch_xml().execute() should not emit DeprecationWarning") + self.assertEqual(len(deprecations), 0, "fetchxml().execute() should not emit DeprecationWarning") def test_execute_pages_returns_iterator_of_query_result(self): """execute_pages() yields QueryResult objects, one per HTTP page.""" @@ -173,7 +173,7 @@ def test_execute_pages_returns_iterator_of_query_result(self): page2 = self._mock_response([{"name": "B"}], more=False) self.client._odata._request.side_effect = [page1, page2] - pages = list(self.client.query.fetch_xml(self._fetch_xml()).execute_pages()) + pages = list(self.client.query.fetchxml(self._fetch_xml()).execute_pages()) self.assertEqual(len(pages), 2) for page in pages: self.assertIsInstance(page, QueryResult) @@ -186,7 +186,7 @@ def test_execute_pages_one_http_call_per_page(self): self.client._odata._request.side_effect = [page1, page2] count = 0 - for _page in self.client.query.fetch_xml(self._fetch_xml()).execute_pages(): + for _page in self.client.query.fetchxml(self._fetch_xml()).execute_pages(): count += 1 self.assertEqual(self.client._odata._request.call_count, 2) self.assertEqual(count, 2) @@ -198,7 +198,7 @@ def test_execute_pages_per_page_records(self): page2 = self._mock_response([{"name": "B"}, {"name": "C"}], more=False) self.client._odata._request.side_effect = [page1, page2] - pages = list(self.client.query.fetch_xml(self._fetch_xml()).execute_pages()) + pages = list(self.client.query.fetchxml(self._fetch_xml()).execute_pages()) self.assertEqual(len(pages[0]), 1) self.assertEqual(len(pages[1]), 2) self.assertEqual(pages[0].first()["name"], "A") From be5be2867cdcfc521788e3ee3f467f8d405f39b1 Mon Sep 17 00:00:00 2001 From: Samson Gebre Date: Wed, 6 May 2026 11:07:53 -0700 Subject: [PATCH 04/16] Enhance FetchXmlQuery and QueryOperations with input validation and paging improvements --- .../Dataverse/models/fetchxml_query.py | 79 ++++++++--- .../Dataverse/operations/query.py | 31 ++++- tests/unit/test_phase4_ga.py | 129 +++++++++++++++++- 3 files changed, 212 insertions(+), 27 deletions(-) diff --git a/src/PowerPlatform/Dataverse/models/fetchxml_query.py b/src/PowerPlatform/Dataverse/models/fetchxml_query.py index dbd05502..3afeb350 100644 --- a/src/PowerPlatform/Dataverse/models/fetchxml_query.py +++ b/src/PowerPlatform/Dataverse/models/fetchxml_query.py @@ -5,10 +5,12 @@ from __future__ import annotations +import warnings import xml.etree.ElementTree as _ET from typing import Iterator, List, TYPE_CHECKING -from urllib.parse import unquote as _url_unquote +from urllib.parse import unquote as _url_unquote, quote as _url_quote +from ..core.errors import ValidationError from .record import QueryResult, Record if TYPE_CHECKING: @@ -21,6 +23,17 @@ "odata.include-annotations=" '"Microsoft.Dynamics.CRM.fetchxmlpagingcookie,' 'Microsoft.Dynamics.CRM.morerecords"' ) +# Documented Dataverse GET request URL limit. See: +# learn.microsoft.com/power-apps/developer/data-platform/webapi/compose-http-requests-handle-errors#maximum-url-length +# FetchXML queries with many attributes or conditions are the most common way to reach it. +# $batch POST doubles this to 64 KB. +_MAX_URL_LENGTH = 32_768 +# Guards against infinite paging loops caused by a bug in cookie propagation or an +# unexpected server response. At the default Dataverse page size of 5,000 rows this +# cap allows up to 50 million records before raising; it is not a practical record +# limit but a circuit-breaker against runaway iteration. +_MAX_PAGES = 10_000 + class FetchXmlQuery: """Inert FetchXML query object. No HTTP request is made until @@ -81,12 +94,27 @@ def execute_pages(self) -> Iterator[QueryResult]: """ current_xml = self._xml page_num = 1 + page_count = 0 with self._client._scoped_odata() as od: entity_set = od._entity_set_from_schema_name(self._entity_name) base_url = f"{od.api}/{entity_set}" while True: + page_count += 1 + if page_count > _MAX_PAGES: + raise ValidationError( + f"FetchXML paging exceeded {_MAX_PAGES} pages. " + "This may indicate a runaway query or a bug in paging cookie propagation." + ) + + encoded_len = len(base_url) + len("?fetchXml=") + len(_url_quote(current_xml, safe="")) + if encoded_len > _MAX_URL_LENGTH: + raise ValidationError( + f"FetchXML request URL exceeds {_MAX_URL_LENGTH} characters after encoding. " + "Simplify the query or reduce attributes/conditions." + ) + r = od._request( "get", base_url, @@ -103,29 +131,46 @@ def execute_pages(self) -> Iterator[QueryResult]: yield QueryResult(page_records) - more = bool(data.get("@Microsoft.Dynamics.CRM.morerecords", False)) if isinstance(data, dict) else False + more_raw = data.get("@Microsoft.Dynamics.CRM.morerecords", False) if isinstance(data, dict) else False + more = more_raw is True or (isinstance(more_raw, str) and more_raw.lower() == "true") if not more: break raw_cookie = ( data.get("@Microsoft.Dynamics.CRM.fetchxmlpagingcookie", "") if isinstance(data, dict) else "" ) - if not raw_cookie: - break - - # The annotation is outer XML: - # The pagingcookie attribute is double URL-encoded; decode twice to get raw cookie XML. - try: - cookie_el = _ET.fromstring(raw_cookie) - except _ET.ParseError: - break - inner_encoded = cookie_el.get("pagingcookie", "") - if not inner_encoded: - break - cookie = _url_unquote(_url_unquote(inner_encoded)) - page_num = int(cookie_el.get("pagenumber", str(page_num + 1))) + if raw_cookie: + try: + cookie_el = _ET.fromstring(raw_cookie) + inner_encoded = cookie_el.get("pagingcookie", "") + if inner_encoded: + cookie = _url_unquote(_url_unquote(inner_encoded)) + page_num = int(cookie_el.get("pagenumber", str(page_num + 1))) + fetch_el = _ET.fromstring(current_xml) + fetch_el.set("paging-cookie", cookie) + fetch_el.set("page", str(page_num)) + current_xml = _ET.tostring(fetch_el, encoding="unicode") + continue + except (_ET.ParseError, ValueError): + pass # Fall through to simple paging + + # Simple paging fallback: server returned morerecords=true but no paging + # cookie. Dataverse omits the cookie when the query cannot use cookie-based + # paging (e.g. FetchXML ordered by a link-entity column). We continue with + # page-number-only paging rather than truncating, but warn because simple + # paging has a 50,000-record server cap and performance degrades at high page + # numbers. The caller may be able to avoid this by reordering on the root + # entity instead. + warnings.warn( + "Dataverse did not return a paging cookie; falling back to simple paging " + "(page-number increment only). Simple paging is capped at 50,000 records " + "and degrades in performance at high page numbers. Consider reordering on " + "a root-entity column to enable cookie-based paging.", + UserWarning, + stacklevel=2, + ) + page_num += 1 fetch_el = _ET.fromstring(current_xml) - fetch_el.set("paging-cookie", cookie) fetch_el.set("page", str(page_num)) current_xml = _ET.tostring(fetch_el, encoding="unicode") diff --git a/src/PowerPlatform/Dataverse/operations/query.py b/src/PowerPlatform/Dataverse/operations/query.py index 84b1ec52..c927bdf0 100644 --- a/src/PowerPlatform/Dataverse/operations/query.py +++ b/src/PowerPlatform/Dataverse/operations/query.py @@ -8,9 +8,10 @@ import warnings import xml.etree.ElementTree as _ET from typing import Any, Dict, List, Optional, TYPE_CHECKING +from urllib.parse import quote as _url_quote -from ..core.errors import MetadataError -from ..models.fetchxml_query import FetchXmlQuery +from ..core.errors import MetadataError, ValidationError +from ..models.fetchxml_query import FetchXmlQuery, _MAX_URL_LENGTH from ..models.record import Record from ..models.query_builder import QueryBuilder @@ -193,15 +194,35 @@ def fetchxml(self, xml: str) -> FetchXmlQuery: for page in query.execute_pages(): process(page.to_dataframe()) """ - root_el = _ET.fromstring(xml.strip()) + if not isinstance(xml, str): + raise ValidationError("xml must be a string") + xml = xml.strip() + if not xml: + raise ValidationError("xml must not be empty") + # Fast-fail before any HTTP is attempted; execute_pages() re-checks the full URL + # (base + encoded XML) on each page. + if len(_url_quote(xml, safe="")) > _MAX_URL_LENGTH: + raise ValidationError( + f"FetchXML exceeds the Dataverse URL length limit ({_MAX_URL_LENGTH:,} characters) when encoded. " + "Use a $batch POST request to send FetchXML in the request body where the limit is 64 KB." + ) + # Parse only to verify well-formedness and extract the entity name needed for the + # request URL. Structural and semantic validation is intentionally left to the server + # to avoid duplicating rules that may diverge from Dataverse's own enforcement. + # ElementTree does not resolve external entities or expand recursive internal entity + # references, so pathological inputs of that kind raise ParseError rather than + # consuming resources. + try: + root_el = _ET.fromstring(xml) + except _ET.ParseError as exc: + raise ValidationError(f"xml is not well-formed: {exc}") from exc entity_el = root_el.find("entity") if entity_el is None: raise ValueError("FetchXML must contain an child element") entity_name = entity_el.get("name", "") if not entity_name: raise ValueError("FetchXML element must have a 'name' attribute") - - return FetchXmlQuery(xml.strip(), entity_name, self._client) + return FetchXmlQuery(xml, entity_name, self._client) # --------------------------------------------------------------- sql_columns diff --git a/tests/unit/test_phase4_ga.py b/tests/unit/test_phase4_ga.py index ed28cbeb..fe843c28 100644 --- a/tests/unit/test_phase4_ga.py +++ b/tests/unit/test_phase4_ga.py @@ -87,7 +87,8 @@ def test_empty_result_returns_empty_query_result(self): def test_pagination_fetches_all_pages(self): """execute_pages() drives the HTTP loop; each page yields one QueryResult.""" - cookie_raw = "%25253Cpagingcookie%252520pagingcookie%25253D%252522" + # Annotation is outer XML; pagingcookie attribute is double URL-encoded inner cookie. + cookie_raw = '' page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) page2 = self._mock_response([{"name": "B"}], more=False) self.client._odata._request.side_effect = [page1, page2] @@ -98,7 +99,8 @@ def test_pagination_fetches_all_pages(self): def test_pagination_second_request_includes_page_and_cookie(self): """execute_pages() injects the decoded paging cookie into the second request.""" - cookie_raw = "%25253Cpagingcookie%252520test%25253D%252522" + # pagingcookie="%253Cc%252F%253E": double URL-decode gives "" (the inner cookie XML). + cookie_raw = '' page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) page2 = self._mock_response([{"name": "B"}], more=False) self.client._odata._request.side_effect = [page1, page2] @@ -168,7 +170,7 @@ def test_no_deprecation_warning_emitted(self): def test_execute_pages_returns_iterator_of_query_result(self): """execute_pages() yields QueryResult objects, one per HTTP page.""" - cookie_raw = "%25253Cpagingcookie%252520pagingcookie%25253D%252522" + cookie_raw = '' page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) page2 = self._mock_response([{"name": "B"}], more=False) self.client._odata._request.side_effect = [page1, page2] @@ -180,7 +182,7 @@ def test_execute_pages_returns_iterator_of_query_result(self): def test_execute_pages_one_http_call_per_page(self): """Each execute_pages() iteration fires exactly one HTTP request.""" - cookie_raw = "%25253Cpagingcookie%252520pagingcookie%25253D%252522" + cookie_raw = '' page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) page2 = self._mock_response([{"name": "B"}], more=False) self.client._odata._request.side_effect = [page1, page2] @@ -193,7 +195,7 @@ def test_execute_pages_one_http_call_per_page(self): def test_execute_pages_per_page_records(self): """Each page yielded by execute_pages() contains only its own records.""" - cookie_raw = "%25253Cpagingcookie%252520pagingcookie%25253D%252522" + cookie_raw = '' page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) page2 = self._mock_response([{"name": "B"}, {"name": "C"}], more=False) self.client._odata._request.side_effect = [page1, page2] @@ -204,6 +206,123 @@ def test_execute_pages_per_page_records(self): self.assertEqual(pages[0].first()["name"], "A") self.assertEqual(pages[1].first()["name"], "B") + # ------------------------------------------------------------------ + # Input validation + # ------------------------------------------------------------------ + + def test_non_string_input_raises_validation_error(self): + from PowerPlatform.Dataverse.core.errors import ValidationError + + with self.assertRaises(ValidationError): + self.client.query.fetchxml(123) + + def test_empty_string_raises_validation_error(self): + from PowerPlatform.Dataverse.core.errors import ValidationError + + with self.assertRaises(ValidationError): + self.client.query.fetchxml("") + + def test_whitespace_only_raises_validation_error(self): + from PowerPlatform.Dataverse.core.errors import ValidationError + + with self.assertRaises(ValidationError): + self.client.query.fetchxml(" ") + + def test_malformed_xml_raises_validation_error(self): + from PowerPlatform.Dataverse.core.errors import ValidationError + + with self.assertRaises(ValidationError): + self.client.query.fetchxml("") + + def test_url_too_long_raises_validation_error(self): + """XML whose URL-encoded form exceeds 32,768 chars is rejected before any HTTP.""" + from PowerPlatform.Dataverse.core.errors import ValidationError + + # Alphanumeric chars are URL-safe and don't expand; a 32,769-char name attribute + # value pushes the encoded XML over the limit. + long_name = "a" * 32_769 + big_xml = f'' + with self.assertRaises(ValidationError): + self.client.query.fetchxml(big_xml) + + # ------------------------------------------------------------------ + # Paging behaviour + # ------------------------------------------------------------------ + + def test_morerecords_string_true_continues_paging(self): + """morerecords annotation as string "true" (not bool) is handled correctly.""" + cookie_raw = '' + page1_payload = { + "value": [{"name": "A"}], + "@Microsoft.Dynamics.CRM.morerecords": "true", + "@Microsoft.Dynamics.CRM.fetchxmlpagingcookie": cookie_raw, + } + page2_payload = { + "value": [{"name": "B"}], + "@Microsoft.Dynamics.CRM.morerecords": False, + } + r1, r2 = MagicMock(), MagicMock() + r1.json.return_value = page1_payload + r2.json.return_value = page2_payload + self.client._odata._request.side_effect = [r1, r2] + + result = self.client.query.fetchxml(self._fetch_xml()).execute() + self.assertEqual(len(result), 2) + self.assertEqual(self.client._odata._request.call_count, 2) + + def test_simple_paging_fallback_emits_user_warning(self): + """No cookie returned with morerecords=True triggers a UserWarning.""" + page1 = self._mock_response([{"name": "A"}], more=True, cookie="") + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + list(self.client.query.fetchxml(self._fetch_xml()).execute_pages()) + + user_warnings = [w for w in caught if issubclass(w.category, UserWarning)] + self.assertEqual(len(user_warnings), 1) + self.assertIn("simple paging", str(user_warnings[0].message).lower()) + + def test_simple_paging_fallback_fetches_all_pages(self): + """Simple paging fallback continues iterating; caller still gets all records.""" + page1 = self._mock_response([{"name": "A"}], more=True, cookie="") + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + result = self.client.query.fetchxml(self._fetch_xml()).execute() + + self.assertEqual(len(result), 2) + self.assertEqual(self.client._odata._request.call_count, 2) + + def test_malformed_cookie_falls_back_to_simple_paging(self): + """A cookie annotation that is not parseable as XML triggers simple paging fallback.""" + page1 = self._mock_response([{"name": "A"}], more=True, cookie="not-valid-xml") + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = self.client.query.fetchxml(self._fetch_xml()).execute() + + self.assertEqual(len(result), 2) + user_warnings = [w for w in caught if issubclass(w.category, UserWarning)] + self.assertEqual(len(user_warnings), 1) + + def test_max_pages_exceeded_raises(self): + """Paging loop raises ValidationError after exceeding _MAX_PAGES.""" + from PowerPlatform.Dataverse.core.errors import ValidationError + + cookie_raw = '' + always_more = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) + self.client._odata._request.return_value = always_more + + with patch("PowerPlatform.Dataverse.models.fetchxml_query._MAX_PAGES", 3): + with self.assertRaises(ValidationError): + list(self.client.query.fetchxml(self._fetch_xml()).execute_pages()) + # --------------------------------------------------------------------------- # Removed SQL helpers — raise AttributeError From c2090f7e65e801a08a736966c1b9b2daee52c8fd Mon Sep 17 00:00:00 2001 From: Samson Gebre Date: Wed, 6 May 2026 13:26:43 -0700 Subject: [PATCH 05/16] Enhance SKILL.md and examples with improved documentation and usage of QueryResult indexing --- .claude/skills/dataverse-sdk-use/SKILL.md | 3 +- examples/advanced/fetchxml.py | 4 + examples/basic/installation_example.py | 5 +- .../claude_skill/dataverse-sdk-use/SKILL.md | 3 +- .../Dataverse/models/query_builder.py | 8 +- src/PowerPlatform/Dataverse/models/record.py | 4 + .../Dataverse/operations/records.py | 72 ++------------- tests/unit/test_phase2_ga.py | 31 +++++++ tests/unit/test_phase3_ga.py | 89 +------------------ tests/unit/test_records_operations.py | 1 - 10 files changed, 60 insertions(+), 160 deletions(-) diff --git a/.claude/skills/dataverse-sdk-use/SKILL.md b/.claude/skills/dataverse-sdk-use/SKILL.md index 00f5a7fa..44a1e861 100644 --- a/.claude/skills/dataverse-sdk-use/SKILL.md +++ b/.claude/skills/dataverse-sdk-use/SKILL.md @@ -38,7 +38,8 @@ The SDK supports Dataverse's native bulk operations: Pass lists to `create()`, ` - Returned by `records.list()`, `records.retrieve()`, `execute()`, and each page from `list_pages()` / `execute_pages()` - Iterable: `for record in result` — each item is a `dict`-like `Record` - `.to_dataframe()` — convert to pandas DataFrame -- `.first()` — return the first record or `None` +- `.first()` — return the first record or `None` (safe: returns `None` on empty result) +- `result[n]` — index access returns a `Record`; `result[n:m]` returns a `QueryResult` - `len(result)` — number of records in this result/page ### DataFrame Support diff --git a/examples/advanced/fetchxml.py b/examples/advanced/fetchxml.py index 9bfecc33..d4ac1e50 100644 --- a/examples/advanced/fetchxml.py +++ b/examples/advanced/fetchxml.py @@ -268,6 +268,10 @@ def _run_examples(client): print(f"[OK] {len(result)} projects:") for r in result: print(f" {r.get('new_code', ''):<10s} Budget={r.get('new_budget')} Active={r.get('new_active')}") + # Index access and first() are equivalent; first() returns None on empty result + if result: + print(f" First by index : {result[0].get('new_code')}") + print(f" First by .first(): {result.first().get('new_code')}") # =============================================================== # 4. operators: eq, like, in, null, not-null, between diff --git a/examples/basic/installation_example.py b/examples/basic/installation_example.py index f7821181..61da149b 100644 --- a/examples/basic/installation_example.py +++ b/examples/basic/installation_example.py @@ -239,9 +239,8 @@ def show_usage_examples(): select=["name", "telephone1"], top=10) -for page in accounts: - for account in page: - print(f"Account: {account['name']}") +for account in accounts: + print(f"Account: {account['name']}") # SQL queries (if enabled) results = client.query.sql("SELECT TOP 5 name FROM account") diff --git a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md index 00f5a7fa..44a1e861 100644 --- a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md +++ b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md @@ -38,7 +38,8 @@ The SDK supports Dataverse's native bulk operations: Pass lists to `create()`, ` - Returned by `records.list()`, `records.retrieve()`, `execute()`, and each page from `list_pages()` / `execute_pages()` - Iterable: `for record in result` — each item is a `dict`-like `Record` - `.to_dataframe()` — convert to pandas DataFrame -- `.first()` — return the first record or `None` +- `.first()` — return the first record or `None` (safe: returns `None` on empty result) +- `result[n]` — index access returns a `Record`; `result[n:m]` returns a `QueryResult` - `len(result)` — number of records in this result/page ### DataFrame Support diff --git a/src/PowerPlatform/Dataverse/models/query_builder.py b/src/PowerPlatform/Dataverse/models/query_builder.py index 52cd8155..e7044d2e 100644 --- a/src/PowerPlatform/Dataverse/models/query_builder.py +++ b/src/PowerPlatform/Dataverse/models/query_builder.py @@ -513,9 +513,9 @@ def execute(self, *, by_page=_BY_PAGE_UNSET) -> Union[QueryResult, Iterator[Quer "Use build() and pass parameters to client.records.list() instead." ) - if not self._select and not self._filter_parts and self._top is None: + if not self._select and not self._filter_parts and self._top is None and self._page_size is None: raise ValueError( - "At least one of select(), where(), or top() must be called before " + "At least one of select(), where(), top(), or page_size() must be called before " "execute() to prevent accidental full-table scans." ) @@ -577,9 +577,9 @@ def execute_pages(self) -> Iterator[QueryResult]: "Use build() and pass parameters to client.records.list() instead." ) - if not self._select and not self._filter_parts and self._top is None: + if not self._select and not self._filter_parts and self._top is None and self._page_size is None: raise ValueError( - "At least one of select(), where(), or top() must be called before " + "At least one of select(), where(), top(), or page_size() must be called before " "execute_pages() to prevent accidental full-table scans." ) diff --git a/src/PowerPlatform/Dataverse/models/record.py b/src/PowerPlatform/Dataverse/models/record.py index f7a47941..e81f412c 100644 --- a/src/PowerPlatform/Dataverse/models/record.py +++ b/src/PowerPlatform/Dataverse/models/record.py @@ -141,6 +141,10 @@ def __bool__(self) -> bool: def __repr__(self) -> str: return f"QueryResult({len(self.records)} records)" + def __getitem__(self, index): + result = self.records[index] + return QueryResult(result) if isinstance(index, slice) else result + def first(self) -> Optional[Record]: """Return the first record, or ``None`` if the result is empty.""" return self.records[0] if self.records else None diff --git a/src/PowerPlatform/Dataverse/operations/records.py b/src/PowerPlatform/Dataverse/operations/records.py index dc8a5129..1eb3274d 100644 --- a/src/PowerPlatform/Dataverse/operations/records.py +++ b/src/PowerPlatform/Dataverse/operations/records.py @@ -8,7 +8,6 @@ import warnings from typing import Any, Dict, Iterable, Iterator, List, Optional, Union, overload, TYPE_CHECKING -from ..models.protocol import DataverseModel from ..models.record import QueryResult, Record from ..models.upsert import UpsertItem @@ -57,16 +56,10 @@ def create(self, table: str, data: Dict[str, Any]) -> str: ... @overload def create(self, table: str, data: List[Dict[str, Any]]) -> List[str]: ... - @overload - def create(self, table_or_entity: DataverseModel, data: None = None) -> str: ... - - @overload - def create(self, table_or_entity: List[DataverseModel], data: None = None) -> List[str]: ... - def create( self, - table_or_entity: Union[str, DataverseModel, List[DataverseModel]], - data: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None, + table: str, + data: Union[Dict[str, Any], List[Dict[str, Any]]], ) -> Union[str, List[str]]: """Create one or more records in a Dataverse table. @@ -87,49 +80,19 @@ def create( :raises TypeError: If ``data`` is not a dict or list[dict]. Example: - Create a single record (dict form):: + Create a single record:: guid = client.records.create("account", {"name": "Contoso"}) print(f"Created: {guid}") - Create multiple records (dict form):: + Create multiple records:: guids = client.records.create("account", [ {"name": "Contoso"}, {"name": "Fabrikam"}, ]) print(f"Created {len(guids)} accounts") - - Create from a DataverseModel instance:: - - guid = client.records.create(Account(name="Contoso")) """ - # DataverseModel dispatch: list of entities - if isinstance(table_or_entity, list) and table_or_entity and isinstance(table_or_entity[0], DataverseModel): - entities = table_or_entity - table = entities[0].__entity_logical_name__ - data_list = [e.to_dict() for e in entities] - with self._client._scoped_odata() as od: - entity_set = od._entity_set_from_schema_name(table) - ids = od._create_multiple(entity_set, table, data_list) - if not isinstance(ids, list) or not all(isinstance(x, str) for x in ids): - raise TypeError("_create (multi) did not return list[str]") - return ids - - # DataverseModel dispatch: single entity - if isinstance(table_or_entity, DataverseModel): - entity = table_or_entity - table = entity.__entity_logical_name__ - record_data = entity.to_dict() - with self._client._scoped_odata() as od: - entity_set = od._entity_set_from_schema_name(table) - rid = od._create(entity_set, table, record_data) - if not isinstance(rid, str): - raise TypeError("_create (single) did not return GUID string") - return rid - - # Existing str/dict path - table = table_or_entity # type: ignore[assignment] with self._client._scoped_odata() as od: entity_set = od._entity_set_from_schema_name(table) if isinstance(data, dict): @@ -148,9 +111,9 @@ def create( def update( self, - table_or_entity: Union[str, DataverseModel], - ids: Optional[Union[str, List[str]]] = None, - changes: Optional[Union[Dict[str, Any], List[Dict[str, Any]]]] = None, + table: str, + ids: Union[str, List[str]], + changes: Union[Dict[str, Any], List[Dict[str, Any]]], ) -> None: """Update one or more records in a Dataverse table. @@ -191,28 +154,7 @@ def update( [{"name": "Name A"}, {"name": "Name B"}], ) - Update from a DataverseModel instance:: - - client.records.update(Account(name="Contoso Updated"), account_id) """ - # DataverseModel dispatch: entity provides table + changes - if isinstance(table_or_entity, DataverseModel): - entity = table_or_entity - table = entity.__entity_logical_name__ - record_data = entity.to_dict() - if ids is None: - raise TypeError("record_id must be provided when updating from a DataverseModel") - with self._client._scoped_odata() as od: - if isinstance(ids, str): - od._update(table, ids, record_data) - return None - if isinstance(ids, list): - od._update_by_ids(table, ids, record_data) - return None - return None - - # Existing str/dict path - table: str = table_or_entity # type: ignore[assignment] with self._client._scoped_odata() as od: if isinstance(ids, str): if not isinstance(changes, dict): diff --git a/tests/unit/test_phase2_ga.py b/tests/unit/test_phase2_ga.py index b8a0aa5d..acd4263e 100644 --- a/tests/unit/test_phase2_ga.py +++ b/tests/unit/test_phase2_ga.py @@ -84,6 +84,37 @@ def test_first_returns_first_record(self): def test_first_returns_none_when_empty(self): self.assertIsNone(QueryResult([]).first()) + # ----- __getitem__ + + def test_getitem_int_returns_record(self): + recs = self._records(3) + qr = QueryResult(recs) + self.assertIs(qr[0], recs[0]) + self.assertIs(qr[2], recs[2]) + + def test_getitem_negative_index(self): + recs = self._records(3) + qr = QueryResult(recs) + self.assertIs(qr[-1], recs[-1]) + + def test_getitem_out_of_range_raises(self): + qr = QueryResult(self._records(2)) + with self.assertRaises(IndexError): + _ = qr[99] + + def test_getitem_slice_returns_query_result(self): + recs = self._records(5) + qr = QueryResult(recs) + sliced = qr[1:3] + self.assertIsInstance(sliced, QueryResult) + self.assertEqual(list(sliced), recs[1:3]) + + def test_getitem_slice_empty(self): + qr = QueryResult(self._records(3)) + sliced = qr[10:] + self.assertIsInstance(sliced, QueryResult) + self.assertEqual(len(sliced), 0) + # ----- to_dataframe() def test_to_dataframe_nonempty(self): diff --git a/tests/unit/test_phase3_ga.py b/tests/unit/test_phase3_ga.py index 34a3cb06..2451da8c 100644 --- a/tests/unit/test_phase3_ga.py +++ b/tests/unit/test_phase3_ga.py @@ -7,11 +7,12 @@ - records.get() deprecation (DeprecationWarning, still functional) - records.retrieve() — single record, None on 404 - records.list() — QueryResult, accepts str/FilterExpression filter -- DataverseModel Protocol and isinstance() check -- records.create(DataverseModel) dispatch -- records.update(DataverseModel, record_id) dispatch +- DataverseModel Protocol definition and isinstance() check - DataverseModel exported from models and package root - execute() emits zero DeprecationWarning (internal records.get() suppressed) + +Note: records.create(DataverseModel) and records.update(DataverseModel) dispatch +are deferred to post-GA and are not covered here. """ import unittest @@ -310,88 +311,6 @@ def test_list_result_to_dataframe(self): self.assertEqual(len(df), 2) -class TestCreateWithDataverseModel(unittest.TestCase): - """records.create() accepts DataverseModel.""" - - def setUp(self): - self.client = _make_client() - self.client._odata._create.return_value = "new-guid-123" - self.client._odata._create_multiple.return_value = ["guid-1", "guid-2"] - - def test_create_single_entity(self): - account = _Account(name="Contoso", telephone1="555-0100") - result = self.client.records.create(account) - self.assertEqual(result, "new-guid-123") - - def test_create_single_entity_uses_logical_name(self): - account = _Account(name="Contoso") - self.client.records.create(account) - self.client._odata._entity_set_from_schema_name.assert_called_with("account") - - def test_create_single_entity_calls_to_dict(self): - account = _Account(name="Contoso", telephone1="555-0100") - self.client.records.create(account) - self.client._odata._create.assert_called_once() - call_args = self.client._odata._create.call_args - self.assertEqual(call_args[0][2]["name"], "Contoso") - - def test_create_list_of_entities(self): - entities = [_Account(name="A"), _Account(name="B")] - result = self.client.records.create(entities) - self.assertEqual(result, ["guid-1", "guid-2"]) - - def test_create_list_uses_first_entity_logical_name(self): - entities = [_Account(name="A"), _Account(name="B")] - self.client.records.create(entities) - self.client._odata._entity_set_from_schema_name.assert_called_with("account") - - def test_create_entity_no_deprecation_warning(self): - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - self.client.records.create(_Account(name="Contoso")) - dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] - self.assertEqual(dep, []) - - def test_create_dict_path_still_works(self): - result = self.client.records.create("account", {"name": "Contoso"}) - self.assertEqual(result, "new-guid-123") - - def test_create_list_dict_path_still_works(self): - result = self.client.records.create("account", [{"name": "A"}, {"name": "B"}]) - self.assertEqual(result, ["guid-1", "guid-2"]) - - -class TestUpdateWithDataverseModel(unittest.TestCase): - """records.update() accepts DataverseModel as first arg.""" - - def setUp(self): - self.client = _make_client() - - def test_update_single_entity_with_id(self): - account = _Account(name="Updated Name") - self.client.records.update(account, "guid-abc") - self.client._odata._update.assert_called_once_with( - "account", "guid-abc", {"name": "Updated Name", "telephone1": ""} - ) - - def test_update_entity_no_id_raises(self): - account = _Account(name="Updated") - with self.assertRaises(TypeError): - self.client.records.update(account) - - def test_update_entity_no_deprecation_warning(self): - account = _Account(name="Updated") - with warnings.catch_warnings(record=True) as caught: - warnings.simplefilter("always") - self.client.records.update(account, "guid-abc") - dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] - self.assertEqual(dep, []) - - def test_update_dict_path_still_works(self): - self.client.records.update("account", "guid-1", {"name": "New Name"}) - self.client._odata._update.assert_called_with("account", "guid-1", {"name": "New Name"}) - - class TestExecuteNoDeprecationFromRecordsGet(unittest.TestCase): """execute() suppresses DeprecationWarning from the internal records.get() call.""" diff --git a/tests/unit/test_records_operations.py b/tests/unit/test_records_operations.py index 104521e2..0618ea5a 100644 --- a/tests/unit/test_records_operations.py +++ b/tests/unit/test_records_operations.py @@ -2,7 +2,6 @@ # Licensed under the MIT license. import unittest -import warnings from unittest.mock import MagicMock from azure.core.credentials import TokenCredential From 8e61fd37f46b1716c1a93c8a2d6d61039feee310 Mon Sep 17 00:00:00 2001 From: Samson Gebre Date: Wed, 6 May 2026 14:15:21 -0700 Subject: [PATCH 06/16] Update CHANGELOG.md with new features, changes, deprecations, and removals; modify prodev_quick_start.py for additional parameter; enhance pyproject.toml for migration tool; refine migrate_v0_to_v1.py documentation and usage instructions. --- CHANGELOG.md | 25 +++++ examples/advanced/prodev_quick_start.py | 4 +- pyproject.toml | 7 +- tools/migrate_v0_to_v1.py | 121 ++++++++++++------------ 4 files changed, 93 insertions(+), 64 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1a1374f4..372d593e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- `client.records.retrieve(table, record_id)` — fetch a single record by GUID; returns `None` on 404 instead of raising (#175) +- `client.records.list(table, filter, select, top)` — eager fetch returning a flat `QueryResult`; GA replacement for `records.get()` without a record ID (#175) +- `client.records.list_pages(table, filter, select, top)` — lazy iterator yielding one `QueryResult` per HTTP page; streaming counterpart to `list()` (#175) +- `client.query.fetchxml(xml)` — FetchXML support returning an inert `FetchXmlQuery`; no HTTP request is made until `.execute()` or `.execute_pages()` is called (#175) +- `FetchXmlQuery` implements the correct Dataverse paging cookie algorithm: annotation parsed as outer XML, `pagingcookie` attribute double URL-decoded, server-supplied `pagenumber` used for next page, `morerecords` handled as both `bool` and `"true"` string, `UserWarning` emitted on simple paging fallback, 32,768-character URL limit enforced (documented Dataverse GET cap), 10,000-page circuit breaker against runaway iteration (#175) +- `QueryBuilder.execute_pages()` — lazy per-page streaming returning one `QueryResult` per HTTP page; replaces deprecated `execute(by_page=True)` (#175) +- `QueryBuilder.where()` — composable filter expressions using `col()` and Python operators (`==`, `>`, `&`, `|`, `~`); replaces deprecated `filter_eq()`, `filter_contains()`, and other `filter_*` helpers (#175) +- `QueryResult.__getitem__` — index access (`result[0]`) returns a `Record`; slice access (`result[1:5]`) returns a new `QueryResult` (#175) +- `DataverseModel` structural `Protocol` (`models/protocol.py`) — implement on any entity class to enable typed integration with CRUD operations without specifying table names or serializing manually (#175) +- `col()`, `raw()`, `QueryResult`, and `DataverseModel` exported from the top-level `PowerPlatform.Dataverse` package (#175) +- v0→v1 migration tool: `tools/migrate_v0_to_v1.py` rewrites v0 call sites to the v1 API with `--dry-run` support; covers `create`, `update`, `delete`, `get`, `list`, `fetchxml`, and query builder patterns (#175) + +### Changed +- `QueryBuilder.execute()` now returns a flat `QueryResult` (all pages collected eagerly) instead of `Iterable[Record]` (#175) +- `records.get()` deprecation extended: calling with a `record_id` emits `DeprecationWarning` directing callers to `retrieve()`; calling without a `record_id` directs callers to `list()` (#175) + +### Deprecated +- `QueryBuilder.execute(by_page=True)` and `execute(by_page=False)` emit `UserWarning`; use `execute_pages()` and `execute()` respectively (#175) +- `client.query.odata_select()`, `client.query.odata_expands()`, `client.query.odata_expand()`, `client.query.odata_bind()` emit `DeprecationWarning`; navigation property helpers are replaced by `QueryBuilder.expand()` (#175) + +### Removed +- All v0 flat methods on `DataverseClient` (`create`, `update`, `delete`, `get`, `list`, `query_sql`, etc.) removed (~570 lines); use the `client.records`, `client.query`, and `client.batch` namespaces (#175) +- `client.query.sql_select()`, `client.query.sql_joins()`, `client.query.sql_join()` removed (#175) + ## [0.1.0b9] - 2026-04-28 ### Added diff --git a/examples/advanced/prodev_quick_start.py b/examples/advanced/prodev_quick_start.py index 15448143..d06e058f 100644 --- a/examples/advanced/prodev_quick_start.py +++ b/examples/advanced/prodev_quick_start.py @@ -116,7 +116,7 @@ def run_demo(client): customer_ids, project_ids, task_ids = step3_populate_data(client, primary_name_col) # -- Step 4: Query and analyze -- - step4_query_and_analyze(client, customer_ids, primary_name_col) + step4_query_and_analyze(client, customer_ids, primary_name_col, primary_id_col) # -- Step 5: Update and delete -- step5_update_and_delete(client, task_ids, primary_name_col, primary_id_col) @@ -385,7 +385,7 @@ def step3_populate_data(client, primary_name_col): # ================================================================ -def step4_query_and_analyze(client, customer_ids, primary_name_col): +def step4_query_and_analyze(client, customer_ids, primary_name_col, primary_id_col): """Query data and demonstrate DataFrame analysis.""" print("\n" + "-" * 60) print("STEP 4: Query and analyze data") diff --git a/pyproject.toml b/pyproject.toml index 97b4a187..387bb21d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dependencies = [ [project.scripts] dataverse-install-claude-skill = "PowerPlatform.Dataverse._skill_installer:main" +dataverse-migrate = "tools.migrate_v0_to_v1:main" [project.optional-dependencies] dev = [ @@ -53,12 +54,12 @@ dev = [ migration = ["libcst>=1.0.0"] [tool.setuptools] -package-dir = {"" = "src"} +package-dir = {"" = "src", "tools" = "tools"} zip-safe = false [tool.setuptools.packages.find] -where = ["src"] -include = ["PowerPlatform*"] +where = ["src", "."] +include = ["PowerPlatform*", "tools"] namespaces = false [tool.setuptools.package-data] diff --git a/tools/migrate_v0_to_v1.py b/tools/migrate_v0_to_v1.py index 553bc026..c879d9d9 100644 --- a/tools/migrate_v0_to_v1.py +++ b/tools/migrate_v0_to_v1.py @@ -3,7 +3,7 @@ # Licensed under the MIT license. """ -DV-Python-SDK v0 → v1 GA migration codemod. +DV-Python-SDK v0 -> v1 GA migration codemod. Mechanically rewrites beta (0.1.0b*) call sites to their GA (1.0) equivalents using LibCST (concrete syntax tree — preserves all whitespace and comments). @@ -11,51 +11,54 @@ Usage:: pip install PowerPlatform-Dataverse-Client[migration] - python -m tools.migrate_v0_to_v1 path/to/scripts/ - python -m tools.migrate_v0_to_v1 examples/ # _codemon.py files only + dataverse-migrate path/to/your/scripts/ + dataverse-migrate path/to/your/scripts/ --dry-run # preview changes without writing + + # Or via module for development installs: + python -m tools.migrate_v0_to_v1 path/to/your/scripts/ Transformations applied ----------------------- -Builder methods (.filter_* → .where(col(...)...)):: - - .filter_eq("col", v) → .where(col("col") == v) - .filter_ne("col", v) → .where(col("col") != v) - .filter_gt("col", v) → .where(col("col") > v) - .filter_ge("col", v) → .where(col("col") >= v) - .filter_lt("col", v) → .where(col("col") < v) - .filter_le("col", v) → .where(col("col") <= v) - .filter_contains("col", v) → .where(col("col").contains(v)) - .filter_startswith("col", v) → .where(col("col").startswith(v)) - .filter_endswith("col", v) → .where(col("col").endswith(v)) - .filter_in("col", vals) → .where(col("col").in_(vals)) - .filter_not_in("col", vals) → .where(col("col").not_in(vals)) - .filter_null("col") → .where(col("col").is_null()) - .filter_not_null("col") → .where(col("col").is_not_null()) - .filter_between("col", lo, hi) → .where(col("col").between(lo, hi)) - .filter_not_between("col", lo, hi) → .where(col("col").not_between(lo, hi)) - .filter_raw("expr") → .where(raw("expr")) - .filter("expr") → .where(raw("expr")) - .execute(by_page=True) → .execute_pages() - .execute(by_page=False) → .execute() (flag removed) +Builder methods (.filter_* -> .where(col(...)...)):: + + .filter_eq("col", v) -> .where(col("col") == v) + .filter_ne("col", v) -> .where(col("col") != v) + .filter_gt("col", v) -> .where(col("col") > v) + .filter_ge("col", v) -> .where(col("col") >= v) + .filter_lt("col", v) -> .where(col("col") < v) + .filter_le("col", v) -> .where(col("col") <= v) + .filter_contains("col", v) -> .where(col("col").contains(v)) + .filter_startswith("col", v) -> .where(col("col").startswith(v)) + .filter_endswith("col", v) -> .where(col("col").endswith(v)) + .filter_in("col", vals) -> .where(col("col").in_(vals)) + .filter_not_in("col", vals) -> .where(col("col").not_in(vals)) + .filter_null("col") -> .where(col("col").is_null()) + .filter_not_null("col") -> .where(col("col").is_not_null()) + .filter_between("col", lo, hi) -> .where(col("col").between(lo, hi)) + .filter_not_between("col", lo, hi) -> .where(col("col").not_between(lo, hi)) + .filter_raw("expr") -> .where(raw("expr")) + .filter("expr") -> .where(raw("expr")) + .execute(by_page=True) -> .execute_pages() + .execute(by_page=False) -> .execute() (flag removed) Record namespace:: - batch.records.get(t, id) → batch.records.retrieve(t, id) + batch.records.get(t, id) -> batch.records.retrieve(t, id) Top-level shortcuts (removed at GA):: - client.create(t, d) → client.records.create(t, d) - client.update(t, id, d) → client.records.update(t, id, d) - client.delete(t, id) → client.records.delete(t, id) - client.get(t, id) → client.records.retrieve(t, id) - client.query_sql(sql) → client.query.sql(sql) - client.get_table_info(t) → client.tables.get(t) - client.create_table(t, …) → client.tables.create(t, …) - client.delete_table(t) → client.tables.delete(t) - client.list_tables() → client.tables.list() - client.create_columns(t, …) → client.tables.add_columns(t, …) - client.delete_columns(t, …) → client.tables.remove_columns(t, …) - client.upload_file(…) → client.files.upload(…) + client.create(t, d) -> client.records.create(t, d) + client.update(t, id, d) -> client.records.update(t, id, d) + client.delete(t, id) -> client.records.delete(t, id) + client.get(t, id) -> client.records.retrieve(t, id) + client.query_sql(sql) -> client.query.sql(sql) + client.get_table_info(t) -> client.tables.get(t) + client.create_table(t, …) -> client.tables.create(t, …) + client.delete_table(t) -> client.tables.delete(t) + client.list_tables() -> client.tables.list() + client.create_columns(t, …) -> client.tables.add_columns(t, …) + client.delete_columns(t, …) -> client.tables.remove_columns(t, …) + client.upload_file(…) -> client.files.upload(…) Import management: Adds ``from PowerPlatform.Dataverse.models.filters import col`` when a @@ -63,17 +66,17 @@ Adds ``raw`` to the same import when .filter_raw or .filter is rewritten. NOT handled by this codemod (manual migration required): - execute(by_page=variable) → manual review required (variable argument, not literal) - client.records.get(t, id) → client.records.retrieve(t, id) + execute(by_page=variable) -> manual review required (variable argument, not literal) + client.records.get(t, id) -> client.records.retrieve(t, id) Return type changes: beta returns Record (raises on 404); GA retrieve() returns Record | None. Callers that do not guard against None will fail silently. - client.records.get(t, kw=…) → client.records.list(t, kw=…) + client.records.get(t, kw=…) -> client.records.list(t, kw=…) Return type changes: beta returns Iterable[List[Record]] (pages); GA list() returns QueryResult (flat iterable over Records). Any ``for page in result: for rec in page:`` iteration pattern breaks after a mechanical rename. - client.dataframe.get() → client.query.builder(…).execute().to_dataframe() + client.dataframe.get() -> client.query.builder(…).execute().to_dataframe() Expression reconstruction requires understanding caller intent. - client.query.sql_select()/sql_join()/sql_joins() → removed (no mechanical replacement) + client.query.sql_select()/sql_join()/sql_joins() -> removed (no mechanical replacement) """ from __future__ import annotations @@ -95,7 +98,7 @@ # --------------------------------------------------------------------------- -# Filter-method → .where(col(...)) mapping +# Filter-method -> .where(col(...)) mapping # --------------------------------------------------------------------------- _UNARY_FILTER_MAP = { @@ -124,8 +127,8 @@ _ALL_FILTER_METHODS: Set[str] = set(_UNARY_FILTER_MAP) | set(_BINARY_OP_MAP) | set(_METHOD_FILTER_MAP) | {"filter_raw"} -# Standalone filter functions from filters module (beta API) → col() equivalents -# eq("f", v) → col("f") == v, between("f", lo, hi) → col("f").between(lo, hi), etc. +# Standalone filter functions from filters module (beta API) -> col() equivalents +# eq("f", v) -> col("f") == v, between("f", lo, hi) -> col("f").between(lo, hi), etc. _FUNC_BINARY_OP_MAP = { "eq": cst.Equal(), "ne": cst.NotEqual(), @@ -149,7 +152,7 @@ } _ALL_FILTER_FUNCS: Set[str] = set(_FUNC_BINARY_OP_MAP) | set(_FUNC_METHOD_MAP) | set(_FUNC_UNARY_MAP) -# Top-level client shortcut → (new_namespace, new_method) +# Top-level client shortcut -> (new_namespace, new_method) _CLIENT_SHORTCUTS = { "create": ("records", "create"), "update": ("records", "update"), @@ -233,7 +236,7 @@ def _positional_count(args: Sequence[cst.Arg]) -> int: class _V1Migrator(cst.CSTTransformer): - """LibCST transformer rewriting DV-Python-SDK beta → v1 GA.""" + """LibCST transformer rewriting DV-Python-SDK beta -> v1 GA.""" def __init__(self, client_var: str = "client") -> None: self._client_var = client_var @@ -271,7 +274,7 @@ def leave_Call(self, original_node: cst.Call, updated_node: cst.Call) -> cst.Bas func = updated_node.func # ---------------------------------------------------------------- - # Standalone filter functions: eq("f", v) → col("f") == v, etc. + # Standalone filter functions: eq("f", v) -> col("f") == v, etc. # Only transform names that were actually imported from filters module. # Wrap Comparison nodes in explicit parentheses so that combining with # & / | doesn't hit Python precedence bugs (& binds tighter than ==/>). @@ -289,7 +292,7 @@ def leave_Call(self, original_node: cst.Call, updated_node: cst.Call) -> cst.Bas method_name = func.attr.value if isinstance(func.attr, cst.Name) else "" # ---------------------------------------------------------------- - # .filter_*(...) → .where(col(...) ...) + # .filter_*(...) -> .where(col(...) ...) # ---------------------------------------------------------------- if method_name in _ALL_FILTER_METHODS: where_arg = self._build_filter_arg(method_name, updated_node.args) @@ -300,7 +303,7 @@ def leave_Call(self, original_node: cst.Call, updated_node: cst.Call) -> cst.Bas ) # ---------------------------------------------------------------- - # .filter("expr") → .where(raw("expr")) + # .filter("expr") -> .where(raw("expr")) # QueryBuilder.filter() was removed at GA (not deprecated). Wrapping # in raw() preserves the OData string exactly for string-literal callers. # ---------------------------------------------------------------- @@ -314,8 +317,8 @@ def leave_Call(self, original_node: cst.Call, updated_node: cst.Call) -> cst.Bas ) # ---------------------------------------------------------------- - # .execute(by_page=True) → .execute_pages() - # .execute(by_page=False) → .execute() (flag removed) + # .execute(by_page=True) -> .execute_pages() + # .execute(by_page=False) -> .execute() (flag removed) # Only literal True/False are codemod-able; variable by_page requires # manual review per section 8.5 of the GA spec. # ---------------------------------------------------------------- @@ -335,7 +338,7 @@ def leave_Call(self, original_node: cst.Call, updated_node: cst.Call) -> cst.Bas return updated_node.with_changes(args=other_args) # ---------------------------------------------------------------- - # batch.records.get(table, id) → batch.records.retrieve(table, id) + # batch.records.get(table, id) -> batch.records.retrieve(table, id) # NOTE: client.records.get() is NOT codemodded — the return type changes # between beta and GA (Record | None vs Record for single-id; QueryResult vs # Iterable[List[Record]] for multi-record). Surrounding iteration patterns @@ -394,18 +397,18 @@ def _build_filter_arg( if field_node is None: return None - # .filter_raw(expr) → raw(expr) + # .filter_raw(expr) -> raw(expr) if method_name == "filter_raw": self._needs_raw = True return _call(_name("raw"), field_node) - # .filter_null / .filter_not_null → col("f").is_null() / .is_not_null() + # .filter_null / .filter_not_null -> col("f").is_null() / .is_not_null() if method_name in _UNARY_FILTER_MAP: self._needs_col = True proxy = _UNARY_FILTER_MAP[method_name] return _call(_attr(_col_call(field_node), proxy)) - # .filter_eq / .filter_ne / ... → col("f") OP val + # .filter_eq / .filter_ne / ... -> col("f") OP val if method_name in _BINARY_OP_MAP: val_node = _pos_arg(args, 1) if val_node is None: @@ -421,7 +424,7 @@ def _build_filter_arg( ], ) - # .filter_between / .filter_not_between → col("f").between(lo, hi) + # .filter_between / .filter_not_between -> col("f").between(lo, hi) if method_name in ("filter_between", "filter_not_between"): lo = _pos_arg(args, 1) hi = _pos_arg(args, 2) @@ -443,7 +446,7 @@ def _build_filter_arg( return None # ------------------------------------------------------------------ - # Standalone filter function: eq("f", v) → col("f") == v, etc. + # Standalone filter function: eq("f", v) -> col("f") == v, etc. # ------------------------------------------------------------------ def _build_filter_func_arg( @@ -642,7 +645,7 @@ def main(argv: Optional[List[str]] = None) -> int: if not remaining: print(__doc__) - print("\nUsage: python -m tools.migrate_v0_to_v1 [--dry-run] [ ...]") + print("\nUsage: dataverse-migrate [--dry-run] [ ...]") return 1 targets = _collect_targets(remaining) From bba45f7d80573df0c4835c372f788cc79d6b212e Mon Sep 17 00:00:00 2001 From: Samson Gebre Date: Wed, 6 May 2026 14:41:00 -0700 Subject: [PATCH 07/16] Improve error handling for missing libcst dependency in migration script --- tools/migrate_v0_to_v1.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/tools/migrate_v0_to_v1.py b/tools/migrate_v0_to_v1.py index c879d9d9..72c41ed1 100644 --- a/tools/migrate_v0_to_v1.py +++ b/tools/migrate_v0_to_v1.py @@ -87,14 +87,12 @@ try: import libcst as cst -except ImportError: - print( - "ERROR: libcst is required. Install with:\n" +except ImportError as _e: + raise ImportError( + "libcst is required. Install with:\n" " pip install PowerPlatform-Dataverse-Client[migration]\n" - " # or: pip install 'libcst>=1.0.0'", - file=sys.stderr, - ) - sys.exit(1) + " # or: pip install 'libcst>=1.0.0'" + ) from _e # --------------------------------------------------------------------------- From d6c1b82ff16025c17d2550657814d97942df89af Mon Sep 17 00:00:00 2001 From: Samson Gebre Date: Wed, 6 May 2026 15:44:19 -0700 Subject: [PATCH 08/16] Enhance FetchXmlQuery and RecordOperations with improved error handling and warnings; update migration tool for client variable support and manual review detection. --- .../Dataverse/models/fetchxml_query.py | 28 ++-- .../Dataverse/models/protocol.py | 16 ++- .../Dataverse/operations/records.py | 16 ++- tests/unit/test_phase3_ga.py | 17 +-- tests/unit/test_phase4_ga.py | 98 ++++++++++++- tools/migrate_v0_to_v1.py | 134 +++++++++++++++--- 6 files changed, 258 insertions(+), 51 deletions(-) diff --git a/src/PowerPlatform/Dataverse/models/fetchxml_query.py b/src/PowerPlatform/Dataverse/models/fetchxml_query.py index 3afeb350..691e2e05 100644 --- a/src/PowerPlatform/Dataverse/models/fetchxml_query.py +++ b/src/PowerPlatform/Dataverse/models/fetchxml_query.py @@ -140,6 +140,7 @@ def execute_pages(self) -> Iterator[QueryResult]: data.get("@Microsoft.Dynamics.CRM.fetchxmlpagingcookie", "") if isinstance(data, dict) else "" ) + _cookie_parse_error = False if raw_cookie: try: cookie_el = _ET.fromstring(raw_cookie) @@ -152,8 +153,14 @@ def execute_pages(self) -> Iterator[QueryResult]: fetch_el.set("page", str(page_num)) current_xml = _ET.tostring(fetch_el, encoding="unicode") continue - except (_ET.ParseError, ValueError): - pass # Fall through to simple paging + except (_ET.ParseError, ValueError) as exc: + warnings.warn( + f"FetchXML paging cookie could not be parsed ({exc}); " + "falling back to simple paging.", + UserWarning, + stacklevel=2, + ) + _cookie_parse_error = True # Simple paging fallback: server returned morerecords=true but no paging # cookie. Dataverse omits the cookie when the query cannot use cookie-based @@ -162,14 +169,15 @@ def execute_pages(self) -> Iterator[QueryResult]: # paging has a 50,000-record server cap and performance degrades at high page # numbers. The caller may be able to avoid this by reordering on the root # entity instead. - warnings.warn( - "Dataverse did not return a paging cookie; falling back to simple paging " - "(page-number increment only). Simple paging is capped at 50,000 records " - "and degrades in performance at high page numbers. Consider reordering on " - "a root-entity column to enable cookie-based paging.", - UserWarning, - stacklevel=2, - ) + if not _cookie_parse_error: + warnings.warn( + "Dataverse did not return a paging cookie; falling back to simple paging " + "(page-number increment only). Simple paging is capped at 50,000 records " + "and degrades in performance at high page numbers. Consider reordering on " + "a root-entity column to enable cookie-based paging.", + UserWarning, + stacklevel=2, + ) page_num += 1 fetch_el = _ET.fromstring(current_xml) fetch_el.set("page", str(page_num)) diff --git a/src/PowerPlatform/Dataverse/models/protocol.py b/src/PowerPlatform/Dataverse/models/protocol.py index a3c48509..d9b0e502 100644 --- a/src/PowerPlatform/Dataverse/models/protocol.py +++ b/src/PowerPlatform/Dataverse/models/protocol.py @@ -32,7 +32,7 @@ class DataverseModel(Protocol): Example:: from dataclasses import dataclass - from PowerPlatform.Dataverse.models.protocol import DataverseModel + from PowerPlatform.Dataverse import DataverseModel @dataclass class Account: @@ -51,9 +51,17 @@ def from_dict(cls, data: dict) -> "Account": telephone1=data.get("telephone1", ""), ) - # Use the entity directly with records operations: - guid = client.records.create(Account(name="Contoso")) - client.records.update(Account(name="Contoso Updated"), guid) + # isinstance() works today — Protocol is runtime_checkable: + assert isinstance(Account(), DataverseModel) + + # Type your own helpers against the Protocol now: + def save(entity: DataverseModel) -> None: + data = entity.to_dict() + client.records.create(entity.__entity_logical_name__, data) + + Note: + Direct dispatch (``client.records.create(entity)`` without a table name + or dict) is not yet supported and will be added in a future release. """ __entity_logical_name__: str diff --git a/src/PowerPlatform/Dataverse/operations/records.py b/src/PowerPlatform/Dataverse/operations/records.py index 1eb3274d..ee800a18 100644 --- a/src/PowerPlatform/Dataverse/operations/records.py +++ b/src/PowerPlatform/Dataverse/operations/records.py @@ -8,6 +8,7 @@ import warnings from typing import Any, Dict, Iterable, Iterator, List, Optional, Union, overload, TYPE_CHECKING +from ..core.errors import HttpError from ..models.record import QueryResult, Record from ..models.upsert import UpsertItem @@ -398,10 +399,12 @@ def get( Only used for multi-record queries. :type include_annotations: :class:`str` or None - :return: A single record dict when ``record_id`` is provided, or a - generator yielding pages (lists of record dicts) when fetching - multiple records. - :rtype: dict or collections.abc.Iterable[list[dict]] + :return: A single :class:`~PowerPlatform.Dataverse.models.record.Record` + when ``record_id`` is provided, or a generator yielding pages + (lists of :class:`~PowerPlatform.Dataverse.models.record.Record`) + when fetching multiple records. + :rtype: ~PowerPlatform.Dataverse.models.record.Record or + collections.abc.Iterable[list[~PowerPlatform.Dataverse.models.record.Record]] :raises TypeError: If ``record_id`` is provided but not a string. :raises ValueError: If query parameters are provided alongside @@ -508,9 +511,8 @@ def retrieve( with self._client._scoped_odata() as od: try: raw = od._get(table, record_id, select=select) - except Exception as exc: - resp = getattr(exc, "response", None) - if resp is not None and getattr(resp, "status_code", None) == 404: + except HttpError as exc: + if exc.status_code == 404: return None raise return Record.from_api_response(table, raw, record_id=record_id) diff --git a/tests/unit/test_phase3_ga.py b/tests/unit/test_phase3_ga.py index 2451da8c..76a34719 100644 --- a/tests/unit/test_phase3_ga.py +++ b/tests/unit/test_phase3_ga.py @@ -22,6 +22,7 @@ from azure.core.credentials import TokenCredential +from PowerPlatform.Dataverse.core.errors import HttpError from PowerPlatform.Dataverse.models.record import QueryResult, Record from PowerPlatform.Dataverse.models.protocol import DataverseModel @@ -192,24 +193,16 @@ def test_retrieve_no_deprecation_warning(self): self.assertEqual(dep, [], f"retrieve() must not emit DeprecationWarning: {dep}") def test_retrieve_returns_none_on_404(self): - exc = Exception("Not Found") - resp_mock = MagicMock() - resp_mock.status_code = 404 - exc.response = resp_mock - self.client._odata._get.side_effect = exc + self.client._odata._get.side_effect = HttpError("Not Found", status_code=404) result = self.client.records.retrieve("account", "nonexistent-guid") self.assertIsNone(result) def test_retrieve_reraises_non_404(self): - exc = Exception("Server Error") - resp_mock = MagicMock() - resp_mock.status_code = 500 - exc.response = resp_mock - self.client._odata._get.side_effect = exc - with self.assertRaises(Exception): + self.client._odata._get.side_effect = HttpError("Server Error", status_code=500) + with self.assertRaises(HttpError): self.client.records.retrieve("account", "some-guid") - def test_retrieve_reraises_when_no_response_attr(self): + def test_retrieve_reraises_non_http_errors(self): self.client._odata._get.side_effect = ValueError("Bad input") with self.assertRaises(ValueError): self.client.records.retrieve("account", "some-guid") diff --git a/tests/unit/test_phase4_ga.py b/tests/unit/test_phase4_ga.py index fe843c28..3cf09f92 100644 --- a/tests/unit/test_phase4_ga.py +++ b/tests/unit/test_phase4_ga.py @@ -297,8 +297,8 @@ def test_simple_paging_fallback_fetches_all_pages(self): self.assertEqual(len(result), 2) self.assertEqual(self.client._odata._request.call_count, 2) - def test_malformed_cookie_falls_back_to_simple_paging(self): - """A cookie annotation that is not parseable as XML triggers simple paging fallback.""" + def test_malformed_cookie_xml_warns_distinctly(self): + """A cookie that is not valid XML emits a 'could not be parsed' warning, not the no-cookie warning.""" page1 = self._mock_response([{"name": "A"}], more=True, cookie="not-valid-xml") page2 = self._mock_response([{"name": "B"}], more=False) self.client._odata._request.side_effect = [page1, page2] @@ -310,6 +310,23 @@ def test_malformed_cookie_falls_back_to_simple_paging(self): self.assertEqual(len(result), 2) user_warnings = [w for w in caught if issubclass(w.category, UserWarning)] self.assertEqual(len(user_warnings), 1) + self.assertIn("could not be parsed", str(user_warnings[0].message).lower()) + + def test_corrupt_pagenumber_warns_distinctly(self): + """Valid XML cookie with non-integer pagenumber emits a 'could not be parsed' warning.""" + bad_cookie = '' + page1 = self._mock_response([{"name": "A"}], more=True, cookie=bad_cookie) + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = self.client.query.fetchxml(self._fetch_xml()).execute() + + self.assertEqual(len(result), 2) + user_warnings = [w for w in caught if issubclass(w.category, UserWarning)] + self.assertEqual(len(user_warnings), 1) + self.assertIn("could not be parsed", str(user_warnings[0].message).lower()) def test_max_pages_exceeded_raises(self): """Paging loop raises ValidationError after exceeding _MAX_PAGES.""" @@ -530,6 +547,83 @@ def test_idempotent_execute_no_flag(self): twice = self.migrate(once) self.assertEqual(once, twice) + def test_client_var_default_rewrites_client(self): + """Default client_var='client' rewrites client.create(...).""" + src = "client.create('account', data)\n" + out = self.migrate(src) + self.assertIn("client.records.create", out) + + def test_client_var_custom_rewrites_matching_name(self): + """custom client_var rewrites that variable name, not 'client'.""" + src = "svc.create('account', data)\n" + out = self.migrate(src, client_var="svc") + self.assertIn("svc.records.create", out) + + def test_client_var_custom_does_not_rewrite_default_name(self): + """When client_var='svc', the literal name 'client' is left untouched.""" + src = "client.create('account', data)\n" + out = self.migrate(src, client_var="svc") + self.assertNotIn("client.records.create", out) + self.assertIn("client.create", out) + + +class TestManualReviewFinder(unittest.TestCase): + """find_manual_patterns() detects patterns that require manual migration.""" + + @classmethod + def setUpClass(cls): + try: + from tools.migrate_v0_to_v1 import find_manual_patterns + + cls.find = staticmethod(find_manual_patterns) + except ImportError: + cls.find = None + + def setUp(self): + if self.find is None: + self.skipTest("libcst not installed or tools package not on path") + + def test_records_get_flagged(self): + src = "result = client.records.get('account', record_id)\n" + findings = self.find(src) + self.assertTrue(any("records.get" in f for f in findings)) + + def test_dataframe_get_flagged(self): + src = "df = client.dataframe.get('account', select=['name'])\n" + findings = self.find(src) + self.assertTrue(any("dataframe.get" in f for f in findings)) + + def test_execute_by_page_variable_flagged(self): + src = "result = builder.execute(by_page=flag)\n" + findings = self.find(src) + self.assertTrue(any("by_page" in f for f in findings)) + + def test_execute_by_page_literal_not_flagged(self): + """Literal True/False is handled by the transformer — not a manual item.""" + src = "result = builder.execute(by_page=True)\n" + findings = self.find(src) + self.assertFalse(any("by_page" in f for f in findings)) + + def test_sql_select_flagged(self): + src = "cols = client.query.sql_select('account')\n" + findings = self.find(src) + self.assertTrue(any("sql_select" in f for f in findings)) + + def test_sql_join_flagged(self): + src = "j = client.query.sql_join('account', 'contact')\n" + findings = self.find(src) + self.assertTrue(any("sql_join" in f for f in findings)) + + def test_clean_code_no_findings(self): + src = "result = client.records.list('account', filter='statecode eq 0')\n" + self.assertEqual(self.find(src), []) + + def test_custom_client_var(self): + src = "svc.records.get('account', guid)\n" + self.assertEqual(self.find(src, client_var="client"), []) + findings = self.find(src, client_var="svc") + self.assertTrue(any("records.get" in f for f in findings)) + if __name__ == "__main__": unittest.main() diff --git a/tools/migrate_v0_to_v1.py b/tools/migrate_v0_to_v1.py index 72c41ed1..0729f6d5 100644 --- a/tools/migrate_v0_to_v1.py +++ b/tools/migrate_v0_to_v1.py @@ -12,7 +12,8 @@ pip install PowerPlatform-Dataverse-Client[migration] dataverse-migrate path/to/your/scripts/ - dataverse-migrate path/to/your/scripts/ --dry-run # preview changes without writing + dataverse-migrate path/to/your/scripts/ --dry-run # preview without writing + dataverse-migrate path/to/your/scripts/ --client-var=svc # if client is named 'svc' # Or via module for development installs: python -m tools.migrate_v0_to_v1 path/to/your/scripts/ @@ -83,7 +84,7 @@ import sys from pathlib import Path -from typing import List, Optional, Sequence, Set +from typing import List, Optional, Sequence, Set, Tuple try: import libcst as cst @@ -587,6 +588,88 @@ def _dotted_name(node: Optional[cst.BaseExpression]) -> str: # --------------------------------------------------------------------------- +# --------------------------------------------------------------------------- +# Manual-review pattern detector +# --------------------------------------------------------------------------- + +_REMOVED_QUERY_METHODS: Set[str] = {"sql_select", "sql_join", "sql_joins"} + + +class _ManualReviewFinder(cst.CSTTransformer): + """Visitor that detects patterns the codemod cannot safely rewrite automatically.""" + + def __init__(self, client_var: str = "client") -> None: + self._client_var = client_var + self.findings: List[str] = [] + + def _receiver_chain(self, node: cst.Attribute) -> List[str]: + """Return the dotted name parts of an Attribute chain, innermost first.""" + parts: List[str] = [] + cur: cst.BaseExpression = node + while isinstance(cur, cst.Attribute): + if isinstance(cur.attr, cst.Name): + parts.append(cur.attr.value) + cur = cur.value + if isinstance(cur, cst.Name): + parts.append(cur.value) + return parts # e.g. ["get", "records", "client"] for client.records.get + + def visit_Call(self, node: cst.Call) -> None: + func = node.func + if not isinstance(func, cst.Attribute): + return + + method = func.attr.value if isinstance(func.attr, cst.Name) else "" + chain = self._receiver_chain(func) # [method, ns, client_var, ...] + + # execute(by_page=) — non-literal by_page cannot be codemodded + if method == "execute": + for a in node.args: + if isinstance(a.keyword, cst.Name) and a.keyword.value == "by_page": + if not (isinstance(a.value, cst.Name) and a.value.value in ("True", "False")): + self.findings.append( + "execute(by_page=) — non-literal by_page requires manual review; " + "replace with execute_pages() or execute() depending on runtime value" + ) + + # client.records.get() — return type changes make a mechanical rename unsafe + if method == "get" and len(chain) >= 3 and chain[1] == "records" and chain[2] == self._client_var: + self.findings.append( + f"{self._client_var}.records.get() — use retrieve() for single-record lookup " + "(return type changes: raises on 404 vs returns None) " + "or list() for multi-record (iteration pattern changes)" + ) + + # client.dataframe.get() — expression reconstruction requires understanding caller intent + if method == "get" and len(chain) >= 3 and chain[1] == "dataframe" and chain[2] == self._client_var: + self.findings.append( + f"{self._client_var}.dataframe.get() — use " + "query.builder(...).execute().to_dataframe(); requires manual reconstruction" + ) + + # client.query.sql_select/sql_join/sql_joins — removed with no mechanical replacement + if method in _REMOVED_QUERY_METHODS and len(chain) >= 3 and chain[1] == "query" and chain[2] == self._client_var: + self.findings.append( + f"{self._client_var}.query.{method}() — removed at GA with no mechanical replacement" + ) + + +def find_manual_patterns(source: str, *, client_var: str = "client") -> List[str]: + """Return descriptions of patterns in *source* that require manual migration.""" + try: + tree = cst.parse_module(source) + except cst.ParserSyntaxError: + return [] + finder = _ManualReviewFinder(client_var=client_var) + tree.visit(finder) + return finder.findings + + +# --------------------------------------------------------------------------- +# File-level migration +# --------------------------------------------------------------------------- + + def migrate_source(source: str, *, client_var: str = "client") -> str: """Parse *source*, apply transformations, return migrated source.""" try: @@ -597,19 +680,21 @@ def migrate_source(source: str, *, client_var: str = "client") -> str: return new_tree.code -def migrate_file(path: Path, *, dry_run: bool = False) -> bool: - """Migrate *path* in place. Returns True if the file was changed.""" +def migrate_file( + path: Path, *, dry_run: bool = False, client_var: str = "client" +) -> Tuple[bool, List[str]]: + """Migrate *path* in place. Returns (was_changed, manual_review_notes).""" original = path.read_text(encoding="utf-8") try: - migrated = migrate_source(original) + migrated = migrate_source(original, client_var=client_var) except ValueError as exc: print(f" [SKIP] {path}: {exc}", file=sys.stderr) - return False - if migrated == original: - return False - if not dry_run: + return False, [] + manual = find_manual_patterns(original, client_var=client_var) + changed = migrated != original + if changed and not dry_run: path.write_text(migrated, encoding="utf-8") - return True + return changed, manual # --------------------------------------------------------------------------- @@ -639,11 +724,19 @@ def _collect_targets(paths: List[str]) -> List[Path]: def main(argv: Optional[List[str]] = None) -> int: args = sys.argv[1:] if argv is None else list(argv) dry_run = "--dry-run" in args - remaining = [a for a in args if a != "--dry-run"] + client_var = "client" + remaining = [] + for a in args: + if a == "--dry-run": + continue + if a.startswith("--client-var="): + client_var = a[len("--client-var="):] + else: + remaining.append(a) if not remaining: print(__doc__) - print("\nUsage: dataverse-migrate [--dry-run] [ ...]") + print("\nUsage: dataverse-migrate [--dry-run] [--client-var=NAME] [ ...]") return 1 targets = _collect_targets(remaining) @@ -651,16 +744,25 @@ def main(argv: Optional[List[str]] = None) -> int: print("[ERROR] No Python files found.", file=sys.stderr) return 1 - changed = skipped = 0 + changed = skipped = manual_total = 0 for path in targets: - if migrate_file(path, dry_run=dry_run): + was_changed, notes = migrate_file(path, dry_run=dry_run, client_var=client_var) + if was_changed: changed += 1 tag = "[DRY-RUN]" if dry_run else "[MIGRATED]" print(f"{tag} {path}") else: skipped += 1 - - print(f"\nDone: {changed} file(s) {'would be ' if dry_run else ''}modified, " f"{skipped} unchanged.") + for note in notes: + print(f" [MANUAL] {path}: {note}") + manual_total += 1 + + suffix = "would be " if dry_run else "" + print(f"\nDone: {changed} file(s) {suffix}modified, {skipped} unchanged.", end="") + if manual_total: + print(f" {manual_total} pattern(s) require manual review.") + else: + print() return 0 From ce2710d0daa5f708b086d8867031d383c47c6147 Mon Sep 17 00:00:00 2001 From: Samson Gebre Date: Wed, 6 May 2026 16:11:36 -0700 Subject: [PATCH 09/16] Fix black format --- .../Dataverse/models/fetchxml_query.py | 3 +-- tools/migrate_v0_to_v1.py | 17 +++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/PowerPlatform/Dataverse/models/fetchxml_query.py b/src/PowerPlatform/Dataverse/models/fetchxml_query.py index 691e2e05..a1f43883 100644 --- a/src/PowerPlatform/Dataverse/models/fetchxml_query.py +++ b/src/PowerPlatform/Dataverse/models/fetchxml_query.py @@ -155,8 +155,7 @@ def execute_pages(self) -> Iterator[QueryResult]: continue except (_ET.ParseError, ValueError) as exc: warnings.warn( - f"FetchXML paging cookie could not be parsed ({exc}); " - "falling back to simple paging.", + f"FetchXML paging cookie could not be parsed ({exc}); " "falling back to simple paging.", UserWarning, stacklevel=2, ) diff --git a/tools/migrate_v0_to_v1.py b/tools/migrate_v0_to_v1.py index 0729f6d5..fecc394b 100644 --- a/tools/migrate_v0_to_v1.py +++ b/tools/migrate_v0_to_v1.py @@ -648,10 +648,13 @@ def visit_Call(self, node: cst.Call) -> None: ) # client.query.sql_select/sql_join/sql_joins — removed with no mechanical replacement - if method in _REMOVED_QUERY_METHODS and len(chain) >= 3 and chain[1] == "query" and chain[2] == self._client_var: - self.findings.append( - f"{self._client_var}.query.{method}() — removed at GA with no mechanical replacement" - ) + if ( + method in _REMOVED_QUERY_METHODS + and len(chain) >= 3 + and chain[1] == "query" + and chain[2] == self._client_var + ): + self.findings.append(f"{self._client_var}.query.{method}() — removed at GA with no mechanical replacement") def find_manual_patterns(source: str, *, client_var: str = "client") -> List[str]: @@ -680,9 +683,7 @@ def migrate_source(source: str, *, client_var: str = "client") -> str: return new_tree.code -def migrate_file( - path: Path, *, dry_run: bool = False, client_var: str = "client" -) -> Tuple[bool, List[str]]: +def migrate_file(path: Path, *, dry_run: bool = False, client_var: str = "client") -> Tuple[bool, List[str]]: """Migrate *path* in place. Returns (was_changed, manual_review_notes).""" original = path.read_text(encoding="utf-8") try: @@ -730,7 +731,7 @@ def main(argv: Optional[List[str]] = None) -> int: if a == "--dry-run": continue if a.startswith("--client-var="): - client_var = a[len("--client-var="):] + client_var = a[len("--client-var=") :] else: remaining.append(a) From 030666c03f6cd3a91591a2eb4b63247e54ff51bd Mon Sep 17 00:00:00 2001 From: Samson Gebre Date: Wed, 6 May 2026 16:41:00 -0700 Subject: [PATCH 10/16] map client.get to client.records.get to automatically migrate it. --- tools/migrate_v0_to_v1.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/migrate_v0_to_v1.py b/tools/migrate_v0_to_v1.py index fecc394b..10ebb925 100644 --- a/tools/migrate_v0_to_v1.py +++ b/tools/migrate_v0_to_v1.py @@ -51,7 +51,7 @@ client.create(t, d) -> client.records.create(t, d) client.update(t, id, d) -> client.records.update(t, id, d) client.delete(t, id) -> client.records.delete(t, id) - client.get(t, id) -> client.records.retrieve(t, id) + client.get(t, id) -> client.records.get(t, id) [deprecated; see manual section] client.query_sql(sql) -> client.query.sql(sql) client.get_table_info(t) -> client.tables.get(t) client.create_table(t, …) -> client.tables.create(t, …) @@ -156,7 +156,7 @@ "create": ("records", "create"), "update": ("records", "update"), "delete": ("records", "delete"), - "get": ("records", "retrieve"), + "get": ("records", "get"), "query_sql": ("query", "sql"), "get_table_info": ("tables", "get"), "create_table": ("tables", "create"), From b7b4a8b89cc55e5cef15853126184ad692209cf3 Mon Sep 17 00:00:00 2001 From: Samson Gebre Date: Wed, 6 May 2026 18:39:32 -0700 Subject: [PATCH 11/16] feat: Enhance OData client with support for include annotations and pagination options - Added `include_annotations` parameter to `_RecordGet` and `_RecordList` classes for OData requests. - Updated `_BatchClient` to handle new parameters in batch operations. - Enhanced `_ODataClient` methods to support `include_annotations`, `expand`, `page_size`, and `count` parameters. - Modified `BatchRecordOperations` to pass new parameters in batch record retrieval and listing methods. - Updated `RecordOperations` to include new parameters for retrieving and listing records. - Added unit tests to validate the new functionality for batch operations and record retrieval. - Implemented migration tool updates to handle changes in method signatures and ensure backward compatibility. --- .claude/skills/dataverse-sdk-use/SKILL.md | 149 +++++---- CHANGELOG.md | 8 +- README.md | 91 ++++-- examples/basic/functional_testing.py | 138 ++++++-- .../claude_skill/dataverse-sdk-dev/SKILL.md | 26 ++ .../claude_skill/dataverse-sdk-use/SKILL.md | 149 +++++---- src/PowerPlatform/Dataverse/data/_batch.py | 13 +- src/PowerPlatform/Dataverse/data/_odata.py | 38 ++- .../Dataverse/operations/batch.py | 55 +++- .../Dataverse/operations/records.py | 88 +++-- tests/unit/data/test_batch_serialization.py | 137 +++++++- tests/unit/test_migration_tool.py | 303 ++++++++++++++++++ tests/unit/test_phase3_ga.py | 44 ++- tests/unit/test_records_operations.py | 31 ++ tools/migrate_v0_to_v1.py | 57 +++- 15 files changed, 1119 insertions(+), 208 deletions(-) create mode 100644 tests/unit/test_migration_tool.py diff --git a/.claude/skills/dataverse-sdk-use/SKILL.md b/.claude/skills/dataverse-sdk-use/SKILL.md index 44a1e861..9e05a401 100644 --- a/.claude/skills/dataverse-sdk-use/SKILL.md +++ b/.claude/skills/dataverse-sdk-use/SKILL.md @@ -28,11 +28,12 @@ Use the PowerPlatform Dataverse Client Python SDK to interact with Microsoft Dat The SDK supports Dataverse's native bulk operations: Pass lists to `create()`, `update()` for automatic bulk processing, for `delete()`, set `use_bulk_delete` when passing lists to use bulk operation ### Paging -- Control page size with `page_size` parameter +- Control page size with `page_size` parameter on `records.list()`, `records.list_pages()`, or `QueryBuilder.page_size()` - Use `top` parameter to limit total records returned -- Simple streaming: `records.list_pages(table, filter, select)` — yields one `QueryResult` per HTTP page (3 params only; use builder for advanced options) -- Advanced streaming: `client.query.builder(table)....execute_pages()` — full builder options, one `QueryResult` per page +- **Preferred**: `client.query.builder(table)....execute_pages()` — composable `where(col(...))` filters, formatted values, expand with nested selects, full pagination control +- Simple streaming shortcut: `records.list_pages(table, *, filter, select, top, orderby, expand, page_size, count, include_annotations)` — string-based OData filter only, yields one `QueryResult` per page - `execute(by_page=True/False)` is **deprecated** and emits `UserWarning`; use `execute_pages()` instead +- `QueryBuilder.to_dataframe()` is **deprecated**; use `.execute().to_dataframe()` instead ### QueryResult - Returned by `records.list()`, `records.retrieve()`, `execute()`, and each page from `list_pages()` / `execute_pages()` @@ -98,48 +99,81 @@ contact_ids = client.records.create("contact", contacts) # Get single record by ID account = client.records.retrieve("account", account_id, select=["name", "telephone1"]) -# Query with filter — follows @odata.nextLink automatically (multiple HTTP requests if needed), -# loads all matching records into memory, returns a single QueryResult. -# Page size is Dataverse's default (~5000/page); use top to bound total records and round-trips. -# For very large sets where memory is a concern, use records.list_pages() or execute_pages() instead. -result = client.records.list( - "account", - select=["accountid", "name"], # select is case-insensitive (automatically lowercased) - filter="statecode eq 0", # filter must use lowercase logical names (not transformed) - top=100, # bounds both total records returned and HTTP round-trips -) +# Simple shortcut — use records.list() only for basic filter + select without composable logic. +# Follows @odata.nextLink automatically and loads all matching records into memory. +# For filtering, sorting, expansion, or formatted values, prefer client.query.builder() (see below). +result = client.records.list("account", filter="statecode eq 0", select=["name", "accountid"]) for record in result: print(record["name"]) +``` -# Simple streaming — page-by-page (3 params only; use builder for ordering/expand/count) -for page in client.records.list_pages( - "account", - select=["accountid", "name"], - filter="statecode eq 0", -): - for record in page: - print(record["name"]) +#### Query Builder (Preferred for Filtering, Sorting, Expand, Formatted Values) -# Advanced streaming — full builder options, one QueryResult per HTTP page +Use `client.query.builder()` for any query that goes beyond simple filter + select. It provides composable `where(col(...))` expressions, formatted value support, nested expansion, and streaming — all with a fluent API. + +```python from PowerPlatform.Dataverse.models.filters import col +from PowerPlatform.Dataverse.models.query_builder import ExpandOption + +# Basic query with composable filter and sort +result = (client.query.builder("account") + .select("accountid", "name", "statecode") + .where(col("statecode") == 0) + .order_by("name asc") + .execute()) +for record in result: + print(record["name"]) + +# Composable filters — AND / OR / NOT using Python operators +result = (client.query.builder("contact") + .select("fullname", "emailaddress1") + .where((col("statecode") == 0) & (col("emailaddress1").contains("@contoso.com"))) + .execute()) + +# Formatted values — display labels for option sets, currency symbols, etc. +result = (client.query.builder("account") + .select("accountid", "name", "industrycode") + .where(col("statecode") == 0) + .include_formatted_values() + .execute()) +for record in result: + label = record.get("industrycode@OData.Community.Display.V1.FormattedValue") + print(record["name"], label) + +# Navigation property expansion with nested column select +result = (client.query.builder("account") + .select("name") + .expand(ExpandOption("primarycontactid").select("fullname", "emailaddress1")) + .where(col("statecode") == 0) + .execute()) +for record in result: + contact = record.get("primarycontactid", {}) + print(f"{record['name']} - {contact.get('fullname', 'N/A')}") + +# Stream large result sets page-by-page (memory-efficient) for page in (client.query.builder("account") .select("accountid", "name") .where(col("statecode") == 0) - .page_size(500) # optional: override Dataverse default page size + .order_by("name asc") + .page_size(500) .execute_pages()): for record in page: print(record["name"]) -# Query with navigation property expansion — use the query builder (records.list() has no expand) -from PowerPlatform.Dataverse.models.query_builder import ExpandOption -from PowerPlatform.Dataverse.models.filters import col -for record in (client.query.builder("account") - .select("name") - .expand(ExpandOption("primarycontactid").select("fullname")) - .where(col("statecode") == 0) - .execute()): - contact = record.get("primarycontactid", {}) - print(f"{record['name']} - {contact.get('fullname', 'N/A')}") +# Convert query results to a DataFrame +df = (client.query.builder("account") + .select("accountid", "name") + .where(col("statecode") == 0) + .execute() + .to_dataframe()) + +# Limit total results +result = client.query.builder("account").select("name").top(100).execute() + +# Simple streaming shortcut via records.list_pages() (string filter only, same params as records.list()) +for page in client.records.list_pages("account", filter="statecode eq 0", select=["name"], page_size=500): + for record in page: + print(record["name"]) ``` #### Create Records with Lookup Bindings (@odata.bind) @@ -212,18 +246,21 @@ client.records.delete("account", [id1, id2, id3], use_bulk_delete=True) The SDK provides DataFrame wrappers for all CRUD operations via the `client.dataframe` namespace, using pandas DataFrames and Series as input/output. -> **Note:** `client.dataframe.get()` is deprecated. Use `client.query.builder(table).select(...).where(...).to_dataframe()` instead. +> **Note:** `client.dataframe.get()` is deprecated. Use `client.query.builder(table).select(...).where(...).execute().to_dataframe()` instead. `QueryBuilder.to_dataframe()` (without `.execute()`) is also deprecated — always call `.execute()` first. ```python import pandas as pd -# Query records -- returns a single DataFrame (GA builder pattern) +# Query records -- returns a single DataFrame (GA pattern: .execute().to_dataframe()) from PowerPlatform.Dataverse.models.filters import col -df = client.query.builder("account").where(col("statecode") == 0).select("name").to_dataframe() +df = client.query.builder("account").where(col("statecode") == 0).select("name").execute().to_dataframe() print(f"Got {len(df)} rows") # Limit results with top -df = client.query.builder("account").select("name").top(100).to_dataframe() +df = client.query.builder("account").select("name").top(100).execute().to_dataframe() + +# Via records.list() (simpler for basic queries) +df = client.records.list("account", filter="statecode eq 0", select=["name"]).to_dataframe() # Fetch single record as one-row DataFrame df = client.records.retrieve("account", account_id, select=["name"]).to_dataframe() @@ -444,8 +481,8 @@ Use `client.batch` to send multiple operations in one HTTP request. All batch me batch = client.batch.new() batch.records.create("account", {"name": "Contoso"}) batch.records.update("account", account_id, {"telephone1": "555-0100"}) -batch.records.retrieve("account", account_id, select=["name"]) # single record (GA) -batch.records.list("account", filter="statecode eq 0", top=50) # multi-record, single page +batch.records.retrieve("account", account_id, select=["name"], include_annotations="OData.Community.Display.V1.FormattedValue") # single record +batch.records.list("account", filter="statecode eq 0", select=["name"], orderby=["name asc"], top=50, page_size=25, count=True) # multi-record, single page batch.query.sql("SELECT TOP 5 name FROM account") result = batch.execute() @@ -530,16 +567,17 @@ except ValidationError as e: ### Performance Optimization -1. **Use bulk operations** - Pass lists to create/update/delete for automatic optimization -2. **Specify select fields** - Limit returned columns to reduce payload size -3. **Control page size** - Use `top` and `page_size` parameters appropriately -4. **Reuse client instances** - Don't create new clients for each operation -5. **Use production credentials** - ClientSecretCredential or CertificateCredential for unattended operations -6. **Error handling** - Implement retry logic for transient errors (`e.is_transient`) -7. **Always include customization prefix** for custom tables/columns -8. **Use lowercase for column names, match `$metadata` for navigation properties** - Column names in `$select`/`$filter`/record payloads use lowercase LogicalNames. Navigation properties in `$expand` and `@odata.bind` keys are case-sensitive and must match the entity's `$metadata` (PascalCase for custom lookups like `new_CustomerId`, lowercase for system lookups like `parentaccountid`) -9. **Test in non-production environments** first -10. **Use named constants** - Import cascade behavior constants from `PowerPlatform.Dataverse.common.constants` +1. **Prefer `client.query.builder()` for any non-trivial query** — use the builder for filtering, sorting, expansion, or formatted values; `records.list()` is a convenience shortcut for simple filter+select only +2. **Use bulk operations** - Pass lists to create/update/delete for automatic optimization +3. **Specify select fields** - Limit returned columns to reduce payload size +4. **Control page size** - Use `top` and `page_size` parameters appropriately; use `execute_pages()` for large sets +5. **Reuse client instances** - Don't create new clients for each operation +6. **Use production credentials** - ClientSecretCredential or CertificateCredential for unattended operations +7. **Error handling** - Implement retry logic for transient errors (`e.is_transient`) +8. **Always include customization prefix** for custom tables/columns +9. **Use lowercase for column names, match `$metadata` for navigation properties** - Column names in `$select`/`$filter`/record payloads use lowercase LogicalNames. Navigation properties in `$expand` and `@odata.bind` keys are case-sensitive and must match the entity's `$metadata` (PascalCase for custom lookups like `new_CustomerId`, lowercase for system lookups like `parentaccountid`) +10. **Test in non-production environments** first +11. **Use named constants** - Import cascade behavior constants from `PowerPlatform.Dataverse.common.constants` ## Additional Resources @@ -552,9 +590,10 @@ Load these resources as needed during development: ## Key Reminders -1. **Schema names are required** - Never use display names -2. **Custom tables need prefixes** - Include customization prefix (e.g., "new_") -3. **Filter is case-sensitive** - Use lowercase logical names -4. **Bulk operations are encouraged** - Pass lists for optimization -5. **No trailing slashes in URLs** - Format: `https://org.crm.dynamics.com` -6. **Structured errors** - Check `is_transient` for retry logic +1. **Use `client.query.builder()` for queries** — it's the primary query pattern; `records.list()` is a shortcut for trivial filter+select only +2. **Schema names are required** - Never use display names +3. **Custom tables need prefixes** - Include customization prefix (e.g., "new_") +4. **Filter is case-sensitive** - Use lowercase logical names +5. **Bulk operations are encouraged** - Pass lists for optimization +6. **No trailing slashes in URLs** - Format: `https://org.crm.dynamics.com` +7. **Structured errors** - Check `is_transient` for retry logic diff --git a/CHANGELOG.md b/CHANGELOG.md index 372d593e..48a9c327 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,9 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- `client.records.retrieve(table, record_id)` — fetch a single record by GUID; returns `None` on 404 instead of raising (#175) -- `client.records.list(table, filter, select, top)` — eager fetch returning a flat `QueryResult`; GA replacement for `records.get()` without a record ID (#175) -- `client.records.list_pages(table, filter, select, top)` — lazy iterator yielding one `QueryResult` per HTTP page; streaming counterpart to `list()` (#175) +- `client.records.retrieve(table, record_id, *, select, include_annotations)` — fetch a single record by GUID; returns `None` on 404 instead of raising; `include_annotations` maps to the `Prefer: odata.include-annotations` header for formatted values and lookup labels (#175) +- `client.records.list(table, *, filter, select, top, orderby, expand, page_size, count, include_annotations)` — eager fetch returning a flat `QueryResult`; GA replacement for `records.get()` without a record ID; `page_size` controls `Prefer: odata.maxpagesize`, `count=True` adds `$count=true`, `include_annotations` requests formatted values (#175) +- `client.records.list_pages(table, *, filter, select, top, orderby, expand, page_size, count, include_annotations)` — lazy iterator yielding one `QueryResult` per HTTP page; streaming counterpart to `list()`; same parameter set (#175) +- `client.batch.records.retrieve()` and `client.batch.records.list()` now accept the same `include_annotations`, `orderby`, `expand`, `page_size`, and `count` parameters as their non-batch counterparts (#175) - `client.query.fetchxml(xml)` — FetchXML support returning an inert `FetchXmlQuery`; no HTTP request is made until `.execute()` or `.execute_pages()` is called (#175) - `FetchXmlQuery` implements the correct Dataverse paging cookie algorithm: annotation parsed as outer XML, `pagingcookie` attribute double URL-decoded, server-supplied `pagenumber` used for next page, `morerecords` handled as both `bool` and `"true"` string, `UserWarning` emitted on simple paging fallback, 32,768-character URL limit enforced (documented Dataverse GET cap), 10,000-page circuit breaker against runaway iteration (#175) - `QueryBuilder.execute_pages()` — lazy per-page streaming returning one `QueryResult` per HTTP page; replaces deprecated `execute(by_page=True)` (#175) @@ -19,6 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `DataverseModel` structural `Protocol` (`models/protocol.py`) — implement on any entity class to enable typed integration with CRUD operations without specifying table names or serializing manually (#175) - `col()`, `raw()`, `QueryResult`, and `DataverseModel` exported from the top-level `PowerPlatform.Dataverse` package (#175) - v0→v1 migration tool: `tools/migrate_v0_to_v1.py` rewrites v0 call sites to the v1 API with `--dry-run` support; covers `create`, `update`, `delete`, `get`, `list`, `fetchxml`, and query builder patterns (#175) +- Migration tool now auto-rewrites `QueryBuilder.to_dataframe()` → `.execute().to_dataframe()` (inserts `.execute()` when receiver is a recognised builder chain); output improved with `[NEEDS-MANUAL]` label for files that have no auto-rewrites but require manual attention, and a trailing note on `[MIGRATED]` lines when manual items remain (#175) ### Changed - `QueryBuilder.execute()` now returns a flat `QueryResult` (all pages collected eagerly) instead of `Iterable[Record]` (#175) diff --git a/README.md b/README.md index 4e72fa5b..99db99fc 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ The SDK provides a simple, pythonic interface for Dataverse operations: | **Records** | Dataverse records represented as Python dictionaries with column schema names | | **Schema names** | Use table schema names (`"account"`, `"new_MyTestTable"`) and column schema names (`"name"`, `"new_MyTestColumn"`). See: [Table definitions in Microsoft Dataverse](https://learn.microsoft.com/en-us/power-apps/developer/data-platform/entity-metadata) | | **Bulk Operations** | Efficient bulk processing for multiple records with automatic optimization | +| **QueryBuilder** | Preferred query API: `client.query.builder()` with composable `where(col(...))` filters, formatted values, expand, and streaming; use `records.list()` only as a shortcut for simple filter+select | | **Paging** | Automatic handling of large result sets with iterators | | **Structured Errors** | Detailed exception hierarchy with retry guidance and diagnostic information | | **Customization prefix values** | Custom tables and columns require a customization prefix value to be included for all operations (e.g., `"new_MyTestTable"`, not `"MyTestTable"`). See: [Table definitions in Microsoft Dataverse](https://learn.microsoft.com/en-us/power-apps/developer/data-platform/entity-metadata) | @@ -357,13 +358,31 @@ for record in (client.query.builder("account") **Formatted values and annotations** -- request localized labels, currency symbols, and display names: ```python -# Get formatted values (choice labels, currency, lookup names) +# Get formatted values (choice labels, currency, lookup names) — via query builder for record in (client.query.builder("account") .select("name", "statecode", "revenue") .include_formatted_values() .execute()): status = record["statecode@OData.Community.Display.V1.FormattedValue"] print(f"{record['name']}: {status}") + +# Get formatted values — via records.list() / records.retrieve() include_annotations param +result = client.records.list( + "account", + select=["name", "statecode"], + include_annotations="OData.Community.Display.V1.FormattedValue", +) +for record in result: + label = record.get("statecode@OData.Community.Display.V1.FormattedValue") + print(f"{record['name']}: {label}") + +record = client.records.retrieve( + "account", account_id, + select=["name", "statuscode"], + include_annotations="OData.Community.Display.V1.FormattedValue", +) +if record: + print(record.get("statuscode@OData.Community.Display.V1.FormattedValue")) ``` **Nested expand with options** -- expand navigation properties with `$select`, `$filter`, `$orderby`, and `$top`: @@ -383,52 +402,58 @@ for record in (client.query.builder("account") print(record["name"], record.get("Account_Tasks")) ``` -**Paging** -- `records.list()` follows `@odata.nextLink` automatically and loads all matching records into a single `QueryResult`. Use `top` to bound total records and round-trips. For very large result sets where memory matters, stream pages explicitly with `execute_pages()` or `records.list_pages()`: +**Paging** -- use `execute_pages()` for streaming large result sets with full builder options (filtering, sorting, formatted values). `records.list()` and `records.list_pages()` are simpler shortcuts for string-based OData filter queries: ```python -# records.list() — automatic paging, all records in memory (good for small-to-medium sets) +# Preferred: query.builder().execute_pages() — stream one page at a time, memory stays flat +# Supports composable filters, sorting, formatted values, and expand with nested selects +for page_num, page in enumerate( + client.query.builder("account") + .select("accountid", "name", "revenue") + .where(col("statecode") == 0) + .order_by("name") + .page_size(500) # optional: override Dataverse default (~5000/page) + .execute_pages() +): + print(f"Page {page_num + 1}: {len(page)} records") + for record in page: + print(f" {record['name']}") + +# Simple shortcut: records.list() — automatic paging, all records in memory +# Use for basic filter+select queries; string OData filter only (no composable expressions) result = client.records.list( "account", filter="statecode eq 0", - select=["name"], - top=500, # bounds total records returned and number of HTTP round-trips + select=["name", "revenue"], + orderby=["name asc"], # optional sort + top=500, # bounds total records returned and number of HTTP round-trips + page_size=200, # optional: hint Dataverse default page size ) for record in result: print(record["name"]) -# records.list_pages() — stream one page at a time without loading all records into memory +# Simple streaming shortcut: records.list_pages() — same params as records.list(), yields one page at a time for page_num, page in enumerate( - client.records.list_pages("account", filter="statecode eq 0", select=["name"]) + client.records.list_pages("account", filter="statecode eq 0", select=["name"], orderby=["name asc"]) ): print(f"Page {page_num + 1}: {len(page)} records") for record in page: print(record["name"]) - -# query.builder().execute_pages() — stream one page at a time, memory stays flat -# Use this for large result sets (exports, bulk processing) where loading all into memory is risky -for page_num, page in enumerate( - client.query.builder("account") - .select("accountid", "name", "revenue") - .where(col("statecode") == 0) - .order_by("name") - .page_size(500) # optional: override Dataverse default (~5000/page) - .execute_pages() -): - print(f"Page {page_num + 1}: {len(page)} records") - for record in page: - print(f" {record['name']}") ``` -> **Deprecation note:** `execute(by_page=True)` and `execute(by_page=False)` are deprecated and emit a `UserWarning`. Replace with `execute_pages()` (streaming) or plain `execute()` (eager). The migration tool (`tools/migrate_v0_to_v1.py`) rewrites these automatically. +> **Deprecation note:** `execute(by_page=True)` and `execute(by_page=False)` are deprecated and emit a `UserWarning`. Replace with `execute_pages()` (streaming) or plain `execute()` (eager). `QueryBuilder.to_dataframe()` is also deprecated; use `.execute().to_dataframe()` instead. The migration tool (`tools/migrate_v0_to_v1.py`) rewrites all of these automatically. **Record count** -- include `$count=true` in the request: ```python -# Request count alongside results +# Via query builder results = (client.query.builder("account") .where(col("statecode") == 0) .count() .execute()) + +# Via records.list() — count=True adds $count=true to the OData request +results = client.records.list("account", filter="statecode eq 0", count=True) ``` **FetchXML queries** -- `client.query.fetchxml()` returns an inert `FetchXmlQuery` object; no HTTP request is made until you call `.execute()` or `.execute_pages()`: @@ -487,15 +512,15 @@ sql = f"SELECT TOP 10 {', '.join(col_names[:5])} FROM account" df = client.dataframe.sql(sql) ``` -**Raw OData filter strings** can be passed directly to `records.list()` or the query builder for cases where you need direct control over the OData filter: +**Simple list shortcut** -- `records.list()` accepts a raw OData filter string for basic queries. For anything beyond simple filter+select, prefer `client.query.builder()` (composable filters, formatted values, nested expand): ```python -# Query with a raw OData filter string — follows @odata.nextLink automatically, -# all records loaded into memory. Use top to bound results and round-trips. +# records.list() shortcut — raw OData filter string, all records loaded into memory +# Column names in filter must be lowercase logical names for record in client.records.list( "account", select=["name"], - filter="statecode eq 0", # Raw OData: column names must be lowercase + filter="statecode eq 0", top=100, ): print(record["name"]) @@ -669,7 +694,14 @@ batch.records.create("account", {"name": "Contoso"}) batch.records.create("account", [{"name": "Fabrikam"}, {"name": "Woodgrove"}]) batch.records.update("account", account_id, {"telephone1": "555-0100"}) batch.records.delete("account", old_id) -batch.records.retrieve("account", account_id, select=["name"]) # single record (GA) +batch.records.retrieve("account", account_id, select=["name"]) # single record +batch.records.list( # multi-record, single page + "account", + filter="statecode eq 0", + select=["name"], + orderby=["name asc"], + top=50, +) result = batch.execute() for item in result.responses: @@ -807,9 +839,10 @@ For optimal performance in production environments: | Best Practice | Description | |---------------|-------------| +| **Prefer QueryBuilder for queries** | Use `client.query.builder()` for filtering, sorting, expansion, or formatted values; use `records.list()` only as a shortcut for simple filter+select | | **Bulk Operations** | Pass lists to `records.create()`, `records.update()` for automatic bulk processing, for `records.delete()`, set `use_bulk_delete` when passing lists to use bulk operation | | **Select Fields** | Specify `select` parameter to limit returned columns and reduce payload size | -| **Page Size Control** | Use `top` and `page_size` parameters to control memory usage | +| **Page Size Control** | Use `top` and `page_size` parameters to control memory usage; use `execute_pages()` for large result sets | | **Connection Reuse** | Reuse `DataverseClient` instances across operations | | **Production Credentials** | Use `ClientSecretCredential` or `CertificateCredential` for unattended operations | | **Error Handling** | Implement retry logic for transient errors (`e.is_transient`) | diff --git a/examples/basic/functional_testing.py b/examples/basic/functional_testing.py index 95603475..c5918ae1 100644 --- a/examples/basic/functional_testing.py +++ b/examples/basic/functional_testing.py @@ -250,24 +250,33 @@ def test_read_record(client: DataverseClient, table_info: Dict[str, Any], record if record is None: raise RuntimeError("Record did not become available in time.") - if record: - print("[OK] Record retrieved successfully!") - print(" Retrieved data:") - - # Display key fields - for field_name in [ - f"{attr_prefix}_name", - f"{attr_prefix}_description", - f"{attr_prefix}_count", - f"{attr_prefix}_amount", - f"{attr_prefix}_is_active", - ]: - if field_name in record: - print(f" {field_name}: {record[field_name]}") - - return record + print("[OK] Record retrieved successfully!") + print(" Retrieved data:") + for field_name in [ + f"{attr_prefix}_name", + f"{attr_prefix}_description", + f"{attr_prefix}_count", + f"{attr_prefix}_amount", + f"{attr_prefix}_is_active", + ]: + if field_name in record: + print(f" {field_name}: {record[field_name]}") + + # -- include_annotations: verify FormattedValue annotations are returned -- + annotation = "OData.Community.Display.V1.FormattedValue" + annotated = client.records.retrieve( + table_schema_name, + record_id, + select=[f"{attr_prefix}_is_active", f"{attr_prefix}_count"], + include_annotations=annotation, + ) + ann_key = f"{attr_prefix}_is_active@{annotation}" + if annotated is not None and ann_key in annotated: + print(f"[OK] include_annotations verified: {ann_key} = '{annotated[ann_key]}'") else: - raise ValueError("Record not found") + print(f"[WARN] include_annotations: expected key '{ann_key}' not present in response") + + return record except HttpError as e: print(f"[ERR] HTTP error during record reading: {e}") @@ -334,6 +343,51 @@ def test_query_records(client: DataverseClient, table_info: Dict[str, Any]) -> N print(f" Page {page_num}: {len(page)} record(s) — {names}") print(f"[OK] records.list_pages() completed! {total_records} records across {page_num} page(s).") + # -- records.list() with orderby, page_size, count, include_annotations -------- + print("\nQuerying records.list() with orderby / page_size / count / include_annotations...") + annotation = "OData.Community.Display.V1.FormattedValue" + annotated_result = client.records.list( + table_schema_name, + select=[f"{attr_prefix}_name", f"{attr_prefix}_is_active"], + filter=active_filter, + orderby=[f"{attr_prefix}_name asc"], + page_size=50, + count=True, + include_annotations=annotation, + ) + names_ordered = [r.get(f"{attr_prefix}_name", "N/A") for r in annotated_result] + ann_key = f"{attr_prefix}_is_active@{annotation}" + ann_present = any(ann_key in r for r in annotated_result) + print(f" Records (ordered): {names_ordered}") + if ann_present: + print(f"[OK] include_annotations verified: '{ann_key}' present in list() results") + else: + print(f"[WARN] include_annotations: '{ann_key}' not found — may not be supported by this environment") + # Verify orderby: names should be non-decreasing + if len(names_ordered) > 1 and all(isinstance(n, str) for n in names_ordered): + assert names_ordered == sorted(names_ordered), f"orderby asc not respected: {names_ordered}" + print(f"[OK] records.list() with extended params completed! {len(annotated_result)} record(s).") + + # -- records.list_pages() with orderby, page_size, include_annotations --------- + print("\nQuerying records.list_pages() with orderby / page_size / include_annotations...") + lp_records = [] + for page in client.records.list_pages( + table_schema_name, + select=[f"{attr_prefix}_name", f"{attr_prefix}_is_active"], + filter=active_filter, + orderby=[f"{attr_prefix}_name asc"], + page_size=50, + include_annotations=annotation, + ): + lp_records.extend(page) + lp_names = [r.get(f"{attr_prefix}_name", "N/A") for r in lp_records] + lp_ann_present = any(ann_key in r for r in lp_records) + if lp_ann_present: + print(f"[OK] include_annotations verified in list_pages() results") + else: + print(f"[WARN] include_annotations: '{ann_key}' not found in list_pages() results") + print(f"[OK] records.list_pages() with extended params completed! {len(lp_records)} record(s).") + except Exception as e: print(f"[WARN] Query test encountered an issue: {e}") print(" This might be expected if the table is very new.") @@ -533,19 +587,38 @@ def test_batch_all_operations(client: DataverseClient, table_info: Dict[str, Any print(f"[OK] {len(result.succeeded)} ops → {len(all_ids)} records created: {all_ids}") # ------------------------------------------------------------------- - # [2/11] READ — get by ID + tables.get + tables.list + query.sql - # All 4 reads in one batch request + # [2/11] READ — records.retrieve + records.list (with extended params) + # + tables.get + tables.list + query.sql — 1 POST $batch # ------------------------------------------------------------------- if all_ids: - print("\n[2/11] Read — records.get + tables.get + tables.list + query.sql (4 ops, 1 POST $batch)") + annotation = "OData.Community.Display.V1.FormattedValue" + print( + "\n[2/11] Read — records.retrieve + records.list(orderby/page_size/count/include_annotations)" + " + tables.get + tables.list + query.sql (5 ops, 1 POST $batch)" + ) batch = client.batch.new() - batch.records.get( + # [0] Single-record retrieve with annotations + batch.records.retrieve( table_schema_name, all_ids[0], - select=[f"{attr_prefix}_name", f"{attr_prefix}_count"], + select=[f"{attr_prefix}_name", f"{attr_prefix}_count", f"{attr_prefix}_is_active"], + include_annotations=annotation, + ) + # [1] Multi-record list with orderby, page_size, count, include_annotations + batch.records.list( + table_schema_name, + select=[f"{attr_prefix}_name", f"{attr_prefix}_is_active"], + filter=f"{attr_prefix}_is_active eq true", + orderby=[f"{attr_prefix}_name asc"], + page_size=50, + count=True, + include_annotations=annotation, ) + # [2] Table metadata batch.tables.get(table_schema_name) + # [3] Table list batch.tables.list() + # [4] SQL batch.query.sql(f"SELECT TOP 3 {attr_prefix}_name FROM {logical_name}") result = batch.execute() print(f"[OK] {len(result.succeeded)} succeeded, {len(result.failed)} failed") @@ -554,16 +627,27 @@ def test_batch_all_operations(client: DataverseClient, table_info: Dict[str, Any print(f" [{i}] FAILED {resp.status_code}: {resp.error_message}") continue if i == 0 and resp.data: - print( - f" records.get → name='{resp.data.get(f'{attr_prefix}_name')}', count={resp.data.get(f'{attr_prefix}_count')}" - ) + name = resp.data.get(f"{attr_prefix}_name") + ann_key = f"{attr_prefix}_is_active@{annotation}" + ann_val = resp.data.get(ann_key, "") + print(f" records.retrieve → name='{name}', {ann_key}='{ann_val}'") elif i == 1 and resp.data: + rows = resp.data.get("value", []) + names_ordered = [r.get(f"{attr_prefix}_name") for r in rows] + ann_key = f"{attr_prefix}_is_active@{annotation}" + ann_present = any(ann_key in r for r in rows) + print(f" records.list → {len(rows)} row(s), ordered: {names_ordered}") + if ann_present: + print(f" [OK] include_annotations '{ann_key}' present in batch.records.list() results") + else: + print(f" [WARN] include_annotations '{ann_key}' not found in batch.records.list() results") + elif i == 2 and resp.data: print( f" tables.get → LogicalName='{resp.data.get('LogicalName')}', EntitySet='{resp.data.get('EntitySetName')}'" ) - elif i == 2 and resp.data: - print(f" tables.list → {len(resp.data.get('value', []))} tables returned") elif i == 3 and resp.data: + print(f" tables.list → {len(resp.data.get('value', []))} tables returned") + elif i == 4 and resp.data: print(f" query.sql → {len(resp.data.get('value', []))} rows returned") # ------------------------------------------------------------------- diff --git a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md index 79d4e342..c3af44c5 100644 --- a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md +++ b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md @@ -20,6 +20,32 @@ This skill provides guidance for developers working on the PowerPlatform Dataver 5. **Consider backwards compatibility** - Avoid breaking changes 6. **Internal vs public naming** - Modules, files, and functions not meant to be part of the public API must use a `_` prefix (e.g., `_odata.py`, `_relationships.py`). Files without the prefix (e.g., `constants.py`, `metadata.py`) are public and importable by SDK consumers +### Dataverse Property Naming Rules + +Dataverse uses two different naming conventions for properties. Getting this wrong causes 400 errors that are hard to debug. + +| Property type | Name convention | Example | When used | +|---|---|---|---| +| **Structural** (columns) | LogicalName (always lowercase) | `new_name`, `new_priority` | `$select`, `$filter`, `$orderby`, record payload keys | +| **Navigation** (relationships / lookups) | Navigation Property Name (usually SchemaName, PascalCase, case-sensitive) | `new_CustomerId`, `new_AgentId` | `$expand`, `@odata.bind` annotation keys | + +Navigation property names are case-sensitive and must match the entity's `$metadata`. Using the logical name instead of the navigation property name results in 400 Bad Request errors. + +**Critical rule:** The OData parser validates `@odata.bind` property names **case-sensitively** against declared navigation properties. Lowercasing `new_CustomerId@odata.bind` to `new_customerid@odata.bind` causes: `ODataException: An undeclared property 'new_customerid' which only has property annotations...` + +**SDK implementation:** + +- `_lowercase_keys()` lowercases all keys EXCEPT those containing `@odata.` (preserves navigation property casing in `@odata.bind` keys) +- `_lowercase_list()` lowercases `$select` and `$orderby` params (structural properties) +- `$expand` params are passed as-is (navigation properties, PascalCase) +- `_convert_labels_to_ints()` skips `@odata.` keys entirely (they are annotations, not attributes) + +**When adding new code that processes record dicts or builds query parameters:** + +- Always use `_lowercase_keys()` for record payloads. Never manually call `.lower()` on all keys +- Never lowercase `$expand` values or `@odata.bind` key prefixes +- If iterating record keys, skip keys containing `@odata.` when doing attribute-level operations + ### Code Style 6. **No emojis** - Do not use emoji in code, comments, or output diff --git a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md index 44a1e861..9e05a401 100644 --- a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md +++ b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md @@ -28,11 +28,12 @@ Use the PowerPlatform Dataverse Client Python SDK to interact with Microsoft Dat The SDK supports Dataverse's native bulk operations: Pass lists to `create()`, `update()` for automatic bulk processing, for `delete()`, set `use_bulk_delete` when passing lists to use bulk operation ### Paging -- Control page size with `page_size` parameter +- Control page size with `page_size` parameter on `records.list()`, `records.list_pages()`, or `QueryBuilder.page_size()` - Use `top` parameter to limit total records returned -- Simple streaming: `records.list_pages(table, filter, select)` — yields one `QueryResult` per HTTP page (3 params only; use builder for advanced options) -- Advanced streaming: `client.query.builder(table)....execute_pages()` — full builder options, one `QueryResult` per page +- **Preferred**: `client.query.builder(table)....execute_pages()` — composable `where(col(...))` filters, formatted values, expand with nested selects, full pagination control +- Simple streaming shortcut: `records.list_pages(table, *, filter, select, top, orderby, expand, page_size, count, include_annotations)` — string-based OData filter only, yields one `QueryResult` per page - `execute(by_page=True/False)` is **deprecated** and emits `UserWarning`; use `execute_pages()` instead +- `QueryBuilder.to_dataframe()` is **deprecated**; use `.execute().to_dataframe()` instead ### QueryResult - Returned by `records.list()`, `records.retrieve()`, `execute()`, and each page from `list_pages()` / `execute_pages()` @@ -98,48 +99,81 @@ contact_ids = client.records.create("contact", contacts) # Get single record by ID account = client.records.retrieve("account", account_id, select=["name", "telephone1"]) -# Query with filter — follows @odata.nextLink automatically (multiple HTTP requests if needed), -# loads all matching records into memory, returns a single QueryResult. -# Page size is Dataverse's default (~5000/page); use top to bound total records and round-trips. -# For very large sets where memory is a concern, use records.list_pages() or execute_pages() instead. -result = client.records.list( - "account", - select=["accountid", "name"], # select is case-insensitive (automatically lowercased) - filter="statecode eq 0", # filter must use lowercase logical names (not transformed) - top=100, # bounds both total records returned and HTTP round-trips -) +# Simple shortcut — use records.list() only for basic filter + select without composable logic. +# Follows @odata.nextLink automatically and loads all matching records into memory. +# For filtering, sorting, expansion, or formatted values, prefer client.query.builder() (see below). +result = client.records.list("account", filter="statecode eq 0", select=["name", "accountid"]) for record in result: print(record["name"]) +``` -# Simple streaming — page-by-page (3 params only; use builder for ordering/expand/count) -for page in client.records.list_pages( - "account", - select=["accountid", "name"], - filter="statecode eq 0", -): - for record in page: - print(record["name"]) +#### Query Builder (Preferred for Filtering, Sorting, Expand, Formatted Values) -# Advanced streaming — full builder options, one QueryResult per HTTP page +Use `client.query.builder()` for any query that goes beyond simple filter + select. It provides composable `where(col(...))` expressions, formatted value support, nested expansion, and streaming — all with a fluent API. + +```python from PowerPlatform.Dataverse.models.filters import col +from PowerPlatform.Dataverse.models.query_builder import ExpandOption + +# Basic query with composable filter and sort +result = (client.query.builder("account") + .select("accountid", "name", "statecode") + .where(col("statecode") == 0) + .order_by("name asc") + .execute()) +for record in result: + print(record["name"]) + +# Composable filters — AND / OR / NOT using Python operators +result = (client.query.builder("contact") + .select("fullname", "emailaddress1") + .where((col("statecode") == 0) & (col("emailaddress1").contains("@contoso.com"))) + .execute()) + +# Formatted values — display labels for option sets, currency symbols, etc. +result = (client.query.builder("account") + .select("accountid", "name", "industrycode") + .where(col("statecode") == 0) + .include_formatted_values() + .execute()) +for record in result: + label = record.get("industrycode@OData.Community.Display.V1.FormattedValue") + print(record["name"], label) + +# Navigation property expansion with nested column select +result = (client.query.builder("account") + .select("name") + .expand(ExpandOption("primarycontactid").select("fullname", "emailaddress1")) + .where(col("statecode") == 0) + .execute()) +for record in result: + contact = record.get("primarycontactid", {}) + print(f"{record['name']} - {contact.get('fullname', 'N/A')}") + +# Stream large result sets page-by-page (memory-efficient) for page in (client.query.builder("account") .select("accountid", "name") .where(col("statecode") == 0) - .page_size(500) # optional: override Dataverse default page size + .order_by("name asc") + .page_size(500) .execute_pages()): for record in page: print(record["name"]) -# Query with navigation property expansion — use the query builder (records.list() has no expand) -from PowerPlatform.Dataverse.models.query_builder import ExpandOption -from PowerPlatform.Dataverse.models.filters import col -for record in (client.query.builder("account") - .select("name") - .expand(ExpandOption("primarycontactid").select("fullname")) - .where(col("statecode") == 0) - .execute()): - contact = record.get("primarycontactid", {}) - print(f"{record['name']} - {contact.get('fullname', 'N/A')}") +# Convert query results to a DataFrame +df = (client.query.builder("account") + .select("accountid", "name") + .where(col("statecode") == 0) + .execute() + .to_dataframe()) + +# Limit total results +result = client.query.builder("account").select("name").top(100).execute() + +# Simple streaming shortcut via records.list_pages() (string filter only, same params as records.list()) +for page in client.records.list_pages("account", filter="statecode eq 0", select=["name"], page_size=500): + for record in page: + print(record["name"]) ``` #### Create Records with Lookup Bindings (@odata.bind) @@ -212,18 +246,21 @@ client.records.delete("account", [id1, id2, id3], use_bulk_delete=True) The SDK provides DataFrame wrappers for all CRUD operations via the `client.dataframe` namespace, using pandas DataFrames and Series as input/output. -> **Note:** `client.dataframe.get()` is deprecated. Use `client.query.builder(table).select(...).where(...).to_dataframe()` instead. +> **Note:** `client.dataframe.get()` is deprecated. Use `client.query.builder(table).select(...).where(...).execute().to_dataframe()` instead. `QueryBuilder.to_dataframe()` (without `.execute()`) is also deprecated — always call `.execute()` first. ```python import pandas as pd -# Query records -- returns a single DataFrame (GA builder pattern) +# Query records -- returns a single DataFrame (GA pattern: .execute().to_dataframe()) from PowerPlatform.Dataverse.models.filters import col -df = client.query.builder("account").where(col("statecode") == 0).select("name").to_dataframe() +df = client.query.builder("account").where(col("statecode") == 0).select("name").execute().to_dataframe() print(f"Got {len(df)} rows") # Limit results with top -df = client.query.builder("account").select("name").top(100).to_dataframe() +df = client.query.builder("account").select("name").top(100).execute().to_dataframe() + +# Via records.list() (simpler for basic queries) +df = client.records.list("account", filter="statecode eq 0", select=["name"]).to_dataframe() # Fetch single record as one-row DataFrame df = client.records.retrieve("account", account_id, select=["name"]).to_dataframe() @@ -444,8 +481,8 @@ Use `client.batch` to send multiple operations in one HTTP request. All batch me batch = client.batch.new() batch.records.create("account", {"name": "Contoso"}) batch.records.update("account", account_id, {"telephone1": "555-0100"}) -batch.records.retrieve("account", account_id, select=["name"]) # single record (GA) -batch.records.list("account", filter="statecode eq 0", top=50) # multi-record, single page +batch.records.retrieve("account", account_id, select=["name"], include_annotations="OData.Community.Display.V1.FormattedValue") # single record +batch.records.list("account", filter="statecode eq 0", select=["name"], orderby=["name asc"], top=50, page_size=25, count=True) # multi-record, single page batch.query.sql("SELECT TOP 5 name FROM account") result = batch.execute() @@ -530,16 +567,17 @@ except ValidationError as e: ### Performance Optimization -1. **Use bulk operations** - Pass lists to create/update/delete for automatic optimization -2. **Specify select fields** - Limit returned columns to reduce payload size -3. **Control page size** - Use `top` and `page_size` parameters appropriately -4. **Reuse client instances** - Don't create new clients for each operation -5. **Use production credentials** - ClientSecretCredential or CertificateCredential for unattended operations -6. **Error handling** - Implement retry logic for transient errors (`e.is_transient`) -7. **Always include customization prefix** for custom tables/columns -8. **Use lowercase for column names, match `$metadata` for navigation properties** - Column names in `$select`/`$filter`/record payloads use lowercase LogicalNames. Navigation properties in `$expand` and `@odata.bind` keys are case-sensitive and must match the entity's `$metadata` (PascalCase for custom lookups like `new_CustomerId`, lowercase for system lookups like `parentaccountid`) -9. **Test in non-production environments** first -10. **Use named constants** - Import cascade behavior constants from `PowerPlatform.Dataverse.common.constants` +1. **Prefer `client.query.builder()` for any non-trivial query** — use the builder for filtering, sorting, expansion, or formatted values; `records.list()` is a convenience shortcut for simple filter+select only +2. **Use bulk operations** - Pass lists to create/update/delete for automatic optimization +3. **Specify select fields** - Limit returned columns to reduce payload size +4. **Control page size** - Use `top` and `page_size` parameters appropriately; use `execute_pages()` for large sets +5. **Reuse client instances** - Don't create new clients for each operation +6. **Use production credentials** - ClientSecretCredential or CertificateCredential for unattended operations +7. **Error handling** - Implement retry logic for transient errors (`e.is_transient`) +8. **Always include customization prefix** for custom tables/columns +9. **Use lowercase for column names, match `$metadata` for navigation properties** - Column names in `$select`/`$filter`/record payloads use lowercase LogicalNames. Navigation properties in `$expand` and `@odata.bind` keys are case-sensitive and must match the entity's `$metadata` (PascalCase for custom lookups like `new_CustomerId`, lowercase for system lookups like `parentaccountid`) +10. **Test in non-production environments** first +11. **Use named constants** - Import cascade behavior constants from `PowerPlatform.Dataverse.common.constants` ## Additional Resources @@ -552,9 +590,10 @@ Load these resources as needed during development: ## Key Reminders -1. **Schema names are required** - Never use display names -2. **Custom tables need prefixes** - Include customization prefix (e.g., "new_") -3. **Filter is case-sensitive** - Use lowercase logical names -4. **Bulk operations are encouraged** - Pass lists for optimization -5. **No trailing slashes in URLs** - Format: `https://org.crm.dynamics.com` -6. **Structured errors** - Check `is_transient` for retry logic +1. **Use `client.query.builder()` for queries** — it's the primary query pattern; `records.list()` is a shortcut for trivial filter+select only +2. **Schema names are required** - Never use display names +3. **Custom tables need prefixes** - Include customization prefix (e.g., "new_") +4. **Filter is case-sensitive** - Use lowercase logical names +5. **Bulk operations are encouraged** - Pass lists for optimization +6. **No trailing slashes in URLs** - Format: `https://org.crm.dynamics.com` +7. **Structured errors** - Check `is_transient` for retry logic diff --git a/src/PowerPlatform/Dataverse/data/_batch.py b/src/PowerPlatform/Dataverse/data/_batch.py index 1d5126e2..c6ce1205 100644 --- a/src/PowerPlatform/Dataverse/data/_batch.py +++ b/src/PowerPlatform/Dataverse/data/_batch.py @@ -69,6 +69,7 @@ class _RecordGet: table: str record_id: str select: Optional[List[str]] = None + include_annotations: Optional[str] = None @dataclass @@ -78,6 +79,10 @@ class _RecordList: filter: Optional[str] = None orderby: Optional[List[str]] = None top: Optional[int] = None + expand: Optional[List[str]] = None + page_size: Optional[int] = None + count: bool = False + include_annotations: Optional[str] = None @dataclass @@ -393,7 +398,9 @@ def _resolve_record_delete(self, op: _RecordDelete) -> List[_RawRequest]: return [self._od._build_delete(op.table, rid) for rid in ids] def _resolve_record_get(self, op: _RecordGet) -> List[_RawRequest]: - return [self._od._build_get(op.table, op.record_id, select=op.select)] + return [ + self._od._build_get(op.table, op.record_id, select=op.select, include_annotations=op.include_annotations) + ] def _resolve_record_list(self, op: _RecordList) -> List[_RawRequest]: return [ @@ -403,6 +410,10 @@ def _resolve_record_list(self, op: _RecordList) -> List[_RawRequest]: filter=op.filter, orderby=op.orderby, top=op.top, + expand=op.expand, + page_size=op.page_size, + count=op.count, + include_annotations=op.include_annotations, ) ] diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index c5cc8fce..2b2145b4 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -684,7 +684,13 @@ def _delete(self, table_schema_name: str, key: str) -> None: """ self._execute_raw(self._build_delete(table_schema_name, key)) - def _get(self, table_schema_name: str, key: str, select: Optional[List[str]] = None) -> Dict[str, Any]: + def _get( + self, + table_schema_name: str, + key: str, + select: Optional[List[str]] = None, + include_annotations: Optional[str] = None, + ) -> Dict[str, Any]: """Retrieve a single record. :param table_schema_name: Schema name of the table. @@ -693,11 +699,15 @@ def _get(self, table_schema_name: str, key: str, select: Optional[List[str]] = N :type key: ``str`` :param select: Columns to select; joined with commas into $select. :type select: ``list[str]`` | ``None`` + :param include_annotations: OData annotation pattern for the ``Prefer: odata.include-annotations`` header, or ``None``. + :type include_annotations: ``str`` | ``None`` :return: Retrieved record dictionary (may be empty if no selected attributes). :rtype: ``dict[str, Any]`` """ - return self._execute_raw(self._build_get(table_schema_name, key, select=select)).json() + return self._execute_raw( + self._build_get(table_schema_name, key, select=select, include_annotations=include_annotations) + ).json() def _get_multiple( self, @@ -2309,13 +2319,17 @@ def _build_get( record_id: str, *, select: Optional[List[str]] = None, + include_annotations: Optional[str] = None, ) -> _RawRequest: """Build a single-record GET request without sending it.""" entity_set = self._entity_set_from_schema_name(table) url = f"{self.api}/{entity_set}{self._format_key(record_id)}" if select: url += "?$select=" + ",".join(self._lowercase_list(select)) - return _RawRequest(method="GET", url=url) + headers = None + if include_annotations: + headers = {"Prefer": f'odata.include-annotations="{include_annotations}"'} + return _RawRequest(method="GET", url=url, headers=headers) def _build_list( self, @@ -2325,6 +2339,10 @@ def _build_list( filter: Optional[str] = None, orderby: Optional[List[str]] = None, top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, ) -> _RawRequest: """Build a multi-record GET request (single page, no pagination) without sending it.""" entity_set = self._entity_set_from_schema_name(table) @@ -2337,10 +2355,22 @@ def _build_list( params.append("$orderby=" + ",".join(orderby)) if top is not None: params.append(f"$top={top}") + if expand: + params.append("$expand=" + ",".join(expand)) + if count: + params.append("$count=true") url = f"{self.api}/{entity_set}" if params: url += "?" + "&".join(params) - return _RawRequest(method="GET", url=url) + prefer_parts: List[str] = [] + if page_size is not None: + ps = int(page_size) + if ps > 0: + prefer_parts.append(f"odata.maxpagesize={ps}") + if include_annotations: + prefer_parts.append(f'odata.include-annotations="{include_annotations}"') + headers = {"Prefer": ",".join(prefer_parts)} if prefer_parts else None + return _RawRequest(method="GET", url=url, headers=headers) def _build_create_entity( self, diff --git a/src/PowerPlatform/Dataverse/operations/batch.py b/src/PowerPlatform/Dataverse/operations/batch.py index c3f040e8..987603bc 100644 --- a/src/PowerPlatform/Dataverse/operations/batch.py +++ b/src/PowerPlatform/Dataverse/operations/batch.py @@ -335,6 +335,7 @@ def retrieve( record_id: str, *, select: Optional[List[str]] = None, + include_annotations: Optional[str] = None, ) -> None: """ Add a single-record retrieve operation to the batch. @@ -350,15 +351,26 @@ def retrieve( :type record_id: :class:`str` :param select: Optional list of column logical names to include. :type select: list[str] or None + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header (e.g. ``"*"`` or + ``"OData.Community.Display.V1.FormattedValue"``), or ``None``. + :type include_annotations: :class:`str` or None Example:: batch = client.batch.new() - batch.records.retrieve("account", account_id, select=["name", "telephone1"]) + batch.records.retrieve( + "account", account_id, + select=["name", "statuscode"], + include_annotations="OData.Community.Display.V1.FormattedValue", + ) result = batch.execute() record = result.responses[0].data + print(record["statuscode@OData.Community.Display.V1.FormattedValue"]) """ - self._batch._items.append(_RecordGet(table=table, record_id=record_id, select=select)) + self._batch._items.append( + _RecordGet(table=table, record_id=record_id, select=select, include_annotations=include_annotations) + ) def list( self, @@ -366,7 +378,12 @@ def list( *, filter: "Optional[Union[str, FilterExpression]]" = None, select: Optional[List[str]] = None, + orderby: Optional[List[str]] = None, top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, ) -> None: """ Add a multi-record list operation to the batch (single page, no pagination). @@ -386,18 +403,48 @@ def list( :type filter: str or FilterExpression or None :param select: Optional list of column logical names to include. :type select: list[str] or None + :param orderby: Optional list of sort expressions (e.g. ``["name asc"]``). + :type orderby: list[str] or None :param top: Maximum number of records to return. :type top: int or None + :param expand: Optional list of navigation properties to expand. + :type expand: list[str] or None + :param page_size: Per-page size hint via ``Prefer: odata.maxpagesize``. + :type page_size: int or None + :param count: If ``True``, adds ``$count=true`` to the request. + :type count: bool + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header, or ``None``. + :type include_annotations: :class:`str` or None Example:: batch = client.batch.new() - batch.records.list("account", filter="statecode eq 0", select=["name"], top=50) + batch.records.list( + "account", + filter="statecode eq 0", + select=["name", "statuscode"], + orderby=["name asc"], + top=50, + include_annotations="OData.Community.Display.V1.FormattedValue", + ) result = batch.execute() records = result.responses[0].data.get("value", []) """ filter_str: Optional[str] = str(filter) if filter is not None else None - self._batch._items.append(_RecordList(table=table, select=select, filter=filter_str, top=top)) + self._batch._items.append( + _RecordList( + table=table, + select=select, + filter=filter_str, + orderby=orderby, + top=top, + expand=expand, + page_size=page_size, + count=count, + include_annotations=include_annotations, + ) + ) class BatchTableOperations: diff --git a/src/PowerPlatform/Dataverse/operations/records.py b/src/PowerPlatform/Dataverse/operations/records.py index ee800a18..29ccd8e2 100644 --- a/src/PowerPlatform/Dataverse/operations/records.py +++ b/src/PowerPlatform/Dataverse/operations/records.py @@ -487,6 +487,7 @@ def retrieve( record_id: str, *, select: Optional[List[str]] = None, + include_annotations: Optional[str] = None, ) -> Optional[Record]: """Fetch a single record by its GUID, returning ``None`` if not found. @@ -499,18 +500,26 @@ def retrieve( :type record_id: :class:`str` :param select: Optional list of column logical names to include. :type select: list[str] or None + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header (e.g. ``"*"`` or + ``"OData.Community.Display.V1.FormattedValue"``), or ``None``. + :type include_annotations: :class:`str` or None :return: Typed record, or ``None`` if not found. :rtype: :class:`~PowerPlatform.Dataverse.models.record.Record` or None Example:: - record = client.records.retrieve("account", account_id, select=["name"]) + record = client.records.retrieve( + "account", account_id, + select=["name", "statuscode"], + include_annotations="OData.Community.Display.V1.FormattedValue", + ) if record is not None: - print(record["name"]) + print(record["statuscode@OData.Community.Display.V1.FormattedValue"]) """ with self._client._scoped_odata() as od: try: - raw = od._get(table, record_id, select=select) + raw = od._get(table, record_id, select=select, include_annotations=include_annotations) except HttpError as exc: if exc.status_code == 404: return None @@ -525,24 +534,37 @@ def list( *, filter: Optional[Union[str, "FilterExpression"]] = None, select: Optional[List[str]] = None, + orderby: Optional[List[str]] = None, top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, ) -> QueryResult: """Fetch multiple records and return them as a :class:`QueryResult`. GA replacement for ``records.get(table, filter=...)``. All pages are collected eagerly and returned as a single :class:`QueryResult`. - For advanced query options (ordering, expand, count, page size, or - OData annotations) use ``client.query.builder()`` instead. - :param table: Schema name of the table (e.g. ``"account"``). :type table: :class:`str` :param filter: Optional OData filter string or :class:`FilterExpression`. :type filter: str or FilterExpression or None :param select: Optional list of column logical names to include. :type select: list[str] or None + :param orderby: Optional list of sort expressions (e.g. ``["name asc", "createdon desc"]``). + :type orderby: list[str] or None :param top: Maximum total number of records to return. :type top: int or None + :param expand: Optional list of navigation properties to expand. + :type expand: list[str] or None + :param page_size: Per-page size hint via ``Prefer: odata.maxpagesize``. + :type page_size: int or None + :param count: If ``True``, adds ``$count=true`` to include a total record count. + :type count: bool + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header, or ``None``. + :type include_annotations: :class:`str` or None :return: All matching records collected into a :class:`QueryResult`. :rtype: :class:`~PowerPlatform.Dataverse.models.record.QueryResult` @@ -553,13 +575,13 @@ def list( result = client.records.list( "account", filter=col("statecode") == 0, - select=["name"], + select=["name", "statuscode"], + orderby=["name asc"], top=100, + include_annotations="OData.Community.Display.V1.FormattedValue", ) for record in result: - print(record["name"]) - - df = result.to_dataframe() + print(record["name"], record.get("statuscode@OData.Community.Display.V1.FormattedValue")) """ filter_str: Optional[str] = str(filter) if filter is not None else None all_records: List[Record] = [] @@ -568,12 +590,12 @@ def list( table, select=select, filter=filter_str, - orderby=None, + orderby=orderby, top=top, - expand=None, - page_size=None, - count=False, - include_annotations=None, + expand=expand, + page_size=page_size, + count=count, + include_annotations=include_annotations, ): all_records.extend(Record.from_api_response(table, row) for row in page) return QueryResult(all_records) @@ -586,7 +608,12 @@ def list_pages( *, filter: Optional[Union[str, "FilterExpression"]] = None, select: Optional[List[str]] = None, + orderby: Optional[List[str]] = None, top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, ) -> Iterator[QueryResult]: """Lazily yield one :class:`QueryResult` per HTTP page. @@ -594,23 +621,36 @@ def list_pages( network request via ``@odata.nextLink``. One-shot — do not iterate more than once. - For advanced query options (ordering, expand, count, page size, or - OData annotations) use ``client.query.builder().execute_pages()`` instead. - :param table: Schema name of the table (e.g. ``"account"``). :type table: :class:`str` :param filter: Optional OData filter string or :class:`FilterExpression`. :type filter: str or FilterExpression or None :param select: Optional list of column logical names to include. :type select: list[str] or None + :param orderby: Optional list of sort expressions (e.g. ``["name asc", "createdon desc"]``). + :type orderby: list[str] or None :param top: Maximum total number of records to return. :type top: int or None + :param expand: Optional list of navigation properties to expand. + :type expand: list[str] or None + :param page_size: Per-page size hint via ``Prefer: odata.maxpagesize``. + :type page_size: int or None + :param count: If ``True``, adds ``$count=true`` to include a total record count. + :type count: bool + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header, or ``None``. + :type include_annotations: :class:`str` or None :return: Iterator of per-page :class:`QueryResult` objects. :rtype: Iterator[:class:`~PowerPlatform.Dataverse.models.record.QueryResult`] Example:: - for page in client.records.list_pages("account", filter="statecode eq 0"): + for page in client.records.list_pages( + "account", + filter="statecode eq 0", + orderby=["name asc"], + page_size=200, + ): process(page.to_dataframe()) """ filter_str: Optional[str] = str(filter) if filter is not None else None @@ -619,12 +659,12 @@ def list_pages( table, select=select, filter=filter_str, - orderby=None, + orderby=orderby, top=top, - expand=None, - page_size=None, - count=False, - include_annotations=None, + expand=expand, + page_size=page_size, + count=count, + include_annotations=include_annotations, ): yield QueryResult([Record.from_api_response(table, row) for row in page]) diff --git a/tests/unit/data/test_batch_serialization.py b/tests/unit/data/test_batch_serialization.py index 9cdfcb24..13ecba56 100644 --- a/tests/unit/data/test_batch_serialization.py +++ b/tests/unit/data/test_batch_serialization.py @@ -14,6 +14,7 @@ _RecordCreate, _RecordDelete, _RecordGet, + _RecordList, _RecordUpdate, _RecordUpsert, _TableCreate, @@ -252,7 +253,72 @@ def test_resolve_record_get(self): op = _RecordGet(table="account", record_id="guid-1", select=["name"]) result = client._resolve_record_get(op) - od._build_get.assert_called_once_with("account", "guid-1", select=["name"]) + od._build_get.assert_called_once_with("account", "guid-1", select=["name"], include_annotations=None) + self.assertEqual(result, [mock_req]) + + def test_resolve_record_get_with_annotations(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_get.return_value = mock_req + + annotation = "OData.Community.Display.V1.FormattedValue" + op = _RecordGet(table="account", record_id="guid-1", select=["name"], include_annotations=annotation) + result = client._resolve_record_get(op) + + od._build_get.assert_called_once_with("account", "guid-1", select=["name"], include_annotations=annotation) + self.assertEqual(result, [mock_req]) + + def test_resolve_record_list_basic(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_list.return_value = mock_req + + op = _RecordList(table="account", select=["name"], filter="statecode eq 0", top=10) + result = client._resolve_record_list(op) + + od._build_list.assert_called_once_with( + "account", + select=["name"], + filter="statecode eq 0", + orderby=None, + top=10, + expand=None, + page_size=None, + count=False, + include_annotations=None, + ) + self.assertEqual(result, [mock_req]) + + def test_resolve_record_list_all_params(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_list.return_value = mock_req + + annotation = "OData.Community.Display.V1.FormattedValue" + op = _RecordList( + table="account", + select=["name"], + filter="statecode eq 0", + orderby=["name asc"], + top=50, + expand=["primarycontactid"], + page_size=100, + count=True, + include_annotations=annotation, + ) + result = client._resolve_record_list(op) + + od._build_list.assert_called_once_with( + "account", + select=["name"], + filter="statecode eq 0", + orderby=["name asc"], + top=50, + expand=["primarycontactid"], + page_size=100, + count=True, + include_annotations=annotation, + ) self.assertEqual(result, [mock_req]) def test_resolve_record_delete_single(self): @@ -599,6 +665,75 @@ def test_upsert_multiple_items_all_normalised(self): self.assertEqual(intent.items[1].alternate_key, {"accountnumber": "B"}) +class TestBatchRecordOperationsList(unittest.TestCase): + """Tests for BatchRecordOperations.list() surface (operations/batch.py).""" + + def _make_batch(self): + from PowerPlatform.Dataverse.operations.batch import BatchRecordOperations + + batch = MagicMock() + batch._items = [] + return BatchRecordOperations(batch), batch + + def test_list_basic_appends_record_list(self): + rec_ops, batch = self._make_batch() + rec_ops.list("account", filter="statecode eq 0", select=["name"], top=10) + + self.assertEqual(len(batch._items), 1) + intent = batch._items[0] + self.assertIsInstance(intent, _RecordList) + self.assertEqual(intent.table, "account") + self.assertEqual(intent.filter, "statecode eq 0") + self.assertEqual(intent.select, ["name"]) + self.assertEqual(intent.top, 10) + + def test_list_passes_orderby(self): + rec_ops, batch = self._make_batch() + rec_ops.list("account", orderby=["name asc"]) + self.assertEqual(batch._items[0].orderby, ["name asc"]) + + def test_list_passes_expand(self): + rec_ops, batch = self._make_batch() + rec_ops.list("account", expand=["primarycontactid"]) + self.assertEqual(batch._items[0].expand, ["primarycontactid"]) + + def test_list_passes_page_size(self): + rec_ops, batch = self._make_batch() + rec_ops.list("account", page_size=200) + self.assertEqual(batch._items[0].page_size, 200) + + def test_list_passes_count(self): + rec_ops, batch = self._make_batch() + rec_ops.list("account", count=True) + self.assertTrue(batch._items[0].count) + + def test_list_passes_include_annotations(self): + annotation = "OData.Community.Display.V1.FormattedValue" + rec_ops, batch = self._make_batch() + rec_ops.list("account", include_annotations=annotation) + self.assertEqual(batch._items[0].include_annotations, annotation) + + def test_list_filter_expression_converted_to_str(self): + from PowerPlatform.Dataverse.models.filters import col + + rec_ops, batch = self._make_batch() + rec_ops.list("account", filter=col("statecode") == 0) + self.assertEqual(batch._items[0].filter, "statecode eq 0") + + def test_list_defaults(self): + rec_ops, batch = self._make_batch() + rec_ops.list("account") + intent = batch._items[0] + self.assertIsNone(intent.filter) + self.assertIsNone(intent.select) + self.assertIsNone(intent.orderby) + self.assertIsNone(intent.top) + self.assertIsNone(intent.expand) + self.assertIsNone(intent.page_size) + self.assertFalse(intent.count) + self.assertIsNone(intent.include_annotations) + + class TestRaiseTopLevelBatchError(unittest.TestCase): """_raise_top_level_batch_error surfaces Dataverse error details as HttpError.""" diff --git a/tests/unit/test_migration_tool.py b/tests/unit/test_migration_tool.py new file mode 100644 index 00000000..88173a8f --- /dev/null +++ b/tests/unit/test_migration_tool.py @@ -0,0 +1,303 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Unit tests for tools/migrate_v0_to_v1.py. + +Covers: +- QueryBuilder.to_dataframe() -> .execute().to_dataframe() (auto-rewrite) +- QueryResult.to_dataframe() left untouched (receiver is .execute()) +- QueryBuilder chain via .select(), .where(), .filter_eq() all trigger the rewrite +- client.get(t, id) -> client.records.get(t, id) (top-level shortcut) +- batch.records.get(t, id) -> batch.records.retrieve(t, id) +- .filter_eq / .filter_ne / .filter_gt -> .where(col(...) OP v) +- .filter_null / .filter_not_null -> .where(col(...).is_null/is_not_null()) +- .filter_raw / .filter -> .where(raw(...)) +- .execute(by_page=True) -> .execute_pages() +- .execute(by_page=False) -> .execute() with flag stripped +- find_manual_patterns: flags client.records.get(), execute(by_page=variable), client.dataframe.get() +""" + +import textwrap +import unittest + +try: + import libcst # noqa: F401 + + _LIBCST_AVAILABLE = True +except ImportError: + _LIBCST_AVAILABLE = False + +_skip_no_libcst = unittest.skipUnless(_LIBCST_AVAILABLE, "libcst not installed") + + +def _migrate(source: str, *, client_var: str = "client") -> str: + from tools.migrate_v0_to_v1 import migrate_source + + return migrate_source(textwrap.dedent(source), client_var=client_var) + + +def _find_manual(source: str, *, client_var: str = "client") -> list: + from tools.migrate_v0_to_v1 import find_manual_patterns + + return find_manual_patterns(textwrap.dedent(source), client_var=client_var) + + +# --------------------------------------------------------------------------- +# QueryBuilder.to_dataframe() -> .execute().to_dataframe() +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestToDataframeRewrite(unittest.TestCase): + """QueryBuilder.to_dataframe() receives .execute() insertion.""" + + def test_builder_chain_gets_execute_inserted(self): + src = "df = client.query.builder('account').select('name').to_dataframe()\n" + out = _migrate(src) + self.assertIn(".execute().to_dataframe()", out) + self.assertNotIn(".to_dataframe().to_dataframe()", out) + + def test_where_chain_triggers_rewrite(self): + src = "df = q.where(col('statecode') == 0).to_dataframe()\n" + out = _migrate(src) + self.assertIn(".execute().to_dataframe()", out) + + def test_filter_eq_chain_triggers_rewrite(self): + src = "df = q.filter_eq('statecode', 0).to_dataframe()\n" + out = _migrate(src) + self.assertIn(".execute().to_dataframe()", out) + + def test_select_alone_triggers_rewrite(self): + src = "df = q.select('name', 'revenue').to_dataframe()\n" + out = _migrate(src) + self.assertIn(".execute().to_dataframe()", out) + + def test_already_executed_not_double_wrapped(self): + src = "df = q.select('name').execute().to_dataframe()\n" + out = _migrate(src) + self.assertNotIn(".execute().execute()", out) + self.assertIn(".execute().to_dataframe()", out) + + def test_unrelated_to_dataframe_not_rewritten(self): + src = "df = some_result.to_dataframe()\n" + out = _migrate(src) + self.assertNotIn(".execute()", out) + self.assertIn("some_result.to_dataframe()", out) + + def test_full_chain_structure_preserved(self): + src = "df = client.query.builder('account')\\\n" " .select('name')\\\n" " .to_dataframe()\n" + out = _migrate(src) + # .execute() is inserted before .to_dataframe(); a line-continuation may separate them + self.assertIn(".execute()", out) + self.assertIn(".to_dataframe()", out) + self.assertNotIn(".get(", out) + + def test_rewrite_inside_assignment(self): + src = "result = builder.select('name').to_dataframe()\n" + out = _migrate(src) + self.assertIn(".execute().to_dataframe()", out) + + +# --------------------------------------------------------------------------- +# Top-level shortcut rewrites +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestClientShortcutRewrites(unittest.TestCase): + def test_client_get_becomes_records_get(self): + src = "r = client.get('account', 'abc')\n" + out = _migrate(src) + self.assertIn("client.records.get(", out) + self.assertNotIn("client.get(", out) + + def test_client_create_becomes_records_create(self): + src = "client.create('account', {'name': 'X'})\n" + out = _migrate(src) + self.assertIn("client.records.create(", out) + + def test_client_delete_becomes_records_delete(self): + src = "client.delete('account', 'abc')\n" + out = _migrate(src) + self.assertIn("client.records.delete(", out) + + def test_client_update_becomes_records_update(self): + src = "client.update('account', 'abc', {'name': 'Y'})\n" + out = _migrate(src) + self.assertIn("client.records.update(", out) + + def test_client_query_sql_becomes_query_sql(self): + src = "rows = client.query_sql('SELECT * FROM account')\n" + out = _migrate(src) + self.assertIn("client.query.sql(", out) + + def test_client_get_table_info_becomes_tables_get(self): + src = "info = client.get_table_info('account')\n" + out = _migrate(src) + self.assertIn("client.tables.get(", out) + + def test_client_list_tables_becomes_tables_list(self): + src = "tables = client.list_tables()\n" + out = _migrate(src) + self.assertIn("client.tables.list(", out) + + def test_client_var_override(self): + src = "r = svc.get('account', 'abc')\n" + out = _migrate(src, client_var="svc") + self.assertIn("svc.records.get(", out) + + def test_client_get_not_matched_on_other_receiver(self): + src = "v = record.get('name')\n" + out = _migrate(src) + self.assertIn("record.get(", out) + self.assertNotIn("record.records.get(", out) + + +# --------------------------------------------------------------------------- +# batch.records.get() -> batch.records.retrieve() +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestBatchRecordsGetRewrite(unittest.TestCase): + def test_batch_records_get_becomes_retrieve(self): + src = "batch.records.get('account', 'abc')\n" + out = _migrate(src) + self.assertIn("batch.records.retrieve(", out) + self.assertNotIn("batch.records.get(", out) + + def test_client_records_get_not_rewritten(self): + src = "client.records.get('account', 'abc')\n" + out = _migrate(src) + self.assertIn("client.records.get(", out) + self.assertNotIn("client.records.retrieve(", out) + + +# --------------------------------------------------------------------------- +# .filter_*() -> .where(col(...) ...) rewrites +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestFilterMethodRewrites(unittest.TestCase): + def test_filter_eq(self): + src = "q.filter_eq('statecode', 0)\n" + out = _migrate(src) + self.assertIn(".where(", out) + self.assertIn("col(", out) + + def test_filter_ne(self): + src = "q.filter_ne('statecode', 0)\n" + out = _migrate(src) + self.assertIn(".where(", out) + + def test_filter_gt(self): + src = "q.filter_gt('revenue', 1000)\n" + out = _migrate(src) + self.assertIn(".where(", out) + + def test_filter_null(self): + src = "q.filter_null('email')\n" + out = _migrate(src) + self.assertIn(".is_null()", out) + + def test_filter_not_null(self): + src = "q.filter_not_null('email')\n" + out = _migrate(src) + self.assertIn(".is_not_null()", out) + + def test_filter_raw(self): + src = "q.filter_raw('statecode eq 0')\n" + out = _migrate(src) + self.assertIn("raw(", out) + + def test_filter_string_literal(self): + src = "q.filter('statecode eq 0')\n" + out = _migrate(src) + self.assertIn(".where(raw(", out) + + def test_filter_between(self): + src = "q.filter_between('revenue', 1000, 5000)\n" + out = _migrate(src) + self.assertIn(".between(", out) + + def test_filter_in(self): + src = "q.filter_in('statecode', [0, 1])\n" + out = _migrate(src) + self.assertIn(".in_(", out) + + +# --------------------------------------------------------------------------- +# .execute(by_page=...) -> .execute_pages() / .execute() +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestExecuteByPageRewrite(unittest.TestCase): + def test_execute_by_page_true_becomes_execute_pages(self): + src = "result = q.execute(by_page=True)\n" + out = _migrate(src) + self.assertIn(".execute_pages()", out) + self.assertNotIn("by_page", out) + + def test_execute_by_page_false_strips_flag(self): + src = "result = q.execute(by_page=False)\n" + out = _migrate(src) + self.assertIn(".execute()", out) + self.assertNotIn("by_page", out) + self.assertNotIn("execute_pages", out) + + def test_execute_no_args_unchanged(self): + src = "result = q.execute()\n" + out = _migrate(src) + self.assertIn(".execute()", out) + self.assertNotIn("execute_pages", out) + + +# --------------------------------------------------------------------------- +# find_manual_patterns +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestFindManualPatterns(unittest.TestCase): + def test_client_records_get_flagged(self): + src = "client.records.get('account', 'abc')\n" + findings = _find_manual(src) + self.assertTrue(any("records.get" in f for f in findings)) + + def test_execute_by_page_variable_flagged(self): + src = "q.execute(by_page=flag)\n" + findings = _find_manual(src) + self.assertTrue(any("by_page" in f for f in findings)) + + def test_execute_by_page_literal_not_flagged(self): + src = "q.execute(by_page=True)\n" + findings = _find_manual(src) + self.assertFalse(any("by_page" in f for f in findings)) + + def test_client_dataframe_get_flagged(self): + src = "client.dataframe.get('account')\n" + findings = _find_manual(src) + self.assertTrue(any("dataframe.get" in f for f in findings)) + + def test_query_sql_select_flagged(self): + src = "client.query.sql_select('account', ['name'])\n" + findings = _find_manual(src) + self.assertTrue(any("sql_select" in f for f in findings)) + + def test_clean_code_has_no_findings(self): + src = ( + "result = client.records.retrieve('account', 'abc')\n" "pages = client.records.list('account').execute()\n" + ) + findings = _find_manual(src) + self.assertEqual(findings, []) + + def test_batch_records_get_not_flagged(self): + src = "batch.records.get('account', 'abc')\n" + findings = _find_manual(src) + self.assertFalse(any("records.get" in f for f in findings)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_phase3_ga.py b/tests/unit/test_phase3_ga.py index 76a34719..c2a7528c 100644 --- a/tests/unit/test_phase3_ga.py +++ b/tests/unit/test_phase3_ga.py @@ -182,7 +182,18 @@ def test_retrieve_returns_record(self): def test_retrieve_passes_select(self): self.client._odata._get.return_value = {"accountid": "abc", "name": "Contoso"} self.client.records.retrieve("account", "abc", select=["name"]) - self.client._odata._get.assert_called_once_with("account", "abc", select=["name"]) + self.client._odata._get.assert_called_once_with("account", "abc", select=["name"], include_annotations=None) + + def test_retrieve_passes_include_annotations(self): + annotation = "OData.Community.Display.V1.FormattedValue" + self.client._odata._get.return_value = { + "accountid": "abc", + "statuscode": 1, + f"statuscode@{annotation}": "Active", + } + record = self.client.records.retrieve("account", "abc", include_annotations=annotation) + self.client._odata._get.assert_called_once_with("account", "abc", select=None, include_annotations=annotation) + self.assertEqual(record[f"statuscode@{annotation}"], "Active") def test_retrieve_no_deprecation_warning(self): self.client._odata._get.return_value = {"accountid": "abc", "name": "Contoso"} @@ -303,6 +314,37 @@ def test_list_result_to_dataframe(self): self.assertIsInstance(df, pd.DataFrame) self.assertEqual(len(df), 2) + def test_list_passes_orderby(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", orderby=["name asc"]) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["orderby"], ["name asc"]) + + def test_list_passes_expand(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", expand=["primarycontactid"]) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["expand"], ["primarycontactid"]) + + def test_list_passes_page_size(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", page_size=200) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["page_size"], 200) + + def test_list_passes_count(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", count=True) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertTrue(call_kwargs["count"]) + + def test_list_passes_include_annotations(self): + annotation = "OData.Community.Display.V1.FormattedValue" + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", include_annotations=annotation) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["include_annotations"], annotation) + class TestExecuteNoDeprecationFromRecordsGet(unittest.TestCase): """execute() suppresses DeprecationWarning from the internal records.get() call.""" diff --git a/tests/unit/test_records_operations.py b/tests/unit/test_records_operations.py index 0618ea5a..b9ae98d8 100644 --- a/tests/unit/test_records_operations.py +++ b/tests/unit/test_records_operations.py @@ -447,6 +447,37 @@ def test_list_pages_no_deprecation_warning(self): deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] self.assertEqual(len(deprecations), 0) + def test_list_pages_passes_orderby(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", orderby=["name asc"])) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("orderby"), ["name asc"]) + + def test_list_pages_passes_expand(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", expand=["primarycontactid"])) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("expand"), ["primarycontactid"]) + + def test_list_pages_passes_page_size(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", page_size=200)) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("page_size"), 200) + + def test_list_pages_passes_count(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", count=True)) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertTrue(call_kwargs.kwargs.get("count")) + + def test_list_pages_passes_include_annotations(self): + annotation = "OData.Community.Display.V1.FormattedValue" + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", include_annotations=annotation)) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("include_annotations"), annotation) + if __name__ == "__main__": unittest.main() diff --git a/tools/migrate_v0_to_v1.py b/tools/migrate_v0_to_v1.py index 10ebb925..62678df8 100644 --- a/tools/migrate_v0_to_v1.py +++ b/tools/migrate_v0_to_v1.py @@ -41,6 +41,9 @@ .filter("expr") -> .where(raw("expr")) .execute(by_page=True) -> .execute_pages() .execute(by_page=False) -> .execute() (flag removed) + .to_dataframe() -> .execute().to_dataframe() + Inserts .execute() when the receiver is a recognised QueryBuilder chain + (contains .builder(), .select(), .where(), or a .filter_*() call). Record namespace:: @@ -151,6 +154,9 @@ } _ALL_FILTER_FUNCS: Set[str] = set(_FUNC_BINARY_OP_MAP) | set(_FUNC_METHOD_MAP) | set(_FUNC_UNARY_MAP) +# Methods that identify a QueryBuilder call chain (used to detect .to_dataframe() callers) +_BUILDER_CHAIN_METHODS: Set[str] = {"builder", "select", "where", "filter", "execute_pages"} | _ALL_FILTER_METHODS + # Top-level client shortcut -> (new_namespace, new_method) _CLIENT_SHORTCUTS = { "create": ("records", "create"), @@ -336,6 +342,25 @@ def leave_Call(self, original_node: cst.Call, updated_node: cst.Call) -> cst.Bas ] return updated_node.with_changes(args=other_args) + # ---------------------------------------------------------------- + # QueryBuilder.to_dataframe() -> .execute().to_dataframe() + # Only rewrites when the receiver is a recognised QueryBuilder chain + # (contains .builder(), .select(), .where(), or a .filter_*() call). + # Skips if receiver is already a .execute() call (QueryResult.to_dataframe() + # is the GA form and must not be touched). + # ---------------------------------------------------------------- + if method_name == "to_dataframe": + receiver = func.value + already_executed = ( + isinstance(receiver, cst.Call) + and isinstance(receiver.func, cst.Attribute) + and isinstance(receiver.func.attr, cst.Name) + and receiver.func.attr.value == "execute" + ) + if not already_executed and self._is_query_builder_chain(receiver): + execute_call = _call(_attr(receiver, "execute")) + return updated_node.with_changes(func=func.with_changes(value=execute_call)) + # ---------------------------------------------------------------- # batch.records.get(table, id) -> batch.records.retrieve(table, id) # NOTE: client.records.get() is NOT codemodded — the return type changes @@ -382,6 +407,20 @@ def _kwarg_bool_literal(args: Sequence[cst.Arg], keyword: str) -> Optional[bool] return False return None + @staticmethod + def _is_query_builder_chain(node: cst.BaseExpression) -> bool: + """Return True if *node* is a call chain that includes a QueryBuilder method.""" + cur: cst.BaseExpression = node + while isinstance(cur, cst.Call): + f = cur.func + if isinstance(f, cst.Attribute) and isinstance(f.attr, cst.Name): + if f.attr.value in _BUILDER_CHAIN_METHODS: + return True + cur = f.value + else: + break + return False + # ------------------------------------------------------------------ # Build the argument for .where() from .filter_*() args # ------------------------------------------------------------------ @@ -745,21 +784,31 @@ def main(argv: Optional[List[str]] = None) -> int: print("[ERROR] No Python files found.", file=sys.stderr) return 1 - changed = skipped = manual_total = 0 + changed = skipped = needs_manual = manual_total = 0 for path in targets: was_changed, notes = migrate_file(path, dry_run=dry_run, client_var=client_var) if was_changed: changed += 1 tag = "[DRY-RUN]" if dry_run else "[MIGRATED]" - print(f"{tag} {path}") + if notes: + print(f"{tag} {path} (auto-rewrites applied; manual review still required)") + else: + print(f"{tag} {path}") + elif notes: + needs_manual += 1 + print(f"[NEEDS-MANUAL] {path} (no auto-rewrites to apply; manual migration required)") else: skipped += 1 for note in notes: - print(f" [MANUAL] {path}: {note}") + print(f" [MANUAL] {note}") manual_total += 1 suffix = "would be " if dry_run else "" - print(f"\nDone: {changed} file(s) {suffix}modified, {skipped} unchanged.", end="") + parts = [f"{changed} file(s) {suffix}auto-migrated"] + if needs_manual: + parts.append(f"{needs_manual} need manual-only migration") + parts.append(f"{skipped} unchanged") + print(f"\nDone: {', '.join(parts)}.", end="") if manual_total: print(f" {manual_total} pattern(s) require manual review.") else: From 65aac7dab4403990a022224d2059d2248d6e5aa3 Mon Sep 17 00:00:00 2001 From: Samson Gebre Date: Thu, 7 May 2026 13:21:11 -0700 Subject: [PATCH 12/16] feat: Add support for $expand in record retrieval methods and update documentation --- .claude/skills/dataverse-sdk-use/SKILL.md | 11 ++++++- CHANGELOG.md | 3 +- README.md | 11 ++++++- examples/basic/functional_testing.py | 21 ++++++++++++- .../claude_skill/dataverse-sdk-use/SKILL.md | 11 ++++++- src/PowerPlatform/Dataverse/data/_batch.py | 9 +++++- src/PowerPlatform/Dataverse/data/_odata.py | 17 +++++++++-- .../Dataverse/operations/batch.py | 17 +++++++++-- .../Dataverse/operations/records.py | 11 +++++-- tests/unit/data/test_batch_serialization.py | 21 +++++++++++-- tests/unit/test_phase3_ga.py | 30 +++++++++++++++++-- 11 files changed, 144 insertions(+), 18 deletions(-) diff --git a/.claude/skills/dataverse-sdk-use/SKILL.md b/.claude/skills/dataverse-sdk-use/SKILL.md index 9e05a401..d25815d7 100644 --- a/.claude/skills/dataverse-sdk-use/SKILL.md +++ b/.claude/skills/dataverse-sdk-use/SKILL.md @@ -99,6 +99,15 @@ contact_ids = client.records.create("contact", contacts) # Get single record by ID account = client.records.retrieve("account", account_id, select=["name", "telephone1"]) +# With expand — fetch a related record in the same HTTP request +account = client.records.retrieve( + "account", account_id, + select=["name"], + expand=["primarycontactid"], +) +contact = (account.get("primarycontactid") or {}) +print(contact.get("fullname")) + # Simple shortcut — use records.list() only for basic filter + select without composable logic. # Follows @odata.nextLink automatically and loads all matching records into memory. # For filtering, sorting, expansion, or formatted values, prefer client.query.builder() (see below). @@ -481,7 +490,7 @@ Use `client.batch` to send multiple operations in one HTTP request. All batch me batch = client.batch.new() batch.records.create("account", {"name": "Contoso"}) batch.records.update("account", account_id, {"telephone1": "555-0100"}) -batch.records.retrieve("account", account_id, select=["name"], include_annotations="OData.Community.Display.V1.FormattedValue") # single record +batch.records.retrieve("account", account_id, select=["name"], expand=["primarycontactid"], include_annotations="OData.Community.Display.V1.FormattedValue") # single record with expand batch.records.list("account", filter="statecode eq 0", select=["name"], orderby=["name asc"], top=50, page_size=25, count=True) # multi-record, single page batch.query.sql("SELECT TOP 5 name FROM account") diff --git a/CHANGELOG.md b/CHANGELOG.md index 48a9c327..d457f070 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,10 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added -- `client.records.retrieve(table, record_id, *, select, include_annotations)` — fetch a single record by GUID; returns `None` on 404 instead of raising; `include_annotations` maps to the `Prefer: odata.include-annotations` header for formatted values and lookup labels (#175) +- `client.records.retrieve(table, record_id, *, select, expand, include_annotations)` — fetch a single record by GUID; returns `None` on 404 instead of raising; `expand` adds `$expand` for navigation property expansion on the single-record GET; `include_annotations` maps to the `Prefer: odata.include-annotations` header for formatted values and lookup labels (#175) - `client.records.list(table, *, filter, select, top, orderby, expand, page_size, count, include_annotations)` — eager fetch returning a flat `QueryResult`; GA replacement for `records.get()` without a record ID; `page_size` controls `Prefer: odata.maxpagesize`, `count=True` adds `$count=true`, `include_annotations` requests formatted values (#175) - `client.records.list_pages(table, *, filter, select, top, orderby, expand, page_size, count, include_annotations)` — lazy iterator yielding one `QueryResult` per HTTP page; streaming counterpart to `list()`; same parameter set (#175) -- `client.batch.records.retrieve()` and `client.batch.records.list()` now accept the same `include_annotations`, `orderby`, `expand`, `page_size`, and `count` parameters as their non-batch counterparts (#175) - `client.query.fetchxml(xml)` — FetchXML support returning an inert `FetchXmlQuery`; no HTTP request is made until `.execute()` or `.execute_pages()` is called (#175) - `FetchXmlQuery` implements the correct Dataverse paging cookie algorithm: annotation parsed as outer XML, `pagingcookie` attribute double URL-decoded, server-supplied `pagenumber` used for next page, `morerecords` handled as both `bool` and `"true"` string, `UserWarning` emitted on simple paging fallback, 32,768-character URL limit enforced (documented Dataverse GET cap), 10,000-page circuit breaker against runaway iteration (#175) - `QueryBuilder.execute_pages()` — lazy per-page streaming returning one `QueryResult` per HTTP page; replaces deprecated `execute(by_page=True)` (#175) diff --git a/README.md b/README.md index 99db99fc..43acb8f4 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,15 @@ account_id = client.records.create("account", {"name": "Contoso Ltd"}) account = client.records.retrieve("account", account_id) print(account["name"]) +# Read with expand — fetch a related record in the same HTTP request +account = client.records.retrieve( + "account", account_id, + select=["name"], + expand=["primarycontactid"], +) +contact = (account.get("primarycontactid") or {}) +print(contact.get("fullname")) + # Update a record client.records.update("account", account_id, {"telephone1": "555-0199"}) @@ -694,7 +703,7 @@ batch.records.create("account", {"name": "Contoso"}) batch.records.create("account", [{"name": "Fabrikam"}, {"name": "Woodgrove"}]) batch.records.update("account", account_id, {"telephone1": "555-0100"}) batch.records.delete("account", old_id) -batch.records.retrieve("account", account_id, select=["name"]) # single record +batch.records.retrieve("account", account_id, select=["name"], expand=["primarycontactid"]) # single record with expand batch.records.list( # multi-record, single page "account", filter="statecode eq 0", diff --git a/examples/basic/functional_testing.py b/examples/basic/functional_testing.py index c5918ae1..ddebd362 100644 --- a/examples/basic/functional_testing.py +++ b/examples/basic/functional_testing.py @@ -276,6 +276,21 @@ def test_read_record(client: DataverseClient, table_info: Dict[str, Any], record else: print(f"[WARN] include_annotations: expected key '{ann_key}' not present in response") + # -- expand: verify navigation property expansion on a single-record GET -- + # owninguser is a system navigation property present on all user-owned tables. + try: + expanded = client.records.retrieve( + table_schema_name, + record_id, + select=[f"{attr_prefix}_name"], + expand=["owninguser"], + ) + owner = (expanded.get("owninguser") or {}) if expanded else {} + owner_name = owner.get("fullname") or owner.get("domainname") or "(unknown)" + print(f"[OK] records.retrieve with expand=['owninguser']: owner='{owner_name}'") + except Exception as e: # noqa: BLE001 + print(f"[WARN] records.retrieve expand skipped: {e}") + return record except HttpError as e: @@ -597,11 +612,12 @@ def test_batch_all_operations(client: DataverseClient, table_info: Dict[str, Any " + tables.get + tables.list + query.sql (5 ops, 1 POST $batch)" ) batch = client.batch.new() - # [0] Single-record retrieve with annotations + # [0] Single-record retrieve with annotations and expand batch.records.retrieve( table_schema_name, all_ids[0], select=[f"{attr_prefix}_name", f"{attr_prefix}_count", f"{attr_prefix}_is_active"], + expand=["owninguser"], include_annotations=annotation, ) # [1] Multi-record list with orderby, page_size, count, include_annotations @@ -630,7 +646,10 @@ def test_batch_all_operations(client: DataverseClient, table_info: Dict[str, Any name = resp.data.get(f"{attr_prefix}_name") ann_key = f"{attr_prefix}_is_active@{annotation}" ann_val = resp.data.get(ann_key, "") + owner = resp.data.get("owninguser") or {} + owner_name = owner.get("fullname") or owner.get("domainname") or "" print(f" records.retrieve → name='{name}', {ann_key}='{ann_val}'") + print(f" records.retrieve expand=['owninguser'] → owner='{owner_name}'") elif i == 1 and resp.data: rows = resp.data.get("value", []) names_ordered = [r.get(f"{attr_prefix}_name") for r in rows] diff --git a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md index 9e05a401..d25815d7 100644 --- a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md +++ b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md @@ -99,6 +99,15 @@ contact_ids = client.records.create("contact", contacts) # Get single record by ID account = client.records.retrieve("account", account_id, select=["name", "telephone1"]) +# With expand — fetch a related record in the same HTTP request +account = client.records.retrieve( + "account", account_id, + select=["name"], + expand=["primarycontactid"], +) +contact = (account.get("primarycontactid") or {}) +print(contact.get("fullname")) + # Simple shortcut — use records.list() only for basic filter + select without composable logic. # Follows @odata.nextLink automatically and loads all matching records into memory. # For filtering, sorting, expansion, or formatted values, prefer client.query.builder() (see below). @@ -481,7 +490,7 @@ Use `client.batch` to send multiple operations in one HTTP request. All batch me batch = client.batch.new() batch.records.create("account", {"name": "Contoso"}) batch.records.update("account", account_id, {"telephone1": "555-0100"}) -batch.records.retrieve("account", account_id, select=["name"], include_annotations="OData.Community.Display.V1.FormattedValue") # single record +batch.records.retrieve("account", account_id, select=["name"], expand=["primarycontactid"], include_annotations="OData.Community.Display.V1.FormattedValue") # single record with expand batch.records.list("account", filter="statecode eq 0", select=["name"], orderby=["name asc"], top=50, page_size=25, count=True) # multi-record, single page batch.query.sql("SELECT TOP 5 name FROM account") diff --git a/src/PowerPlatform/Dataverse/data/_batch.py b/src/PowerPlatform/Dataverse/data/_batch.py index c6ce1205..b848c5a0 100644 --- a/src/PowerPlatform/Dataverse/data/_batch.py +++ b/src/PowerPlatform/Dataverse/data/_batch.py @@ -69,6 +69,7 @@ class _RecordGet: table: str record_id: str select: Optional[List[str]] = None + expand: Optional[List[str]] = None include_annotations: Optional[str] = None @@ -399,7 +400,13 @@ def _resolve_record_delete(self, op: _RecordDelete) -> List[_RawRequest]: def _resolve_record_get(self, op: _RecordGet) -> List[_RawRequest]: return [ - self._od._build_get(op.table, op.record_id, select=op.select, include_annotations=op.include_annotations) + self._od._build_get( + op.table, + op.record_id, + select=op.select, + expand=op.expand, + include_annotations=op.include_annotations, + ) ] def _resolve_record_list(self, op: _RecordList) -> List[_RawRequest]: diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index 2b2145b4..8d79a9aa 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -689,6 +689,7 @@ def _get( table_schema_name: str, key: str, select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, include_annotations: Optional[str] = None, ) -> Dict[str, Any]: """Retrieve a single record. @@ -699,6 +700,8 @@ def _get( :type key: ``str`` :param select: Columns to select; joined with commas into $select. :type select: ``list[str]`` | ``None`` + :param expand: Navigation properties to expand (``$expand``); passed as-is (case-sensitive). + :type expand: ``list[str]`` | ``None`` :param include_annotations: OData annotation pattern for the ``Prefer: odata.include-annotations`` header, or ``None``. :type include_annotations: ``str`` | ``None`` @@ -706,7 +709,9 @@ def _get( :rtype: ``dict[str, Any]`` """ return self._execute_raw( - self._build_get(table_schema_name, key, select=select, include_annotations=include_annotations) + self._build_get( + table_schema_name, key, select=select, expand=expand, include_annotations=include_annotations + ) ).json() def _get_multiple( @@ -2319,13 +2324,19 @@ def _build_get( record_id: str, *, select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, include_annotations: Optional[str] = None, ) -> _RawRequest: """Build a single-record GET request without sending it.""" entity_set = self._entity_set_from_schema_name(table) - url = f"{self.api}/{entity_set}{self._format_key(record_id)}" + params: List[str] = [] if select: - url += "?$select=" + ",".join(self._lowercase_list(select)) + params.append("$select=" + ",".join(self._lowercase_list(select))) + if expand: + params.append("$expand=" + ",".join(expand)) + url = f"{self.api}/{entity_set}{self._format_key(record_id)}" + if params: + url += "?" + "&".join(params) headers = None if include_annotations: headers = {"Prefer": f'odata.include-annotations="{include_annotations}"'} diff --git a/src/PowerPlatform/Dataverse/operations/batch.py b/src/PowerPlatform/Dataverse/operations/batch.py index 987603bc..aa5d8391 100644 --- a/src/PowerPlatform/Dataverse/operations/batch.py +++ b/src/PowerPlatform/Dataverse/operations/batch.py @@ -335,6 +335,7 @@ def retrieve( record_id: str, *, select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, include_annotations: Optional[str] = None, ) -> None: """ @@ -351,6 +352,10 @@ def retrieve( :type record_id: :class:`str` :param select: Optional list of column logical names to include. :type select: list[str] or None + :param expand: Optional list of navigation properties to expand. + Navigation property names are case-sensitive and must match the + entity's ``$metadata``. + :type expand: list[str] or None :param include_annotations: OData annotation pattern for the ``Prefer: odata.include-annotations`` header (e.g. ``"*"`` or ``"OData.Community.Display.V1.FormattedValue"``), or ``None``. @@ -362,14 +367,22 @@ def retrieve( batch.records.retrieve( "account", account_id, select=["name", "statuscode"], + expand=["primarycontactid"], include_annotations="OData.Community.Display.V1.FormattedValue", ) result = batch.execute() record = result.responses[0].data - print(record["statuscode@OData.Community.Display.V1.FormattedValue"]) + contact = (record.get("primarycontactid") or {}) + print(contact.get("fullname")) """ self._batch._items.append( - _RecordGet(table=table, record_id=record_id, select=select, include_annotations=include_annotations) + _RecordGet( + table=table, + record_id=record_id, + select=select, + expand=expand, + include_annotations=include_annotations, + ) ) def list( diff --git a/src/PowerPlatform/Dataverse/operations/records.py b/src/PowerPlatform/Dataverse/operations/records.py index 29ccd8e2..c9c66119 100644 --- a/src/PowerPlatform/Dataverse/operations/records.py +++ b/src/PowerPlatform/Dataverse/operations/records.py @@ -487,6 +487,7 @@ def retrieve( record_id: str, *, select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, include_annotations: Optional[str] = None, ) -> Optional[Record]: """Fetch a single record by its GUID, returning ``None`` if not found. @@ -500,6 +501,10 @@ def retrieve( :type record_id: :class:`str` :param select: Optional list of column logical names to include. :type select: list[str] or None + :param expand: Optional list of navigation properties to expand (e.g. + ``["primarycontactid"]``). Navigation property names are + case-sensitive and must match the entity's ``$metadata``. + :type expand: list[str] or None :param include_annotations: OData annotation pattern for the ``Prefer: odata.include-annotations`` header (e.g. ``"*"`` or ``"OData.Community.Display.V1.FormattedValue"``), or ``None``. @@ -512,14 +517,16 @@ def retrieve( record = client.records.retrieve( "account", account_id, select=["name", "statuscode"], + expand=["primarycontactid"], include_annotations="OData.Community.Display.V1.FormattedValue", ) if record is not None: - print(record["statuscode@OData.Community.Display.V1.FormattedValue"]) + contact = record.get("primarycontactid") or {} + print(contact.get("fullname")) """ with self._client._scoped_odata() as od: try: - raw = od._get(table, record_id, select=select, include_annotations=include_annotations) + raw = od._get(table, record_id, select=select, expand=expand, include_annotations=include_annotations) except HttpError as exc: if exc.status_code == 404: return None diff --git a/tests/unit/data/test_batch_serialization.py b/tests/unit/data/test_batch_serialization.py index 13ecba56..a1c293df 100644 --- a/tests/unit/data/test_batch_serialization.py +++ b/tests/unit/data/test_batch_serialization.py @@ -253,7 +253,22 @@ def test_resolve_record_get(self): op = _RecordGet(table="account", record_id="guid-1", select=["name"]) result = client._resolve_record_get(op) - od._build_get.assert_called_once_with("account", "guid-1", select=["name"], include_annotations=None) + od._build_get.assert_called_once_with( + "account", "guid-1", select=["name"], expand=None, include_annotations=None + ) + self.assertEqual(result, [mock_req]) + + def test_resolve_record_get_with_expand(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_get.return_value = mock_req + + op = _RecordGet(table="account", record_id="guid-1", select=["name"], expand=["primarycontactid"]) + result = client._resolve_record_get(op) + + od._build_get.assert_called_once_with( + "account", "guid-1", select=["name"], expand=["primarycontactid"], include_annotations=None + ) self.assertEqual(result, [mock_req]) def test_resolve_record_get_with_annotations(self): @@ -265,7 +280,9 @@ def test_resolve_record_get_with_annotations(self): op = _RecordGet(table="account", record_id="guid-1", select=["name"], include_annotations=annotation) result = client._resolve_record_get(op) - od._build_get.assert_called_once_with("account", "guid-1", select=["name"], include_annotations=annotation) + od._build_get.assert_called_once_with( + "account", "guid-1", select=["name"], expand=None, include_annotations=annotation + ) self.assertEqual(result, [mock_req]) def test_resolve_record_list_basic(self): diff --git a/tests/unit/test_phase3_ga.py b/tests/unit/test_phase3_ga.py index c2a7528c..f9e53f9f 100644 --- a/tests/unit/test_phase3_ga.py +++ b/tests/unit/test_phase3_ga.py @@ -182,7 +182,31 @@ def test_retrieve_returns_record(self): def test_retrieve_passes_select(self): self.client._odata._get.return_value = {"accountid": "abc", "name": "Contoso"} self.client.records.retrieve("account", "abc", select=["name"]) - self.client._odata._get.assert_called_once_with("account", "abc", select=["name"], include_annotations=None) + self.client._odata._get.assert_called_once_with( + "account", "abc", select=["name"], expand=None, include_annotations=None + ) + + def test_retrieve_passes_expand(self): + self.client._odata._get.return_value = { + "accountid": "abc", + "name": "Contoso", + "primarycontactid": {"contactid": "cid", "fullname": "John Doe"}, + } + record = self.client.records.retrieve("account", "abc", expand=["primarycontactid"]) + self.client._odata._get.assert_called_once_with( + "account", "abc", select=None, expand=["primarycontactid"], include_annotations=None + ) + self.assertEqual(record["primarycontactid"]["fullname"], "John Doe") + + def test_retrieve_passes_select_and_expand(self): + self.client._odata._get.return_value = { + "name": "Contoso", + "primarycontactid": {"fullname": "John Doe"}, + } + self.client.records.retrieve("account", "abc", select=["name"], expand=["primarycontactid"]) + self.client._odata._get.assert_called_once_with( + "account", "abc", select=["name"], expand=["primarycontactid"], include_annotations=None + ) def test_retrieve_passes_include_annotations(self): annotation = "OData.Community.Display.V1.FormattedValue" @@ -192,7 +216,9 @@ def test_retrieve_passes_include_annotations(self): f"statuscode@{annotation}": "Active", } record = self.client.records.retrieve("account", "abc", include_annotations=annotation) - self.client._odata._get.assert_called_once_with("account", "abc", select=None, include_annotations=annotation) + self.client._odata._get.assert_called_once_with( + "account", "abc", select=None, expand=None, include_annotations=annotation + ) self.assertEqual(record[f"statuscode@{annotation}"], "Active") def test_retrieve_no_deprecation_warning(self): From fc590c25842fba8243b4ce0b8af61afee65d07b3 Mon Sep 17 00:00:00 2001 From: Samson Gebre Date: Sun, 17 May 2026 21:28:01 -0700 Subject: [PATCH 13/16] Enhance QueryResult class with list-like behavior and add contract tests --- src/PowerPlatform/Dataverse/models/record.py | 4 +- .../Dataverse/operations/query.py | 16 +-- tests/unit/test_phase2_ga.py | 99 +++++++++++++++++++ 3 files changed, 111 insertions(+), 8 deletions(-) diff --git a/src/PowerPlatform/Dataverse/models/record.py b/src/PowerPlatform/Dataverse/models/record.py index e81f412c..da65e7a5 100644 --- a/src/PowerPlatform/Dataverse/models/record.py +++ b/src/PowerPlatform/Dataverse/models/record.py @@ -6,7 +6,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any, Dict, Iterator, KeysView, List, Optional, ValuesView, ItemsView +from typing import Any, Dict, Iterator, KeysView, List, Optional, Union, ValuesView, ItemsView __all__ = ["Record", "QueryResult"] @@ -141,7 +141,7 @@ def __bool__(self) -> bool: def __repr__(self) -> str: return f"QueryResult({len(self.records)} records)" - def __getitem__(self, index): + def __getitem__(self, index: Union[int, slice]) -> Union[Record, "QueryResult"]: result = self.records[index] return QueryResult(result) if isinstance(index, slice) else result diff --git a/src/PowerPlatform/Dataverse/operations/query.py b/src/PowerPlatform/Dataverse/operations/query.py index c927bdf0..1f3c8ef2 100644 --- a/src/PowerPlatform/Dataverse/operations/query.py +++ b/src/PowerPlatform/Dataverse/operations/query.py @@ -33,12 +33,14 @@ class QueryOperations: Example:: + from PowerPlatform.Dataverse.models.filters import col + client = DataverseClient(base_url, credential) # Fluent query builder (recommended) for record in (client.query.builder("account") .select("name", "revenue") - .filter_eq("statecode", 0) + .where(col("statecode") == 0) .order_by("revenue", descending=True) .top(100) .execute()): @@ -70,10 +72,12 @@ def builder(self, table: str) -> QueryBuilder: Example: Build and execute a query fluently:: + from PowerPlatform.Dataverse.models.filters import col + for record in (client.query.builder("account") .select("name", "revenue") - .filter_eq("statecode", 0) - .filter_gt("revenue", 1000000) + .where(col("statecode") == 0) + .where(col("revenue") > 1_000_000) .order_by("revenue", descending=True) .top(100) .page_size(50) @@ -82,11 +86,11 @@ def builder(self, table: str) -> QueryBuilder: With composable expression tree:: - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col for record in (client.query.builder("account") - .where((eq("statecode", 0) | eq("statecode", 1)) - & gt("revenue", 100000)) + .where((col("statecode") == 0) | (col("statecode") == 1)) + .where(col("revenue") > 100_000) .execute()): print(record["name"]) """ diff --git a/tests/unit/test_phase2_ga.py b/tests/unit/test_phase2_ga.py index acd4263e..5809e1be 100644 --- a/tests/unit/test_phase2_ga.py +++ b/tests/unit/test_phase2_ga.py @@ -165,6 +165,105 @@ def test_list_conversion(self): self.assertEqual(list(qr), recs) +class TestQueryResultListLikeContract(unittest.TestCase): + """Contract tests: QueryResult must be substitutable for ``list[Record]`` + in the patterns shown in the class docstring, examples, and skill docs. + + These tests exist to catch the gap from PR #175 where ``__getitem__`` was + missing despite docs and examples treating ``QueryResult`` as list-like. + They drive the contract from the *caller's* perspective rather than + introspecting which dunders are implemented, so removing any single dunder + breaks at least one assertion here with a clear signal. + """ + + def _records(self, n=3): + return [Record(id=f"id-{i}", table="account", data={"name": f"R{i}"}) for i in range(n)] + + def test_contract_index_then_field_access(self): + """Pattern from examples/advanced/fetchxml.py: ``row = result[0]; row.get(...)``.""" + qr = QueryResult(self._records(3)) + row = qr[0] + self.assertEqual(row.get("name"), "R0") + + def test_contract_single_loop_field_access(self): + """Pattern from examples/basic/installation_example.py: ``for r in result: r["name"]``.""" + qr = QueryResult(self._records(3)) + names = [r["name"] for r in qr] + self.assertEqual(names, ["R0", "R1", "R2"]) + + def test_contract_first_with_none_guard(self): + """Pattern recommended by Copilot review: ``result.first()`` with None-check.""" + empty = QueryResult([]) + nonempty = QueryResult(self._records(2)) + self.assertIsNone(empty.first()) + first = nonempty.first() + self.assertIsNotNone(first) + self.assertEqual(first.get("name"), "R0") # type: ignore[union-attr] + + def test_contract_truthy_guard(self): + """Pattern: ``if result: ...`` to skip empty results before indexing.""" + if QueryResult(self._records(1)): + ok = True + else: + ok = False + self.assertTrue(ok) + self.assertFalse(bool(QueryResult([]))) + + def test_contract_len_for_size_check(self): + """Pattern: ``f"{len(result)} rows"`` in log/print statements.""" + self.assertEqual(len(QueryResult(self._records(7))), 7) + self.assertEqual(len(QueryResult([])), 0) + + def test_contract_slice_returns_list_like(self): + """Slicing must yield something that supports iteration and len().""" + qr = QueryResult(self._records(5)) + page = qr[1:4] + self.assertEqual(len(page), 3) + self.assertEqual([r.get("name") for r in page], ["R1", "R2", "R3"]) + + def test_contract_negative_index(self): + """``result[-1]`` for "last record" is a common Python idiom.""" + qr = QueryResult(self._records(3)) + self.assertEqual(qr[-1].get("name"), "R2") + + def test_contract_list_conversion_round_trip(self): + """``list(result)`` must yield the same records iteration yields.""" + recs = self._records(4) + qr = QueryResult(recs) + self.assertEqual(list(qr), recs) + self.assertEqual(list(qr), [r for r in qr]) + + def test_contract_iteration_does_not_consume(self): + """Multiple ``for`` loops over the same result must all see records. + + Guards against an accidental refactor to a single-shot iterator. + """ + qr = QueryResult(self._records(3)) + first_pass = list(qr) + second_pass = list(qr) + self.assertEqual(first_pass, second_pass) + self.assertEqual(len(first_pass), 3) + + def test_contract_nested_loop_iterates_records_not_fields(self): + """Regression guard for the bug in installation_example.py (PR #175 review #3). + + ``for page in result: for r in page: ...`` would iterate Record keys + if QueryResult were a flat iterable of Records (each Record is also + iterable over its keys). This test makes it explicit that the outer + loop yields Records, not pages — so callers know to use a single loop. + """ + qr = QueryResult(self._records(2)) + outer = list(qr) + self.assertTrue(all(isinstance(r, Record) for r in outer)) + + def test_contract_records_attribute_is_underlying_list(self): + """``result.records`` is the documented escape hatch for list-only APIs.""" + recs = self._records(3) + qr = QueryResult(recs) + self.assertIsInstance(qr.records, list) + self.assertIs(qr.records, recs) + + class TestExecuteReturnsQueryResult(unittest.TestCase): """execute() flat mode returns QueryResult.""" From f8dc75c16476feb575f0a56292c73edb5e740d1a Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 18 May 2026 04:33:33 +0000 Subject: [PATCH 14/16] style: simplify config/context ValueError message Co-authored-by: sagebree <6541424+sagebree@users.noreply.github.com> --- src/PowerPlatform/Dataverse/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/PowerPlatform/Dataverse/client.py b/src/PowerPlatform/Dataverse/client.py index c9a1364f..12ceaaac 100644 --- a/src/PowerPlatform/Dataverse/client.py +++ b/src/PowerPlatform/Dataverse/client.py @@ -105,7 +105,7 @@ def __init__( ) -> None: if config is not None and context is not None: raise ValueError( - "Cannot specify both 'config' and 'context'. " "Pass operation_context via DataverseConfig instead." + "Cannot specify both 'config' and 'context'. Pass operation_context via DataverseConfig instead." ) self.auth = _AuthManager(credential) self._base_url = (base_url or "").rstrip("/") From 2b456329c2b6740db67d22438915fd2c5d38b773 Mon Sep 17 00:00:00 2001 From: Samson Gebre Date: Sun, 17 May 2026 21:58:09 -0700 Subject: [PATCH 15/16] Update migration tool and update related documentation --- CHANGELOG.md | 2 +- README.md | 2 +- pyproject.toml | 8 ++++---- .../PowerPlatform/Dataverse/migration}/__init__.py | 0 .../Dataverse/migration}/migrate_v0_to_v1.py | 6 +++++- tests/unit/test_migration_tool.py | 6 +++--- tests/unit/test_phase4_ga.py | 8 ++++---- 7 files changed, 18 insertions(+), 14 deletions(-) rename {tools => src/PowerPlatform/Dataverse/migration}/__init__.py (100%) rename {tools => src/PowerPlatform/Dataverse/migration}/migrate_v0_to_v1.py (99%) diff --git a/CHANGELOG.md b/CHANGELOG.md index d457f070..39bb3029 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `QueryResult.__getitem__` — index access (`result[0]`) returns a `Record`; slice access (`result[1:5]`) returns a new `QueryResult` (#175) - `DataverseModel` structural `Protocol` (`models/protocol.py`) — implement on any entity class to enable typed integration with CRUD operations without specifying table names or serializing manually (#175) - `col()`, `raw()`, `QueryResult`, and `DataverseModel` exported from the top-level `PowerPlatform.Dataverse` package (#175) -- v0→v1 migration tool: `tools/migrate_v0_to_v1.py` rewrites v0 call sites to the v1 API with `--dry-run` support; covers `create`, `update`, `delete`, `get`, `list`, `fetchxml`, and query builder patterns (#175) +- v0→v1 migration tool: installed as the `dataverse-migrate` console script (also runnable via `python -m PowerPlatform.Dataverse.migration.migrate_v0_to_v1`); rewrites v0 call sites to the v1 API with `--dry-run` support; covers `create`, `update`, `delete`, `get`, `list`, `fetchxml`, and query builder patterns; requires the `[migration]` optional extra (`pip install PowerPlatform-Dataverse-Client[migration]`) (#175) - Migration tool now auto-rewrites `QueryBuilder.to_dataframe()` → `.execute().to_dataframe()` (inserts `.execute()` when receiver is a recognised builder chain); output improved with `[NEEDS-MANUAL]` label for files that have no auto-rewrites but require manual attention, and a trailing note on `[MIGRATED]` lines when manual items remain (#175) ### Changed diff --git a/README.md b/README.md index 43acb8f4..21abb591 100644 --- a/README.md +++ b/README.md @@ -450,7 +450,7 @@ for page_num, page in enumerate( print(record["name"]) ``` -> **Deprecation note:** `execute(by_page=True)` and `execute(by_page=False)` are deprecated and emit a `UserWarning`. Replace with `execute_pages()` (streaming) or plain `execute()` (eager). `QueryBuilder.to_dataframe()` is also deprecated; use `.execute().to_dataframe()` instead. The migration tool (`tools/migrate_v0_to_v1.py`) rewrites all of these automatically. +> **Deprecation note:** `execute(by_page=True)` and `execute(by_page=False)` are deprecated and emit a `UserWarning`. Replace with `execute_pages()` (streaming) or plain `execute()` (eager). `QueryBuilder.to_dataframe()` is also deprecated; use `.execute().to_dataframe()` instead. The migration tool rewrites all of these automatically — install with `pip install PowerPlatform-Dataverse-Client[migration]` and run `dataverse-migrate path/to/your/scripts/` (or `python -m PowerPlatform.Dataverse.migration.migrate_v0_to_v1` for development checkouts). **Record count** -- include `$count=true` in the request: diff --git a/pyproject.toml b/pyproject.toml index 387bb21d..88124511 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dependencies = [ [project.scripts] dataverse-install-claude-skill = "PowerPlatform.Dataverse._skill_installer:main" -dataverse-migrate = "tools.migrate_v0_to_v1:main" +dataverse-migrate = "PowerPlatform.Dataverse.migration.migrate_v0_to_v1:main" [project.optional-dependencies] dev = [ @@ -54,12 +54,12 @@ dev = [ migration = ["libcst>=1.0.0"] [tool.setuptools] -package-dir = {"" = "src", "tools" = "tools"} +package-dir = {"" = "src"} zip-safe = false [tool.setuptools.packages.find] -where = ["src", "."] -include = ["PowerPlatform*", "tools"] +where = ["src"] +include = ["PowerPlatform*"] namespaces = false [tool.setuptools.package-data] diff --git a/tools/__init__.py b/src/PowerPlatform/Dataverse/migration/__init__.py similarity index 100% rename from tools/__init__.py rename to src/PowerPlatform/Dataverse/migration/__init__.py diff --git a/tools/migrate_v0_to_v1.py b/src/PowerPlatform/Dataverse/migration/migrate_v0_to_v1.py similarity index 99% rename from tools/migrate_v0_to_v1.py rename to src/PowerPlatform/Dataverse/migration/migrate_v0_to_v1.py index 62678df8..bf9782ec 100644 --- a/tools/migrate_v0_to_v1.py +++ b/src/PowerPlatform/Dataverse/migration/migrate_v0_to_v1.py @@ -16,7 +16,7 @@ dataverse-migrate path/to/your/scripts/ --client-var=svc # if client is named 'svc' # Or via module for development installs: - python -m tools.migrate_v0_to_v1 path/to/your/scripts/ + python -m PowerPlatform.Dataverse.migration.migrate_v0_to_v1 path/to/your/scripts/ Transformations applied ----------------------- @@ -763,6 +763,10 @@ def _collect_targets(paths: List[str]) -> List[Path]: def main(argv: Optional[List[str]] = None) -> int: args = sys.argv[1:] if argv is None else list(argv) + if "--help" in args or "-h" in args: + print(__doc__) + print("\nUsage: dataverse-migrate [--dry-run] [--client-var=NAME] [ ...]") + return 0 dry_run = "--dry-run" in args client_var = "client" remaining = [] diff --git a/tests/unit/test_migration_tool.py b/tests/unit/test_migration_tool.py index 88173a8f..4f79bb63 100644 --- a/tests/unit/test_migration_tool.py +++ b/tests/unit/test_migration_tool.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Unit tests for tools/migrate_v0_to_v1.py. +"""Unit tests for PowerPlatform/Dataverse/migration/migrate_v0_to_v1.py. Covers: - QueryBuilder.to_dataframe() -> .execute().to_dataframe() (auto-rewrite) @@ -31,13 +31,13 @@ def _migrate(source: str, *, client_var: str = "client") -> str: - from tools.migrate_v0_to_v1 import migrate_source + from PowerPlatform.Dataverse.migration.migrate_v0_to_v1 import migrate_source return migrate_source(textwrap.dedent(source), client_var=client_var) def _find_manual(source: str, *, client_var: str = "client") -> list: - from tools.migrate_v0_to_v1 import find_manual_patterns + from PowerPlatform.Dataverse.migration.migrate_v0_to_v1 import find_manual_patterns return find_manual_patterns(textwrap.dedent(source), client_var=client_var) diff --git a/tests/unit/test_phase4_ga.py b/tests/unit/test_phase4_ga.py index 3cf09f92..879a9f78 100644 --- a/tests/unit/test_phase4_ga.py +++ b/tests/unit/test_phase4_ga.py @@ -505,7 +505,7 @@ class TestCodemodByPage(unittest.TestCase): @classmethod def setUpClass(cls): try: - from tools.migrate_v0_to_v1 import migrate_source + from PowerPlatform.Dataverse.migration.migrate_v0_to_v1 import migrate_source cls.migrate = staticmethod(migrate_source) except ImportError: @@ -513,7 +513,7 @@ def setUpClass(cls): def setUp(self): if self.migrate is None: - self.skipTest("libcst not installed or tools package not on path") + self.skipTest("libcst not installed") def test_execute_by_page_true_becomes_execute_pages(self): src = "result = builder.execute(by_page=True)\n" @@ -573,7 +573,7 @@ class TestManualReviewFinder(unittest.TestCase): @classmethod def setUpClass(cls): try: - from tools.migrate_v0_to_v1 import find_manual_patterns + from PowerPlatform.Dataverse.migration.migrate_v0_to_v1 import find_manual_patterns cls.find = staticmethod(find_manual_patterns) except ImportError: @@ -581,7 +581,7 @@ def setUpClass(cls): def setUp(self): if self.find is None: - self.skipTest("libcst not installed or tools package not on path") + self.skipTest("libcst not installed") def test_records_get_flagged(self): src = "result = client.records.get('account', record_id)\n" From d683c3648e939a8314ae31272548d9df9efa4d6b Mon Sep 17 00:00:00 2001 From: Samson Gebre Date: Sun, 17 May 2026 22:11:46 -0700 Subject: [PATCH 16/16] add CLI help handling and tests for migration tool --- pyproject.toml | 7 +++++ tests/unit/test_migration_tool.py | 48 +++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index cd5fe7e4..77a6d492 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ dev = [ "isort>=5.12.0", "mypy>=1.0.0", "ruff>=0.1.0", + "libcst>=1.0.0", ] migration = ["libcst>=1.0.0"] @@ -98,6 +99,12 @@ testpaths = ["tests/unit"] [tool.coverage.run] source = ["src/PowerPlatform"] +# Migration codemod is an opt-in tool (requires the [migration] extra). +# It is exercised by tests/unit/test_migration_tool.py and tests/unit/test_phase4_ga.py, +# but its CLI plumbing and libcst error-path branches are not meaningfully testable +# under the default coverage scope. Exclude from coverage reporting so the metric +# reflects core SDK code only. +omit = ["src/PowerPlatform/Dataverse/migration/*"] [tool.coverage.report] fail_under = 90 diff --git a/tests/unit/test_migration_tool.py b/tests/unit/test_migration_tool.py index 4f79bb63..8f7af8ff 100644 --- a/tests/unit/test_migration_tool.py +++ b/tests/unit/test_migration_tool.py @@ -299,5 +299,53 @@ def test_batch_records_get_not_flagged(self): self.assertFalse(any("records.get" in f for f in findings)) +# --------------------------------------------------------------------------- +# CLI: --help / -h handling +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestMainHelp(unittest.TestCase): + """``main()`` returns 0 and prints usage when --help / -h is passed. + + Regression guard for the UX gap where ``--help`` was treated as a positional + path argument and produced ``[WARN] Not a file or directory: --help``. + """ + + def _run_main_capture(self, argv): + import io + import contextlib + from PowerPlatform.Dataverse.migration.migrate_v0_to_v1 import main + + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + rc = main(argv) + return rc, buf.getvalue() + + def test_long_help_flag_returns_zero(self): + rc, _ = self._run_main_capture(["--help"]) + self.assertEqual(rc, 0) + + def test_short_help_flag_returns_zero(self): + rc, _ = self._run_main_capture(["-h"]) + self.assertEqual(rc, 0) + + def test_help_prints_usage_line(self): + _, out = self._run_main_capture(["--help"]) + self.assertIn("Usage:", out) + self.assertIn("dataverse-migrate", out) + + def test_help_takes_precedence_over_other_flags(self): + """--help with other flags still exits 0 without processing paths.""" + rc, _ = self._run_main_capture(["--dry-run", "--help", "/nonexistent/path"]) + self.assertEqual(rc, 0) + + def test_no_args_returns_one(self): + """No arguments still prints usage but returns 1 (error).""" + rc, out = self._run_main_capture([]) + self.assertEqual(rc, 1) + self.assertIn("Usage:", out) + + if __name__ == "__main__": unittest.main()