diff --git a/.claude/skills/dataverse-sdk-use/SKILL.md b/.claude/skills/dataverse-sdk-use/SKILL.md index 72677468..d25815d7 100644 --- a/.claude/skills/dataverse-sdk-use/SKILL.md +++ b/.claude/skills/dataverse-sdk-use/SKILL.md @@ -28,11 +28,23 @@ Use the PowerPlatform Dataverse Client Python SDK to interact with Microsoft Dat The SDK supports Dataverse's native bulk operations: Pass lists to `create()`, `update()` for automatic bulk processing, for `delete()`, set `use_bulk_delete` when passing lists to use bulk operation ### Paging -- Control page size with `page_size` parameter +- Control page size with `page_size` parameter on `records.list()`, `records.list_pages()`, or `QueryBuilder.page_size()` - Use `top` parameter to limit total records returned +- **Preferred**: `client.query.builder(table)....execute_pages()` — composable `where(col(...))` filters, formatted values, expand with nested selects, full pagination control +- Simple streaming shortcut: `records.list_pages(table, *, filter, select, top, orderby, expand, page_size, count, include_annotations)` — string-based OData filter only, yields one `QueryResult` per page +- `execute(by_page=True/False)` is **deprecated** and emits `UserWarning`; use `execute_pages()` instead +- `QueryBuilder.to_dataframe()` is **deprecated**; use `.execute().to_dataframe()` instead + +### QueryResult +- Returned by `records.list()`, `records.retrieve()`, `execute()`, and each page from `list_pages()` / `execute_pages()` +- Iterable: `for record in result` — each item is a `dict`-like `Record` +- `.to_dataframe()` — convert to pandas DataFrame +- `.first()` — return the first record or `None` (safe: returns `None` on empty result) +- `result[n]` — index access returns a `Record`; `result[n:m]` returns a `QueryResult` +- `len(result)` — number of records in this result/page ### DataFrame Support -- DataFrame operations are accessed via the `client.dataframe` namespace: `client.dataframe.get()`, `client.dataframe.create()`, `client.dataframe.update()`, `client.dataframe.delete()` +- DataFrame operations are accessed via the `client.dataframe` namespace: `client.dataframe.create()`, `client.dataframe.update()`, `client.dataframe.delete()` — `client.dataframe.get()` is deprecated; use `client.query.builder(table).where(...).execute().to_dataframe()` instead ## Common Operations @@ -85,28 +97,92 @@ contact_ids = client.records.create("contact", contacts) #### Read Records ```python # Get single record by ID -account = client.records.get("account", account_id, select=["name", "telephone1"]) - -# Query with filter (paginated) -for page in client.records.get( - "account", - select=["accountid", "name"], # select is case-insensitive (automatically lowercased) - filter="statecode eq 0", # filter must use lowercase logical names (not transformed) - top=100, -): +account = client.records.retrieve("account", account_id, select=["name", "telephone1"]) + +# With expand — fetch a related record in the same HTTP request +account = client.records.retrieve( + "account", account_id, + select=["name"], + expand=["primarycontactid"], +) +contact = (account.get("primarycontactid") or {}) +print(contact.get("fullname")) + +# Simple shortcut — use records.list() only for basic filter + select without composable logic. +# Follows @odata.nextLink automatically and loads all matching records into memory. +# For filtering, sorting, expansion, or formatted values, prefer client.query.builder() (see below). +result = client.records.list("account", filter="statecode eq 0", select=["name", "accountid"]) +for record in result: + print(record["name"]) +``` + +#### Query Builder (Preferred for Filtering, Sorting, Expand, Formatted Values) + +Use `client.query.builder()` for any query that goes beyond simple filter + select. It provides composable `where(col(...))` expressions, formatted value support, nested expansion, and streaming — all with a fluent API. + +```python +from PowerPlatform.Dataverse.models.filters import col +from PowerPlatform.Dataverse.models.query_builder import ExpandOption + +# Basic query with composable filter and sort +result = (client.query.builder("account") + .select("accountid", "name", "statecode") + .where(col("statecode") == 0) + .order_by("name asc") + .execute()) +for record in result: + print(record["name"]) + +# Composable filters — AND / OR / NOT using Python operators +result = (client.query.builder("contact") + .select("fullname", "emailaddress1") + .where((col("statecode") == 0) & (col("emailaddress1").contains("@contoso.com"))) + .execute()) + +# Formatted values — display labels for option sets, currency symbols, etc. +result = (client.query.builder("account") + .select("accountid", "name", "industrycode") + .where(col("statecode") == 0) + .include_formatted_values() + .execute()) +for record in result: + label = record.get("industrycode@OData.Community.Display.V1.FormattedValue") + print(record["name"], label) + +# Navigation property expansion with nested column select +result = (client.query.builder("account") + .select("name") + .expand(ExpandOption("primarycontactid").select("fullname", "emailaddress1")) + .where(col("statecode") == 0) + .execute()) +for record in result: + contact = record.get("primarycontactid", {}) + print(f"{record['name']} - {contact.get('fullname', 'N/A')}") + +# Stream large result sets page-by-page (memory-efficient) +for page in (client.query.builder("account") + .select("accountid", "name") + .where(col("statecode") == 0) + .order_by("name asc") + .page_size(500) + .execute_pages()): for record in page: print(record["name"]) -# Query with navigation property expansion (case-sensitive!) -for page in client.records.get( - "account", - select=["name"], - expand=["primarycontactid"], # Navigation properties are case-sensitive! - filter="statecode eq 0", # Column names must be lowercase logical names -): - for account in page: - contact = account.get("primarycontactid", {}) - print(f"{account['name']} - {contact.get('fullname', 'N/A')}") +# Convert query results to a DataFrame +df = (client.query.builder("account") + .select("accountid", "name") + .where(col("statecode") == 0) + .execute() + .to_dataframe()) + +# Limit total results +result = client.query.builder("account").select("name").top(100).execute() + +# Simple streaming shortcut via records.list_pages() (string filter only, same params as records.list()) +for page in client.records.list_pages("account", filter="statecode eq 0", select=["name"], page_size=500): + for record in page: + print(record["name"]) ``` #### Create Records with Lookup Bindings (@odata.bind) @@ -179,18 +255,24 @@ client.records.delete("account", [id1, id2, id3], use_bulk_delete=True) The SDK provides DataFrame wrappers for all CRUD operations via the `client.dataframe` namespace, using pandas DataFrames and Series as input/output. +> **Note:** `client.dataframe.get()` is deprecated. Use `client.query.builder(table).select(...).where(...).execute().to_dataframe()` instead. `QueryBuilder.to_dataframe()` (without `.execute()`) is also deprecated — always call `.execute()` first. + ```python import pandas as pd -# Query records -- returns a single DataFrame -df = client.dataframe.get("account", filter="statecode eq 0", select=["name"]) +# Query records -- returns a single DataFrame (GA pattern: .execute().to_dataframe()) +from PowerPlatform.Dataverse.models.filters import col +df = client.query.builder("account").where(col("statecode") == 0).select("name").execute().to_dataframe() print(f"Got {len(df)} rows") -# Limit results with top for large tables -df = client.dataframe.get("account", select=["name"], top=100) +# Limit results with top +df = client.query.builder("account").select("name").top(100).execute().to_dataframe() + +# Via records.list() (simpler for basic queries) +df = client.records.list("account", filter="statecode eq 0", select=["name"]).to_dataframe() # Fetch single record as one-row DataFrame -df = client.dataframe.get("account", record_id=account_id, select=["name"]) +df = client.records.retrieve("account", account_id, select=["name"]).to_dataframe() # Create records from a DataFrame (returns a Series of GUIDs) new_accounts = pd.DataFrame([ @@ -223,6 +305,34 @@ for record in results: print(record["name"]) ``` +### FetchXML Queries + +`client.query.fetchxml(xml)` returns an inert `FetchXmlQuery` object — **no HTTP request is made** until `.execute()` or `.execute_pages()` is called. + +```python +xml = """ + + + + + + + + + +""" + +# Load all results into memory (simple, small-to-medium sets) +query = client.query.fetchxml(xml) +result = query.execute() # returns QueryResult — all pages fetched upfront +for record in result: + print(record["name"]) + +# Stream page-by-page (large sets or early exit) +for page in query.execute_pages(): # yields one QueryResult per HTTP page + process(page.to_dataframe()) +``` + ### Table Management #### Create Custom Tables @@ -380,7 +490,8 @@ Use `client.batch` to send multiple operations in one HTTP request. All batch me batch = client.batch.new() batch.records.create("account", {"name": "Contoso"}) batch.records.update("account", account_id, {"telephone1": "555-0100"}) -batch.records.get("account", account_id, select=["name"]) +batch.records.retrieve("account", account_id, select=["name"], expand=["primarycontactid"], include_annotations="OData.Community.Display.V1.FormattedValue") # single record with expand +batch.records.list("account", filter="statecode eq 0", select=["name"], orderby=["name asc"], top=50, page_size=25, count=True) # multi-record, single page batch.query.sql("SELECT TOP 5 name FROM account") result = batch.execute() @@ -412,7 +523,8 @@ print(f"Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") **Batch limitations:** - Maximum 1000 operations per batch -- Paginated `records.get()` (without `record_id`) is not supported in batch +- `batch.records.get()` is deprecated; use `batch.records.retrieve()` for single records +- `batch.records.list()` returns a single page (no pagination); use `top` to bound results - `flush_cache()` is not supported in batch ## Error Handling @@ -430,7 +542,7 @@ from PowerPlatform.Dataverse.core.errors import ( from PowerPlatform.Dataverse.client import DataverseClient try: - client.records.get("account", "invalid-id") + client.records.retrieve("account", "invalid-id") except HttpError as e: print(f"HTTP {e.status_code}: {e.message}") print(f"Error code: {e.code}") @@ -464,16 +576,17 @@ except ValidationError as e: ### Performance Optimization -1. **Use bulk operations** - Pass lists to create/update/delete for automatic optimization -2. **Specify select fields** - Limit returned columns to reduce payload size -3. **Control page size** - Use `top` and `page_size` parameters appropriately -4. **Reuse client instances** - Don't create new clients for each operation -5. **Use production credentials** - ClientSecretCredential or CertificateCredential for unattended operations -6. **Error handling** - Implement retry logic for transient errors (`e.is_transient`) -7. **Always include customization prefix** for custom tables/columns -8. **Use lowercase for column names, match `$metadata` for navigation properties** - Column names in `$select`/`$filter`/record payloads use lowercase LogicalNames. Navigation properties in `$expand` and `@odata.bind` keys are case-sensitive and must match the entity's `$metadata` (PascalCase for custom lookups like `new_CustomerId`, lowercase for system lookups like `parentaccountid`) -9. **Test in non-production environments** first -10. **Use named constants** - Import cascade behavior constants from `PowerPlatform.Dataverse.common.constants` +1. **Prefer `client.query.builder()` for any non-trivial query** — use the builder for filtering, sorting, expansion, or formatted values; `records.list()` is a convenience shortcut for simple filter+select only +2. **Use bulk operations** - Pass lists to create/update/delete for automatic optimization +3. **Specify select fields** - Limit returned columns to reduce payload size +4. **Control page size** - Use `top` and `page_size` parameters appropriately; use `execute_pages()` for large sets +5. **Reuse client instances** - Don't create new clients for each operation +6. **Use production credentials** - ClientSecretCredential or CertificateCredential for unattended operations +7. **Error handling** - Implement retry logic for transient errors (`e.is_transient`) +8. **Always include customization prefix** for custom tables/columns +9. **Use lowercase for column names, match `$metadata` for navigation properties** - Column names in `$select`/`$filter`/record payloads use lowercase LogicalNames. Navigation properties in `$expand` and `@odata.bind` keys are case-sensitive and must match the entity's `$metadata` (PascalCase for custom lookups like `new_CustomerId`, lowercase for system lookups like `parentaccountid`) +10. **Test in non-production environments** first +11. **Use named constants** - Import cascade behavior constants from `PowerPlatform.Dataverse.common.constants` ## Additional Resources @@ -486,9 +599,10 @@ Load these resources as needed during development: ## Key Reminders -1. **Schema names are required** - Never use display names -2. **Custom tables need prefixes** - Include customization prefix (e.g., "new_") -3. **Filter is case-sensitive** - Use lowercase logical names -4. **Bulk operations are encouraged** - Pass lists for optimization -5. **No trailing slashes in URLs** - Format: `https://org.crm.dynamics.com` -6. **Structured errors** - Check `is_transient` for retry logic +1. **Use `client.query.builder()` for queries** — it's the primary query pattern; `records.list()` is a shortcut for trivial filter+select only +2. **Schema names are required** - Never use display names +3. **Custom tables need prefixes** - Include customization prefix (e.g., "new_") +4. **Filter is case-sensitive** - Use lowercase logical names +5. **Bulk operations are encouraged** - Pass lists for optimization +6. **No trailing slashes in URLs** - Format: `https://org.crm.dynamics.com` +7. **Structured errors** - Check `is_transient` for retry logic diff --git a/CHANGELOG.md b/CHANGELOG.md index cf36ae04..1ae58c5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- `client.records.retrieve(table, record_id, *, select, expand, include_annotations)` — fetch a single record by GUID; returns `None` on 404 instead of raising; `expand` adds `$expand` for navigation property expansion on the single-record GET; `include_annotations` maps to the `Prefer: odata.include-annotations` header for formatted values and lookup labels (#175) +- `client.records.list(table, *, filter, select, top, orderby, expand, page_size, count, include_annotations)` — eager fetch returning a flat `QueryResult`; GA replacement for `records.get()` without a record ID; `page_size` controls `Prefer: odata.maxpagesize`, `count=True` adds `$count=true`, `include_annotations` requests formatted values (#175) +- `client.records.list_pages(table, *, filter, select, top, orderby, expand, page_size, count, include_annotations)` — lazy iterator yielding one `QueryResult` per HTTP page; streaming counterpart to `list()`; same parameter set (#175) +- `client.query.fetchxml(xml)` — FetchXML support returning an inert `FetchXmlQuery`; no HTTP request is made until `.execute()` or `.execute_pages()` is called (#175) +- `FetchXmlQuery` implements the correct Dataverse paging cookie algorithm: annotation parsed as outer XML, `pagingcookie` attribute double URL-decoded, server-supplied `pagenumber` used for next page, `morerecords` handled as both `bool` and `"true"` string, `UserWarning` emitted on simple paging fallback, 32,768-character URL limit enforced (documented Dataverse GET cap), 10,000-page circuit breaker against runaway iteration (#175) +- `QueryBuilder.execute_pages()` — lazy per-page streaming returning one `QueryResult` per HTTP page; replaces deprecated `execute(by_page=True)` (#175) +- `QueryBuilder.where()` — composable filter expressions using `col()` and Python operators (`==`, `>`, `&`, `|`, `~`); replaces deprecated `filter_eq()`, `filter_contains()`, and other `filter_*` helpers (#175) +- `QueryResult.__getitem__` — index access (`result[0]`) returns a `Record`; slice access (`result[1:5]`) returns a new `QueryResult` (#175) +- `DataverseModel` structural `Protocol` (`models/protocol.py`) — implement on any entity class to enable typed integration with CRUD operations without specifying table names or serializing manually (#175) +- `col()`, `raw()`, `QueryResult`, and `DataverseModel` exported from the top-level `PowerPlatform.Dataverse` package (#175) +- v0→v1 migration tool: installed as the `dataverse-migrate` console script (also runnable via `python -m PowerPlatform.Dataverse.migration.migrate_v0_to_v1`); rewrites v0 call sites to the v1 API with `--dry-run` support; covers `create`, `update`, `delete`, `get`, `list`, `fetchxml`, and query builder patterns; requires the `[migration]` optional extra (`pip install PowerPlatform-Dataverse-Client[migration]`) (#175) +- Migration tool now auto-rewrites `QueryBuilder.to_dataframe()` → `.execute().to_dataframe()` (inserts `.execute()` when receiver is a recognised builder chain); output improved with `[NEEDS-MANUAL]` label for files that have no auto-rewrites but require manual attention, and a trailing note on `[MIGRATED]` lines when manual items remain (#175) + +### Changed +- `QueryBuilder.execute()` now returns a flat `QueryResult` (all pages collected eagerly) instead of `Iterable[Record]` (#175) +- `records.get()` deprecation extended: calling with a `record_id` emits `DeprecationWarning` directing callers to `retrieve()`; calling without a `record_id` directs callers to `list()` (#175) + +### Deprecated +- `QueryBuilder.execute(by_page=True)` and `execute(by_page=False)` emit `UserWarning`; use `execute_pages()` and `execute()` respectively (#175) +- `client.query.odata_select()`, `client.query.odata_expands()`, `client.query.odata_expand()`, `client.query.odata_bind()` emit `DeprecationWarning`; navigation property helpers are replaced by `QueryBuilder.expand()` (#175) + +### Removed +- All v0 flat methods on `DataverseClient` (`create`, `update`, `delete`, `get`, `list`, `query_sql`, etc.) removed (~570 lines); use the `client.records`, `client.query`, and `client.batch` namespaces (#175) +- `client.query.sql_select()`, `client.query.sql_joins()`, `client.query.sql_join()` removed (#175) + ## [0.1.0b10] - 2026-05-12 ### Added diff --git a/README.md b/README.md index eab08734..21abb591 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ The SDK provides a simple, pythonic interface for Dataverse operations: | **Records** | Dataverse records represented as Python dictionaries with column schema names | | **Schema names** | Use table schema names (`"account"`, `"new_MyTestTable"`) and column schema names (`"name"`, `"new_MyTestColumn"`). See: [Table definitions in Microsoft Dataverse](https://learn.microsoft.com/en-us/power-apps/developer/data-platform/entity-metadata) | | **Bulk Operations** | Efficient bulk processing for multiple records with automatic optimization | +| **QueryBuilder** | Preferred query API: `client.query.builder()` with composable `where(col(...))` filters, formatted values, expand, and streaming; use `records.list()` only as a shortcut for simple filter+select | | **Paging** | Automatic handling of large result sets with iterators | | **Structured Errors** | Detailed exception hierarchy with retry guidance and diagnostic information | | **Customization prefix values** | Custom tables and columns require a customization prefix value to be included for all operations (e.g., `"new_MyTestTable"`, not `"MyTestTable"`). See: [Table definitions in Microsoft Dataverse](https://learn.microsoft.com/en-us/power-apps/developer/data-platform/entity-metadata) | @@ -144,7 +145,7 @@ with DataverseClient("https://yourorg.crm.dynamics.com", credential) as client: contact_id = client.records.create("contact", {"firstname": "John", "lastname": "Doe"}) # Read the contact back - contact = client.records.get("contact", contact_id, select=["firstname", "lastname"]) + contact = client.records.retrieve("contact", contact_id, select=["firstname", "lastname"]) print(f"Created: {contact['firstname']} {contact['lastname']}") # Clean up @@ -159,9 +160,18 @@ with DataverseClient("https://yourorg.crm.dynamics.com", credential) as client: account_id = client.records.create("account", {"name": "Contoso Ltd"}) # Read a record -account = client.records.get("account", account_id) +account = client.records.retrieve("account", account_id) print(account["name"]) +# Read with expand — fetch a related record in the same HTTP request +account = client.records.retrieve( + "account", account_id, + select=["name"], + expand=["primarycontactid"], +) +contact = (account.get("primarycontactid") or {}) +print(contact.get("fullname")) + # Update a record client.records.update("account", account_id, {"telephone1": "555-0199"}) @@ -242,18 +252,25 @@ client.records.upsert("account", [ The SDK provides pandas wrappers for all CRUD operations via the `client.dataframe` namespace, using DataFrames and Series for input and output. +> **Note:** `client.dataframe.get()` is deprecated. Use the GA patterns shown below instead. + ```python import pandas as pd +from PowerPlatform.Dataverse.models.filters import col -# Query records as a single DataFrame -df = client.dataframe.get("account", filter="statecode eq 0", select=["name", "telephone1"]) +# Query records as a single DataFrame (GA builder pattern) +df = (client.query.builder("account") + .select("name", "telephone1") + .where(col("statecode") == 0) + .execute() + .to_dataframe()) print(f"Found {len(df)} accounts") # Limit results with top for large tables -df = client.dataframe.get("account", select=["name"], top=100) +df = client.query.builder("account").select("name").top(100).execute().to_dataframe() # Fetch a single record as a one-row DataFrame -df = client.dataframe.get("account", record_id=account_id, select=["name"]) +df = client.records.retrieve("account", account_id, select=["name"]).to_dataframe() # Create records from a DataFrame (returns a Series of GUIDs) new_accounts = pd.DataFrame([ @@ -288,10 +305,12 @@ The **QueryBuilder** is the recommended way to query records. It provides a flue ```python # Fluent query builder (recommended) +from PowerPlatform.Dataverse.models.filters import col + for record in (client.query.builder("account") .select("name", "revenue") - .filter_eq("statecode", 0) - .filter_gt("revenue", 1000000) + .where(col("statecode") == 0) + .where(col("revenue") > 1000000) .order_by("revenue", descending=True) .top(100) .page_size(50) @@ -299,47 +318,48 @@ for record in (client.query.builder("account") print(f"{record['name']}: {record['revenue']}") ``` -The QueryBuilder handles value formatting, column name casing, and OData syntax automatically. All filter methods are discoverable via IDE autocomplete: +The QueryBuilder handles value formatting, column name casing, and OData syntax automatically. Filter expressions are built with `col()` and standard Python operators: ```python # Get results as a pandas DataFrame (consolidates all pages) df = (client.query.builder("account") .select("name", "telephone1") - .filter_eq("statecode", 0) + .where(col("statecode") == 0) .top(100) + .execute() .to_dataframe()) print(f"Got {len(df)} accounts") ``` ```python -# Comparison filters +# Comparison filters using col() expressions query = (client.query.builder("contact") - .filter_eq("statecode", 0) # statecode eq 0 - .filter_gt("revenue", 1000000) # revenue gt 1000000 - .filter_contains("name", "Corp") # contains(name, 'Corp') - .filter_in("statecode", [0, 1]) # Microsoft.Dynamics.CRM.In(...) - .filter_between("revenue", 100000, 500000) # (revenue ge 100000 and revenue le 500000) - .filter_null("telephone1") # telephone1 eq null + .where(col("statecode") == 0) # statecode eq 0 + .where(col("revenue") > 1000000) # revenue gt 1000000 + .where(col("name").contains("Corp")) # contains(name, 'Corp') + .where(col("statecode").in_([0, 1])) # Microsoft.Dynamics.CRM.In(...) + .where(col("revenue").between(100000, 500000)) # revenue ge 100000 and revenue le 500000 + .where(col("telephone1").is_null()) # telephone1 eq null ) ``` -For complex logic (OR, NOT, grouping), use the composable expression tree with `where()`: +For complex logic (OR, NOT, grouping), compose expressions with `&`, `|`, `~`: ```python -from PowerPlatform.Dataverse.models.filters import eq, gt, filter_in, between +from PowerPlatform.Dataverse.models.filters import col # OR conditions: (statecode = 0 OR statecode = 1) AND revenue > 100k for record in (client.query.builder("account") .select("name", "revenue") - .where((eq("statecode", 0) | eq("statecode", 1)) - & gt("revenue", 100000)) + .where(((col("statecode") == 0) | (col("statecode") == 1)) + & (col("revenue") > 100000)) .execute()): print(record["name"]) # NOT, between, and in operators for record in (client.query.builder("account") - .where(~eq("statecode", 2)) # NOT inactive - .where(between("revenue", 100000, 500000)) # revenue in range + .where(col("statecode") != 2) # NOT inactive + .where(col("revenue").between(100000, 500000)) # revenue in range .execute()): print(record["name"]) ``` @@ -347,13 +367,31 @@ for record in (client.query.builder("account") **Formatted values and annotations** -- request localized labels, currency symbols, and display names: ```python -# Get formatted values (choice labels, currency, lookup names) +# Get formatted values (choice labels, currency, lookup names) — via query builder for record in (client.query.builder("account") .select("name", "statecode", "revenue") .include_formatted_values() .execute()): status = record["statecode@OData.Community.Display.V1.FormattedValue"] print(f"{record['name']}: {status}") + +# Get formatted values — via records.list() / records.retrieve() include_annotations param +result = client.records.list( + "account", + select=["name", "statecode"], + include_annotations="OData.Community.Display.V1.FormattedValue", +) +for record in result: + label = record.get("statecode@OData.Community.Display.V1.FormattedValue") + print(f"{record['name']}: {label}") + +record = client.records.retrieve( + "account", account_id, + select=["name", "statuscode"], + include_annotations="OData.Community.Display.V1.FormattedValue", +) +if record: + print(record.get("statuscode@OData.Community.Display.V1.FormattedValue")) ``` **Nested expand with options** -- expand navigation properties with `$select`, `$filter`, `$orderby`, and `$top`: @@ -373,14 +411,83 @@ for record in (client.query.builder("account") print(record["name"], record.get("Account_Tasks")) ``` +**Paging** -- use `execute_pages()` for streaming large result sets with full builder options (filtering, sorting, formatted values). `records.list()` and `records.list_pages()` are simpler shortcuts for string-based OData filter queries: + +```python +# Preferred: query.builder().execute_pages() — stream one page at a time, memory stays flat +# Supports composable filters, sorting, formatted values, and expand with nested selects +for page_num, page in enumerate( + client.query.builder("account") + .select("accountid", "name", "revenue") + .where(col("statecode") == 0) + .order_by("name") + .page_size(500) # optional: override Dataverse default (~5000/page) + .execute_pages() +): + print(f"Page {page_num + 1}: {len(page)} records") + for record in page: + print(f" {record['name']}") + +# Simple shortcut: records.list() — automatic paging, all records in memory +# Use for basic filter+select queries; string OData filter only (no composable expressions) +result = client.records.list( + "account", + filter="statecode eq 0", + select=["name", "revenue"], + orderby=["name asc"], # optional sort + top=500, # bounds total records returned and number of HTTP round-trips + page_size=200, # optional: hint Dataverse default page size +) +for record in result: + print(record["name"]) + +# Simple streaming shortcut: records.list_pages() — same params as records.list(), yields one page at a time +for page_num, page in enumerate( + client.records.list_pages("account", filter="statecode eq 0", select=["name"], orderby=["name asc"]) +): + print(f"Page {page_num + 1}: {len(page)} records") + for record in page: + print(record["name"]) +``` + +> **Deprecation note:** `execute(by_page=True)` and `execute(by_page=False)` are deprecated and emit a `UserWarning`. Replace with `execute_pages()` (streaming) or plain `execute()` (eager). `QueryBuilder.to_dataframe()` is also deprecated; use `.execute().to_dataframe()` instead. The migration tool rewrites all of these automatically — install with `pip install PowerPlatform-Dataverse-Client[migration]` and run `dataverse-migrate path/to/your/scripts/` (or `python -m PowerPlatform.Dataverse.migration.migrate_v0_to_v1` for development checkouts). + **Record count** -- include `$count=true` in the request: ```python -# Request count alongside results +# Via query builder results = (client.query.builder("account") - .filter_eq("statecode", 0) + .where(col("statecode") == 0) .count() .execute()) + +# Via records.list() — count=True adds $count=true to the OData request +results = client.records.list("account", filter="statecode eq 0", count=True) +``` + +**FetchXML queries** -- `client.query.fetchxml()` returns an inert `FetchXmlQuery` object; no HTTP request is made until you call `.execute()` or `.execute_pages()`: + +```python +xml = """ + + + + + + + +""" + +# .execute() — blocking, fetches all pages and returns a single QueryResult +result = client.query.fetchxml(xml).execute() +df = result.to_dataframe() + +# .execute_pages() — streaming, yields one QueryResult per HTTP page +# Use count="N" in the FetchXML element to set page size +for page_num, page in enumerate(client.query.fetchxml(xml).execute_pages()): + print(f"Page {page_num + 1}: {len(page)} records") + for record in page: + print(record["name"]) ``` **SQL queries** provide an alternative read-only query syntax with support for @@ -405,52 +512,44 @@ df = client.dataframe.sql( "SELECT name, revenue FROM account ORDER BY revenue DESC" ) -# SQL helpers: discover columns and JOINs from metadata -cols = client.query.sql_select("account") # "accountid, name, revenue, ..." -join = client.query.sql_join("contact", "account", from_alias="c", to_alias="a") -# Returns: "JOIN account a ON c.parentcustomerid = a.accountid" +# Discover columns from metadata (schema-discovery helper, kept at GA) +cols_meta = client.query.sql_columns("account") +col_names = [c["LogicalName"] for c in cols_meta] -# Build queries using helpers -- no OData knowledge needed -sql = f"SELECT TOP 10 c.fullname, a.name FROM contact c {join}" +# Build queries using the discovered column names +sql = f"SELECT TOP 10 {', '.join(col_names[:5])} FROM account" df = client.dataframe.sql(sql) - -# Discover all possible JOINs from a table (including polymorphic) -joins = client.query.sql_joins("opportunity") -for j in joins: - print(f"{j['column']:30s} -> {j['target']}.{j['target_pk']}") ``` -**Raw OData queries** are available via `records.get()` for cases where you need direct control over the OData filter string. The SDK provides helpers to eliminate the most error-prone parts: +**Simple list shortcut** -- `records.list()` accepts a raw OData filter string for basic queries. For anything beyond simple filter+select, prefer `client.query.builder()` (composable filters, formatted values, nested expand): ```python -# Discover columns for $select (returns list ready for select= parameter) -cols = client.query.odata_select("account") -for page in client.records.get("account", select=cols, top=10): - ... - -# Discover $expand navigation properties (auto-resolves PascalCase names) -nav = client.query.odata_expand("contact", "account") -# Returns: "parentcustomerid_account" -for page in client.records.get("contact", select=["fullname"], expand=[nav], top=5): - for r in page: - acct = r.get(nav) or {} - print(f"{r['fullname']} -> {acct.get('name')}") - -# Build @odata.bind for lookup fields (no manual name construction) -bind = client.query.odata_bind("contact", "account", account_id) -# Returns: {"parentcustomerid_account@odata.bind": "/accounts(guid)"} -client.records.create("contact", {"firstname": "Jane", **bind}) - -# Raw OData query with manual parameters -for page in client.records.get( +# records.list() shortcut — raw OData filter string, all records loaded into memory +# Column names in filter must be lowercase logical names +for record in client.records.list( "account", select=["name"], - filter="statecode eq 0", # Raw OData: column names must be lowercase - expand=["primarycontactid"], # Navigation properties are case-sensitive + filter="statecode eq 0", top=100, ): - for record in page: - print(record["name"]) + print(record["name"]) + +# Discover navigation property names for $expand (metadata-discovery helper, kept at GA) +nav_props = client.query.odata_expands("account") # → list of navigation property metadata + +# Expand navigation properties using the query builder +from PowerPlatform.Dataverse.models.query_builder import ExpandOption +for record in (client.query.builder("contact") + .select("fullname") + .expand(ExpandOption("parentcustomerid_account").select("name")) + .execute()): + acct = record.get("parentcustomerid_account") or {} + print(f"{record['fullname']} -> {acct.get('name')}") + +# Build @odata.bind for lookup fields (deprecated helper, still functional with DeprecationWarning) +bind = client.query.odata_bind("contact", "account", account_id) +# Returns: {"parentcustomerid_account@odata.bind": "/accounts(guid)"} +client.records.create("contact", {"firstname": "Jane", **bind}) ``` ### Table management @@ -604,7 +703,14 @@ batch.records.create("account", {"name": "Contoso"}) batch.records.create("account", [{"name": "Fabrikam"}, {"name": "Woodgrove"}]) batch.records.update("account", account_id, {"telephone1": "555-0100"}) batch.records.delete("account", old_id) -batch.records.get("account", account_id, select=["name"]) +batch.records.retrieve("account", account_id, select=["name"], expand=["primarycontactid"]) # single record with expand +batch.records.list( # multi-record, single page + "account", + filter="statecode eq 0", + select=["name"], + orderby=["name asc"], + top=50, +) result = batch.execute() for item in result.responses: @@ -718,7 +824,7 @@ from PowerPlatform.Dataverse.client import DataverseClient from PowerPlatform.Dataverse.core.errors import HttpError, ValidationError try: - client.records.get("account", "invalid-id") + client.records.retrieve("account", "invalid-id") except HttpError as e: print(f"HTTP {e.status_code}: {e.message}") print(f"Error code: {e.code}") @@ -742,9 +848,10 @@ For optimal performance in production environments: | Best Practice | Description | |---------------|-------------| +| **Prefer QueryBuilder for queries** | Use `client.query.builder()` for filtering, sorting, expansion, or formatted values; use `records.list()` only as a shortcut for simple filter+select | | **Bulk Operations** | Pass lists to `records.create()`, `records.update()` for automatic bulk processing, for `records.delete()`, set `use_bulk_delete` when passing lists to use bulk operation | | **Select Fields** | Specify `select` parameter to limit returned columns and reduce payload size | -| **Page Size Control** | Use `top` and `page_size` parameters to control memory usage | +| **Page Size Control** | Use `top` and `page_size` parameters to control memory usage; use `execute_pages()` for large result sets | | **Connection Reuse** | Reuse `DataverseClient` instances across operations | | **Production Credentials** | Use `ClientSecretCredential` or `CertificateCredential` for unattended operations | | **Error Handling** | Implement retry logic for transient errors (`e.is_transient`) | diff --git a/examples/advanced/alternate_keys_upsert.py b/examples/advanced/alternate_keys_upsert.py index 67e8a43e..3248282a 100644 --- a/examples/advanced/alternate_keys_upsert.py +++ b/examples/advanced/alternate_keys_upsert.py @@ -100,29 +100,37 @@ def main(): client = DataverseClient(base_url, credential) # ------------------------------------------------------------------ - # Step 1: Create table + # Step 1: Create table (skip if already exists) # ------------------------------------------------------------------ print("\n1. Creating table...") - table_info = backoff( - lambda: client.tables.create( - TABLE_NAME, - columns={ - KEY_COLUMN: "string", - "new_ProductName": "string", - "new_Price": "decimal", - }, + table_info = client.tables.get(TABLE_NAME) + if table_info: + print(f" Table already exists: {TABLE_NAME} (skipped)") + else: + table_info = backoff( + lambda: client.tables.create( + TABLE_NAME, + columns={ + KEY_COLUMN: "string", + "new_ProductName": "string", + "new_Price": "decimal", + }, + ) ) - ) - print(f" Created: {table_info.get('table_schema_name', TABLE_NAME)}") - - time.sleep(10) # Wait for metadata propagation + print(f" Created: {table_info.get('table_schema_name', TABLE_NAME)}") + time.sleep(10) # Wait for metadata propagation # ------------------------------------------------------------------ - # Step 2: Create alternate key + # Step 2: Create alternate key (skip if already exists) # ------------------------------------------------------------------ print("\n2. Creating alternate key...") - key_info = backoff(lambda: client.tables.create_alternate_key(TABLE_NAME, KEY_NAME, [KEY_COLUMN.lower()])) - print(f" Key created: {key_info.schema_name} (id={key_info.metadata_id})") + existing_keys = client.tables.get_alternate_keys(TABLE_NAME) + existing_key = next((k for k in existing_keys if k.schema_name == KEY_NAME), None) + if existing_key: + print(f" Alternate key already exists: {KEY_NAME} (skipped)") + else: + key_info = backoff(lambda: client.tables.create_alternate_key(TABLE_NAME, KEY_NAME, [KEY_COLUMN.lower()])) + print(f" Key created: {key_info.schema_name} (id={key_info.metadata_id})") # ------------------------------------------------------------------ # Step 3: Wait for key to become Active @@ -212,15 +220,14 @@ def main(): # Step 6: Verify # ------------------------------------------------------------------ print("\n6. Verifying records...") - for page in client.records.get( + for record in client.records.list( TABLE_NAME, select=["new_productname", "new_price", KEY_COLUMN.lower()], ): - for record in page: - ext_id = record.get(KEY_COLUMN.lower(), "?") - name = record.get("new_productname", "?") - price = record.get("new_price", "?") - print(f" {ext_id}: {name} @ ${price}") + ext_id = record.get(KEY_COLUMN.lower(), "?") + name = record.get("new_productname", "?") + price = record.get("new_price", "?") + print(f" {ext_id}: {name} @ ${price}") # ------------------------------------------------------------------ # Step 7: List alternate keys diff --git a/examples/advanced/batch.py b/examples/advanced/batch.py index a95aa303..bb180bfc 100644 --- a/examples/advanced/batch.py +++ b/examples/advanced/batch.py @@ -13,248 +13,258 @@ from __future__ import annotations -# --------------------------------------------------------------------------- -# Setup — replace with your environment URL and credential -# --------------------------------------------------------------------------- +import sys from azure.identity import InteractiveBrowserCredential from PowerPlatform.Dataverse.client import DataverseClient -credential = InteractiveBrowserCredential() -with DataverseClient("https://org.crm.dynamics.com", credential) as client: +def main(): + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") - # --------------------------------------------------------------------------- - # Example 1: Record CRUD in a single batch - # --------------------------------------------------------------------------- + credential = InteractiveBrowserCredential() - print("\n[INFO] Example 1: Record CRUD in a single batch") + with DataverseClient(base_url=base_url, credential=credential) as client: - batch = client.batch.new() + # --------------------------------------------------------------------------- + # Example 1: Record CRUD in a single batch + # --------------------------------------------------------------------------- - # Create a single record - batch.records.create("account", {"name": "Contoso Ltd", "telephone1": "555-0100"}) + print("\n[INFO] Example 1: Record CRUD in a single batch") - # Create multiple records via CreateMultiple (one batch item) - batch.records.create( - "contact", - [ - {"firstname": "Alice", "lastname": "Smith"}, - {"firstname": "Bob", "lastname": "Jones"}, - ], - ) + batch = client.batch.new() - # Assume we have an existing account_id from a prior operation - # batch.records.update("account", account_id, {"telephone1": "555-9999"}) - # batch.records.delete("account", old_id) + # Create a single record + batch.records.create("account", {"name": "Contoso Ltd", "telephone1": "555-0100"}) - result = batch.execute() + # Create multiple records via CreateMultiple (one batch item) + batch.records.create( + "contact", + [ + {"firstname": "Alice", "lastname": "Smith"}, + {"firstname": "Bob", "lastname": "Jones"}, + ], + ) - print(f"[OK] Total: {len(result.responses)}, Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") - for guid in result.entity_ids: - print(f"[OK] Created: {guid}") - for item in result.failed: - print(f"[ERR] {item.status_code}: {item.error_message}") + # Assume we have an existing account_id from a prior operation + # batch.records.update("account", account_id, {"telephone1": "555-9999"}) + # batch.records.delete("account", old_id) - # --------------------------------------------------------------------------- - # Example 2: Transactional changeset with content-ID chaining - # --------------------------------------------------------------------------- + result = batch.execute() - print("\n[INFO] Example 2: Transactional changeset") + print(f"[OK] Total: {len(result.responses)}, Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") + for guid in result.entity_ids: + print(f"[OK] Created: {guid}") + for item in result.failed: + print(f"[ERR] {item.status_code}: {item.error_message}") - batch = client.batch.new() + # --------------------------------------------------------------------------- + # Example 2: Transactional changeset with content-ID chaining + # --------------------------------------------------------------------------- - with batch.changeset() as cs: - # Each create() returns a "$n" reference usable in subsequent operations - lead_ref = cs.records.create( - "lead", - {"firstname": "Ada", "lastname": "Lovelace"}, - ) - contact_ref = cs.records.create("contact", {"firstname": "Ada"}) - - # Reference the newly created lead and contact in the account - cs.records.create( - "account", - { - "name": "Babbage & Co.", - "originatingleadid@odata.bind": lead_ref, - "primarycontactid@odata.bind": contact_ref, - }, + print("\n[INFO] Example 2: Transactional changeset") + + batch = client.batch.new() + + with batch.changeset() as cs: + # Each create() returns a "$n" reference usable in subsequent operations + lead_ref = cs.records.create( + "lead", + {"firstname": "Ada", "lastname": "Lovelace"}, + ) + contact_ref = cs.records.create("contact", {"firstname": "Ada"}) + + # Reference the newly created lead and contact in the account + cs.records.create( + "account", + { + "name": "Babbage & Co.", + "originatingleadid@odata.bind": lead_ref, + "primarycontactid@odata.bind": contact_ref, + }, + ) + + # Update using a content-ID reference as the record_id + cs.records.update("contact", contact_ref, {"lastname": "Lovelace"}) + + result = batch.execute() + + if result.has_errors: + print("[ERR] Changeset rolled back") + for item in result.failed: + print(f" {item.status_code}: {item.error_message}") + else: + print(f"[OK] {len(result.entity_ids)} records created atomically") + + # --------------------------------------------------------------------------- + # Example 3: Table metadata operations in a batch + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 3: Table metadata operations") + + batch = client.batch.new() + + # Create a new custom table + batch.tables.create( + "new_Product", + {"new_Price": "decimal", "new_InStock": "bool"}, + solution="MySolution", ) - # Update using a content-ID reference as the record_id - cs.records.update("contact", contact_ref, {"lastname": "Lovelace"}) + # Read table metadata + batch.tables.get("new_Product") - result = batch.execute() + # List all non-private tables + batch.tables.list() - if result.has_errors: - print("[ERR] Changeset rolled back") + result = batch.execute() + print(f"[OK] Table ops: {[(r.status_code, r.is_success) for r in result.responses]}") + + # --------------------------------------------------------------------------- + # Example 4: SQL query in a batch + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 4: SQL query in batch") + + batch = client.batch.new() + batch.query.sql("SELECT TOP 5 accountid, name FROM account ORDER BY name") + + result = batch.execute() + if result.responses and result.responses[0].is_success and result.responses[0].data: + rows = result.responses[0].data.get("value", []) + print(f"[OK] Retrieved {len(rows)} accounts") + for row in rows: + print(f" {row.get('name')}") + + # --------------------------------------------------------------------------- + # Example 5: Mixed batch — changeset writes + standalone GETs + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 5: Mixed batch") + + # NOTE: Commented out because it requires a pre-existing account_id. + # Uncomment and set account_id to run this example. + # batch = client.batch.new() + # + # with batch.changeset() as cs: + # cs.records.update("account", account_id, {"statecode": 0}) + # + # batch.records.get("account", account_id, select=["name", "statecode"]) + # + # result = batch.execute() + # update_response = result.responses[0] + # account_data = result.responses[1] + # if account_data.is_success and account_data.data: + # print(f"Account: {account_data.data.get('name')}") + + # --------------------------------------------------------------------------- + # Example 6: Continue on error + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 6: Continue on error") + + batch = client.batch.new() + batch.records.retrieve("account", "00000000-0000-0000-0000-000000000000") + batch.query.sql("SELECT TOP 1 name FROM account") + + result = batch.execute(continue_on_error=True) + print(f"[OK] Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") for item in result.failed: - print(f" {item.status_code}: {item.error_message}") - else: - print(f"[OK] {len(result.entity_ids)} records created atomically") - - # --------------------------------------------------------------------------- - # Example 3: Table metadata operations in a batch - # --------------------------------------------------------------------------- - - print("\n[INFO] Example 3: Table metadata operations") - - batch = client.batch.new() - - # Create a new custom table - batch.tables.create( - "new_Product", - {"new_Price": "decimal", "new_InStock": "bool"}, - solution="MySolution", - ) - - # Read table metadata - batch.tables.get("new_Product") - - # List all non-private tables - batch.tables.list() - - result = batch.execute() - print(f"[OK] Table ops: {[(r.status_code, r.is_success) for r in result.responses]}") - - # --------------------------------------------------------------------------- - # Example 4: SQL query in a batch - # --------------------------------------------------------------------------- - - print("\n[INFO] Example 4: SQL query in batch") - - batch = client.batch.new() - batch.query.sql("SELECT TOP 5 accountid, name FROM account ORDER BY name") - - result = batch.execute() - if result.responses and result.responses[0].is_success and result.responses[0].data: - rows = result.responses[0].data.get("value", []) - print(f"[OK] Retrieved {len(rows)} accounts") - for row in rows: - print(f" {row.get('name')}") - - # --------------------------------------------------------------------------- - # Example 5: Mixed batch — changeset writes + standalone GETs - # --------------------------------------------------------------------------- - - print("\n[INFO] Example 5: Mixed batch") - - # NOTE: Commented out because it requires a pre-existing account_id. - # Uncomment and set account_id to run this example. - # batch = client.batch.new() - # - # with batch.changeset() as cs: - # cs.records.update("account", account_id, {"statecode": 0}) - # - # batch.records.get("account", account_id, select=["name", "statecode"]) - # - # result = batch.execute() - # update_response = result.responses[0] - # account_data = result.responses[1] - # if account_data.is_success and account_data.data: - # print(f"Account: {account_data.data.get('name')}") - - # --------------------------------------------------------------------------- - # Example 6: Continue on error - # --------------------------------------------------------------------------- - - print("\n[INFO] Example 6: Continue on error") - - batch = client.batch.new() - batch.records.get("account", "00000000-0000-0000-0000-000000000000") - batch.query.sql("SELECT TOP 1 name FROM account") - - result = batch.execute(continue_on_error=True) - print(f"[OK] Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") - for item in result.failed: - print(f"[ERR] {item.status_code}: {item.error_message}") - - # --------------------------------------------------------------------------- - # Example 7: DataFrame integration - # --------------------------------------------------------------------------- - - print("\n[INFO] Example 7: DataFrame batch operations") - - import pandas as pd - - # Create records from a DataFrame - df = pd.DataFrame( - [ - {"name": "DF-Batch-A", "telephone1": "555-0100"}, - {"name": "DF-Batch-B", "telephone1": "555-0200"}, - ] - ) - batch = client.batch.new() - batch.dataframe.create("account", df) - result = batch.execute() - print(f"[OK] DataFrame create: {len(result.succeeded)} succeeded") - created_ids = list(result.entity_ids) - - # Update records from a DataFrame - if len(created_ids) >= 2: - update_df = pd.DataFrame( + print(f"[ERR] {item.status_code}: {item.error_message}") + + # --------------------------------------------------------------------------- + # Example 7: DataFrame integration + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 7: DataFrame batch operations") + + import pandas as pd + + # Create records from a DataFrame + df = pd.DataFrame( [ - {"accountid": created_ids[0], "telephone1": "555-9990"}, - {"accountid": created_ids[1], "telephone1": "555-9991"}, + {"name": "DF-Batch-A", "telephone1": "555-0100"}, + {"name": "DF-Batch-B", "telephone1": "555-0200"}, ] ) batch = client.batch.new() - batch.dataframe.update("account", update_df, id_column="accountid") + batch.dataframe.create("account", df) result = batch.execute() - print(f"[OK] DataFrame update: {len(result.succeeded)} succeeded") + print(f"[OK] DataFrame create: {len(result.succeeded)} succeeded") + created_ids = list(result.entity_ids) + + # Update records from a DataFrame + if len(created_ids) >= 2: + update_df = pd.DataFrame( + [ + {"accountid": created_ids[0], "telephone1": "555-9990"}, + {"accountid": created_ids[1], "telephone1": "555-9991"}, + ] + ) + batch = client.batch.new() + batch.dataframe.update("account", update_df, id_column="accountid") + result = batch.execute() + print(f"[OK] DataFrame update: {len(result.succeeded)} succeeded") + + # Delete records from a Series + if created_ids: + batch = client.batch.new() + batch.dataframe.delete("account", pd.Series(created_ids), use_bulk_delete=False) + result = batch.execute() + print(f"[OK] DataFrame delete: {len(result.succeeded)} succeeded") + + # --------------------------------------------------------------------------- + # Example 8: Understanding response data patterns + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 8: Response data patterns") + + # Every batch result maps 1:1 with the operations you added. + # Different operations return different response shapes: - # Delete records from a Series - if created_ids: batch = client.batch.new() - batch.dataframe.delete("account", pd.Series(created_ids), use_bulk_delete=False) - result = batch.execute() - print(f"[OK] DataFrame delete: {len(result.succeeded)} succeeded") - - # --------------------------------------------------------------------------- - # Example 8: Understanding response data patterns - # --------------------------------------------------------------------------- + # Op 0: single create -> 204 No Content, entity_id in OData-EntityId header + batch.records.create("account", {"name": "Pattern-Demo"}) + # Op 1: bulk create -> 200 OK, IDs in body as {"Ids": [...]} + batch.records.create("account", [{"name": "Bulk-A"}, {"name": "Bulk-B"}]) + # Op 2: SQL query -> 200 OK, rows in body as {"value": [...]} + batch.query.sql("SELECT TOP 3 name FROM account") - print("\n[INFO] Example 8: Response data patterns") + result = batch.execute() - # Every batch result maps 1:1 with the operations you added. - # Different operations return different response shapes: + for i, resp in enumerate(result.responses): + if not resp.is_success: + print(f" Op {i}: [FAIL] {resp.status_code}: {resp.error_message}") + continue - batch = client.batch.new() - # Op 0: single create -> 204 No Content, entity_id in OData-EntityId header - batch.records.create("account", {"name": "Pattern-Demo"}) - # Op 1: bulk create -> 200 OK, IDs in body as {"Ids": [...]} - batch.records.create("account", [{"name": "Bulk-A"}, {"name": "Bulk-B"}]) - # Op 2: SQL query -> 200 OK, rows in body as {"value": [...]} - batch.query.sql("SELECT TOP 3 name FROM account") + # Single create: entity_id from OData-EntityId header + if resp.entity_id: + print(f" Op {i}: [CREATE] entity_id={resp.entity_id}") - result = batch.execute() + # Bulk action (CreateMultiple/UpsertMultiple): IDs in body + elif resp.data and "Ids" in resp.data: + print(f" Op {i}: [BULK] {len(resp.data['Ids'])} IDs: {resp.data['Ids']}") - for i, resp in enumerate(result.responses): - if not resp.is_success: - print(f" Op {i}: [FAIL] {resp.status_code}: {resp.error_message}") - continue + # Query: rows in body + elif resp.data and "value" in resp.data: + print(f" Op {i}: [QUERY] {len(resp.data['value'])} rows") - # Single create: entity_id from OData-EntityId header - if resp.entity_id: - print(f" Op {i}: [CREATE] entity_id={resp.entity_id}") + # Delete or metadata operation: 204, no data + else: + print(f" Op {i}: [OK] {resp.status_code}") - # Bulk action (CreateMultiple/UpsertMultiple): IDs in body - elif resp.data and "Ids" in resp.data: - print(f" Op {i}: [BULK] {len(resp.data['Ids'])} IDs: {resp.data['Ids']}") + # Clean up demo records + for rid in result.entity_ids: + client.records.delete("account", rid) + for resp in result.succeeded: + if resp.data and "Ids" in resp.data: + for rid in resp.data["Ids"]: + client.records.delete("account", rid) - # Query: rows in body - elif resp.data and "value" in resp.data: - print(f" Op {i}: [QUERY] {len(resp.data['value'])} rows") - # Delete or metadata operation: 204, no data - else: - print(f" Op {i}: [OK] {resp.status_code}") - - # Clean up demo records - for rid in result.entity_ids: - client.records.delete("account", rid) - for resp in result.succeeded: - if resp.data and "Ids" in resp.data: - for rid in resp.data["Ids"]: - client.records.delete("account", rid) +if __name__ == "__main__": + main() diff --git a/examples/advanced/dataframe_operations.py b/examples/advanced/dataframe_operations.py index 7c0b6010..0a51b4c7 100644 --- a/examples/advanced/dataframe_operations.py +++ b/examples/advanced/dataframe_operations.py @@ -19,6 +19,7 @@ from azure.identity import InteractiveBrowserCredential from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.models.filters import col, raw def main(): @@ -81,7 +82,7 @@ def _run_walkthrough(client): print("2. Query records as a DataFrame") print("-" * 60) - df_all = client.dataframe.get(table, select=select_cols, filter=test_filter) + df_all = client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute().to_dataframe() print(f"[OK] Got {len(df_all)} records in one DataFrame") print(f" Columns: {list(df_all.columns)}") print(f"{df_all.to_string(index=False)}") @@ -91,7 +92,7 @@ def _run_walkthrough(client): print("3. Limit results with top") print("-" * 60) - df_top2 = client.dataframe.get(table, select=select_cols, filter=test_filter, top=2) + df_top2 = client.query.builder(table).select(*select_cols).where(raw(test_filter)).top(2).execute().to_dataframe() print(f"[OK] Got {len(df_top2)} records with top=2") print(f"{df_top2.to_string(index=False)}") @@ -102,7 +103,9 @@ def _run_walkthrough(client): first_id = new_accounts["accountid"].iloc[0] print(f" Fetching record {first_id}...") - single = client.dataframe.get(table, record_id=first_id, select=select_cols) + single = ( + client.query.builder(table).select(*select_cols).where(col("accountid") == first_id).execute().to_dataframe() + ) print(f"[OK] Single record DataFrame:\n{single.to_string(index=False)}") # -- 5. Update records from a DataFrame ------------------------ @@ -116,7 +119,7 @@ def _run_walkthrough(client): print("[OK] Updated 3 records") # Verify the updates - verified = client.dataframe.get(table, select=select_cols, filter=test_filter) + verified = client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute().to_dataframe() print(f" Verified:\n{verified.to_string(index=False)}") # -- 6. Broadcast update (same value to all records) ----------- @@ -131,7 +134,7 @@ def _run_walkthrough(client): print("[OK] Broadcast update complete") # Verify all records have the same websiteurl - verified = client.dataframe.get(table, select=select_cols, filter=test_filter) + verified = client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute().to_dataframe() print(f" Verified:\n{verified.to_string(index=False)}") # Default: NaN/None fields are skipped (not overridden on server) @@ -142,14 +145,14 @@ def _run_walkthrough(client): ] ) client.dataframe.update(table, sparse_df, id_column="accountid") - verified = client.dataframe.get(table, select=select_cols, filter=test_filter) + verified = client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute().to_dataframe() print(f" Verified (Contoso telephone1 updated, websiteurl unchanged):\n{verified.to_string(index=False)}") # Opt-in: clear_nulls=True sends None as null to clear the field print("\n Clearing websiteurl for Contoso with clear_nulls=True...") clear_df = pd.DataFrame([{"accountid": new_accounts["accountid"].iloc[0], "websiteurl": None}]) client.dataframe.update(table, clear_df, id_column="accountid", clear_nulls=True) - verified = client.dataframe.get(table, select=select_cols, filter=test_filter) + verified = client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute().to_dataframe() print(f" Verified (Contoso websiteurl should be empty):\n{verified.to_string(index=False)}") # -- 7. Delete records by passing a Series of GUIDs ------------ @@ -162,7 +165,7 @@ def _run_walkthrough(client): print(f"[OK] Deleted {len(new_accounts)} records") # Verify deletions - filter for our tagged records should return 0 - remaining = client.dataframe.get(table, select=select_cols, filter=test_filter) + remaining = client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute().to_dataframe() print(f" Verified: {len(remaining)} test records remaining (expected 0)") print("\n" + "=" * 60) diff --git a/examples/advanced/datascience_risk_assessment.py b/examples/advanced/datascience_risk_assessment.py index 338713e4..80dafdc4 100644 --- a/examples/advanced/datascience_risk_assessment.py +++ b/examples/advanced/datascience_risk_assessment.py @@ -50,6 +50,7 @@ from azure.identity import InteractiveBrowserCredential from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.models.filters import col, raw # -- Optional imports (graceful degradation if not installed) ------ @@ -272,43 +273,42 @@ def step1_extract(client): print("=" * 60) # Pull accounts - accounts = client.dataframe.get( - TABLE_ACCOUNTS, - select=["accountid", "name", "revenue", "numberofemployees", "industrycode"], - filter="statecode eq 0", - top=200, + accounts = ( + client.query.builder(TABLE_ACCOUNTS) + .select("accountid", "name", "revenue", "numberofemployees", "industrycode") + .where(col("statecode") == 0) + .top(200) + .execute() + .to_dataframe() ) print(f"[OK] Extracted {len(accounts)} active accounts") # Pull open cases (service incidents) - cases = client.dataframe.get( - TABLE_CASES, - select=[ - "incidentid", - "_customerid_value", - "title", - "severitycode", - "prioritycode", - "createdon", - ], - filter="statecode eq 0", - top=1000, + cases = ( + client.query.builder(TABLE_CASES) + .select("incidentid", "_customerid_value", "title", "severitycode", "prioritycode", "createdon") + .where(raw("statecode eq 0")) + .top(1000) + .execute() + .to_dataframe() ) print(f"[OK] Extracted {len(cases)} open cases") # Pull active opportunities - opportunities = client.dataframe.get( - TABLE_OPPORTUNITIES, - select=[ + opportunities = ( + client.query.builder(TABLE_OPPORTUNITIES) + .select( "opportunityid", "_parentaccountid_value", "name", "estimatedvalue", "closeprobability", "estimatedclosedate", - ], - filter="statecode eq 0", - top=1000, + ) + .where(col("statecode") == 0) + .top(1000) + .execute() + .to_dataframe() ) print(f"[OK] Extracted {len(opportunities)} active opportunities") diff --git a/examples/advanced/fetchxml.py b/examples/advanced/fetchxml.py new file mode 100644 index 00000000..d4ac1e50 --- /dev/null +++ b/examples/advanced/fetchxml.py @@ -0,0 +1,582 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +End-to-end FetchXML examples for Dataverse. + +Demonstrates ``client.query.fetchxml()`` across the scenarios where FetchXML +is required or preferred over OData/SQL: + +- Basic attribute queries +- operators (eq, like, in, null, not-null, between) +- (inner and outer joins) +- Ordering +- Top N with automatic paging-cookie propagation +- Aggregate queries (count, sum, avg, min, max, group-by) +- Built-in system tables (account → contact join) + +FetchXML is the right tool when: +- You need a JOIN type OData $expand cannot express (many-to-many, outer link) +- You need server-side aggregates (count, sum, avg) without GROUP BY SQL +- You need ```` operators unavailable in OData ($filter) + +Prerequisites: +- pip install PowerPlatform-Dataverse-Client azure-identity +""" + +import sys +import time + +from azure.identity import InteractiveBrowserCredential +from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.core.errors import MetadataError +import requests + +# --------------------------------------------------------------------------- +# Helpers (same pattern as sql_examples.py) +# --------------------------------------------------------------------------- + + +def log_call(description): + print(f"\n-> {description}") + + +def heading(section_num, title): + print(f"\n{'=' * 80}") + print(f"{section_num}. {title}") + print("=" * 80) + + +def backoff(op, *, delays=(0, 2, 5, 10, 20, 20)): + """Retry an operation with exponential back-off.""" + last = None + total_delay = 0 + attempts = 0 + for d in delays: + if d: + time.sleep(d) + total_delay += d + attempts += 1 + try: + result = op() + if attempts > 1: + print(f" [INFO] Backoff succeeded after {attempts - 1} retry(s); waited {total_delay}s total.") + return result + except Exception as ex: + last = ex + continue + if last: + if attempts: + print( + f" [WARN] Backoff exhausted after {max(attempts - 1, 0)} retry(s); waited {total_delay}s total." + f"\n [ERROR] {last}" + ) + raise last + + +def main(): + print("=" * 80) + print("Dataverse SDK -- FetchXML End-to-End Examples") + print("=" * 80) + + heading(1, "Setup & Authentication") + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") + + log_call("InteractiveBrowserCredential()") + credential = InteractiveBrowserCredential() + + log_call(f"DataverseClient(base_url='{base_url}', credential=...)") + with DataverseClient(base_url=base_url, credential=credential) as client: + print(f"[OK] Connected to: {base_url}") + _run_examples(client) + + +def _run_examples(client): + project_table = "new_FXDemoProject" + task_table = "new_FXDemoTask" + + # =================================================================== + # 2. Create tables and seed data + # =================================================================== + heading(2, "Create Tables & Seed Data") + + log_call(f"client.tables.get('{project_table}')") + if client.tables.get(project_table): + print(f"[OK] Table already exists: {project_table}") + else: + log_call(f"client.tables.create('{project_table}', ...)") + try: + backoff( + lambda: client.tables.create( + project_table, + { + "new_Code": "string", + "new_Budget": "decimal", + "new_Active": "bool", + "new_Region": "int", + }, + ) + ) + print(f"[OK] Created table: {project_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {project_table} (skipped)") + else: + raise + + log_call(f"client.tables.get('{task_table}')") + if client.tables.get(task_table): + print(f"[OK] Table already exists: {task_table}") + else: + log_call(f"client.tables.create('{task_table}', ...)") + try: + backoff( + lambda: client.tables.create( + task_table, + { + "new_Title": "string", + "new_Hours": "int", + "new_Done": "bool", + "new_Priority": "int", + }, + ) + ) + print(f"[OK] Created table: {task_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {task_table} (skipped)") + else: + raise + + print("\n[INFO] Creating lookup field: tasks → projects ...") + try: + client.tables.create_lookup_field( + referencing_table=task_table, + lookup_field_name="new_ProjectId", + referenced_table=project_table, + display_name="Project", + ) + print("[OK] Created lookup: new_ProjectId on tasks → projects") + except Exception as e: + msg = str(e).lower() + if "already exists" in msg or "duplicate" in msg or "not unique" in msg: + print("[OK] Lookup already exists (skipped)") + else: + raise + + # Resolve entity set name for @odata.bind + project_set = f"{project_table.lower()}s" + try: + tinfo = client.tables.get(project_table) + if tinfo: + project_set = tinfo.get("entity_set_name", project_set) + except Exception: + pass + + log_call(f"client.records.create('{project_table}', [...])") + projects = [ + {"new_Code": "ALPHA", "new_Budget": 50000, "new_Active": True, "new_Region": 1}, + {"new_Code": "BRAVO", "new_Budget": 75000, "new_Active": True, "new_Region": 2}, + {"new_Code": "CHARLIE", "new_Budget": 30000, "new_Active": False, "new_Region": 3}, + {"new_Code": "DELTA", "new_Budget": 90000, "new_Active": True, "new_Region": 1}, + {"new_Code": "ECHO", "new_Budget": 42000, "new_Active": True, "new_Region": 2}, + ] + project_ids = backoff(lambda: client.records.create(project_table, projects)) + print(f"[OK] Seeded {len(project_ids)} projects") + + log_call(f"client.records.create('{task_table}', [...])") + tasks = [ + { + "new_Title": "Design mockups", + "new_Hours": 8, + "new_Done": True, + "new_Priority": 2, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[0]})", + }, + { + "new_Title": "Write unit tests", + "new_Hours": 12, + "new_Done": False, + "new_Priority": 3, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[0]})", + }, + { + "new_Title": "Code review", + "new_Hours": 3, + "new_Done": True, + "new_Priority": 1, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[1]})", + }, + { + "new_Title": "Deploy to staging", + "new_Hours": 5, + "new_Done": False, + "new_Priority": 3, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[1]})", + }, + { + "new_Title": "Update docs", + "new_Hours": 4, + "new_Done": True, + "new_Priority": 1, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[2]})", + }, + { + "new_Title": "Performance tuning", + "new_Hours": 10, + "new_Done": False, + "new_Priority": 2, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[3]})", + }, + { + "new_Title": "Security audit", + "new_Hours": 6, + "new_Done": False, + "new_Priority": 3, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[4]})", + }, + ] + task_ids = backoff(lambda: client.records.create(task_table, tasks)) + print(f"[OK] Seeded {len(task_ids)} tasks") + + project_logical = project_table.lower() # new_fxdemoproject + task_logical = task_table.lower() # new_fxdemotask + project_pk = f"{project_logical}id" # new_fxdemoprojectid + task_pk = f"{task_logical}id" # new_fxdemoproject id + lookup_attr = "new_projectid" # lookup logical name on task + + try: + # =============================================================== + # 3. Basic attribute query + # =============================================================== + heading(3, "Basic Attribute Query") + xml = f""" + + + + + + + + """ + log_call("client.query.fetchxml(basic attribute query)") + result = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] {len(result)} projects:") + for r in result: + print(f" {r.get('new_code', ''):<10s} Budget={r.get('new_budget')} Active={r.get('new_active')}") + # Index access and first() are equivalent; first() returns None on empty result + if result: + print(f" First by index : {result[0].get('new_code')}") + print(f" First by .first(): {result.first().get('new_code')}") + + # =============================================================== + # 4. operators: eq, like, in, null, not-null, between + # =============================================================== + heading(4, " Operators") + + # eq + xml = f""" + + + + + + + + + """ + log_call('operator="eq" value="ALPHA"') + r = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] eq: {[x.get('new_code') for x in r]}") + + # like + xml = f""" + + + + + + + + + """ + log_call('operator="like" value="%test%"') + r = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] like: {len(r)} matches -> {[x.get('new_title') for x in r]}") + + # in + xml = f""" + + + + + + ALPHA + DELTA + + + + + """ + log_call('operator="in" values=[ALPHA, DELTA]') + r = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] in: {[x.get('new_code') for x in r]}") + + # null / not-null + xml = f""" + + + + + + + + + """ + log_call('operator="not-null"') + r = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] not-null: {len(r)} tasks have priority set") + + # between + xml = f""" + + + + + + + 40000 + 80000 + + + + + """ + log_call('operator="between" 40000 and 80000') + r = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] between: {len(r)} projects -> {[(x.get('new_code'), x.get('new_budget')) for x in r]}") + + # =============================================================== + # 5. — inner join (tasks → projects) + # =============================================================== + heading(5, " Inner Join (Tasks → Projects)") + xml = f""" + + + + + + + + + + + """ + log_call("client.query.fetchxml(link-entity inner join)") + try: + result = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] {len(result)} rows:") + for r in result: + print( + f" Task={r.get('new_title', ''):<25s} " + f"Hours={r.get('new_hours')} " + f"Project={r.get('p.new_code', '')} " + f"Budget={r.get('p.new_budget')}" + ) + except Exception as e: + print(f"[WARN] link-entity join failed: {e}") + + # =============================================================== + # 6. — outer join (projects with or without tasks) + # =============================================================== + heading(6, " Outer Join (Projects With or Without Tasks)") + xml = f""" + + + + + + + + + """ + log_call("client.query.fetchxml(link-entity outer join)") + try: + result = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] {len(result)} rows (includes projects with no tasks):") + for r in result[:8]: + print(f" Project={r.get('new_code', ''):<10s} Task={r.get('t.new_title', '(none)')}") + except Exception as e: + print(f"[WARN] outer join failed: {e}") + + # =============================================================== + # 7. Ordering + # =============================================================== + heading(7, "Ordering ( element)") + + xml = f""" + + + + + + + + """ + log_call("client.query.fetchxml(order by hours DESC)") + result = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] Tasks by hours DESC:") + for r in result: + print(f" {r.get('new_title', ''):<25s} Hours={r.get('new_hours')}") + + # =============================================================== + # 8. Top N + paging-cookie propagation + # =============================================================== + heading(8, "Paging-Cookie Propagation") + print( + "[INFO] 'count' sets the page size in FetchXML (not 'top' — 'top' is a total-result limit).\n" + "With count='2' and 7 seeded tasks the server returns pages of 2, 2, 2, 1.\n" + ".execute() collects all pages eagerly; .execute_pages() yields one QueryResult per HTTP page." + ) + xml_paged = f""" + + + + + + + + """ + log_call("client.query.fetchxml(xml).execute() — eager, all pages collected") + result = backoff(lambda: client.query.fetchxml(xml_paged).execute()) + print(f"[OK] execute(): {len(result)} total tasks across all pages (seeded {len(task_ids)}):") + for r in result: + print(f" {r.get('new_title', ''):<25s} Hours={r.get('new_hours')}") + + log_call("client.query.fetchxml(xml).execute_pages() — lazy, one QueryResult per HTTP page") + page_num = 0 + page_record_count = 0 + for page in backoff(lambda: client.query.fetchxml(xml_paged).execute_pages()): + page_num += 1 + page_record_count += len(page) + print(f" Page {page_num}: {len(page)} record(s) — {[r.get('new_title') for r in page]}") + print(f"[OK] execute_pages(): {page_record_count} total tasks across {page_num} page(s)") + + # =============================================================== + # 9. Aggregates (count, sum, avg, min, max) + # =============================================================== + heading(9, "Aggregate Queries ()") + + # Global aggregates + xml = f""" + + + + + + + + + + """ + log_call("client.query.fetchxml(aggregate: count, sum, avg, min, max)") + try: + result = backoff(lambda: client.query.fetchxml(xml).execute()) + if result: + row = result.first() + print( + f"[OK] count={row.get('task_count')} sum={row.get('total_hours')} " + f"avg={row.get('avg_hours')} min={row.get('min_hours')} max={row.get('max_hours')}" + ) + except Exception as e: + print(f"[WARN] aggregate failed: {e}") + + # Group-by aggregate: total hours per project + xml = f""" + + + + + + + + + + """ + log_call("client.query.fetchxml(aggregate group-by project)") + try: + result = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] Hours per project ({len(result)} groups):") + for r in result: + print( + f" {r.get('project_code', ''):<10s} " + f"Tasks={r.get('task_count')} " + f"Hours={r.get('total_hours')}" + ) + except Exception as e: + print(f"[WARN] group-by aggregate failed: {e}") + + # =============================================================== + # 10. Built-in system tables (account → contact) + # =============================================================== + heading(10, "Built-In System Tables (account → contact Join)") + xml = """ + + + + + + + + + """ + log_call("client.query.fetchxml(account → contact inner join)") + try: + result = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] {len(result)} account-contact pairs:") + for r in result: + print(f" Account={r.get('name', ''):<25s} Contact={r.get('c.fullname', '')}") + except Exception as e: + print(f"[INFO] No account-contact data in this org: {e}") + + finally: + heading(11, "Cleanup") + for tbl in [task_table, project_table]: + log_call(f"client.tables.delete('{tbl}')") + try: + backoff(lambda tbl=tbl: client.tables.delete(tbl)) + print(f"[OK] Deleted table: {tbl}") + except Exception as ex: + code = getattr(getattr(ex, "response", None), "status_code", None) + if isinstance(ex, (requests.exceptions.HTTPError, MetadataError)) and code == 404: + print(f"[OK] Table already removed: {tbl}") + else: + print(f"[WARN] Could not delete {tbl}: {ex}") + + print("\n" + "=" * 80) + print("FetchXML Examples Complete!") + print("=" * 80) + + +if __name__ == "__main__": + main() diff --git a/examples/advanced/prodev_quick_start.py b/examples/advanced/prodev_quick_start.py index e28d1575..d06e058f 100644 --- a/examples/advanced/prodev_quick_start.py +++ b/examples/advanced/prodev_quick_start.py @@ -56,6 +56,7 @@ from azure.identity import InteractiveBrowserCredential from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.models.filters import col # -- Table schema names -- # Uses the standard 'new_' publisher prefix (default Dataverse publisher). @@ -115,7 +116,7 @@ def run_demo(client): customer_ids, project_ids, task_ids = step3_populate_data(client, primary_name_col) # -- Step 4: Query and analyze -- - step4_query_and_analyze(client, customer_ids, primary_name_col) + step4_query_and_analyze(client, customer_ids, primary_name_col, primary_id_col) # -- Step 5: Update and delete -- step5_update_and_delete(client, task_ids, primary_name_col, primary_id_col) @@ -298,10 +299,11 @@ def step3_populate_data(client, primary_name_col): print(f"[OK] Created {len(customers_df)} customers") # -- Projects (linked to customers via lookup) -- - # @odata.bind keys use the navigation property logical name (lowercase) - # and the entity set name (also lowercase) in the value. - customer_lookup = f"{TABLE_PROJECT}_CustomerId".lower() + "@odata.bind" - customer_set = TABLE_CUSTOMER.lower() + "s" + # @odata.bind keys use the lookup field schema name (case-sensitive) + # and the entity set name (from table metadata) in the value. + customer_lookup = f"{TABLE_PROJECT}_CustomerId@odata.bind" + customer_info = client.tables.get(TABLE_CUSTOMER) + customer_set = customer_info.get("entity_set_name") if customer_info else TABLE_CUSTOMER.lower() + "s" projects_df = pd.DataFrame( [ { @@ -352,8 +354,9 @@ def step3_populate_data(client, primary_name_col): for i, (task_name, priority, status, hours) in enumerate(task_names): proj_idx = project_assignment[i] - project_lookup = f"{TABLE_TASK}_ProjectId".lower() + "@odata.bind" - project_set = TABLE_PROJECT.lower() + "s" + project_lookup = f"{TABLE_TASK}_ProjectId@odata.bind" + project_info = client.tables.get(TABLE_PROJECT) + project_set = project_info.get("entity_set_name") if project_info else TABLE_PROJECT.lower() + "s" tasks_data.append( { name_col: task_name, @@ -382,7 +385,7 @@ def step3_populate_data(client, primary_name_col): # ================================================================ -def step4_query_and_analyze(client, customer_ids, primary_name_col): +def step4_query_and_analyze(client, customer_ids, primary_name_col, primary_id_col): """Query data and demonstrate DataFrame analysis.""" print("\n" + "-" * 60) print("STEP 4: Query and analyze data") @@ -392,26 +395,21 @@ def step4_query_and_analyze(client, customer_ids, primary_name_col): # Note: The SDK lowercases $select values automatically, so schema-name # casing (e.g., new_DemoProject_Budget) works -- it becomes the logical name. name_attr = primary_name_col - projects = client.dataframe.get( - TABLE_PROJECT, - select=[ - name_attr, - f"{TABLE_PROJECT}_Budget", - f"{TABLE_PROJECT}_Status", - ], + projects = ( + client.query.builder(TABLE_PROJECT) + .select(name_attr, f"{TABLE_PROJECT}_Budget", f"{TABLE_PROJECT}_Status") + .execute() + .to_dataframe() ) print(f"\n All projects ({len(projects)} rows):") print(f"{projects.to_string(index=False)}") # Query tasks and analyze - tasks = client.dataframe.get( - TABLE_TASK, - select=[ - name_attr, - f"{TABLE_TASK}_Priority", - f"{TABLE_TASK}_Status", - f"{TABLE_TASK}_EstimatedHours", - ], + tasks = ( + client.query.builder(TABLE_TASK) + .select(name_attr, f"{TABLE_TASK}_Priority", f"{TABLE_TASK}_Status", f"{TABLE_TASK}_EstimatedHours") + .execute() + .to_dataframe() ) print(f"\n All tasks ({len(tasks)} rows):") print(f"{tasks.to_string(index=False)}") @@ -440,7 +438,7 @@ def step4_query_and_analyze(client, customer_ids, primary_name_col): # Fetch single record by ID first_id = customer_ids.iloc[0] - single = client.dataframe.get(TABLE_CUSTOMER, record_id=first_id) + single = client.query.builder(TABLE_CUSTOMER).where(col(primary_id_col) == first_id).execute().to_dataframe() print(f"\n Single customer record (by ID):") print(f"{single.to_string(index=False)}") @@ -481,10 +479,7 @@ def step5_update_and_delete(client, task_ids, primary_name_col, primary_id_col): print(f"[OK] Deleted 1 task") # Verify - remaining = client.dataframe.get( - TABLE_TASK, - select=[primary_name_col, status_col], - ) + remaining = client.query.builder(TABLE_TASK).select(primary_name_col, status_col).execute().to_dataframe() print(f"\n Remaining tasks ({len(remaining)}):") print(f"{remaining.to_string(index=False)}") diff --git a/examples/advanced/sql_examples.py b/examples/advanced/sql_examples.py index d9fd9627..372a3567 100644 --- a/examples/advanced/sql_examples.py +++ b/examples/advanced/sql_examples.py @@ -22,7 +22,7 @@ - SQL read -> DataFrame transform -> SDK write-back (full round-trip) - AND/OR, NOT IN, NOT LIKE boolean logic - Deep JOINs (5-8 tables) with no server depth limit -- SQL helper functions: sql_columns, sql_select, sql_joins, sql_join +- SQL helper functions: sql_columns (sql_select/sql_join/sql_joins removed at GA) - OData helper functions: odata_select, odata_expands, odata_expand, odata_bind - SQL vs OData side-by-side comparison @@ -133,58 +133,66 @@ def _run_examples(client): ) log_call(f"client.tables.get('{parent_table}')") - info = backoff(lambda: client.tables.get(parent_table)) - if info: + if client.tables.get(parent_table): print(f"[OK] Table already exists: {parent_table}") else: log_call(f"client.tables.create('{parent_table}', ...)") - info = backoff( - lambda: client.tables.create( - parent_table, - { - "new_Code": "string", - "new_Region": Region, - "new_Budget": "decimal", - "new_Active": "bool", - }, + try: + backoff( + lambda: client.tables.create( + parent_table, + { + "new_Code": "string", + "new_Region": Region, + "new_Budget": "decimal", + "new_Active": "bool", + }, + ) ) - ) - print(f"[OK] Created table: {parent_table}") + print(f"[OK] Created table: {parent_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {parent_table} (skipped)") + else: + raise log_call(f"client.tables.get('{child_table}')") - info2 = backoff(lambda: client.tables.get(child_table)) - if info2: + if client.tables.get(child_table): print(f"[OK] Table already exists: {child_table}") else: log_call(f"client.tables.create('{child_table}', ...)") - info2 = backoff( - lambda: client.tables.create( - child_table, - { - "new_Title": "string", - "new_Hours": "int", - "new_Done": "bool", - "new_Priority": "int", - }, + try: + backoff( + lambda: client.tables.create( + child_table, + { + "new_Title": "string", + "new_Hours": "int", + "new_Done": "bool", + "new_Priority": "int", + }, + ) ) - ) - print(f"[OK] Created table: {child_table}") + print(f"[OK] Created table: {child_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {child_table} (skipped)") + else: + raise # Create lookup so tasks reference teams via JOIN print("\n[INFO] Creating lookup field so tasks reference teams via JOIN...") try: - backoff( - lambda: client.tables.create_lookup_field( - referencing_table=child_table, - lookup_field_name="new_TeamId", - referenced_table=parent_table, - display_name="Team", - ) + client.tables.create_lookup_field( + referencing_table=child_table, + lookup_field_name="new_TeamId", + referenced_table=parent_table, + display_name="Team", ) print("[OK] Created lookup: new_TeamId on tasks -> teams") except Exception as e: msg = str(e).lower() - if "already exists" in msg or "duplicate" in msg: + if "already exists" in msg or "duplicate" in msg or "not unique" in msg: print("[OK] Lookup already exists (skipped)") else: raise @@ -419,19 +427,14 @@ def _run_examples(client): heading(13, "SQL -- INNER JOIN") print("Use the lookup attribute's logical name (e.g. new_teamid) for JOINs.") - # Use sql_join() to auto-discover the relationship and build - # the JOIN clause with proper aliases. + # Build the JOIN clause manually (sql_join() was removed at GA). + # Use the lookup attribute's logical name (not _..._value) as the join column. lookup_col = "new_teamid" # Lookup logical name, NOT _..._value - join_clause = client.query.sql_join( - from_table=child_table, - to_table=parent_table, - from_alias="tk", - to_alias="t", - ) + join_clause = f"JOIN {parent_table} t ON tk.{lookup_col} = t.{parent_logical}id" print(f"[INFO] Lookup column: {lookup_col}") - print(f"[INFO] Generated JOIN: {join_clause}") + print(f"[INFO] JOIN clause: {join_clause}") - sql = f"SELECT t.new_code, tk.new_title, tk.new_hours " f"FROM {child_table} tk " f"{join_clause}" + sql = f"SELECT t.new_code, tk.new_title, tk.new_hours FROM {child_table} tk {join_clause}" log_call('client.query.sql("...INNER JOIN...")') try: results = backoff(lambda: client.query.sql(sql)) @@ -912,71 +915,26 @@ def _run_examples(client): # ============================================================== heading(29, "SQL Helper Functions (query.sql_*)") print( - "The SDK provides helper functions that auto-discover column\n" - "names and JOIN clauses from metadata -- no guessing needed." + "At GA, sql_columns() is the only retained SQL schema-discovery helper.\n" + "sql_select(), sql_join(), and sql_joins() were removed — write JOIN\n" + "clauses directly or use client.query.fetchxml() for complex queries." ) - # sql_columns + # sql_columns — still available at GA log_call(f"client.query.sql_columns('{parent_table}')") cols = client.query.sql_columns(parent_table) print(f"[OK] {len(cols)} columns:") for c in cols[:5]: print(f" {c['name']:30s} Type: {c['type']:15s} PK={c['is_pk']}") - # sql_select - log_call(f"client.query.sql_select('{parent_table}')") - select_str = client.query.sql_select(parent_table) - print(f"[OK] SELECT list: {select_str[:60]}...") - - # sql_joins - log_call(f"client.query.sql_joins('{child_table}')") - joins = client.query.sql_joins(child_table) - print(f"[OK] {len(joins)} possible JOINs:") - for j in joins[:5]: - print(f" {j['column']:25s} -> {j['target']}.{j['target_pk']}") - - # sql_joins -- alias uniqueness: multiple lookups to the same target - # table (e.g. ownerid + createdby + modifiedby all point to systemuser) - # must each get a distinct alias so the combined SQL is valid. - # Expected output: - # ownerid -> systemuser alias=s - # createdby -> systemuser alias=s2 - # modifiedby -> systemuser alias=s3 - log_call("client.query.sql_joins('contact') -- distinct aliases for same target table") - try: - contact_joins = client.query.sql_joins("contact") - systemuser_joins = [j for j in contact_joins if j["target"] == "systemuser"] - print(f"[OK] {len(systemuser_joins)} lookup(s) from contact -> systemuser:") - for j in systemuser_joins: - alias = j["join_clause"].split()[2] - print(f" {j['column']:30s} -> {j['target']} alias={alias}") - aliases = [j["join_clause"].split()[2] for j in contact_joins] - if len(aliases) != len(set(aliases)): - print("[WARN] Duplicate aliases detected") - else: - print(f"[OK] All {len(contact_joins)} aliases unique") - except Exception as e: - print(f"[INFO] Alias check skipped: {e}") - - # sql_join (auto-generate JOIN clause) - log_call(f"client.query.sql_join('{child_table}', '{parent_table}', ...)") - try: - join_clause = client.query.sql_join(child_table, parent_table, from_alias="tk", to_alias="t") - print(f"[OK] {join_clause}") - - sql = f"SELECT TOP 3 tk.new_title, t.new_code FROM {child_table} tk {join_clause}" - results = backoff(lambda: client.query.sql(sql)) - print(f"[OK] Live query with sql_join(): {len(results)} rows") - except Exception as e: - print(f"[WARN] {e}") - # ============================================================== # 30. OData Helper Functions # ============================================================== - heading(30, "OData Helper Functions (query.odata_*)") + heading(30, "OData Helper Functions (query.odata_* — deprecated at GA)") print( - "Parallel helpers for OData/records.get() users -- auto-discover\n" - "navigation properties and build @odata.bind payloads." + "odata_select(), odata_expand(), and odata_bind() still work at GA\n" + "but emit DeprecationWarning. Use the typed query builder instead.\n" + "odata_expands() is kept without deprecation." ) # odata_select @@ -998,7 +956,7 @@ def _run_examples(client): try: nav = client.query.odata_expand(child_table, parent_table) print(f"\n[OK] odata_expand('{child_table}', '{parent_table}') = '{nav}'") - print(" Usage: client.records.get('" + child_table + "', expand=['" + nav + "'])") + print(" Usage: client.query.builder('" + child_table + "').expand('" + nav + "').execute()") except Exception as e: print(f"[WARN] {e}") @@ -1030,8 +988,8 @@ def _run_examples(client): | DISTINCT | YES | Not directly | | Pagination | OFFSET FETCH | @odata.nextLink | | Max results | 5000 per query | 5000 per page | -| Column discovery | sql_columns/sql_select | odata_select | -| JOIN discovery | sql_joins/sql_join | odata_expands/expand | +| Column discovery | sql_columns | odata_expands (kept) | +| JOIN discovery | write manually/fetchxml | odata_expand (deprecated)| | Lookup binding | N/A (read-only) | odata_bind | | SELECT * | YES (SDK auto-expands) | Not applicable | | Polymorphic lookups | Separate JOINs | $expand by nav prop | @@ -1077,21 +1035,20 @@ def _run_examples(client): # OData version (expand) t0 = _time.time() try: - odata_rows = [] - for page in backoff( - lambda: client.records.get( - "account", - select=["name"], - expand=["contact_customer_accounts"], - top=5, + odata_rows = list( + backoff( + lambda: client.records.list( + "account", + select=["name"], + top=5, + ) ) - ): - odata_rows.extend(page) + ) odata_time = _time.time() - t0 - print(f" OData $expand: {len(odata_rows)} rows in {odata_time:.2f}s") + print(f" OData records.list: {len(odata_rows)} rows in {odata_time:.2f}s") except Exception as e: odata_time = _time.time() - t0 - print(f" OData $expand: error ({odata_time:.2f}s): {e}") + print(f" OData records.list: error ({odata_time:.2f}s): {e}") # ============================================================== # 32. Anti-Patterns & Best Practices @@ -1144,8 +1101,8 @@ def _run_examples(client): -> ValidationError (blocked). - Pattern #2 (cartesian FROM a, b) -> UserWarning (advisory). - Server enforces 5000-row cap on all queries (#3, #5). - - Use sql_columns() or sql_select() to discover valid column names. - - Use sql_joins() or sql_join() to discover valid JOIN clauses. + - Use sql_columns() to discover valid column names. + - Write JOIN clauses manually or use fetchxml() for complex queries. """) # ============================================================== @@ -1177,8 +1134,8 @@ def _run_examples(client): | Nested polymorphic chains | YES | e.g. opp -> acct -> contact -> owner | | Audit trail (createdby, etc.) | YES | JOIN to systemuser | | SQL read -> DF write-back | YES | dataframe.sql() + .update()/.create() | -| SQL column discovery | YES | query.sql_columns() / sql_select() | -| SQL JOIN discovery | YES | query.sql_joins() / sql_join() | +| SQL column discovery | YES | query.sql_columns() | +| SQL JOIN clause | manual | write directly or use fetchxml() | | OData column discovery | YES | query.odata_select() | | OData expand discovery | YES | query.odata_expands() / odata_expand() | | OData bind builder | YES | query.odata_bind() | @@ -1195,12 +1152,11 @@ def _run_examples(client): SQL-First Workflow (no OData knowledge needed): 1. Discover schema: cols = client.query.sql_columns("account") - 2. Discover JOINs: joins = client.query.sql_joins("contact") - 3. Build JOIN: j = client.query.sql_join("contact", "account", from_alias="c", to_alias="a") - 4. Query with SQL: df = client.dataframe.sql(f"SELECT c.fullname, a.name FROM contact c {j}") - 5. Transform: df["col"] = df["col"] * 1.1 - 6. Write back: client.dataframe.update("account", df, id_column="accountid") - 7. Verify: df2 = client.dataframe.sql("SELECT ...") + 2. Write JOIN: j = "JOIN account a ON c.parentcustomerid = a.accountid" + 3. Query with SQL: df = client.dataframe.sql(f"SELECT c.fullname, a.name FROM contact c {j}") + 4. Transform: df["col"] = df["col"] * 1.1 + 5. Write back: client.dataframe.update("account", df, id_column="accountid") + 6. Verify: df2 = client.dataframe.sql("SELECT ...") """) finally: diff --git a/examples/advanced/walkthrough.py b/examples/advanced/walkthrough.py index 6c0a6184..d2cc4ff9 100644 --- a/examples/advanced/walkthrough.py +++ b/examples/advanced/walkthrough.py @@ -26,7 +26,7 @@ from azure.identity import InteractiveBrowserCredential from PowerPlatform.Dataverse.client import DataverseClient from PowerPlatform.Dataverse.core.errors import MetadataError -from PowerPlatform.Dataverse.models.filters import eq, gt, between +from PowerPlatform.Dataverse.models.filters import col from PowerPlatform.Dataverse.models.query_builder import ExpandOption import requests @@ -183,8 +183,8 @@ def _run_walkthrough(client): print("=" * 80) # Single read by ID - log_call(f"client.records.get('{table_name}', '{id1}')") - record = backoff(lambda: client.records.get(table_name, id1)) + log_call(f"client.records.retrieve('{table_name}', '{id1}')") + record = backoff(lambda: client.records.retrieve(table_name, id1)) print("[OK] Retrieved single record:") print( json.dumps( @@ -203,11 +203,8 @@ def _run_walkthrough(client): ) # Multiple read with filter - log_call(f"client.records.get('{table_name}', filter='new_quantity gt 5')") - all_records = [] - records_iterator = backoff(lambda: client.records.get(table_name, filter="new_quantity gt 5")) - for page in records_iterator: - all_records.extend(page) + log_call(f"client.records.list('{table_name}', filter='new_quantity gt 5')") + all_records = list(backoff(lambda: client.records.list(table_name, filter="new_quantity gt 5"))) print(f"[OK] Found {len(all_records)} records with new_quantity > 5") for rec in all_records: print(f" - new_Title='{rec.get('new_title')}', new_Quantity={rec.get('new_quantity')}") @@ -231,7 +228,7 @@ def _run_walkthrough(client): }, ) ) - updated = backoff(lambda: client.records.get(table_name, id1)) + updated = backoff(lambda: client.records.retrieve(table_name, id1)) print(f"[OK] Updated single record new_Quantity: {updated.get('new_quantity')}") print(f" new_Notes: {repr(updated.get('new_notes'))}") @@ -263,9 +260,11 @@ def _run_walkthrough(client): print(f"[OK] Created {len(paging_ids)} records for paging demo") # Query with paging - log_call(f"client.records.get('{table_name}', page_size=5)") + log_call(f"client.query.builder('{table_name}').order_by().page_size(5).execute_pages()") print("Fetching records with page_size=5...") - paging_iterator = backoff(lambda: client.records.get(table_name, orderby=["new_Quantity"], page_size=5)) + paging_iterator = backoff( + lambda: client.query.builder(table_name).order_by("new_Quantity").page_size(5).execute_pages() + ) for page_num, page in enumerate(paging_iterator, start=1): record_ids = [r.get("new_walkthroughdemoid")[:8] + "..." for r in page] print(f" Page {page_num}: {len(page)} records - IDs: {record_ids}") @@ -278,13 +277,13 @@ def _run_walkthrough(client): print("=" * 80) # Basic fluent query: active records sorted by amount (flat iteration) - log_call("client.query.builder(...).select().filter_eq().order_by().execute()") + log_call("client.query.builder(...).select().where(col(...)==...).order_by().execute()") print("Querying incomplete records ordered by amount (fluent builder)...") qb_records = list( backoff( lambda: client.query.builder(table_name) .select("new_Title", "new_Amount", "new_Priority") - .filter_eq("new_Completed", False) + .where(col("new_Completed") == False) .order_by("new_Amount", descending=True) .top(10) .execute() @@ -294,14 +293,14 @@ def _run_walkthrough(client): for rec in qb_records[:5]: print(f" - '{rec.get('new_title')}' Amount={rec.get('new_amount')}") - # filter_in: records with specific priorities - log_call("client.query.builder(...).filter_in('new_Priority', [HIGH, LOW]).execute()") - print("Querying records with HIGH or LOW priority (filter_in)...") + # col().in_(): records with specific priorities + log_call("client.query.builder(...).where(col('new_Priority').in_([HIGH, LOW])).execute()") + print("Querying records with HIGH or LOW priority (col().in_())...") priority_records = list( backoff( lambda: client.query.builder(table_name) .select("new_Title", "new_Priority") - .filter_in("new_Priority", [Priority.HIGH, Priority.LOW]) + .where(col("new_Priority").in_([Priority.HIGH, Priority.LOW])) .execute() ) ) @@ -309,14 +308,14 @@ def _run_walkthrough(client): for rec in priority_records[:5]: print(f" - '{rec.get('new_title')}' Priority={rec.get('new_priority')}") - # filter_between: amount in a range - log_call("client.query.builder(...).filter_between('new_Amount', 500, 1500).execute()") - print("Querying records with amount between 500 and 1500 (filter_between)...") + # col().between(): amount in a range + log_call("client.query.builder(...).where(col('new_Amount').between(500, 1500)).execute()") + print("Querying records with amount between 500 and 1500 (col().between())...") range_records = list( backoff( lambda: client.query.builder(table_name) .select("new_Title", "new_Amount") - .filter_between("new_Amount", 500, 1500) + .where(col("new_Amount").between(500, 1500)) .execute() ) ) @@ -325,13 +324,13 @@ def _run_walkthrough(client): print(f" - '{rec.get('new_title')}' Amount={rec.get('new_amount')}") # Composable expression tree with where() - log_call("client.query.builder(...).where((eq(...) | eq(...)) & gt(...)).execute()") + log_call("client.query.builder(...).where((col(...) == ...) & (col(...) > ...)).execute()") print("Querying with composable expression tree (where)...") expr_records = list( backoff( lambda: client.query.builder(table_name) .select("new_Title", "new_Amount", "new_Quantity") - .where((eq("new_Completed", False) & gt("new_Amount", 100))) + .where((col("new_Completed") == False) & (col("new_Amount") > 100)) .order_by("new_Amount", descending=True) .top(5) .execute() @@ -341,19 +340,19 @@ def _run_walkthrough(client): for rec in expr_records: print(f" - '{rec.get('new_title')}' Amount={rec.get('new_amount')} Qty={rec.get('new_quantity')}") - # Combined: fluent filters + expression tree + paging (by_page=True) - log_call("client.query.builder(...).filter_eq().where(between()).page_size().execute(by_page=True)") - print("Querying with combined fluent + expression filters and paging...") + # Multiple where() clauses + lazy paging (execute_pages) + log_call("client.query.builder(...).where(col(...)==...).where(col().between()).page_size().execute_pages()") + print("Querying with combined expression filters and paging...") combined_page_count = 0 combined_record_count = 0 for page in backoff( lambda: client.query.builder(table_name) .select("new_Title", "new_Quantity") - .filter_eq("new_Completed", False) - .where(between("new_Quantity", 1, 15)) + .where(col("new_Completed") == False) + .where(col("new_Quantity").between(1, 15)) .order_by("new_Quantity") .page_size(3) - .execute(by_page=True) + .execute_pages() ): combined_page_count += 1 combined_record_count += len(page) @@ -362,13 +361,14 @@ def _run_walkthrough(client): print(f"[OK] Combined query: {combined_record_count} records across {combined_page_count} page(s)") # to_dataframe: get results as a pandas DataFrame - log_call(f"client.query.builder('{table_name}').select(...).filter_eq(...).to_dataframe()") + log_call(f"client.query.builder('{table_name}').select(...).where(col(...)==...).execute().to_dataframe()") print("Querying completed records as a pandas DataFrame (to_dataframe)...") df = backoff( lambda: ( client.query.builder(table_name) .select("new_title", "new_quantity") - .filter_eq("new_completed", True) + .where(col("new_completed") == True) + .execute() .to_dataframe() ) ) @@ -453,7 +453,7 @@ def _run_walkthrough(client): "new_Priority": "High", # String label instead of int } label_id = backoff(lambda: client.records.create(table_name, label_record)) - retrieved = backoff(lambda: client.records.get(table_name, label_id)) + retrieved = backoff(lambda: client.records.retrieve(table_name, label_id)) print(f"[OK] Created record with string label 'High' for new_Priority") print(f" new_Priority stored as integer: {retrieved.get('new_priority')}") print(f" new_Priority@FormattedValue: {retrieved.get('new_priority@OData.Community.Display.V1.FormattedValue')}") @@ -461,7 +461,7 @@ def _run_walkthrough(client): # Update with a string label log_call(f"client.records.update('{table_name}', label_id, {{'new_Priority': 'Low'}})") backoff(lambda: client.records.update(table_name, label_id, {"new_Priority": "Low"})) - updated_label = backoff(lambda: client.records.get(table_name, label_id)) + updated_label = backoff(lambda: client.records.retrieve(table_name, label_id)) print(f"[OK] Updated record with string label 'Low' for new_Priority") print(f" new_Priority stored as integer: {updated_label.get('new_priority')}") print( @@ -615,7 +615,7 @@ def _run_walkthrough(client): print(" [OK] Reading records by ID and with filters") print(" [OK] Single and multiple record updates") print(" [OK] Paging through large result sets") - print(" [OK] QueryBuilder fluent queries (filter_eq, filter_in, filter_between, where, to_dataframe)") + print(" [OK] QueryBuilder fluent queries (where + col(), col().in_(), col().between(), to_dataframe)") print(" [OK] Expand navigation properties (simple + nested ExpandOption)") print(" [OK] SQL queries") print(" [OK] Picklist label-to-value conversion") diff --git a/examples/basic/functional_testing.py b/examples/basic/functional_testing.py index 1ea0d5f0..ddebd362 100644 --- a/examples/basic/functional_testing.py +++ b/examples/basic/functional_testing.py @@ -236,7 +236,7 @@ def test_read_record(client: DataverseClient, table_info: Dict[str, Any], record record = None for attempt in range(1, retries + 1): try: - record = client.records.get(table_schema_name, record_id) + record = client.records.retrieve(table_schema_name, record_id) if attempt > 1: print(f" [OK] Record read succeeded after {attempt} attempts.") break @@ -250,24 +250,48 @@ def test_read_record(client: DataverseClient, table_info: Dict[str, Any], record if record is None: raise RuntimeError("Record did not become available in time.") - if record: - print("[OK] Record retrieved successfully!") - print(" Retrieved data:") - - # Display key fields - for field_name in [ - f"{attr_prefix}_name", - f"{attr_prefix}_description", - f"{attr_prefix}_count", - f"{attr_prefix}_amount", - f"{attr_prefix}_is_active", - ]: - if field_name in record: - print(f" {field_name}: {record[field_name]}") - - return record + print("[OK] Record retrieved successfully!") + print(" Retrieved data:") + for field_name in [ + f"{attr_prefix}_name", + f"{attr_prefix}_description", + f"{attr_prefix}_count", + f"{attr_prefix}_amount", + f"{attr_prefix}_is_active", + ]: + if field_name in record: + print(f" {field_name}: {record[field_name]}") + + # -- include_annotations: verify FormattedValue annotations are returned -- + annotation = "OData.Community.Display.V1.FormattedValue" + annotated = client.records.retrieve( + table_schema_name, + record_id, + select=[f"{attr_prefix}_is_active", f"{attr_prefix}_count"], + include_annotations=annotation, + ) + ann_key = f"{attr_prefix}_is_active@{annotation}" + if annotated is not None and ann_key in annotated: + print(f"[OK] include_annotations verified: {ann_key} = '{annotated[ann_key]}'") else: - raise ValueError("Record not found") + print(f"[WARN] include_annotations: expected key '{ann_key}' not present in response") + + # -- expand: verify navigation property expansion on a single-record GET -- + # owninguser is a system navigation property present on all user-owned tables. + try: + expanded = client.records.retrieve( + table_schema_name, + record_id, + select=[f"{attr_prefix}_name"], + expand=["owninguser"], + ) + owner = (expanded.get("owninguser") or {}) if expanded else {} + owner_name = owner.get("fullname") or owner.get("domainname") or "(unknown)" + print(f"[OK] records.retrieve with expand=['owninguser']: owner='{owner_name}'") + except Exception as e: # noqa: BLE001 + print(f"[WARN] records.retrieve expand skipped: {e}") + + return record except HttpError as e: print(f"[ERR] HTTP error during record reading: {e}") @@ -287,28 +311,30 @@ def test_query_records(client: DataverseClient, table_info: Dict[str, Any]) -> N retries = 5 delay_seconds = 3 + select_cols = [f"{attr_prefix}_name", f"{attr_prefix}_count", f"{attr_prefix}_amount"] + active_filter = f"{attr_prefix}_is_active eq true" + try: - print("Querying records from test table...") + # -- records.list() — eager, all pages collected into one QueryResult ---------- + print("Querying records with records.list()...") for attempt in range(1, retries + 1): try: - records_iterator = client.records.get( + result = client.records.list( table_schema_name, - select=[f"{attr_prefix}_name", f"{attr_prefix}_count", f"{attr_prefix}_amount"], - filter=f"{attr_prefix}_is_active eq true", + select=select_cols, + filter=active_filter, top=5, - orderby=[f"{attr_prefix}_name asc"], ) record_count = 0 - for batch in records_iterator: - for record in batch: - record_count += 1 - name = record.get(f"{attr_prefix}_name", "N/A") - count = record.get(f"{attr_prefix}_count", "N/A") - amount = record.get(f"{attr_prefix}_amount", "N/A") - print(f" Record {record_count}: {name} (Count: {count}, Amount: {amount})") - - print(f"[OK] Query completed! Found {record_count} active records.") + for record in result: + record_count += 1 + name = record.get(f"{attr_prefix}_name", "N/A") + count = record.get(f"{attr_prefix}_count", "N/A") + amount = record.get(f"{attr_prefix}_amount", "N/A") + print(f" Record {record_count}: {name} (Count: {count}, Amount: {amount})") + + print(f"[OK] records.list() completed! Found {record_count} active records.") break except HttpError as err: if getattr(err, "status_code", None) == 404 and attempt < retries: @@ -317,6 +343,66 @@ def test_query_records(client: DataverseClient, table_info: Dict[str, Any]) -> N continue raise + # -- records.list_pages() — lazy, one QueryResult per HTTP page --------------- + print("\nQuerying records with records.list_pages() (paged)...") + page_num = 0 + total_records = 0 + for page in client.records.list_pages( + table_schema_name, + select=select_cols, + filter=active_filter, + ): + page_num += 1 + total_records += len(page) + names = [r.get(f"{attr_prefix}_name", "N/A") for r in page] + print(f" Page {page_num}: {len(page)} record(s) — {names}") + print(f"[OK] records.list_pages() completed! {total_records} records across {page_num} page(s).") + + # -- records.list() with orderby, page_size, count, include_annotations -------- + print("\nQuerying records.list() with orderby / page_size / count / include_annotations...") + annotation = "OData.Community.Display.V1.FormattedValue" + annotated_result = client.records.list( + table_schema_name, + select=[f"{attr_prefix}_name", f"{attr_prefix}_is_active"], + filter=active_filter, + orderby=[f"{attr_prefix}_name asc"], + page_size=50, + count=True, + include_annotations=annotation, + ) + names_ordered = [r.get(f"{attr_prefix}_name", "N/A") for r in annotated_result] + ann_key = f"{attr_prefix}_is_active@{annotation}" + ann_present = any(ann_key in r for r in annotated_result) + print(f" Records (ordered): {names_ordered}") + if ann_present: + print(f"[OK] include_annotations verified: '{ann_key}' present in list() results") + else: + print(f"[WARN] include_annotations: '{ann_key}' not found — may not be supported by this environment") + # Verify orderby: names should be non-decreasing + if len(names_ordered) > 1 and all(isinstance(n, str) for n in names_ordered): + assert names_ordered == sorted(names_ordered), f"orderby asc not respected: {names_ordered}" + print(f"[OK] records.list() with extended params completed! {len(annotated_result)} record(s).") + + # -- records.list_pages() with orderby, page_size, include_annotations --------- + print("\nQuerying records.list_pages() with orderby / page_size / include_annotations...") + lp_records = [] + for page in client.records.list_pages( + table_schema_name, + select=[f"{attr_prefix}_name", f"{attr_prefix}_is_active"], + filter=active_filter, + orderby=[f"{attr_prefix}_name asc"], + page_size=50, + include_annotations=annotation, + ): + lp_records.extend(page) + lp_names = [r.get(f"{attr_prefix}_name", "N/A") for r in lp_records] + lp_ann_present = any(ann_key in r for r in lp_records) + if lp_ann_present: + print(f"[OK] include_annotations verified in list_pages() results") + else: + print(f"[WARN] include_annotations: '{ann_key}' not found in list_pages() results") + print(f"[OK] records.list_pages() with extended params completed! {len(lp_records)} record(s).") + except Exception as e: print(f"[WARN] Query test encountered an issue: {e}") print(" This might be expected if the table is very new.") @@ -516,19 +602,39 @@ def test_batch_all_operations(client: DataverseClient, table_info: Dict[str, Any print(f"[OK] {len(result.succeeded)} ops → {len(all_ids)} records created: {all_ids}") # ------------------------------------------------------------------- - # [2/11] READ — get by ID + tables.get + tables.list + query.sql - # All 4 reads in one batch request + # [2/11] READ — records.retrieve + records.list (with extended params) + # + tables.get + tables.list + query.sql — 1 POST $batch # ------------------------------------------------------------------- if all_ids: - print("\n[2/11] Read — records.get + tables.get + tables.list + query.sql (4 ops, 1 POST $batch)") + annotation = "OData.Community.Display.V1.FormattedValue" + print( + "\n[2/11] Read — records.retrieve + records.list(orderby/page_size/count/include_annotations)" + " + tables.get + tables.list + query.sql (5 ops, 1 POST $batch)" + ) batch = client.batch.new() - batch.records.get( + # [0] Single-record retrieve with annotations and expand + batch.records.retrieve( table_schema_name, all_ids[0], - select=[f"{attr_prefix}_name", f"{attr_prefix}_count"], + select=[f"{attr_prefix}_name", f"{attr_prefix}_count", f"{attr_prefix}_is_active"], + expand=["owninguser"], + include_annotations=annotation, + ) + # [1] Multi-record list with orderby, page_size, count, include_annotations + batch.records.list( + table_schema_name, + select=[f"{attr_prefix}_name", f"{attr_prefix}_is_active"], + filter=f"{attr_prefix}_is_active eq true", + orderby=[f"{attr_prefix}_name asc"], + page_size=50, + count=True, + include_annotations=annotation, ) + # [2] Table metadata batch.tables.get(table_schema_name) + # [3] Table list batch.tables.list() + # [4] SQL batch.query.sql(f"SELECT TOP 3 {attr_prefix}_name FROM {logical_name}") result = batch.execute() print(f"[OK] {len(result.succeeded)} succeeded, {len(result.failed)} failed") @@ -537,16 +643,30 @@ def test_batch_all_operations(client: DataverseClient, table_info: Dict[str, Any print(f" [{i}] FAILED {resp.status_code}: {resp.error_message}") continue if i == 0 and resp.data: - print( - f" records.get → name='{resp.data.get(f'{attr_prefix}_name')}', count={resp.data.get(f'{attr_prefix}_count')}" - ) + name = resp.data.get(f"{attr_prefix}_name") + ann_key = f"{attr_prefix}_is_active@{annotation}" + ann_val = resp.data.get(ann_key, "") + owner = resp.data.get("owninguser") or {} + owner_name = owner.get("fullname") or owner.get("domainname") or "" + print(f" records.retrieve → name='{name}', {ann_key}='{ann_val}'") + print(f" records.retrieve expand=['owninguser'] → owner='{owner_name}'") elif i == 1 and resp.data: + rows = resp.data.get("value", []) + names_ordered = [r.get(f"{attr_prefix}_name") for r in rows] + ann_key = f"{attr_prefix}_is_active@{annotation}" + ann_present = any(ann_key in r for r in rows) + print(f" records.list → {len(rows)} row(s), ordered: {names_ordered}") + if ann_present: + print(f" [OK] include_annotations '{ann_key}' present in batch.records.list() results") + else: + print(f" [WARN] include_annotations '{ann_key}' not found in batch.records.list() results") + elif i == 2 and resp.data: print( f" tables.get → LogicalName='{resp.data.get('LogicalName')}', EntitySet='{resp.data.get('EntitySetName')}'" ) - elif i == 2 and resp.data: - print(f" tables.list → {len(resp.data.get('value', []))} tables returned") elif i == 3 and resp.data: + print(f" tables.list → {len(resp.data.get('value', []))} tables returned") + elif i == 4 and resp.data: print(f" query.sql → {len(resp.data.get('value', []))} rows returned") # ------------------------------------------------------------------- @@ -954,32 +1074,29 @@ def test_relationships(client: DataverseClient) -> None: # --- Create parent and child tables --- print("\nCreating relationship test tables...") - parent_info = backoff( - lambda: client.tables.create( - rel_parent_schema, - {"test_Code": "string"}, - ) - ) + def _get_or_create(schema, columns, label): + info = client.tables.get(schema) + if info: + print(f"[OK] Table already exists: {schema} (skipped)") + return info + try: + result = backoff(lambda: client.tables.create(schema, columns)) + print(f"[OK] Created {label}: {schema}") + return result + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {schema} (skipped)") + return client.tables.get(schema) + raise + + parent_info = _get_or_create(rel_parent_schema, {"test_Code": "string"}, "parent table") created_tables.append(rel_parent_schema) - print(f"[OK] Created parent table: {parent_info['table_schema_name']}") - child_info = backoff( - lambda: client.tables.create( - rel_child_schema, - {"test_Number": "string"}, - ) - ) + child_info = _get_or_create(rel_child_schema, {"test_Number": "string"}, "child table") created_tables.append(rel_child_schema) - print(f"[OK] Created child table: {child_info['table_schema_name']}") - proj_info = backoff( - lambda: client.tables.create( - rel_m2m_schema, - {"test_ProjectCode": "string"}, - ) - ) + proj_info = _get_or_create(rel_m2m_schema, {"test_ProjectCode": "string"}, "M:N table") created_tables.append(rel_m2m_schema) - print(f"[OK] Created M:N table: {proj_info['table_schema_name']}") # --- Wait for table metadata to propagate --- wait_for_table_metadata(client, rel_parent_schema) @@ -1008,42 +1125,52 @@ def test_relationships(client: DataverseClient) -> None: ), ) - result_1n = backoff( - lambda: client.tables.create_one_to_many_relationship( - lookup=lookup, - relationship=relationship, + existing_1n = client.tables.get_relationship("test_RelParent_RelChild") + if existing_1n: + result_1n = existing_1n + rel_id_1n = result_1n.relationship_id + print(f" [OK] Relationship already exists: {result_1n.relationship_schema_name} (skipped)") + else: + result_1n = backoff( + lambda: client.tables.create_one_to_many_relationship( + lookup=lookup, + relationship=relationship, + ) ) - ) - - assert result_1n.relationship_schema_name == "test_RelParent_RelChild" - assert result_1n.relationship_type == "one_to_many" - assert result_1n.lookup_schema_name is not None - rel_id_1n = result_1n.relationship_id - print(f" [OK] Created 1:N relationship: {result_1n.relationship_schema_name}") - print(f" Lookup: {result_1n.lookup_schema_name}") - print(f" ID: {rel_id_1n}") + assert result_1n.relationship_schema_name == "test_RelParent_RelChild" + assert result_1n.relationship_type == "one_to_many" + assert result_1n.lookup_schema_name is not None + rel_id_1n = result_1n.relationship_id + print(f" [OK] Created 1:N relationship: {result_1n.relationship_schema_name}") + print(f" Lookup: {result_1n.lookup_schema_name}") + print(f" ID: {rel_id_1n}") # --- Test 2: Create lookup field (convenience API) --- print("\n Test 2: Create lookup field (convenience API)") print(" " + "-" * 45) - result_lookup = backoff( - lambda: client.tables.create_lookup_field( - referencing_table=child_info["table_logical_name"], - lookup_field_name="test_ManagerId", - referenced_table="contact", - display_name="Manager", - description="The record's manager contact", - required=False, - cascade_delete=CASCADE_BEHAVIOR_REMOVE_LINK, + existing_lookup = client.tables.get_relationship("contact_test_relchild_test_ManagerId") + if existing_lookup: + result_lookup = existing_lookup + rel_id_lookup = result_lookup.relationship_id + print(f" [OK] Lookup already exists: {result_lookup.relationship_schema_name} (skipped)") + else: + result_lookup = backoff( + lambda: client.tables.create_lookup_field( + referencing_table=child_info["table_logical_name"], + lookup_field_name="test_ManagerId", + referenced_table="contact", + display_name="Manager", + description="The record's manager contact", + required=False, + cascade_delete=CASCADE_BEHAVIOR_REMOVE_LINK, + ) ) - ) - - assert result_lookup.relationship_type == "one_to_many" - assert result_lookup.lookup_schema_name is not None - rel_id_lookup = result_lookup.relationship_id - print(f" [OK] Created lookup: {result_lookup.lookup_schema_name}") - print(f" Relationship: {result_lookup.relationship_schema_name}") + assert result_lookup.relationship_type == "one_to_many" + assert result_lookup.lookup_schema_name is not None + rel_id_lookup = result_lookup.relationship_id + print(f" [OK] Created lookup: {result_lookup.lookup_schema_name}") + print(f" Relationship: {result_lookup.relationship_schema_name}") # --- Test 3: Create N:N relationship --- print("\n Test 3: Create N:N relationship") @@ -1055,13 +1182,18 @@ def test_relationships(client: DataverseClient) -> None: entity2_logical_name=proj_info["table_logical_name"], ) - result_nn = backoff(lambda: client.tables.create_many_to_many_relationship(relationship=m2m)) - - assert result_nn.relationship_schema_name == "test_relchild_relproject" - assert result_nn.relationship_type == "many_to_many" - rel_id_nn = result_nn.relationship_id - print(f" [OK] Created N:N relationship: {result_nn.relationship_schema_name}") - print(f" ID: {rel_id_nn}") + existing_nn = client.tables.get_relationship("test_relchild_relproject") + if existing_nn: + result_nn = existing_nn + rel_id_nn = result_nn.relationship_id + print(f" [OK] Relationship already exists: {result_nn.relationship_schema_name} (skipped)") + else: + result_nn = backoff(lambda: client.tables.create_many_to_many_relationship(relationship=m2m)) + assert result_nn.relationship_schema_name == "test_relchild_relproject" + assert result_nn.relationship_type == "many_to_many" + rel_id_nn = result_nn.relationship_id + print(f" [OK] Created N:N relationship: {result_nn.relationship_schema_name}") + print(f" ID: {rel_id_nn}") # --- Test 4: Get relationship metadata --- print("\n Test 4: Query relationship metadata") diff --git a/examples/basic/installation_example.py b/examples/basic/installation_example.py index 98189e58..61da149b 100644 --- a/examples/basic/installation_example.py +++ b/examples/basic/installation_example.py @@ -221,7 +221,7 @@ def show_usage_examples(): print(f"Created account: {account_id}") # Read a single record by ID -account = client.records.get("account", account_id) +account = client.records.retrieve("account", account_id) print(f"Account name: {account['name']}") # Update a record @@ -234,14 +234,13 @@ def show_usage_examples(): Querying Data: ```python # Query with OData filter -accounts = client.records.get("account", +accounts = client.records.list("account", filter="name eq 'Contoso Ltd'", select=["name", "telephone1"], top=10) -for batch in accounts: - for account in batch: - print(f"Account: {account['name']}") +for account in accounts: + print(f"Account: {account['name']}") # SQL queries (if enabled) results = client.query.sql("SELECT TOP 5 name FROM account") diff --git a/pyproject.toml b/pyproject.toml index 723e045d..77a6d492 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,6 +40,7 @@ dependencies = [ [project.scripts] dataverse-install-claude-skill = "PowerPlatform.Dataverse._skill_installer:main" +dataverse-migrate = "PowerPlatform.Dataverse.migration.migrate_v0_to_v1:main" [project.optional-dependencies] dev = [ @@ -49,7 +50,9 @@ dev = [ "isort>=5.12.0", "mypy>=1.0.0", "ruff>=0.1.0", + "libcst>=1.0.0", ] +migration = ["libcst>=1.0.0"] [tool.setuptools] package-dir = {"" = "src"} @@ -96,6 +99,12 @@ testpaths = ["tests/unit"] [tool.coverage.run] source = ["src/PowerPlatform"] +# Migration codemod is an opt-in tool (requires the [migration] extra). +# It is exercised by tests/unit/test_migration_tool.py and tests/unit/test_phase4_ga.py, +# but its CLI plumbing and libcst error-path branches are not meaningfully testable +# under the default coverage scope. Exclude from coverage reporting so the metric +# reflects core SDK code only. +omit = ["src/PowerPlatform/Dataverse/migration/*"] [tool.coverage.report] fail_under = 90 diff --git a/src/PowerPlatform/Dataverse/__init__.py b/src/PowerPlatform/Dataverse/__init__.py index 95b5171c..0d4ed6e2 100644 --- a/src/PowerPlatform/Dataverse/__init__.py +++ b/src/PowerPlatform/Dataverse/__init__.py @@ -3,6 +3,10 @@ from importlib.metadata import version +from .models.filters import col, raw +from .models.protocol import DataverseModel +from .models.record import QueryResult + __version__ = version("PowerPlatform-Dataverse-Client") -__all__ = ["__version__"] +__all__ = ["__version__", "col", "raw", "DataverseModel", "QueryResult"] diff --git a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md index 79d4e342..c3af44c5 100644 --- a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md +++ b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md @@ -20,6 +20,32 @@ This skill provides guidance for developers working on the PowerPlatform Dataver 5. **Consider backwards compatibility** - Avoid breaking changes 6. **Internal vs public naming** - Modules, files, and functions not meant to be part of the public API must use a `_` prefix (e.g., `_odata.py`, `_relationships.py`). Files without the prefix (e.g., `constants.py`, `metadata.py`) are public and importable by SDK consumers +### Dataverse Property Naming Rules + +Dataverse uses two different naming conventions for properties. Getting this wrong causes 400 errors that are hard to debug. + +| Property type | Name convention | Example | When used | +|---|---|---|---| +| **Structural** (columns) | LogicalName (always lowercase) | `new_name`, `new_priority` | `$select`, `$filter`, `$orderby`, record payload keys | +| **Navigation** (relationships / lookups) | Navigation Property Name (usually SchemaName, PascalCase, case-sensitive) | `new_CustomerId`, `new_AgentId` | `$expand`, `@odata.bind` annotation keys | + +Navigation property names are case-sensitive and must match the entity's `$metadata`. Using the logical name instead of the navigation property name results in 400 Bad Request errors. + +**Critical rule:** The OData parser validates `@odata.bind` property names **case-sensitively** against declared navigation properties. Lowercasing `new_CustomerId@odata.bind` to `new_customerid@odata.bind` causes: `ODataException: An undeclared property 'new_customerid' which only has property annotations...` + +**SDK implementation:** + +- `_lowercase_keys()` lowercases all keys EXCEPT those containing `@odata.` (preserves navigation property casing in `@odata.bind` keys) +- `_lowercase_list()` lowercases `$select` and `$orderby` params (structural properties) +- `$expand` params are passed as-is (navigation properties, PascalCase) +- `_convert_labels_to_ints()` skips `@odata.` keys entirely (they are annotations, not attributes) + +**When adding new code that processes record dicts or builds query parameters:** + +- Always use `_lowercase_keys()` for record payloads. Never manually call `.lower()` on all keys +- Never lowercase `$expand` values or `@odata.bind` key prefixes +- If iterating record keys, skip keys containing `@odata.` when doing attribute-level operations + ### Code Style 6. **No emojis** - Do not use emoji in code, comments, or output diff --git a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md index 72677468..d25815d7 100644 --- a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md +++ b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md @@ -28,11 +28,23 @@ Use the PowerPlatform Dataverse Client Python SDK to interact with Microsoft Dat The SDK supports Dataverse's native bulk operations: Pass lists to `create()`, `update()` for automatic bulk processing, for `delete()`, set `use_bulk_delete` when passing lists to use bulk operation ### Paging -- Control page size with `page_size` parameter +- Control page size with `page_size` parameter on `records.list()`, `records.list_pages()`, or `QueryBuilder.page_size()` - Use `top` parameter to limit total records returned +- **Preferred**: `client.query.builder(table)....execute_pages()` — composable `where(col(...))` filters, formatted values, expand with nested selects, full pagination control +- Simple streaming shortcut: `records.list_pages(table, *, filter, select, top, orderby, expand, page_size, count, include_annotations)` — string-based OData filter only, yields one `QueryResult` per page +- `execute(by_page=True/False)` is **deprecated** and emits `UserWarning`; use `execute_pages()` instead +- `QueryBuilder.to_dataframe()` is **deprecated**; use `.execute().to_dataframe()` instead + +### QueryResult +- Returned by `records.list()`, `records.retrieve()`, `execute()`, and each page from `list_pages()` / `execute_pages()` +- Iterable: `for record in result` — each item is a `dict`-like `Record` +- `.to_dataframe()` — convert to pandas DataFrame +- `.first()` — return the first record or `None` (safe: returns `None` on empty result) +- `result[n]` — index access returns a `Record`; `result[n:m]` returns a `QueryResult` +- `len(result)` — number of records in this result/page ### DataFrame Support -- DataFrame operations are accessed via the `client.dataframe` namespace: `client.dataframe.get()`, `client.dataframe.create()`, `client.dataframe.update()`, `client.dataframe.delete()` +- DataFrame operations are accessed via the `client.dataframe` namespace: `client.dataframe.create()`, `client.dataframe.update()`, `client.dataframe.delete()` — `client.dataframe.get()` is deprecated; use `client.query.builder(table).where(...).execute().to_dataframe()` instead ## Common Operations @@ -85,28 +97,92 @@ contact_ids = client.records.create("contact", contacts) #### Read Records ```python # Get single record by ID -account = client.records.get("account", account_id, select=["name", "telephone1"]) - -# Query with filter (paginated) -for page in client.records.get( - "account", - select=["accountid", "name"], # select is case-insensitive (automatically lowercased) - filter="statecode eq 0", # filter must use lowercase logical names (not transformed) - top=100, -): +account = client.records.retrieve("account", account_id, select=["name", "telephone1"]) + +# With expand — fetch a related record in the same HTTP request +account = client.records.retrieve( + "account", account_id, + select=["name"], + expand=["primarycontactid"], +) +contact = (account.get("primarycontactid") or {}) +print(contact.get("fullname")) + +# Simple shortcut — use records.list() only for basic filter + select without composable logic. +# Follows @odata.nextLink automatically and loads all matching records into memory. +# For filtering, sorting, expansion, or formatted values, prefer client.query.builder() (see below). +result = client.records.list("account", filter="statecode eq 0", select=["name", "accountid"]) +for record in result: + print(record["name"]) +``` + +#### Query Builder (Preferred for Filtering, Sorting, Expand, Formatted Values) + +Use `client.query.builder()` for any query that goes beyond simple filter + select. It provides composable `where(col(...))` expressions, formatted value support, nested expansion, and streaming — all with a fluent API. + +```python +from PowerPlatform.Dataverse.models.filters import col +from PowerPlatform.Dataverse.models.query_builder import ExpandOption + +# Basic query with composable filter and sort +result = (client.query.builder("account") + .select("accountid", "name", "statecode") + .where(col("statecode") == 0) + .order_by("name asc") + .execute()) +for record in result: + print(record["name"]) + +# Composable filters — AND / OR / NOT using Python operators +result = (client.query.builder("contact") + .select("fullname", "emailaddress1") + .where((col("statecode") == 0) & (col("emailaddress1").contains("@contoso.com"))) + .execute()) + +# Formatted values — display labels for option sets, currency symbols, etc. +result = (client.query.builder("account") + .select("accountid", "name", "industrycode") + .where(col("statecode") == 0) + .include_formatted_values() + .execute()) +for record in result: + label = record.get("industrycode@OData.Community.Display.V1.FormattedValue") + print(record["name"], label) + +# Navigation property expansion with nested column select +result = (client.query.builder("account") + .select("name") + .expand(ExpandOption("primarycontactid").select("fullname", "emailaddress1")) + .where(col("statecode") == 0) + .execute()) +for record in result: + contact = record.get("primarycontactid", {}) + print(f"{record['name']} - {contact.get('fullname', 'N/A')}") + +# Stream large result sets page-by-page (memory-efficient) +for page in (client.query.builder("account") + .select("accountid", "name") + .where(col("statecode") == 0) + .order_by("name asc") + .page_size(500) + .execute_pages()): for record in page: print(record["name"]) -# Query with navigation property expansion (case-sensitive!) -for page in client.records.get( - "account", - select=["name"], - expand=["primarycontactid"], # Navigation properties are case-sensitive! - filter="statecode eq 0", # Column names must be lowercase logical names -): - for account in page: - contact = account.get("primarycontactid", {}) - print(f"{account['name']} - {contact.get('fullname', 'N/A')}") +# Convert query results to a DataFrame +df = (client.query.builder("account") + .select("accountid", "name") + .where(col("statecode") == 0) + .execute() + .to_dataframe()) + +# Limit total results +result = client.query.builder("account").select("name").top(100).execute() + +# Simple streaming shortcut via records.list_pages() (string filter only, same params as records.list()) +for page in client.records.list_pages("account", filter="statecode eq 0", select=["name"], page_size=500): + for record in page: + print(record["name"]) ``` #### Create Records with Lookup Bindings (@odata.bind) @@ -179,18 +255,24 @@ client.records.delete("account", [id1, id2, id3], use_bulk_delete=True) The SDK provides DataFrame wrappers for all CRUD operations via the `client.dataframe` namespace, using pandas DataFrames and Series as input/output. +> **Note:** `client.dataframe.get()` is deprecated. Use `client.query.builder(table).select(...).where(...).execute().to_dataframe()` instead. `QueryBuilder.to_dataframe()` (without `.execute()`) is also deprecated — always call `.execute()` first. + ```python import pandas as pd -# Query records -- returns a single DataFrame -df = client.dataframe.get("account", filter="statecode eq 0", select=["name"]) +# Query records -- returns a single DataFrame (GA pattern: .execute().to_dataframe()) +from PowerPlatform.Dataverse.models.filters import col +df = client.query.builder("account").where(col("statecode") == 0).select("name").execute().to_dataframe() print(f"Got {len(df)} rows") -# Limit results with top for large tables -df = client.dataframe.get("account", select=["name"], top=100) +# Limit results with top +df = client.query.builder("account").select("name").top(100).execute().to_dataframe() + +# Via records.list() (simpler for basic queries) +df = client.records.list("account", filter="statecode eq 0", select=["name"]).to_dataframe() # Fetch single record as one-row DataFrame -df = client.dataframe.get("account", record_id=account_id, select=["name"]) +df = client.records.retrieve("account", account_id, select=["name"]).to_dataframe() # Create records from a DataFrame (returns a Series of GUIDs) new_accounts = pd.DataFrame([ @@ -223,6 +305,34 @@ for record in results: print(record["name"]) ``` +### FetchXML Queries + +`client.query.fetchxml(xml)` returns an inert `FetchXmlQuery` object — **no HTTP request is made** until `.execute()` or `.execute_pages()` is called. + +```python +xml = """ + + + + + + + + + +""" + +# Load all results into memory (simple, small-to-medium sets) +query = client.query.fetchxml(xml) +result = query.execute() # returns QueryResult — all pages fetched upfront +for record in result: + print(record["name"]) + +# Stream page-by-page (large sets or early exit) +for page in query.execute_pages(): # yields one QueryResult per HTTP page + process(page.to_dataframe()) +``` + ### Table Management #### Create Custom Tables @@ -380,7 +490,8 @@ Use `client.batch` to send multiple operations in one HTTP request. All batch me batch = client.batch.new() batch.records.create("account", {"name": "Contoso"}) batch.records.update("account", account_id, {"telephone1": "555-0100"}) -batch.records.get("account", account_id, select=["name"]) +batch.records.retrieve("account", account_id, select=["name"], expand=["primarycontactid"], include_annotations="OData.Community.Display.V1.FormattedValue") # single record with expand +batch.records.list("account", filter="statecode eq 0", select=["name"], orderby=["name asc"], top=50, page_size=25, count=True) # multi-record, single page batch.query.sql("SELECT TOP 5 name FROM account") result = batch.execute() @@ -412,7 +523,8 @@ print(f"Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") **Batch limitations:** - Maximum 1000 operations per batch -- Paginated `records.get()` (without `record_id`) is not supported in batch +- `batch.records.get()` is deprecated; use `batch.records.retrieve()` for single records +- `batch.records.list()` returns a single page (no pagination); use `top` to bound results - `flush_cache()` is not supported in batch ## Error Handling @@ -430,7 +542,7 @@ from PowerPlatform.Dataverse.core.errors import ( from PowerPlatform.Dataverse.client import DataverseClient try: - client.records.get("account", "invalid-id") + client.records.retrieve("account", "invalid-id") except HttpError as e: print(f"HTTP {e.status_code}: {e.message}") print(f"Error code: {e.code}") @@ -464,16 +576,17 @@ except ValidationError as e: ### Performance Optimization -1. **Use bulk operations** - Pass lists to create/update/delete for automatic optimization -2. **Specify select fields** - Limit returned columns to reduce payload size -3. **Control page size** - Use `top` and `page_size` parameters appropriately -4. **Reuse client instances** - Don't create new clients for each operation -5. **Use production credentials** - ClientSecretCredential or CertificateCredential for unattended operations -6. **Error handling** - Implement retry logic for transient errors (`e.is_transient`) -7. **Always include customization prefix** for custom tables/columns -8. **Use lowercase for column names, match `$metadata` for navigation properties** - Column names in `$select`/`$filter`/record payloads use lowercase LogicalNames. Navigation properties in `$expand` and `@odata.bind` keys are case-sensitive and must match the entity's `$metadata` (PascalCase for custom lookups like `new_CustomerId`, lowercase for system lookups like `parentaccountid`) -9. **Test in non-production environments** first -10. **Use named constants** - Import cascade behavior constants from `PowerPlatform.Dataverse.common.constants` +1. **Prefer `client.query.builder()` for any non-trivial query** — use the builder for filtering, sorting, expansion, or formatted values; `records.list()` is a convenience shortcut for simple filter+select only +2. **Use bulk operations** - Pass lists to create/update/delete for automatic optimization +3. **Specify select fields** - Limit returned columns to reduce payload size +4. **Control page size** - Use `top` and `page_size` parameters appropriately; use `execute_pages()` for large sets +5. **Reuse client instances** - Don't create new clients for each operation +6. **Use production credentials** - ClientSecretCredential or CertificateCredential for unattended operations +7. **Error handling** - Implement retry logic for transient errors (`e.is_transient`) +8. **Always include customization prefix** for custom tables/columns +9. **Use lowercase for column names, match `$metadata` for navigation properties** - Column names in `$select`/`$filter`/record payloads use lowercase LogicalNames. Navigation properties in `$expand` and `@odata.bind` keys are case-sensitive and must match the entity's `$metadata` (PascalCase for custom lookups like `new_CustomerId`, lowercase for system lookups like `parentaccountid`) +10. **Test in non-production environments** first +11. **Use named constants** - Import cascade behavior constants from `PowerPlatform.Dataverse.common.constants` ## Additional Resources @@ -486,9 +599,10 @@ Load these resources as needed during development: ## Key Reminders -1. **Schema names are required** - Never use display names -2. **Custom tables need prefixes** - Include customization prefix (e.g., "new_") -3. **Filter is case-sensitive** - Use lowercase logical names -4. **Bulk operations are encouraged** - Pass lists for optimization -5. **No trailing slashes in URLs** - Format: `https://org.crm.dynamics.com` -6. **Structured errors** - Check `is_transient` for retry logic +1. **Use `client.query.builder()` for queries** — it's the primary query pattern; `records.list()` is a shortcut for trivial filter+select only +2. **Schema names are required** - Never use display names +3. **Custom tables need prefixes** - Include customization prefix (e.g., "new_") +4. **Filter is case-sensitive** - Use lowercase logical names +5. **Bulk operations are encouraged** - Pass lists for optimization +6. **No trailing slashes in URLs** - Format: `https://org.crm.dynamics.com` +7. **Structured errors** - Check `is_transient` for retry logic diff --git a/src/PowerPlatform/Dataverse/client.py b/src/PowerPlatform/Dataverse/client.py index 29be479d..12ceaaac 100644 --- a/src/PowerPlatform/Dataverse/client.py +++ b/src/PowerPlatform/Dataverse/client.py @@ -3,9 +3,8 @@ from __future__ import annotations -import warnings from contextlib import contextmanager -from typing import Any, Dict, Iterable, Iterator, List, Optional, Union +from typing import Iterator, Optional import requests @@ -106,7 +105,7 @@ def __init__( ) -> None: if config is not None and context is not None: raise ValueError( - "Cannot specify both 'config' and 'context'. " "Pass operation_context via DataverseConfig instead." + "Cannot specify both 'config' and 'context'. Pass operation_context via DataverseConfig instead." ) self.auth = _AuthManager(credential) self._base_url = (base_url or "").rstrip("/") @@ -216,581 +215,8 @@ def _check_closed(self) -> None: if self._closed: raise RuntimeError("DataverseClient is closed") - # ---------------- Unified CRUD: create/update/delete ---------------- - def create(self, table_schema_name: str, records: Union[Dict[str, Any], List[Dict[str, Any]]]) -> List[str]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.records.RecordOperations.create` instead. - - Create one or more records by table name. - - :param table_schema_name: Schema name of the table (e.g. ``"account"``, ``"contact"``, or ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param records: A single record dictionary or a list of record dictionaries. - Each dictionary should contain column schema names as keys. - :type records: dict or list[dict] - - :return: List of created record GUIDs. Returns a single-element list for a single input. - :rtype: list[str] - - :raises TypeError: If ``records`` is not a dict or list[dict], or if the internal - client returns an unexpected type. - - Example: - Create a single record:: - - client = DataverseClient(base_url, credential) - ids = client.create("account", {"name": "Contoso"}) - print(f"Created: {ids[0]}") - - Create multiple records:: - - records = [ - {"name": "Contoso"}, - {"name": "Fabrikam"} - ] - ids = client.create("account", records) - print(f"Created {len(ids)} accounts") - """ - warnings.warn( - "client.create() is deprecated. Use client.records.create() instead.", - DeprecationWarning, - stacklevel=2, - ) - # Old API always returned list[str]; new returns str for single - if isinstance(records, dict): - return [self.records.create(table_schema_name, records)] - return self.records.create(table_schema_name, records) - - def update( - self, table_schema_name: str, ids: Union[str, List[str]], changes: Union[Dict[str, Any], List[Dict[str, Any]]] - ) -> None: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.records.RecordOperations.update` instead. - - Update one or more records. - - This method supports three usage patterns: - - 1. Single record update: ``update("account", "guid", {"name": "New Name"})`` - 2. Broadcast update: ``update("account", [id1, id2], {"status": 1})`` - applies same changes to all IDs - 3. Paired updates: ``update("account", [id1, id2], [changes1, changes2])`` - one-to-one mapping - - :param table_schema_name: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param ids: Single GUID string or list of GUID strings to update. - :type ids: str or list[str] - :param changes: Dictionary of changes for single/broadcast mode, or list of dictionaries - for paired mode. When ``ids`` is a list and ``changes`` is a single dict, - the same changes are broadcast to all records. When both are lists, they must - have equal length for one-to-one mapping. - :type changes: dict or list[dict] - - :raises TypeError: If ``ids`` is not str or list[str], or if ``changes`` type doesn't match usage pattern. - - .. note:: - Single updates discard the response representation for better performance. For broadcast or paired updates, the method delegates to the internal client's batch update logic. - - Example: - Single record update:: - - client.update("account", account_id, {"telephone1": "555-0100"}) - - Broadcast same changes to multiple records:: - - client.update("account", [id1, id2, id3], {"statecode": 1}) - - Update multiple records with different values:: - - ids = [id1, id2] - changes = [ - {"name": "Updated Name 1"}, - {"name": "Updated Name 2"} - ] - client.update("account", ids, changes) - """ - warnings.warn( - "client.update() is deprecated. Use client.records.update() instead.", - DeprecationWarning, - stacklevel=2, - ) - self.records.update(table_schema_name, ids, changes) - - def delete( - self, - table_schema_name: str, - ids: Union[str, List[str]], - use_bulk_delete: bool = True, - ) -> Optional[str]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.records.RecordOperations.delete` instead. - - Delete one or more records by GUID. - - :param table_schema_name: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param ids: Single GUID string or list of GUID strings to delete. - :type ids: str or list[str] - :param use_bulk_delete: When ``True`` (default) and ``ids`` is a list, execute the BulkDelete action and - return its async job identifier. When ``False`` each record is deleted sequentially. - :type use_bulk_delete: :class:`bool` - - :raises TypeError: If ``ids`` is not str or list[str]. - :raises HttpError: If the underlying Web API delete request fails. - - :return: BulkDelete job ID when deleting multiple records via BulkDelete; otherwise ``None``. - :rtype: :class:`str` or None - - Example: - Delete a single record:: - - client.delete("account", account_id) - - Delete multiple records:: - - job_id = client.delete("account", [id1, id2, id3]) - """ - warnings.warn( - "client.delete() is deprecated. Use client.records.delete() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.records.delete(table_schema_name, ids, use_bulk_delete=use_bulk_delete) - - def get( - self, - table_schema_name: str, - record_id: Optional[str] = None, - select: Optional[List[str]] = None, - filter: Optional[str] = None, - orderby: Optional[List[str]] = None, - top: Optional[int] = None, - expand: Optional[List[str]] = None, - page_size: Optional[int] = None, - ) -> Union[Dict[str, Any], Iterable[List[Dict[str, Any]]]]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.records.RecordOperations.get` instead. - - - **Single record by ID** -- ``client.records.get(table, record_id)`` - - **Query / filter multiple records** -- ``client.records.get(table, filter=..., select=...)`` - - Fetch a single record by ID or query multiple records. - - When ``record_id`` is provided, returns a single record dictionary. - When ``record_id`` is None, returns a generator yielding batches of records. - - :param table_schema_name: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param record_id: Optional GUID to fetch a specific record. If None, queries multiple records. - :type record_id: :class:`str` or None - :param select: Optional list of attribute logical names to retrieve. Column names are case-insensitive and automatically lowercased (e.g. ``["new_Title", "new_Amount"]`` becomes ``"new_title,new_amount"``). - :type select: list[str] or None - :param filter: Optional OData filter string, e.g. ``"name eq 'Contoso'"`` or ``"new_quantity gt 5"``. Column names in filter expressions must use exact lowercase logical names (e.g. ``"new_quantity"``, not ``"new_Quantity"``). The filter string is passed directly to the Dataverse Web API without transformation. - :type filter: :class:`str` or None - :param orderby: Optional list of attributes to sort by, e.g. ``["name asc", "createdon desc"]``. Column names are automatically lowercased. - :type orderby: list[str] or None - :param top: Optional maximum number of records to return. - :type top: :class:`int` or None - :param expand: Optional list of navigation properties to expand, e.g. ``["primarycontactid"]``. Navigation property names are case-sensitive and must match the server-defined names exactly. These are NOT automatically transformed. Consult entity metadata for correct casing. - :type expand: list[str] or None - :param page_size: Optional number of records per page for pagination. - :type page_size: :class:`int` or None - - :return: Single record dict if ``record_id`` is provided, otherwise a generator - yielding lists of record dictionaries (one list per page). - :rtype: dict or collections.abc.Iterable[list[dict]] - - :raises TypeError: If ``record_id`` is provided but not a string. - - Example: - Fetch a single record:: - - record = client.get("account", record_id=account_id, select=["name", "telephone1"]) - print(record["name"]) - - Query multiple records with filtering (note: exact logical names in filter):: - - for batch in client.get( - "account", - filter="statecode eq 0 and name eq 'Contoso'", # Must use exact logical names (lower-case) - select=["name", "telephone1"] - ): - for account in batch: - print(account["name"]) - - Query with navigation property expansion (note: case-sensitive property name):: - - for batch in client.get( - "account", - select=["name"], - expand=["primarycontactid"], # Case-sensitive! Check metadata for exact name - filter="statecode eq 0" - ): - for account in batch: - print(f"{account['name']} - Contact: {account.get('primarycontactid', {}).get('fullname')}") - - Query with sorting and pagination:: - - for batch in client.get( - "account", - orderby=["createdon desc"], - top=100, - page_size=50 - ): - print(f"Batch size: {len(batch)}") - """ - warnings.warn( - "client.get() is deprecated. Use client.records.get() instead.", - DeprecationWarning, - stacklevel=2, - ) - if record_id is not None: - return self.records.get(table_schema_name, record_id, select=select) - else: - return self.records.get( - table_schema_name, - select=select, - filter=filter, - orderby=orderby, - top=top, - expand=expand, - page_size=page_size, - ) - - # SQL via Web API sql parameter - def query_sql(self, sql: str) -> List[Dict[str, Any]]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.query.QueryOperations.sql` instead. - - Execute a read-only SQL query using the Dataverse Web API ``?sql`` capability. - - The SQL query must follow the supported subset: a single SELECT statement with - optional WHERE, TOP (integer literal), ORDER BY (column names only), and a simple - table alias after FROM. - - :param sql: Supported SQL SELECT statement. - :type sql: :class:`str` - - :return: List of result row dictionaries. Returns an empty list if no rows match. - :rtype: list[dict] - - :raises ~PowerPlatform.Dataverse.core.errors.SQLParseError: If the SQL query uses unsupported syntax. - :raises ~PowerPlatform.Dataverse.core.errors.HttpError: If the Web API returns an error. - - .. note:: - The SQL support is limited to read-only queries. Complex joins, subqueries, and certain SQL functions may not be supported. Consult the Dataverse documentation for the current feature set. - - Example: - Basic SQL query:: - - sql = "SELECT TOP 10 accountid, name FROM account WHERE name LIKE 'C%' ORDER BY name" - results = client.query_sql(sql) - for row in results: - print(row["name"]) - - Query with alias:: + # ---------------- Cache utilities ---------------- - sql = "SELECT a.name, a.telephone1 FROM account AS a WHERE a.statecode = 0" - results = client.query_sql(sql) - """ - warnings.warn( - "client.query_sql() is deprecated. Use client.query.sql() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.query.sql(sql) - - # Table metadata helpers - def get_table_info(self, table_schema_name: str) -> Optional[Dict[str, Any]]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.tables.TableOperations.get` instead. - - Get basic metadata for a table if it exists. - - :param table_schema_name: Schema name of the table (e.g. ``"new_MyTestTable"`` or ``"account"``). - :type table_schema_name: :class:`str` - - :return: Dictionary containing table metadata with keys ``table_schema_name``, - ``table_logical_name``, ``entity_set_name``, and ``metadata_id``. - Returns None if the table is not found. - :rtype: :class:`dict` or None - - Example: - Retrieve table metadata:: - - info = client.get_table_info("new_MyTestTable") - if info: - print(f"Logical name: {info['table_logical_name']}") - print(f"Entity set: {info['entity_set_name']}") - """ - warnings.warn( - "client.get_table_info() is deprecated. Use client.tables.get() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.tables.get(table_schema_name) - - def create_table( - self, - table_schema_name: str, - columns: Dict[str, Any], - solution_unique_name: Optional[str] = None, - primary_column_schema_name: Optional[str] = None, - ) -> Dict[str, Any]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.tables.TableOperations.create` instead. - - Create a simple custom table with specified columns. - - :param table_schema_name: Schema name of the table with customization prefix value (e.g. ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param columns: Dictionary mapping column names (with customization prefix value) to their types. All custom column names must include the customization prefix value (e.g. ``"new_Title"``). - Supported types: - - - Primitive types: ``"string"`` (alias: ``"text"``), ``"int"`` (alias: ``"integer"``), ``"decimal"`` (alias: ``"money"``), ``"float"`` (alias: ``"double"``), ``"datetime"`` (alias: ``"date"``), ``"bool"`` (alias: ``"boolean"``), and ``"file"`` - - Enum subclass (IntEnum preferred): Creates a local option set. Optional multilingual - labels can be provided via ``__labels__`` class attribute, defined inside the Enum subclass:: - - class ItemStatus(IntEnum): - ACTIVE = 1 - INACTIVE = 2 - __labels__ = { - 1033: {"Active": "Active", "Inactive": "Inactive"}, - 1036: {"Active": "Actif", "Inactive": "Inactif"} - } - - :type columns: dict[str, typing.Any] - :param solution_unique_name: Optional solution unique name that should own the new table. When omitted the table is created in the default solution. - :type solution_unique_name: :class:`str` or None - :param primary_column_schema_name: Optional primary name column schema name with customization prefix value (e.g. ``"new_MyTestTable"``). If not provided, defaults to ``"{customization prefix value}_Name"``. - :type primary_column_schema_name: :class:`str` or None - - :return: Dictionary containing table metadata including ``table_schema_name``, - ``entity_set_name``, ``table_logical_name``, ``metadata_id``, and ``columns_created``. - :rtype: :class:`dict` - - :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: If table creation fails or the schema is invalid. - - Example: - Create a table with simple columns:: - - from enum import IntEnum - - class ItemStatus(IntEnum): - ACTIVE = 1 - INACTIVE = 2 - - columns = { - "new_Title": "string", # Note: includes 'new_' customization prefix value - "new_Quantity": "int", - "new_Price": "decimal", - "new_Available": "bool", - "new_Status": ItemStatus - } - - result = client.create_table("new_MyTestTable", columns) - print(f"Created table: {result['table_schema_name']}") - print(f"Columns: {result['columns_created']}") - - Create a table with a custom primary column name:: - - result = client.create_table( - "new_Product", - {"new_Price": "decimal"}, - primary_column_schema_name="new_ProductName" - ) - """ - warnings.warn( - "client.create_table() is deprecated. Use client.tables.create() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.tables.create( - table_schema_name, - columns, - solution=solution_unique_name, - primary_column=primary_column_schema_name, - ) - - def delete_table(self, table_schema_name: str) -> None: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.tables.TableOperations.delete` instead. - - Delete a custom table by name. - - :param table_schema_name: Schema name of the table (e.g. ``"new_MyTestTable"`` or ``"account"``). - :type table_schema_name: :class:`str` - - :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: If the table does not exist or deletion fails. - - .. warning:: - This operation is irreversible and will delete all records in the table along - with the table definition. Use with caution. - - Example: - Delete a custom table:: - - client.delete_table("new_MyTestTable") - """ - warnings.warn( - "client.delete_table() is deprecated. Use client.tables.delete() instead.", - DeprecationWarning, - stacklevel=2, - ) - self.tables.delete(table_schema_name) - - def list_tables(self) -> list[dict[str, Any]]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.tables.TableOperations.list` instead. - - List all non-private tables in the Dataverse environment. - - :return: List of EntityDefinition metadata dictionaries. - :rtype: list[dict] - - Example: - List all non-private tables and print their logical names:: - - tables = client.list_tables() - for table in tables: - print(table["LogicalName"]) - """ - warnings.warn( - "client.list_tables() is deprecated. Use client.tables.list() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.tables.list() - - def create_columns( - self, - table_schema_name: str, - columns: Dict[str, Any], - ) -> List[str]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.tables.TableOperations.add_columns` instead. - - Create one or more columns on an existing table using a schema-style mapping. - - :param table_schema_name: Schema name of the table (e.g. ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param columns: Mapping of column schema names (with customization prefix value) to supported types. All custom column names must include the customization prefix value** (e.g. ``"new_Notes"``). Primitive types include - ``"string"`` (alias: ``"text"``), ``"int"`` (alias: ``"integer"``), ``"decimal"`` (alias: ``"money"``), ``"float"`` (alias: ``"double"``), ``"datetime"`` (alias: ``"date"``), ``"bool"`` (alias: ``"boolean"``), and ``"file"``. Enum subclasses (IntEnum preferred) - generate a local option set and can specify localized labels via ``__labels__``. - :type columns: dict[str, typing.Any] - :returns: Schema names for the columns that were created. - :rtype: list[str] - Example: - Create multiple columns on the custom table:: - - created = client.create_columns( - "new_MyTestTable", - { - "new_Scratch": "string", - "new_Flags": "bool", - "new_Document": "file", - }, - ) - print(created) # ['new_Scratch', 'new_Flags', 'new_Document'] - """ - warnings.warn( - "client.create_columns() is deprecated. Use client.tables.add_columns() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.tables.add_columns(table_schema_name, columns) - - def delete_columns( - self, - table_schema_name: str, - columns: Union[str, List[str]], - ) -> List[str]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.tables.TableOperations.remove_columns` instead. - - Delete one or more columns from a table. - - :param table_schema_name: Schema name of the table (e.g. ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param columns: Column name or list of column names to remove. Must include customization prefix value (e.g. ``"new_TestColumn"``). - :type columns: str or list[str] - :returns: Schema names for the columns that were removed. - :rtype: list[str] - Example: - Remove two custom columns by schema name: - - removed = client.delete_columns( - "new_MyTestTable", - ["new_Scratch", "new_Flags"], - ) - print(removed) # ['new_Scratch', 'new_Flags'] - """ - warnings.warn( - "client.delete_columns() is deprecated. Use client.tables.remove_columns() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.tables.remove_columns(table_schema_name, columns) - - # File upload - def upload_file( - self, - table_schema_name: str, - record_id: str, - file_name_attribute: str, - path: str, - mode: Optional[str] = None, - mime_type: Optional[str] = None, - if_none_match: bool = True, - ) -> None: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.files.FileOperations.upload` instead. - - Upload a file to a Dataverse file column. - - :param table_schema_name: Schema name of the table. - :type table_schema_name: :class:`str` - :param record_id: GUID of the target record. - :type record_id: :class:`str` - :param file_name_attribute: Schema name of the file column attribute. - :type file_name_attribute: :class:`str` - :param path: Local filesystem path to the file. - :type path: :class:`str` - :param mode: Upload strategy: ``"auto"`` (default), ``"small"``, or ``"chunk"``. - :type mode: :class:`str` or None - :param mime_type: Explicit MIME type to store with the file. - :type mime_type: :class:`str` or None - :param if_none_match: When True (default), only succeed if the column is - currently empty. - :type if_none_match: :class:`bool` - """ - warnings.warn( - "client.upload_file() is deprecated. Use client.files.upload() instead.", - DeprecationWarning, - stacklevel=2, - ) - self.files.upload( - table_schema_name, - record_id, - file_name_attribute, - path, - mode=mode, - mime_type=mime_type, - if_none_match=if_none_match, - ) - - # Cache utilities def flush_cache(self, kind) -> int: """ Flush cached client metadata or state. diff --git a/src/PowerPlatform/Dataverse/data/_batch.py b/src/PowerPlatform/Dataverse/data/_batch.py index 2e880c01..b848c5a0 100644 --- a/src/PowerPlatform/Dataverse/data/_batch.py +++ b/src/PowerPlatform/Dataverse/data/_batch.py @@ -69,6 +69,21 @@ class _RecordGet: table: str record_id: str select: Optional[List[str]] = None + expand: Optional[List[str]] = None + include_annotations: Optional[str] = None + + +@dataclass +class _RecordList: + table: str + select: Optional[List[str]] = None + filter: Optional[str] = None + orderby: Optional[List[str]] = None + top: Optional[int] = None + expand: Optional[List[str]] = None + page_size: Optional[int] = None + count: bool = False + include_annotations: Optional[str] = None @dataclass @@ -312,6 +327,8 @@ def _resolve_item(self, item: Any) -> List[_RawRequest]: return self._resolve_record_delete(item) if isinstance(item, _RecordGet): return self._resolve_record_get(item) + if isinstance(item, _RecordList): + return self._resolve_record_list(item) if isinstance(item, _RecordUpsert): return self._resolve_record_upsert(item) if isinstance(item, _TableCreate): @@ -382,7 +399,30 @@ def _resolve_record_delete(self, op: _RecordDelete) -> List[_RawRequest]: return [self._od._build_delete(op.table, rid) for rid in ids] def _resolve_record_get(self, op: _RecordGet) -> List[_RawRequest]: - return [self._od._build_get(op.table, op.record_id, select=op.select)] + return [ + self._od._build_get( + op.table, + op.record_id, + select=op.select, + expand=op.expand, + include_annotations=op.include_annotations, + ) + ] + + def _resolve_record_list(self, op: _RecordList) -> List[_RawRequest]: + return [ + self._od._build_list( + op.table, + select=op.select, + filter=op.filter, + orderby=op.orderby, + top=op.top, + expand=op.expand, + page_size=op.page_size, + count=op.count, + include_annotations=op.include_annotations, + ) + ] def _resolve_record_upsert(self, op: _RecordUpsert) -> List[_RawRequest]: entity_set = self._od._entity_set_from_schema_name(op.table) diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index b6cdf29a..8b27dd20 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -689,7 +689,14 @@ def _delete(self, table_schema_name: str, key: str) -> None: """ self._execute_raw(self._build_delete(table_schema_name, key)) - def _get(self, table_schema_name: str, key: str, select: Optional[List[str]] = None) -> Dict[str, Any]: + def _get( + self, + table_schema_name: str, + key: str, + select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, + include_annotations: Optional[str] = None, + ) -> Dict[str, Any]: """Retrieve a single record. :param table_schema_name: Schema name of the table. @@ -698,11 +705,19 @@ def _get(self, table_schema_name: str, key: str, select: Optional[List[str]] = N :type key: ``str`` :param select: Columns to select; joined with commas into $select. :type select: ``list[str]`` | ``None`` + :param expand: Navigation properties to expand (``$expand``); passed as-is (case-sensitive). + :type expand: ``list[str]`` | ``None`` + :param include_annotations: OData annotation pattern for the ``Prefer: odata.include-annotations`` header, or ``None``. + :type include_annotations: ``str`` | ``None`` :return: Retrieved record dictionary (may be empty if no selected attributes). :rtype: ``dict[str, Any]`` """ - return self._execute_raw(self._build_get(table_schema_name, key, select=select)).json() + return self._execute_raw( + self._build_get( + table_schema_name, key, select=select, expand=expand, include_annotations=include_annotations + ) + ).json() def _get_multiple( self, @@ -2314,13 +2329,64 @@ def _build_get( record_id: str, *, select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, + include_annotations: Optional[str] = None, ) -> _RawRequest: """Build a single-record GET request without sending it.""" entity_set = self._entity_set_from_schema_name(table) + params: List[str] = [] + if select: + params.append("$select=" + ",".join(self._lowercase_list(select))) + if expand: + params.append("$expand=" + ",".join(expand)) url = f"{self.api}/{entity_set}{self._format_key(record_id)}" + if params: + url += "?" + "&".join(params) + headers = None + if include_annotations: + headers = {"Prefer": f'odata.include-annotations="{include_annotations}"'} + return _RawRequest(method="GET", url=url, headers=headers) + + def _build_list( + self, + table: str, + *, + select: Optional[List[str]] = None, + filter: Optional[str] = None, + orderby: Optional[List[str]] = None, + top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, + ) -> _RawRequest: + """Build a multi-record GET request (single page, no pagination) without sending it.""" + entity_set = self._entity_set_from_schema_name(table) + params: List[str] = [] if select: - url += "?$select=" + ",".join(self._lowercase_list(select)) - return _RawRequest(method="GET", url=url) + params.append("$select=" + ",".join(self._lowercase_list(select))) + if filter: + params.append("$filter=" + filter) + if orderby: + params.append("$orderby=" + ",".join(orderby)) + if top is not None: + params.append(f"$top={top}") + if expand: + params.append("$expand=" + ",".join(expand)) + if count: + params.append("$count=true") + url = f"{self.api}/{entity_set}" + if params: + url += "?" + "&".join(params) + prefer_parts: List[str] = [] + if page_size is not None: + ps = int(page_size) + if ps > 0: + prefer_parts.append(f"odata.maxpagesize={ps}") + if include_annotations: + prefer_parts.append(f'odata.include-annotations="{include_annotations}"') + headers = {"Prefer": ",".join(prefer_parts)} if prefer_parts else None + return _RawRequest(method="GET", url=url, headers=headers) def _build_create_entity( self, diff --git a/src/PowerPlatform/Dataverse/migration/__init__.py b/src/PowerPlatform/Dataverse/migration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/PowerPlatform/Dataverse/migration/migrate_v0_to_v1.py b/src/PowerPlatform/Dataverse/migration/migrate_v0_to_v1.py new file mode 100644 index 00000000..bf9782ec --- /dev/null +++ b/src/PowerPlatform/Dataverse/migration/migrate_v0_to_v1.py @@ -0,0 +1,824 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +DV-Python-SDK v0 -> v1 GA migration codemod. + +Mechanically rewrites beta (0.1.0b*) call sites to their GA (1.0) equivalents +using LibCST (concrete syntax tree — preserves all whitespace and comments). + +Usage:: + + pip install PowerPlatform-Dataverse-Client[migration] + dataverse-migrate path/to/your/scripts/ + dataverse-migrate path/to/your/scripts/ --dry-run # preview without writing + dataverse-migrate path/to/your/scripts/ --client-var=svc # if client is named 'svc' + + # Or via module for development installs: + python -m PowerPlatform.Dataverse.migration.migrate_v0_to_v1 path/to/your/scripts/ + +Transformations applied +----------------------- +Builder methods (.filter_* -> .where(col(...)...)):: + + .filter_eq("col", v) -> .where(col("col") == v) + .filter_ne("col", v) -> .where(col("col") != v) + .filter_gt("col", v) -> .where(col("col") > v) + .filter_ge("col", v) -> .where(col("col") >= v) + .filter_lt("col", v) -> .where(col("col") < v) + .filter_le("col", v) -> .where(col("col") <= v) + .filter_contains("col", v) -> .where(col("col").contains(v)) + .filter_startswith("col", v) -> .where(col("col").startswith(v)) + .filter_endswith("col", v) -> .where(col("col").endswith(v)) + .filter_in("col", vals) -> .where(col("col").in_(vals)) + .filter_not_in("col", vals) -> .where(col("col").not_in(vals)) + .filter_null("col") -> .where(col("col").is_null()) + .filter_not_null("col") -> .where(col("col").is_not_null()) + .filter_between("col", lo, hi) -> .where(col("col").between(lo, hi)) + .filter_not_between("col", lo, hi) -> .where(col("col").not_between(lo, hi)) + .filter_raw("expr") -> .where(raw("expr")) + .filter("expr") -> .where(raw("expr")) + .execute(by_page=True) -> .execute_pages() + .execute(by_page=False) -> .execute() (flag removed) + .to_dataframe() -> .execute().to_dataframe() + Inserts .execute() when the receiver is a recognised QueryBuilder chain + (contains .builder(), .select(), .where(), or a .filter_*() call). + +Record namespace:: + + batch.records.get(t, id) -> batch.records.retrieve(t, id) + +Top-level shortcuts (removed at GA):: + + client.create(t, d) -> client.records.create(t, d) + client.update(t, id, d) -> client.records.update(t, id, d) + client.delete(t, id) -> client.records.delete(t, id) + client.get(t, id) -> client.records.get(t, id) [deprecated; see manual section] + client.query_sql(sql) -> client.query.sql(sql) + client.get_table_info(t) -> client.tables.get(t) + client.create_table(t, …) -> client.tables.create(t, …) + client.delete_table(t) -> client.tables.delete(t) + client.list_tables() -> client.tables.list() + client.create_columns(t, …) -> client.tables.add_columns(t, …) + client.delete_columns(t, …) -> client.tables.remove_columns(t, …) + client.upload_file(…) -> client.files.upload(…) + +Import management: + Adds ``from PowerPlatform.Dataverse.models.filters import col`` when a + .filter_* method is rewritten (if col is not already imported). + Adds ``raw`` to the same import when .filter_raw or .filter is rewritten. + +NOT handled by this codemod (manual migration required): + execute(by_page=variable) -> manual review required (variable argument, not literal) + client.records.get(t, id) -> client.records.retrieve(t, id) + Return type changes: beta returns Record (raises on 404); GA retrieve() returns + Record | None. Callers that do not guard against None will fail silently. + client.records.get(t, kw=…) -> client.records.list(t, kw=…) + Return type changes: beta returns Iterable[List[Record]] (pages); GA list() + returns QueryResult (flat iterable over Records). Any ``for page in result: + for rec in page:`` iteration pattern breaks after a mechanical rename. + client.dataframe.get() -> client.query.builder(…).execute().to_dataframe() + Expression reconstruction requires understanding caller intent. + client.query.sql_select()/sql_join()/sql_joins() -> removed (no mechanical replacement) +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import List, Optional, Sequence, Set, Tuple + +try: + import libcst as cst +except ImportError as _e: + raise ImportError( + "libcst is required. Install with:\n" + " pip install PowerPlatform-Dataverse-Client[migration]\n" + " # or: pip install 'libcst>=1.0.0'" + ) from _e + + +# --------------------------------------------------------------------------- +# Filter-method -> .where(col(...)) mapping +# --------------------------------------------------------------------------- + +_UNARY_FILTER_MAP = { + "filter_null": "is_null", + "filter_not_null": "is_not_null", +} + +_BINARY_OP_MAP = { + "filter_eq": cst.Equal(), + "filter_ne": cst.NotEqual(), + "filter_gt": cst.GreaterThan(), + "filter_ge": cst.GreaterThanEqual(), + "filter_lt": cst.LessThan(), + "filter_le": cst.LessThanEqual(), +} + +_METHOD_FILTER_MAP = { + "filter_contains": "contains", + "filter_startswith": "startswith", + "filter_endswith": "endswith", + "filter_in": "in_", + "filter_not_in": "not_in", + "filter_between": "between", + "filter_not_between": "not_between", +} + +_ALL_FILTER_METHODS: Set[str] = set(_UNARY_FILTER_MAP) | set(_BINARY_OP_MAP) | set(_METHOD_FILTER_MAP) | {"filter_raw"} + +# Standalone filter functions from filters module (beta API) -> col() equivalents +# eq("f", v) -> col("f") == v, between("f", lo, hi) -> col("f").between(lo, hi), etc. +_FUNC_BINARY_OP_MAP = { + "eq": cst.Equal(), + "ne": cst.NotEqual(), + "gt": cst.GreaterThan(), + "ge": cst.GreaterThanEqual(), + "lt": cst.LessThan(), + "le": cst.LessThanEqual(), +} +_FUNC_METHOD_MAP = { + "contains": "contains", + "startswith": "startswith", + "endswith": "endswith", + "filter_in": "in_", + "not_in": "not_in", + "between": "between", + "not_between": "not_between", +} +_FUNC_UNARY_MAP = { + "is_null": "is_null", + "is_not_null": "is_not_null", +} +_ALL_FILTER_FUNCS: Set[str] = set(_FUNC_BINARY_OP_MAP) | set(_FUNC_METHOD_MAP) | set(_FUNC_UNARY_MAP) + +# Methods that identify a QueryBuilder call chain (used to detect .to_dataframe() callers) +_BUILDER_CHAIN_METHODS: Set[str] = {"builder", "select", "where", "filter", "execute_pages"} | _ALL_FILTER_METHODS + +# Top-level client shortcut -> (new_namespace, new_method) +_CLIENT_SHORTCUTS = { + "create": ("records", "create"), + "update": ("records", "update"), + "delete": ("records", "delete"), + "get": ("records", "get"), + "query_sql": ("query", "sql"), + "get_table_info": ("tables", "get"), + "create_table": ("tables", "create"), + "delete_table": ("tables", "delete"), + "list_tables": ("tables", "list"), + "create_columns": ("tables", "add_columns"), + "delete_columns": ("tables", "remove_columns"), + "upload_file": ("files", "upload"), +} + +_FILTERS_MODULE = "PowerPlatform.Dataverse.models.filters" + + +# --------------------------------------------------------------------------- +# Node helpers +# --------------------------------------------------------------------------- + + +def _name(s: str) -> cst.Name: + return cst.Name(s) + + +def _attr(obj: cst.BaseExpression, attr: str) -> cst.Attribute: + return cst.Attribute(value=obj, attr=cst.Name(attr)) + + +def _call(func: cst.BaseExpression, *args: cst.BaseExpression) -> cst.Call: + cst_args = [] + for i, a in enumerate(args): + comma = ( + cst.MaybeSentinel.DEFAULT if i == len(args) - 1 else cst.Comma(whitespace_after=cst.SimpleWhitespace(" ")) + ) + cst_args.append(cst.Arg(value=a, comma=comma)) + return cst.Call(func=func, args=cst_args) + + +def _col_call(col_name_node: cst.BaseExpression) -> cst.Call: + """col("field_name") call node.""" + return _call(_name("col"), col_name_node) + + +def _filters_module_attr() -> cst.Attribute: + """Build the Attribute chain for PowerPlatform.Dataverse.models.filters.""" + return _attr( + _attr( + _attr(_name("PowerPlatform"), "Dataverse"), + "models", + ), + "filters", + ) + + +# --------------------------------------------------------------------------- +# Positional argument helpers +# --------------------------------------------------------------------------- + + +def _pos_arg(args: Sequence[cst.Arg], n: int) -> Optional[cst.BaseExpression]: + """Return the n-th (0-indexed) positional argument value, or None.""" + count = 0 + for a in args: + if a.keyword is None: + if count == n: + return a.value + count += 1 + return None + + +def _positional_count(args: Sequence[cst.Arg]) -> int: + return sum(1 for a in args if a.keyword is None) + + +# --------------------------------------------------------------------------- +# Main transformer +# --------------------------------------------------------------------------- + + +class _V1Migrator(cst.CSTTransformer): + """LibCST transformer rewriting DV-Python-SDK beta -> v1 GA.""" + + def __init__(self, client_var: str = "client") -> None: + self._client_var = client_var + self._needs_col = False + self._needs_raw = False + self._has_col = False + self._has_raw = False + # Names imported from filters module in this file (e.g. eq, gt, between) + self._imported_filter_funcs: Set[str] = set() + + # ------------------------------------------------------------------ + # Track existing col / raw imports + # ------------------------------------------------------------------ + + def visit_ImportFrom(self, node: cst.ImportFrom) -> None: + if isinstance(node.names, cst.ImportStar): + return + module_str = _dotted_name(node.module) + if module_str != _FILTERS_MODULE: + return + for alias in node.names: + name = alias.name.value if isinstance(alias.name, cst.Name) else "" + if name == "col": + self._has_col = True + elif name == "raw": + self._has_raw = True + elif name in _ALL_FILTER_FUNCS: + self._imported_filter_funcs.add(name) + + # ------------------------------------------------------------------ + # Rewrite call nodes + # ------------------------------------------------------------------ + + def leave_Call(self, original_node: cst.Call, updated_node: cst.Call) -> cst.BaseExpression: + func = updated_node.func + + # ---------------------------------------------------------------- + # Standalone filter functions: eq("f", v) -> col("f") == v, etc. + # Only transform names that were actually imported from filters module. + # Wrap Comparison nodes in explicit parentheses so that combining with + # & / | doesn't hit Python precedence bugs (& binds tighter than ==/>). + # ---------------------------------------------------------------- + if isinstance(func, cst.Name) and func.value in self._imported_filter_funcs: + result = self._build_filter_func_arg(func.value, updated_node.args) + if result is not None: + if isinstance(result, cst.Comparison): + result = result.with_changes(lpar=[cst.LeftParen()], rpar=[cst.RightParen()]) + return result + + if not isinstance(func, cst.Attribute): + return updated_node + + method_name = func.attr.value if isinstance(func.attr, cst.Name) else "" + + # ---------------------------------------------------------------- + # .filter_*(...) -> .where(col(...) ...) + # ---------------------------------------------------------------- + if method_name in _ALL_FILTER_METHODS: + where_arg = self._build_filter_arg(method_name, updated_node.args) + if where_arg is not None: + return updated_node.with_changes( + func=func.with_changes(attr=_name("where")), + args=[cst.Arg(value=where_arg)], + ) + + # ---------------------------------------------------------------- + # .filter("expr") -> .where(raw("expr")) + # QueryBuilder.filter() was removed at GA (not deprecated). Wrapping + # in raw() preserves the OData string exactly for string-literal callers. + # ---------------------------------------------------------------- + if method_name == "filter": + expr_node = _pos_arg(updated_node.args, 0) + if expr_node is not None and _positional_count(updated_node.args) == 1: + self._needs_raw = True + return updated_node.with_changes( + func=func.with_changes(attr=_name("where")), + args=[cst.Arg(value=_call(_name("raw"), expr_node))], + ) + + # ---------------------------------------------------------------- + # .execute(by_page=True) -> .execute_pages() + # .execute(by_page=False) -> .execute() (flag removed) + # Only literal True/False are codemod-able; variable by_page requires + # manual review per section 8.5 of the GA spec. + # ---------------------------------------------------------------- + if method_name == "execute": + by_page_val = self._kwarg_bool_literal(updated_node.args, "by_page") + if by_page_val is True: + return updated_node.with_changes( + func=func.with_changes(attr=_name("execute_pages")), + args=[], + ) + if by_page_val is False: + other_args = [ + a + for a in updated_node.args + if not (isinstance(a.keyword, cst.Name) and a.keyword.value == "by_page") + ] + return updated_node.with_changes(args=other_args) + + # ---------------------------------------------------------------- + # QueryBuilder.to_dataframe() -> .execute().to_dataframe() + # Only rewrites when the receiver is a recognised QueryBuilder chain + # (contains .builder(), .select(), .where(), or a .filter_*() call). + # Skips if receiver is already a .execute() call (QueryResult.to_dataframe() + # is the GA form and must not be touched). + # ---------------------------------------------------------------- + if method_name == "to_dataframe": + receiver = func.value + already_executed = ( + isinstance(receiver, cst.Call) + and isinstance(receiver.func, cst.Attribute) + and isinstance(receiver.func.attr, cst.Name) + and receiver.func.attr.value == "execute" + ) + if not already_executed and self._is_query_builder_chain(receiver): + execute_call = _call(_attr(receiver, "execute")) + return updated_node.with_changes(func=func.with_changes(value=execute_call)) + + # ---------------------------------------------------------------- + # batch.records.get(table, id) -> batch.records.retrieve(table, id) + # NOTE: client.records.get() is NOT codemodded — the return type changes + # between beta and GA (Record | None vs Record for single-id; QueryResult vs + # Iterable[List[Record]] for multi-record). Surrounding iteration patterns + # would silently break after a mechanical rename. + # ---------------------------------------------------------------- + if method_name == "get" and isinstance(func.value, cst.Attribute): + inner = func.value + if isinstance(inner.attr, cst.Name) and inner.attr.value == "records": + if isinstance(inner.value, cst.Name) and inner.value.value == "batch": + # batch.records.get() returns None in both versions — safe to rename + return updated_node.with_changes(func=func.with_changes(attr=_name("retrieve"))) + + # ---------------------------------------------------------------- + # client.(...) top-level shortcuts removed at GA + # Only match when receiver is the known client variable name to avoid + # false positives on record.get("field"), table_info.get("field"), etc. + # ---------------------------------------------------------------- + if ( + isinstance(func.value, cst.Name) + and func.value.value == self._client_var + and method_name in _CLIENT_SHORTCUTS + ): + new_ns, new_method = _CLIENT_SHORTCUTS[method_name] + new_func = _attr(_attr(func.value, new_ns), new_method) + return updated_node.with_changes(func=new_func) + + return updated_node + + # ------------------------------------------------------------------ + # Keyword-argument helpers + # ------------------------------------------------------------------ + + @staticmethod + def _kwarg_bool_literal(args: Sequence[cst.Arg], keyword: str) -> Optional[bool]: + """Return True/False if *keyword* is a literal bool kwarg, else None.""" + for a in args: + if isinstance(a.keyword, cst.Name) and a.keyword.value == keyword: + if isinstance(a.value, cst.Name): + if a.value.value == "True": + return True + if a.value.value == "False": + return False + return None + + @staticmethod + def _is_query_builder_chain(node: cst.BaseExpression) -> bool: + """Return True if *node* is a call chain that includes a QueryBuilder method.""" + cur: cst.BaseExpression = node + while isinstance(cur, cst.Call): + f = cur.func + if isinstance(f, cst.Attribute) and isinstance(f.attr, cst.Name): + if f.attr.value in _BUILDER_CHAIN_METHODS: + return True + cur = f.value + else: + break + return False + + # ------------------------------------------------------------------ + # Build the argument for .where() from .filter_*() args + # ------------------------------------------------------------------ + + def _build_filter_arg( + self, + method_name: str, + args: Sequence[cst.Arg], + ) -> Optional[cst.BaseExpression]: + + field_node = _pos_arg(args, 0) + if field_node is None: + return None + + # .filter_raw(expr) -> raw(expr) + if method_name == "filter_raw": + self._needs_raw = True + return _call(_name("raw"), field_node) + + # .filter_null / .filter_not_null -> col("f").is_null() / .is_not_null() + if method_name in _UNARY_FILTER_MAP: + self._needs_col = True + proxy = _UNARY_FILTER_MAP[method_name] + return _call(_attr(_col_call(field_node), proxy)) + + # .filter_eq / .filter_ne / ... -> col("f") OP val + if method_name in _BINARY_OP_MAP: + val_node = _pos_arg(args, 1) + if val_node is None: + return None + self._needs_col = True + return cst.Comparison( + left=_col_call(field_node), + comparisons=[ + cst.ComparisonTarget( + operator=_BINARY_OP_MAP[method_name], + comparator=val_node, + ) + ], + ) + + # .filter_between / .filter_not_between -> col("f").between(lo, hi) + if method_name in ("filter_between", "filter_not_between"): + lo = _pos_arg(args, 1) + hi = _pos_arg(args, 2) + if lo is None or hi is None: + return None + self._needs_col = True + proxy = _METHOD_FILTER_MAP[method_name] + return _call(_attr(_col_call(field_node), proxy), lo, hi) + + # .filter_in / .filter_not_in / .filter_contains / etc. + if method_name in _METHOD_FILTER_MAP: + val_node = _pos_arg(args, 1) + if val_node is None: + return None + self._needs_col = True + proxy = _METHOD_FILTER_MAP[method_name] + return _call(_attr(_col_call(field_node), proxy), val_node) + + return None + + # ------------------------------------------------------------------ + # Standalone filter function: eq("f", v) -> col("f") == v, etc. + # ------------------------------------------------------------------ + + def _build_filter_func_arg( + self, + func_name: str, + args: Sequence[cst.Arg], + ) -> Optional[cst.BaseExpression]: + """Return the replacement expression node for a standalone filter call.""" + field_node = _pos_arg(args, 0) + if field_node is None: + return None + + if func_name in _FUNC_UNARY_MAP: + self._needs_col = True + proxy = _FUNC_UNARY_MAP[func_name] + return _call(_attr(_col_call(field_node), proxy)) + + if func_name in _FUNC_BINARY_OP_MAP: + val_node = _pos_arg(args, 1) + if val_node is None: + return None + self._needs_col = True + return cst.Comparison( + left=_col_call(field_node), + comparisons=[ + cst.ComparisonTarget( + operator=_FUNC_BINARY_OP_MAP[func_name], + comparator=val_node, + ) + ], + ) + + if func_name in ("between", "not_between"): + lo = _pos_arg(args, 1) + hi = _pos_arg(args, 2) + if lo is None or hi is None: + return None + self._needs_col = True + proxy = _FUNC_METHOD_MAP[func_name] + return _call(_attr(_col_call(field_node), proxy), lo, hi) + + if func_name in _FUNC_METHOD_MAP: + val_node = _pos_arg(args, 1) + if val_node is None: + return None + self._needs_col = True + proxy = _FUNC_METHOD_MAP[func_name] + return _call(_attr(_col_call(field_node), proxy), val_node) + + return None + + # ------------------------------------------------------------------ + # Inject missing col / raw imports at module level + # ------------------------------------------------------------------ + + def leave_Module(self, original_node: cst.Module, updated_node: cst.Module) -> cst.Module: + to_add: List[str] = [] + if self._needs_col and not self._has_col: + to_add.append("col") + if self._needs_raw and not self._has_raw: + to_add.append("raw") + if not to_add: + return updated_node + + new_body = list(updated_node.body) + + # Try to augment an existing filters import line + for i, stmt in enumerate(new_body): + if not ( + isinstance(stmt, cst.SimpleStatementLine) + and len(stmt.body) == 1 + and isinstance(stmt.body[0], cst.ImportFrom) + ): + continue + imp = stmt.body[0] + if isinstance(imp.names, cst.ImportStar): + continue + if _dotted_name(imp.module) != _FILTERS_MODULE: + continue + existing_names = {alias.name.value for alias in imp.names if isinstance(alias.name, cst.Name)} + need = [n for n in to_add if n not in existing_names] + if not need: + return updated_node # already present + all_aliases = list(imp.names) + [cst.ImportAlias(name=_name(n)) for n in need] + # Re-apply commas + fixed = _comma_separated(all_aliases) + new_imp = imp.with_changes(names=fixed) + new_body[i] = stmt.with_changes(body=[new_imp]) + return updated_node.with_changes(body=new_body) + + # No existing filters import — insert a new one after the last import block + new_import_stmt = cst.SimpleStatementLine( + body=[ + cst.ImportFrom( + module=_filters_module_attr(), + names=_comma_separated([cst.ImportAlias(name=_name(n)) for n in to_add]), + ) + ] + ) + last_import_idx = 0 + for i, stmt in enumerate(new_body): + if isinstance(stmt, cst.SimpleStatementLine) and any( + isinstance(s, (cst.Import, cst.ImportFrom)) for s in stmt.body + ): + last_import_idx = i + new_body.insert(last_import_idx + 1, new_import_stmt) + return updated_node.with_changes(body=new_body) + + +def _comma_separated( + aliases: List[cst.ImportAlias], +) -> List[cst.ImportAlias]: + """Return aliases with commas between each, last one without.""" + result = [] + for i, alias in enumerate(aliases): + if i < len(aliases) - 1: + result.append(alias.with_changes(comma=cst.Comma(whitespace_after=cst.SimpleWhitespace(" ")))) + else: + result.append(alias.with_changes(comma=cst.MaybeSentinel.DEFAULT)) + return result + + +# --------------------------------------------------------------------------- +# Utility: dotted-name string from libcst Attribute / Name tree +# --------------------------------------------------------------------------- + + +def _dotted_name(node: Optional[cst.BaseExpression]) -> str: + if node is None: + return "" + if isinstance(node, cst.Name): + return node.value + if isinstance(node, cst.Attribute): + return f"{_dotted_name(node.value)}.{node.attr.value}" + return "" + + +# --------------------------------------------------------------------------- +# File-level migration +# --------------------------------------------------------------------------- + + +# --------------------------------------------------------------------------- +# Manual-review pattern detector +# --------------------------------------------------------------------------- + +_REMOVED_QUERY_METHODS: Set[str] = {"sql_select", "sql_join", "sql_joins"} + + +class _ManualReviewFinder(cst.CSTTransformer): + """Visitor that detects patterns the codemod cannot safely rewrite automatically.""" + + def __init__(self, client_var: str = "client") -> None: + self._client_var = client_var + self.findings: List[str] = [] + + def _receiver_chain(self, node: cst.Attribute) -> List[str]: + """Return the dotted name parts of an Attribute chain, innermost first.""" + parts: List[str] = [] + cur: cst.BaseExpression = node + while isinstance(cur, cst.Attribute): + if isinstance(cur.attr, cst.Name): + parts.append(cur.attr.value) + cur = cur.value + if isinstance(cur, cst.Name): + parts.append(cur.value) + return parts # e.g. ["get", "records", "client"] for client.records.get + + def visit_Call(self, node: cst.Call) -> None: + func = node.func + if not isinstance(func, cst.Attribute): + return + + method = func.attr.value if isinstance(func.attr, cst.Name) else "" + chain = self._receiver_chain(func) # [method, ns, client_var, ...] + + # execute(by_page=) — non-literal by_page cannot be codemodded + if method == "execute": + for a in node.args: + if isinstance(a.keyword, cst.Name) and a.keyword.value == "by_page": + if not (isinstance(a.value, cst.Name) and a.value.value in ("True", "False")): + self.findings.append( + "execute(by_page=) — non-literal by_page requires manual review; " + "replace with execute_pages() or execute() depending on runtime value" + ) + + # client.records.get() — return type changes make a mechanical rename unsafe + if method == "get" and len(chain) >= 3 and chain[1] == "records" and chain[2] == self._client_var: + self.findings.append( + f"{self._client_var}.records.get() — use retrieve() for single-record lookup " + "(return type changes: raises on 404 vs returns None) " + "or list() for multi-record (iteration pattern changes)" + ) + + # client.dataframe.get() — expression reconstruction requires understanding caller intent + if method == "get" and len(chain) >= 3 and chain[1] == "dataframe" and chain[2] == self._client_var: + self.findings.append( + f"{self._client_var}.dataframe.get() — use " + "query.builder(...).execute().to_dataframe(); requires manual reconstruction" + ) + + # client.query.sql_select/sql_join/sql_joins — removed with no mechanical replacement + if ( + method in _REMOVED_QUERY_METHODS + and len(chain) >= 3 + and chain[1] == "query" + and chain[2] == self._client_var + ): + self.findings.append(f"{self._client_var}.query.{method}() — removed at GA with no mechanical replacement") + + +def find_manual_patterns(source: str, *, client_var: str = "client") -> List[str]: + """Return descriptions of patterns in *source* that require manual migration.""" + try: + tree = cst.parse_module(source) + except cst.ParserSyntaxError: + return [] + finder = _ManualReviewFinder(client_var=client_var) + tree.visit(finder) + return finder.findings + + +# --------------------------------------------------------------------------- +# File-level migration +# --------------------------------------------------------------------------- + + +def migrate_source(source: str, *, client_var: str = "client") -> str: + """Parse *source*, apply transformations, return migrated source.""" + try: + tree = cst.parse_module(source) + except cst.ParserSyntaxError as exc: + raise ValueError(f"Parse error: {exc}") from exc + new_tree = tree.visit(_V1Migrator(client_var=client_var)) + return new_tree.code + + +def migrate_file(path: Path, *, dry_run: bool = False, client_var: str = "client") -> Tuple[bool, List[str]]: + """Migrate *path* in place. Returns (was_changed, manual_review_notes).""" + original = path.read_text(encoding="utf-8") + try: + migrated = migrate_source(original, client_var=client_var) + except ValueError as exc: + print(f" [SKIP] {path}: {exc}", file=sys.stderr) + return False, [] + manual = find_manual_patterns(original, client_var=client_var) + changed = migrated != original + if changed and not dry_run: + path.write_text(migrated, encoding="utf-8") + return changed, manual + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def _collect_targets(paths: List[str]) -> List[Path]: + targets: List[Path] = [] + for p_str in paths: + p = Path(p_str) + if p.is_dir(): + root = p.resolve() + for candidate in sorted(p.rglob("*.py")): + resolved = candidate.resolve() + if root == resolved or root in resolved.parents: + targets.append(candidate) + else: + print(f"[WARN] Skipping symlink outside target directory: {candidate}", file=sys.stderr) + elif p.is_file() and p.suffix == ".py": + targets.append(p) + else: + print(f"[WARN] Not a file or directory: {p}", file=sys.stderr) + return targets + + +def main(argv: Optional[List[str]] = None) -> int: + args = sys.argv[1:] if argv is None else list(argv) + if "--help" in args or "-h" in args: + print(__doc__) + print("\nUsage: dataverse-migrate [--dry-run] [--client-var=NAME] [ ...]") + return 0 + dry_run = "--dry-run" in args + client_var = "client" + remaining = [] + for a in args: + if a == "--dry-run": + continue + if a.startswith("--client-var="): + client_var = a[len("--client-var=") :] + else: + remaining.append(a) + + if not remaining: + print(__doc__) + print("\nUsage: dataverse-migrate [--dry-run] [--client-var=NAME] [ ...]") + return 1 + + targets = _collect_targets(remaining) + if not targets: + print("[ERROR] No Python files found.", file=sys.stderr) + return 1 + + changed = skipped = needs_manual = manual_total = 0 + for path in targets: + was_changed, notes = migrate_file(path, dry_run=dry_run, client_var=client_var) + if was_changed: + changed += 1 + tag = "[DRY-RUN]" if dry_run else "[MIGRATED]" + if notes: + print(f"{tag} {path} (auto-rewrites applied; manual review still required)") + else: + print(f"{tag} {path}") + elif notes: + needs_manual += 1 + print(f"[NEEDS-MANUAL] {path} (no auto-rewrites to apply; manual migration required)") + else: + skipped += 1 + for note in notes: + print(f" [MANUAL] {note}") + manual_total += 1 + + suffix = "would be " if dry_run else "" + parts = [f"{changed} file(s) {suffix}auto-migrated"] + if needs_manual: + parts.append(f"{needs_manual} need manual-only migration") + parts.append(f"{skipped} unchanged") + print(f"\nDone: {', '.join(parts)}.", end="") + if manual_total: + print(f" {manual_total} pattern(s) require manual review.") + else: + print() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/PowerPlatform/Dataverse/models/__init__.py b/src/PowerPlatform/Dataverse/models/__init__.py index dc10a4c0..50bd7326 100644 --- a/src/PowerPlatform/Dataverse/models/__init__.py +++ b/src/PowerPlatform/Dataverse/models/__init__.py @@ -8,12 +8,18 @@ - :class:`~PowerPlatform.Dataverse.models.query_builder.QueryBuilder`: Fluent query builder. - :mod:`~PowerPlatform.Dataverse.models.filters`: Composable OData filter expressions. +- :class:`~PowerPlatform.Dataverse.models.record.QueryResult`: Iterable result wrapper. - :class:`~PowerPlatform.Dataverse.models.upsert.UpsertItem`: Upsert operation item. Import directly from the specific module, e.g.:: from PowerPlatform.Dataverse.models.query_builder import QueryBuilder - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col, raw + from PowerPlatform.Dataverse.models.record import QueryResult """ -__all__ = [] +from .filters import col, raw +from .protocol import DataverseModel +from .record import QueryResult + +__all__ = ["col", "raw", "DataverseModel", "QueryResult"] diff --git a/src/PowerPlatform/Dataverse/models/fetchxml_query.py b/src/PowerPlatform/Dataverse/models/fetchxml_query.py new file mode 100644 index 00000000..a1f43883 --- /dev/null +++ b/src/PowerPlatform/Dataverse/models/fetchxml_query.py @@ -0,0 +1,183 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""FetchXmlQuery — inert query object returned by QueryOperations.fetchxml().""" + +from __future__ import annotations + +import warnings +import xml.etree.ElementTree as _ET +from typing import Iterator, List, TYPE_CHECKING +from urllib.parse import unquote as _url_unquote, quote as _url_quote + +from ..core.errors import ValidationError +from .record import QueryResult, Record + +if TYPE_CHECKING: + from ..client import DataverseClient + + +__all__ = ["FetchXmlQuery"] + +_PREFER_HEADER = ( + "odata.include-annotations=" '"Microsoft.Dynamics.CRM.fetchxmlpagingcookie,' 'Microsoft.Dynamics.CRM.morerecords"' +) + +# Documented Dataverse GET request URL limit. See: +# learn.microsoft.com/power-apps/developer/data-platform/webapi/compose-http-requests-handle-errors#maximum-url-length +# FetchXML queries with many attributes or conditions are the most common way to reach it. +# $batch POST doubles this to 64 KB. +_MAX_URL_LENGTH = 32_768 +# Guards against infinite paging loops caused by a bug in cookie propagation or an +# unexpected server response. At the default Dataverse page size of 5,000 rows this +# cap allows up to 50 million records before raising; it is not a practical record +# limit but a circuit-breaker against runaway iteration. +_MAX_PAGES = 10_000 + + +class FetchXmlQuery: + """Inert FetchXML query object. No HTTP request is made until + :meth:`execute` or :meth:`execute_pages` is called. + + Obtained via ``client.query.fetchxml(xml)``. + + :param xml: Stripped, well-formed FetchXML string. + :param entity_name: Entity schema name from the ```` element. + :param client: Parent :class:`~PowerPlatform.Dataverse.client.DataverseClient`. + """ + + def __init__(self, xml: str, entity_name: str, client: "DataverseClient") -> None: + self._xml = xml + self._entity_name = entity_name + self._client = client + + def execute(self) -> QueryResult: + """Execute the FetchXML query and return all results as a :class:`QueryResult`. + + Blocking — fetches all pages upfront and holds every record in memory before + returning. Simple for small-to-medium result sets; use :meth:`execute_pages` + when the result set may be large or you want to process records as they arrive. + + :return: All matching records across all pages. + :rtype: :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + + Example:: + + rows = client.query.fetchxml(xml).execute() + df = rows.to_dataframe() + """ + all_records: List[Record] = [] + for page in self.execute_pages(): + all_records.extend(page.records) + return QueryResult(all_records) + + def execute_pages(self) -> Iterator[QueryResult]: + """Lazily yield one :class:`QueryResult` per HTTP page. + + Streaming — each iteration fires one HTTP request and yields one page. + Prefer over :meth:`execute` when: + + - The result set may be large and you do not want all records in memory at once. + - You want early exit: stop iterating once you find what you need and the + remaining HTTP round-trips are skipped automatically. + - You need per-page progress reporting or batched downstream writes. + + One-shot — do not iterate more than once. + + :return: Iterator of per-page :class:`QueryResult` objects. + :rtype: Iterator[:class:`~PowerPlatform.Dataverse.models.record.QueryResult`] + + Example:: + + for page in client.query.fetchxml(xml).execute_pages(): + process(page.to_dataframe()) + """ + current_xml = self._xml + page_num = 1 + page_count = 0 + + with self._client._scoped_odata() as od: + entity_set = od._entity_set_from_schema_name(self._entity_name) + base_url = f"{od.api}/{entity_set}" + + while True: + page_count += 1 + if page_count > _MAX_PAGES: + raise ValidationError( + f"FetchXML paging exceeded {_MAX_PAGES} pages. " + "This may indicate a runaway query or a bug in paging cookie propagation." + ) + + encoded_len = len(base_url) + len("?fetchXml=") + len(_url_quote(current_xml, safe="")) + if encoded_len > _MAX_URL_LENGTH: + raise ValidationError( + f"FetchXML request URL exceeds {_MAX_URL_LENGTH} characters after encoding. " + "Simplify the query or reduce attributes/conditions." + ) + + r = od._request( + "get", + base_url, + headers={"Prefer": _PREFER_HEADER}, + params={"fetchXml": current_xml}, + ) + data = r.json() if hasattr(r, "json") else {} + items = data.get("value") if isinstance(data, dict) else None + page_records: List[Record] = [] + if isinstance(items, list): + for item in items: + if isinstance(item, dict): + page_records.append(Record.from_api_response(self._entity_name, item)) + + yield QueryResult(page_records) + + more_raw = data.get("@Microsoft.Dynamics.CRM.morerecords", False) if isinstance(data, dict) else False + more = more_raw is True or (isinstance(more_raw, str) and more_raw.lower() == "true") + if not more: + break + + raw_cookie = ( + data.get("@Microsoft.Dynamics.CRM.fetchxmlpagingcookie", "") if isinstance(data, dict) else "" + ) + + _cookie_parse_error = False + if raw_cookie: + try: + cookie_el = _ET.fromstring(raw_cookie) + inner_encoded = cookie_el.get("pagingcookie", "") + if inner_encoded: + cookie = _url_unquote(_url_unquote(inner_encoded)) + page_num = int(cookie_el.get("pagenumber", str(page_num + 1))) + fetch_el = _ET.fromstring(current_xml) + fetch_el.set("paging-cookie", cookie) + fetch_el.set("page", str(page_num)) + current_xml = _ET.tostring(fetch_el, encoding="unicode") + continue + except (_ET.ParseError, ValueError) as exc: + warnings.warn( + f"FetchXML paging cookie could not be parsed ({exc}); " "falling back to simple paging.", + UserWarning, + stacklevel=2, + ) + _cookie_parse_error = True + + # Simple paging fallback: server returned morerecords=true but no paging + # cookie. Dataverse omits the cookie when the query cannot use cookie-based + # paging (e.g. FetchXML ordered by a link-entity column). We continue with + # page-number-only paging rather than truncating, but warn because simple + # paging has a 50,000-record server cap and performance degrades at high page + # numbers. The caller may be able to avoid this by reordering on the root + # entity instead. + if not _cookie_parse_error: + warnings.warn( + "Dataverse did not return a paging cookie; falling back to simple paging " + "(page-number increment only). Simple paging is capped at 50,000 records " + "and degrades in performance at high page numbers. Consider reordering on " + "a root-entity column to enable cookie-based paging.", + UserWarning, + stacklevel=2, + ) + page_num += 1 + fetch_el = _ET.fromstring(current_xml) + fetch_el.set("page", str(page_num)) + current_xml = _ET.tostring(fetch_el, encoding="unicode") diff --git a/src/PowerPlatform/Dataverse/models/filters.py b/src/PowerPlatform/Dataverse/models/filters.py index c5de258c..2a7929cc 100644 --- a/src/PowerPlatform/Dataverse/models/filters.py +++ b/src/PowerPlatform/Dataverse/models/filters.py @@ -10,24 +10,26 @@ Example:: - from PowerPlatform.Dataverse.models.filters import eq, gt, filter_in + from PowerPlatform.Dataverse.models.filters import col, raw - # Simple comparison - expr = eq("statecode", 0) + # Preferred GA idiom — col() proxy + expr = col("statecode") == 0 print(expr.to_odata()) # statecode eq 0 # Complex composition with OR and AND - expr = (eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000) + expr = (col("statecode") == 0) | (col("statecode") == 1) & (col("revenue") > 100000) print(expr.to_odata()) - # ((statecode eq 0 or statecode eq 1) and revenue gt 100000) - # In operator (Dataverse function) - expr = filter_in("statecode", [0, 1, 2]) + # In / not-in + expr = col("statecode").in_([0, 1, 2]) print(expr.to_odata()) # Microsoft.Dynamics.CRM.In(PropertyName='statecode',PropertyValues=["0","1","2"]) + # Raw OData escape hatch (no deprecation warning) + expr = raw("Microsoft.Dynamics.CRM.Today(PropertyName='createdon')") + # Negation - expr = ~eq("statecode", 1) + expr = ~(col("statecode") == 1) print(expr.to_odata()) # not (statecode eq 1) """ @@ -35,11 +37,16 @@ import enum import uuid +import warnings from datetime import date, datetime, timezone -from typing import Any, Collection, Sequence +from typing import Any, Collection, List __all__ = [ "FilterExpression", + "ColumnProxy", + "col", + "raw", + # Deprecated factories — still functional, fire DeprecationWarning on call: "eq", "ne", "gt", @@ -55,7 +62,6 @@ "filter_in", "not_in", "not_between", - "raw", ] @@ -264,138 +270,458 @@ def to_odata(self) -> str: # --------------------------------------------------------------------------- -# Public factory functions +# Private implementation helpers (no warnings — used internally and by col()) # --------------------------------------------------------------------------- -def eq(column: str, value: Any) -> FilterExpression: - """Equality filter: ``column eq value``. +def _eq_impl(column: str, value: Any) -> FilterExpression: + return _ComparisonFilter(column, "eq", value) + + +def _ne_impl(column: str, value: Any) -> FilterExpression: + return _ComparisonFilter(column, "ne", value) + + +def _gt_impl(column: str, value: Any) -> FilterExpression: + return _ComparisonFilter(column, "gt", value) + + +def _ge_impl(column: str, value: Any) -> FilterExpression: + return _ComparisonFilter(column, "ge", value) + + +def _lt_impl(column: str, value: Any) -> FilterExpression: + return _ComparisonFilter(column, "lt", value) + + +def _le_impl(column: str, value: Any) -> FilterExpression: + return _ComparisonFilter(column, "le", value) + + +def _contains_impl(column: str, value: str) -> FilterExpression: + return _FunctionFilter("contains", column, value) + + +def _startswith_impl(column: str, value: str) -> FilterExpression: + return _FunctionFilter("startswith", column, value) + + +def _endswith_impl(column: str, value: str) -> FilterExpression: + return _FunctionFilter("endswith", column, value) + + +def _in_impl(column: str, values: Collection[Any]) -> FilterExpression: + return _InFilter(column, values) + + +def _not_in_impl(column: str, values: Collection[Any]) -> FilterExpression: + return _NotInFilter(column, values) + + +# --------------------------------------------------------------------------- +# ColumnProxy — GA idiom for building filter expressions +# --------------------------------------------------------------------------- - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: A filter expression. +_LIKE_WILDCARD = "%" + + +def _compile_like(column: str, pattern: str) -> FilterExpression: + """Compile a LIKE-style pattern to an OData FilterExpression. + + Pattern rules: + - ``val%`` → ``startswith(column, 'val')`` + - ``%val`` → ``endswith(column, 'val')`` + - ``%val%`` → ``contains(column, 'val')`` + - ``val`` (no wildcard) → ``column eq 'val'`` (equality) + - Anything else → :class:`ValueError` + + :param column: Lowercased column name. + :param pattern: The LIKE pattern string. + :raises ValueError: If the pattern contains wildcards in unsupported positions. + """ + has_start = pattern.startswith(_LIKE_WILDCARD) + has_end = pattern.endswith(_LIKE_WILDCARD) + inner = pattern.strip(_LIKE_WILDCARD) + + # Detect non-reducible interior wildcards: after stripping the leading/trailing + # % the inner value must contain no further % characters. + if _LIKE_WILDCARD in inner: + raise ValueError( + f"like() pattern {pattern!r} is not reducible to a single OData function. " + "Use raw(), fetchxml(), or query.sql() for complex wildcard patterns." + ) + + if not has_start and has_end: + # "val%" — startswith + return _startswith_impl(column, inner) + if has_start and not has_end: + # "%val" — endswith + return _endswith_impl(column, inner) + if has_start and has_end: + # "%val%" — contains + return _contains_impl(column, inner) + # No wildcard at all — exact equality + return _eq_impl(column, pattern) + + +class ColumnProxy: + """Fluent proxy for building OData filter expressions from a column name. + + Returned by :func:`col`. Operator overloads and methods produce + :class:`FilterExpression` instances that can be passed to + ``QueryBuilder.where()``. Example:: - eq("statecode", 0).to_odata() # "statecode eq 0" + from PowerPlatform.Dataverse.models.filters import col + + expr = col("statecode") == 0 # equality + expr = col("revenue") > 1_000_000 # comparison + expr = col("name").like("Contoso%") # startswith + expr = col("name").is_null() # null check + expr = col("statecode").in_([0, 1]) # in """ - return _ComparisonFilter(column, "eq", value) + + __slots__ = ("_column",) + + def __init__(self, name: str) -> None: + if not name or not name.strip(): + raise ValueError("col() requires a non-empty column name") + self._column = name.strip().lower() + + # ---------------------------------------------------------------- comparisons + + def __eq__(self, other: Any) -> FilterExpression: # type: ignore[override] + return _eq_impl(self._column, other) + + def __ne__(self, other: Any) -> FilterExpression: # type: ignore[override] + return _ne_impl(self._column, other) + + def __gt__(self, other: Any) -> FilterExpression: + return _gt_impl(self._column, other) + + def __ge__(self, other: Any) -> FilterExpression: + return _ge_impl(self._column, other) + + def __lt__(self, other: Any) -> FilterExpression: + return _lt_impl(self._column, other) + + def __le__(self, other: Any) -> FilterExpression: + return _le_impl(self._column, other) + + # ---------------------------------------------------------------- null checks + + def is_null(self) -> FilterExpression: + """Column equals null: ``column eq null``.""" + return _eq_impl(self._column, None) + + def is_not_null(self) -> FilterExpression: + """Column not null: ``column ne null``.""" + return _ne_impl(self._column, None) + + # ---------------------------------------------------------------- in / not-in + + def in_(self, values: Collection[Any]) -> FilterExpression: + """In filter using ``Microsoft.Dynamics.CRM.In``. + + :param values: Non-empty collection of values. + :raises ValueError: If ``values`` is empty. + """ + return _in_impl(self._column, values) + + def not_in(self, values: Collection[Any]) -> FilterExpression: + """Not-in filter using ``Microsoft.Dynamics.CRM.NotIn``. + + :param values: Non-empty collection of values. + :raises ValueError: If ``values`` is empty. + """ + return _not_in_impl(self._column, values) + + # ---------------------------------------------------------------- range + + def between(self, lo: Any, hi: Any) -> FilterExpression: + """Between filter: ``(column ge lo and column le hi)``.""" + return _ge_impl(self._column, lo) & _le_impl(self._column, hi) + + def not_between(self, lo: Any, hi: Any) -> FilterExpression: + """Not-between filter: ``not (column ge lo and column le hi)``.""" + return ~(self.between(lo, hi)) + + # ---------------------------------------------------------------- string functions + + def contains(self, value: str) -> FilterExpression: + """Contains filter: ``contains(column, value)``.""" + return _contains_impl(self._column, value) + + def startswith(self, value: str) -> FilterExpression: + """Startswith filter: ``startswith(column, value)``.""" + return _startswith_impl(self._column, value) + + def endswith(self, value: str) -> FilterExpression: + """Endswith filter: ``endswith(column, value)``.""" + return _endswith_impl(self._column, value) + + # ---------------------------------------------------------------- like / not_like + + def like(self, pattern: str) -> FilterExpression: + """Pattern-match filter compiled to the closest OData equivalent. + + +-----------------+-----------------------------+-------------------------------------+ + | Pattern form | Example | Compiles to | + +=================+=============================+=====================================+ + | ``val%`` | ``like("Contoso%")`` | ``startswith(column,'Contoso')`` | + +-----------------+-----------------------------+-------------------------------------+ + | ``%val`` | ``like("%Ltd")`` | ``endswith(column,'Ltd')`` | + +-----------------+-----------------------------+-------------------------------------+ + | ``%val%`` | ``like("%Corp%")`` | ``contains(column,'Corp')`` | + +-----------------+-----------------------------+-------------------------------------+ + | No wildcard | ``like("Contoso")`` | ``column eq 'Contoso'`` | + +-----------------+-----------------------------+-------------------------------------+ + | Other | ``like("Con%oso")`` | :class:`ValueError` | + +-----------------+-----------------------------+-------------------------------------+ + + :param pattern: LIKE-style pattern string. + :raises ValueError: If the pattern cannot be reduced to a single OData function. + """ + return _compile_like(self._column, pattern) + + def not_like(self, pattern: str) -> FilterExpression: + """Negated pattern-match filter; mirrors :meth:`like` rules then negates. + + :param pattern: LIKE-style pattern string (same rules as :meth:`like`). + :raises ValueError: If the pattern cannot be reduced to a single OData function. + """ + return ~_compile_like(self._column, pattern) + + # ---------------------------------------------------------------- hash / repr + + def __hash__(self) -> int: + return hash(self._column) + + def __repr__(self) -> str: + return f"ColumnProxy({self._column!r})" + + +# --------------------------------------------------------------------------- +# Public factory: col() — no deprecation warning +# --------------------------------------------------------------------------- + + +def col(name: str) -> ColumnProxy: + """Return a :class:`ColumnProxy` for building filter expressions. + + This is the preferred GA idiom for constructing filter expressions:: + + from PowerPlatform.Dataverse.models.filters import col + + expr = col("statecode") == 0 + expr = col("revenue") > 1_000_000 + expr = col("name").like("Contoso%") + expr = col("statecode").in_([0, 1]) + expr = col("parentaccountid").is_null() + + :param name: Column logical name (case-insensitive, will be lowercased). + :return: A :class:`ColumnProxy` bound to the column. + :raises ValueError: If ``name`` is empty. + """ + return ColumnProxy(name) + + +# --------------------------------------------------------------------------- +# Public factory: raw() — no deprecation warning (OData escape hatch) +# --------------------------------------------------------------------------- + + +def raw(filter_string: str) -> FilterExpression: + """Verbatim OData filter expression (passed through unchanged). + + This function is **not** deprecated — it is the OData escape hatch with + no typed replacement. + + :param filter_string: Raw OData filter string. + :return: A :class:`FilterExpression`. + + Example:: + + raw("Microsoft.Dynamics.CRM.Today(PropertyName='createdon')") + """ + return _RawFilter(filter_string) + + +# --------------------------------------------------------------------------- +# Deprecated public factory functions — fire DeprecationWarning on CALL +# --------------------------------------------------------------------------- + +_DEP_MSG = "'{name}' is deprecated and will be removed in a future release. " "Use {replacement} instead." + + +def eq(column: str, value: Any) -> FilterExpression: + """Equality filter: ``column eq value``. + + .. deprecated:: + Use ``col(column) == value`` instead. + """ + warnings.warn( + _DEP_MSG.format(name="eq", replacement="col('column') == value"), + DeprecationWarning, + stacklevel=2, + ) + return _eq_impl(column, value) def ne(column: str, value: Any) -> FilterExpression: """Not-equal filter: ``column ne value``. - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: A filter expression. + .. deprecated:: + Use ``col(column) != value`` instead. """ - return _ComparisonFilter(column, "ne", value) + warnings.warn( + _DEP_MSG.format(name="ne", replacement="col('column') != value"), + DeprecationWarning, + stacklevel=2, + ) + return _ne_impl(column, value) def gt(column: str, value: Any) -> FilterExpression: """Greater-than filter: ``column gt value``. - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: A filter expression. + .. deprecated:: + Use ``col(column) > value`` instead. """ - return _ComparisonFilter(column, "gt", value) + warnings.warn( + _DEP_MSG.format(name="gt", replacement="col('column') > value"), + DeprecationWarning, + stacklevel=2, + ) + return _gt_impl(column, value) def ge(column: str, value: Any) -> FilterExpression: """Greater-than-or-equal filter: ``column ge value``. - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: A filter expression. + .. deprecated:: + Use ``col(column) >= value`` instead. """ - return _ComparisonFilter(column, "ge", value) + warnings.warn( + _DEP_MSG.format(name="ge", replacement="col('column') >= value"), + DeprecationWarning, + stacklevel=2, + ) + return _ge_impl(column, value) def lt(column: str, value: Any) -> FilterExpression: """Less-than filter: ``column lt value``. - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: A filter expression. + .. deprecated:: + Use ``col(column) < value`` instead. """ - return _ComparisonFilter(column, "lt", value) + warnings.warn( + _DEP_MSG.format(name="lt", replacement="col('column') < value"), + DeprecationWarning, + stacklevel=2, + ) + return _lt_impl(column, value) def le(column: str, value: Any) -> FilterExpression: """Less-than-or-equal filter: ``column le value``. - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: A filter expression. + .. deprecated:: + Use ``col(column) <= value`` instead. """ - return _ComparisonFilter(column, "le", value) + warnings.warn( + _DEP_MSG.format(name="le", replacement="col('column') <= value"), + DeprecationWarning, + stacklevel=2, + ) + return _le_impl(column, value) def contains(column: str, value: str) -> FilterExpression: """Contains filter: ``contains(column, value)``. - :param column: Column name (will be lowercased). - :param value: Substring to search for. - :return: A filter expression. + .. deprecated:: + Use ``col(column).contains(value)`` instead. """ - return _FunctionFilter("contains", column, value) + warnings.warn( + _DEP_MSG.format(name="contains", replacement="col('column').contains(value)"), + DeprecationWarning, + stacklevel=2, + ) + return _contains_impl(column, value) def startswith(column: str, value: str) -> FilterExpression: """Startswith filter: ``startswith(column, value)``. - :param column: Column name (will be lowercased). - :param value: Prefix to match. - :return: A filter expression. + .. deprecated:: + Use ``col(column).startswith(value)`` instead. """ - return _FunctionFilter("startswith", column, value) + warnings.warn( + _DEP_MSG.format(name="startswith", replacement="col('column').startswith(value)"), + DeprecationWarning, + stacklevel=2, + ) + return _startswith_impl(column, value) def endswith(column: str, value: str) -> FilterExpression: """Endswith filter: ``endswith(column, value)``. - :param column: Column name (will be lowercased). - :param value: Suffix to match. - :return: A filter expression. + .. deprecated:: + Use ``col(column).endswith(value)`` instead. """ - return _FunctionFilter("endswith", column, value) + warnings.warn( + _DEP_MSG.format(name="endswith", replacement="col('column').endswith(value)"), + DeprecationWarning, + stacklevel=2, + ) + return _endswith_impl(column, value) def between(column: str, low: Any, high: Any) -> FilterExpression: """Between filter: ``(column ge low and column le high)``. - Syntactic sugar that composes :func:`ge` and :func:`le` with ``&``. - - :param column: Column name (will be lowercased). - :param low: Lower bound (inclusive). - :param high: Upper bound (inclusive). - :return: A composed filter expression. - - Example:: - - between("revenue", 100000, 500000).to_odata() - # "(revenue ge 100000 and revenue le 500000)" + .. deprecated:: + Use ``col(column).between(low, high)`` instead. """ - return ge(column, low) & le(column, high) + warnings.warn( + _DEP_MSG.format(name="between", replacement="col('column').between(low, high)"), + DeprecationWarning, + stacklevel=2, + ) + # Use private helpers to avoid chaining through the deprecated ge/le wrappers + return _ge_impl(column, low) & _le_impl(column, high) def is_null(column: str) -> FilterExpression: """Null check: ``column eq null``. - :param column: Column name (will be lowercased). - :return: A filter expression. + .. deprecated:: + Use ``col(column).is_null()`` instead. """ - return _ComparisonFilter(column, "eq", None) + warnings.warn( + _DEP_MSG.format(name="is_null", replacement="col('column').is_null()"), + DeprecationWarning, + stacklevel=2, + ) + return _eq_impl(column, None) def is_not_null(column: str) -> FilterExpression: """Not-null check: ``column ne null``. - :param column: Column name (will be lowercased). - :return: A filter expression. + .. deprecated:: + Use ``col(column).is_not_null()`` instead. """ - return _ComparisonFilter(column, "ne", None) + warnings.warn( + _DEP_MSG.format(name="is_not_null", replacement="col('column').is_not_null()"), + DeprecationWarning, + stacklevel=2, + ) + return _ne_impl(column, None) def filter_in(column: str, values: Collection[Any]) -> FilterExpression: @@ -403,63 +729,45 @@ def filter_in(column: str, values: Collection[Any]) -> FilterExpression: Named ``filter_in`` because ``in`` is a Python keyword. - :param column: Column name (will be lowercased). - :param values: Non-empty sequence of values. - :return: A filter expression. - :raises ValueError: If ``values`` is empty. - - Example:: + .. deprecated:: + Use ``col(column).in_(values)`` instead. - filter_in("statecode", [0, 1, 2]).to_odata() - # "Microsoft.Dynamics.CRM.In(PropertyName='statecode',PropertyValues=["0","1","2"])" + :raises ValueError: If ``values`` is empty. """ - return _InFilter(column, values) + warnings.warn( + _DEP_MSG.format(name="filter_in", replacement="col('column').in_(values)"), + DeprecationWarning, + stacklevel=2, + ) + return _in_impl(column, values) def not_in(column: str, values: Collection[Any]) -> FilterExpression: """Not-in filter using ``Microsoft.Dynamics.CRM.NotIn``. - Named ``not_in`` to parallel :func:`filter_in`. + .. deprecated:: + Use ``col(column).not_in(values)`` instead. - :param column: Column name (will be lowercased). - :param values: Non-empty sequence of values. - :return: A filter expression. :raises ValueError: If ``values`` is empty. - - Example:: - - not_in("statecode", [0, 1]).to_odata() - # "Microsoft.Dynamics.CRM.NotIn(PropertyName='statecode',PropertyValues=[\"0\",\"1\"])" """ - return _NotInFilter(column, values) + warnings.warn( + _DEP_MSG.format(name="not_in", replacement="col('column').not_in(values)"), + DeprecationWarning, + stacklevel=2, + ) + return _not_in_impl(column, values) def not_between(column: str, low: Any, high: Any) -> FilterExpression: """Not-between filter: ``not (column ge low and column le high)``. - Syntactic sugar that negates :func:`between` with ``~``. - - :param column: Column name (will be lowercased). - :param low: Lower bound (inclusive, will be excluded). - :param high: Upper bound (inclusive, will be excluded). - :return: A composed filter expression. - - Example:: - - not_between("revenue", 100000, 500000).to_odata() - # "not ((revenue ge 100000 and revenue le 500000))" + .. deprecated:: + Use ``col(column).not_between(low, high)`` instead. """ - return ~between(column, low, high) - - -def raw(filter_string: str) -> FilterExpression: - """Verbatim OData filter expression (passed through unchanged). - - :param filter_string: Raw OData filter string. - :return: A filter expression. - - Example:: - - raw("Microsoft.Dynamics.CRM.Today(PropertyName='createdon')") - """ - return _RawFilter(filter_string) + warnings.warn( + _DEP_MSG.format(name="not_between", replacement="col('column').not_between(low, high)"), + DeprecationWarning, + stacklevel=2, + ) + # Use private helpers to avoid chaining through deprecated ge/le wrappers + return ~(_ge_impl(column, low) & _le_impl(column, high)) diff --git a/src/PowerPlatform/Dataverse/models/protocol.py b/src/PowerPlatform/Dataverse/models/protocol.py new file mode 100644 index 00000000..d9b0e502 --- /dev/null +++ b/src/PowerPlatform/Dataverse/models/protocol.py @@ -0,0 +1,77 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""DataverseModel structural Protocol for typed entity integration.""" + +from __future__ import annotations + +from typing import Protocol, runtime_checkable + +__all__ = ["DataverseModel"] + + +@runtime_checkable +class DataverseModel(Protocol): + """Structural Protocol enabling typed entity instances to be passed to + ``records.create()`` and ``records.update()``. + + Implement this Protocol on any entity class (dataclass, Pydantic model, + hand-rolled) to enable it to be passed directly to CRUD operations without + specifying the table name or converting to dict manually. + + Required class variables: + + - ``__entity_logical_name__`` — Dataverse logical entity name (e.g. ``"account"``) + - ``__entity_set_name__`` — OData entity set name (e.g. ``"accounts"``) + + Required instance methods: + + - ``to_dict()`` — return record payload as ``dict`` + - ``from_dict(data)`` — classmethod to reconstruct from a response ``dict`` + + Example:: + + from dataclasses import dataclass + from PowerPlatform.Dataverse import DataverseModel + + @dataclass + class Account: + __entity_logical_name__ = "account" + __entity_set_name__ = "accounts" + name: str = "" + telephone1: str = "" + + def to_dict(self) -> dict: + return {"name": self.name, "telephone1": self.telephone1} + + @classmethod + def from_dict(cls, data: dict) -> "Account": + return cls( + name=data.get("name", ""), + telephone1=data.get("telephone1", ""), + ) + + # isinstance() works today — Protocol is runtime_checkable: + assert isinstance(Account(), DataverseModel) + + # Type your own helpers against the Protocol now: + def save(entity: DataverseModel) -> None: + data = entity.to_dict() + client.records.create(entity.__entity_logical_name__, data) + + Note: + Direct dispatch (``client.records.create(entity)`` without a table name + or dict) is not yet supported and will be added in a future release. + """ + + __entity_logical_name__: str + __entity_set_name__: str + + def to_dict(self) -> dict: + """Return the record payload as a plain dictionary.""" + ... + + @classmethod + def from_dict(cls, data: dict) -> DataverseModel: + """Reconstruct an instance from a response dictionary.""" + ... diff --git a/src/PowerPlatform/Dataverse/models/query_builder.py b/src/PowerPlatform/Dataverse/models/query_builder.py index dbd79b36..e7044d2e 100644 --- a/src/PowerPlatform/Dataverse/models/query_builder.py +++ b/src/PowerPlatform/Dataverse/models/query_builder.py @@ -10,57 +10,64 @@ Example:: # Via client (recommended) -- flat iteration over records + from PowerPlatform.Dataverse.models.filters import col + for record in (client.query.builder("account") .select("name", "revenue") - .filter_eq("statecode", 0) - .filter_gt("revenue", 1000000) + .where(col("statecode") == 0) + .where(col("revenue") > 1_000_000) .order_by("revenue", descending=True) .top(100) .execute()): print(record["name"]) # With composable expression tree - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col, raw for record in (client.query.builder("account") .select("name", "revenue") - .where((eq("statecode", 0) | eq("statecode", 1)) - & gt("revenue", 100000)) + .where((col("statecode") == 0) | (col("statecode") == 1)) + .where(col("revenue") > 100000) .top(100) .execute()): print(record["name"]) - # Opt-in paged iteration (for batch processing) + # Lazy paged iteration (one QueryResult per HTTP page) for page in (client.query.builder("account") .select("name") - .execute(by_page=True)): + .execute_pages()): process_batch(page) # Get results as a pandas DataFrame df = (client.query.builder("account") .select("name", "telephone1") - .filter_eq("statecode", 0) + .where(col("statecode") == 0) .top(100) + .execute() .to_dataframe()) """ from __future__ import annotations -from typing import Any, Collection, Dict, Iterable, List, Optional, Sequence, TypedDict, Union +import warnings +from typing import Any, Dict, Iterator, List, Optional, TypedDict, Union import pandas as pd from . import filters -from .record import Record +from .record import QueryResult, Record __all__ = ["QueryBuilder", "QueryParams", "ExpandOption"] +# Sentinel for detecting when by_page is explicitly passed to execute() +_BY_PAGE_UNSET = object() + class QueryParams(TypedDict, total=False): """Typed dictionary returned by :meth:`QueryBuilder.build`. Provides IDE autocomplete when passing build results to - ``client.records.get()`` manually. + ``client.records.list()`` manually. """ table: str @@ -168,7 +175,7 @@ class QueryBuilder: """Fluent interface for building OData queries. Provides method chaining for constructing complex queries with - type-safe filter operations. Can be used standalone (via :meth:`build`) + composable filter expressions. Can be used standalone (via :meth:`build`) or bound to a client (via :meth:`execute`). :param table: Table schema name to query. @@ -178,9 +185,11 @@ class QueryBuilder: Example: Standalone query construction:: + from PowerPlatform.Dataverse.models.filters import col + query = (QueryBuilder("account") .select("name") - .filter_eq("statecode", 0) + .where(col("statecode") == 0) .top(10)) params = query.build() # {"table": "account", "select": ["name"], @@ -220,222 +229,16 @@ def select(self, *columns: str) -> QueryBuilder: self._select.extend(columns) return self - # ----------------------------------------------------------- filter: comparison - - def filter_eq(self, column: str, value: Any) -> QueryBuilder: - """Add equality filter: ``column eq value``. - - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.eq(column, value)) - return self - - def filter_ne(self, column: str, value: Any) -> QueryBuilder: - """Add not-equal filter: ``column ne value``. - - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.ne(column, value)) - return self - - def filter_gt(self, column: str, value: Any) -> QueryBuilder: - """Add greater-than filter: ``column gt value``. - - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.gt(column, value)) - return self - - def filter_ge(self, column: str, value: Any) -> QueryBuilder: - """Add greater-than-or-equal filter: ``column ge value``. - - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.ge(column, value)) - return self - - def filter_lt(self, column: str, value: Any) -> QueryBuilder: - """Add less-than filter: ``column lt value``. - - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.lt(column, value)) - return self - - def filter_le(self, column: str, value: Any) -> QueryBuilder: - """Add less-than-or-equal filter: ``column le value``. - - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.le(column, value)) - return self - - # --------------------------------------------------------- filter: string functions - - def filter_contains(self, column: str, value: str) -> QueryBuilder: - """Add contains filter: ``contains(column, value)``. - - :param column: Column name (will be lowercased). - :param value: Substring to search for. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.contains(column, value)) - return self - - def filter_startswith(self, column: str, value: str) -> QueryBuilder: - """Add startswith filter: ``startswith(column, value)``. - - :param column: Column name (will be lowercased). - :param value: Prefix to match. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.startswith(column, value)) - return self - - def filter_endswith(self, column: str, value: str) -> QueryBuilder: - """Add endswith filter: ``endswith(column, value)``. - - :param column: Column name (will be lowercased). - :param value: Suffix to match. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.endswith(column, value)) - return self - - # --------------------------------------------------------- filter: null checks - - def filter_null(self, column: str) -> QueryBuilder: - """Add null check: ``column eq null``. - - :param column: Column name (will be lowercased). - :return: Self for method chaining. - """ - self._filter_parts.append(filters.is_null(column)) - return self - - def filter_not_null(self, column: str) -> QueryBuilder: - """Add not-null check: ``column ne null``. - - :param column: Column name (will be lowercased). - :return: Self for method chaining. - """ - self._filter_parts.append(filters.is_not_null(column)) - return self - - # --------------------------------------------------------- filter: special - - def filter_in(self, column: str, values: Collection[Any]) -> QueryBuilder: - """Add an ``in`` filter using ``Microsoft.Dynamics.CRM.In``. - - :param column: Column name (will be lowercased). - :param values: Non-empty list of values for the ``in`` clause. - :return: Self for method chaining. - :raises ValueError: If ``values`` is empty. - - Example:: - - query = QueryBuilder("account").filter_in("statecode", [0, 1, 2]) - # Produces: Microsoft.Dynamics.CRM.In( - # PropertyName='statecode',PropertyValues=["0","1","2"]) - """ - self._filter_parts.append(filters.filter_in(column, values)) - return self - - def filter_not_in(self, column: str, values: Collection[Any]) -> QueryBuilder: - """Add a ``not in`` filter using ``Microsoft.Dynamics.CRM.NotIn``. - - :param column: Column name (will be lowercased). - :param values: Non-empty list of values to exclude. - :return: Self for method chaining. - :raises ValueError: If ``values`` is empty. - - Example:: - - query = QueryBuilder("account").filter_not_in("statecode", [2, 3]) - # Produces: Microsoft.Dynamics.CRM.NotIn( - # PropertyName='statecode',PropertyValues=["2","3"]) - """ - self._filter_parts.append(filters.not_in(column, values)) - return self - - def filter_between(self, column: str, low: Any, high: Any) -> QueryBuilder: - """Add a between filter: ``(column ge low and column le high)``. - - :param column: Column name (will be lowercased). - :param low: Lower bound (inclusive). - :param high: Upper bound (inclusive). - :return: Self for method chaining. - - Example:: - - query = QueryBuilder("account").filter_between("revenue", 100000, 500000) - # Produces: (revenue ge 100000 and revenue le 500000) - """ - self._filter_parts.append(filters.between(column, low, high)) - return self - - def filter_not_between(self, column: str, low: Any, high: Any) -> QueryBuilder: - """Add a not-between filter: ``not (column ge low and column le high)``. - - :param column: Column name (will be lowercased). - :param low: Lower bound (inclusive, will be excluded). - :param high: Upper bound (inclusive, will be excluded). - :return: Self for method chaining. - - Example:: - - query = QueryBuilder("account").filter_not_between("revenue", 100000, 500000) - # Produces: not ((revenue ge 100000 and revenue le 500000)) - """ - self._filter_parts.append(filters.not_between(column, low, high)) - return self - - def filter_raw(self, filter_string: str) -> QueryBuilder: - """Add a raw OData filter string. - - Use this for complex filters not covered by other methods. - Column names in the filter string should be lowercase. - - .. warning:: - The filter string is passed directly to Dataverse without validation. - Ensure it follows OData filter syntax; a malformed expression will result - in a ``400 Bad Request`` error from the server. - - :param filter_string: Raw OData filter expression. - :return: Self for method chaining. - - Example:: - - query = QueryBuilder("account").filter_raw( - "(statecode eq 0 or statecode eq 1)" - ) - """ - self._filter_parts.append(filters.raw(filter_string)) - return self - # ------------------------------------------------------ filter: expression tree def where(self, expression: filters.FilterExpression) -> QueryBuilder: """Add a composable filter expression. Accepts a :class:`~PowerPlatform.Dataverse.models.filters.FilterExpression` - built using the convenience functions from - :mod:`~PowerPlatform.Dataverse.models.filters`. + built using :func:`~PowerPlatform.Dataverse.models.filters.col` or + :func:`~PowerPlatform.Dataverse.models.filters.raw`. - Multiple ``where()`` calls and ``filter_*()`` calls are all - AND-joined together in the order they were called. + Multiple ``where()`` calls are AND-joined together in call order. :param expression: A composable filter expression. :type expression: FilterExpression @@ -444,11 +247,11 @@ def where(self, expression: filters.FilterExpression) -> QueryBuilder: Example:: - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col query = (QueryBuilder("account") - .where((eq("statecode", 0) | eq("statecode", 1)) - & gt("revenue", 100000))) + .where((col("statecode") == 0) | (col("statecode") == 1)) + .where(col("revenue") > 100000)) """ if not isinstance(expression, filters.FilterExpression): raise TypeError(f"where() requires a FilterExpression, got {type(expression).__name__}") @@ -511,7 +314,7 @@ def count(self) -> QueryBuilder: Example:: results = (client.query.builder("account") - .filter_eq("statecode", 0) + .where(col("statecode") == 0) .count() .execute()) """ @@ -612,8 +415,8 @@ def build(self) -> QueryParams: """Build query parameters dictionary. Returns a :class:`QueryParams` dictionary suitable for passing to - the OData layer. All ``filter_*()`` and ``where()`` clauses are - AND-joined into a single ``filter`` string in call order. + the OData layer. All ``where()`` clauses are AND-joined into a + single ``filter`` string in call order. :return: Dictionary with ``table`` and optionally ``select``, ``filter``, ``orderby``, ``expand``, ``top``, ``page_size``, @@ -645,148 +448,207 @@ def build(self) -> QueryParams: params["include_annotations"] = self._include_annotations return params - # --------------------------------------------------------------- guards - - def _validate_constraints(self) -> None: - """Raise if the query has no limiting constraints. - - At least one of ``select``, ``filter``, or ``top`` must be set - before executing a query to prevent accidental full-table scans. - - :raises ValueError: If none of ``select()``, ``filter_*()``, - ``where()``, or ``top()`` has been called. - """ - if not (self._select or self._filter_parts or self._top is not None): - raise ValueError( - "Unbounded query: set at least one of select(), filter_*(), " - "where(), or top() before calling execute() or to_dataframe()." - ) - # --------------------------------------------------------------- execute - def execute(self, *, by_page: bool = False) -> Union[Iterable[Record], Iterable[List[Record]]]: + def execute(self, *, by_page=_BY_PAGE_UNSET) -> Union[QueryResult, Iterator[QueryResult]]: """Execute the query and return results. - By default, returns a flat iterator over individual records, - abstracting away OData paging. Pass ``by_page=True`` to get - page-level iteration instead (useful for batch processing). + Returns a :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + with all pages collected. Use :meth:`execute_pages` for lazy per-page + iteration. This method is only available when the QueryBuilder was created via ``client.query.builder(table)``. Standalone ``QueryBuilder`` instances should use :meth:`build` to get parameters and pass them - to ``client.records.get()`` manually. - - At least one of ``select()``, ``filter_*()``, ``where()``, or - ``top()`` must be called before ``execute()``; otherwise a - :class:`ValueError` is raised to prevent accidental full-table - scans. - - :param by_page: If ``True``, yield pages (lists of - :class:`~PowerPlatform.Dataverse.models.record.Record` objects) - instead of individual records. Defaults to ``False``. - :type by_page: bool - :return: Generator yielding individual - :class:`~PowerPlatform.Dataverse.models.record.Record` objects - (default) or pages of records (when ``by_page=True``). - :rtype: Iterable[Record] or Iterable[List[Record]] - :raises ValueError: If no ``select``, ``filter``, or ``top`` + to ``client.records.list()`` manually. + + At least one of ``select()``, ``where()``, or ``top()`` must be + called before ``execute()``; otherwise a :class:`ValueError` is + raised to prevent accidental full-table scans. + + .. deprecated:: + The ``by_page`` parameter is deprecated. Use :meth:`execute_pages` + for lazy per-page iteration, or plain ``execute()`` (no flag) for + the default eager result. + + :return: :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + with all pages collected (default), or page iterator (deprecated + ``by_page=True``). + :rtype: QueryResult or Iterator[QueryResult] + :raises ValueError: If no ``select``, ``where``, or ``top`` constraint has been set. :raises RuntimeError: If the query was not created via ``client.query.builder()``. - Example: - Flat iteration (default):: + Example:: - for record in (client.query.builder("account") - .select("name") - .filter_eq("statecode", 0) - .execute()): - print(record["name"]) + from PowerPlatform.Dataverse.models.filters import col - Paged iteration:: + for record in (client.query.builder("account") + .select("name") + .where(col("statecode") == 0) + .execute()): + print(record["name"]) + """ + use_by_page = False + if by_page is not _BY_PAGE_UNSET: + use_by_page = bool(by_page) + if use_by_page: + warnings.warn( + "'execute(by_page=True)' is deprecated; use 'execute_pages()' instead.", + UserWarning, + stacklevel=2, + ) + else: + warnings.warn( + "'execute(by_page=False)' is deprecated; " + "the by_page flag is redundant — use plain 'execute()' instead.", + UserWarning, + stacklevel=2, + ) - for page in (client.query.builder("account") - .select("name") - .execute(by_page=True)): - process_batch(page) - """ if self._query_ops is None: raise RuntimeError( "Cannot execute: query was not created via client.query.builder(). " - "Use build() and pass parameters to client.records.get() instead." + "Use build() and pass parameters to client.records.list() instead." ) - self._validate_constraints() + + if not self._select and not self._filter_parts and self._top is None and self._page_size is None: + raise ValueError( + "At least one of select(), where(), top(), or page_size() must be called before " + "execute() to prevent accidental full-table scans." + ) + params = self.build() client = self._query_ops._client - pages = client.records.get( - params["table"], - select=params.get("select"), - filter=params.get("filter"), - orderby=params.get("orderby"), - top=params.get("top"), - expand=params.get("expand"), - page_size=params.get("page_size"), - count=params.get("count", False), - include_annotations=params.get("include_annotations"), - ) + if use_by_page: + return self.execute_pages() + + all_records: List[Record] = [] + with client._scoped_odata() as od: + for page in od._get_multiple( + params["table"], + select=params.get("select"), + filter=params.get("filter"), + orderby=params.get("orderby"), + top=params.get("top"), + expand=params.get("expand"), + page_size=params.get("page_size"), + count=params.get("count", False), + include_annotations=params.get("include_annotations"), + ): + all_records.extend(Record.from_api_response(params["table"], row) for row in page) + return QueryResult(all_records) + + # ---------------------------------------------------------- execute_pages + + def execute_pages(self) -> Iterator[QueryResult]: + """Lazily yield one :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + per HTTP page. + + Each iteration triggers a network request via ``@odata.nextLink``. + One-shot — do not iterate more than once. + + At least one of ``select()``, ``where()``, or ``top()`` must be + called before ``execute_pages()``; otherwise a :class:`ValueError` is + raised to prevent accidental full-table scans. + + :return: Iterator of per-page :class:`~PowerPlatform.Dataverse.models.record.QueryResult`. + :rtype: Iterator[QueryResult] + :raises ValueError: If no ``select``, ``where``, or ``top`` + constraint has been set. + :raises RuntimeError: If the query was not created via + ``client.query.builder()``. + + Example:: + + from PowerPlatform.Dataverse.models.filters import col - if by_page: - return pages + for page in (client.query.builder("account") + .select("name") + .where(col("statecode") == 0) + .execute_pages()): + process(page.to_dataframe()) + """ + if self._query_ops is None: + raise RuntimeError( + "Cannot execute: query was not created via client.query.builder(). " + "Use build() and pass parameters to client.records.list() instead." + ) + + if not self._select and not self._filter_parts and self._top is None and self._page_size is None: + raise ValueError( + "At least one of select(), where(), top(), or page_size() must be called before " + "execute_pages() to prevent accidental full-table scans." + ) - def _flat() -> Iterable[Record]: - for page in pages: - yield from page + params = self.build() + client = self._query_ops._client - return _flat() + with client._scoped_odata() as od: + for page in od._get_multiple( + params["table"], + select=params.get("select"), + filter=params.get("filter"), + orderby=params.get("orderby"), + top=params.get("top"), + expand=params.get("expand"), + page_size=params.get("page_size"), + count=params.get("count", False), + include_annotations=params.get("include_annotations"), + ): + yield QueryResult([Record.from_api_response(params["table"], row) for row in page]) # ----------------------------------------------------------- to_dataframe def to_dataframe(self) -> pd.DataFrame: """Execute the query and return results as a pandas DataFrame. - All pages are consolidated into a single DataFrame, matching - the behavior of ``client.dataframe.get()``. + .. deprecated:: + Use ``QueryBuilder.execute().to_dataframe()`` instead. + ``QueryBuilder.to_dataframe()`` will be removed in a future release. + + All pages are consolidated into a single DataFrame. This method is only available when the QueryBuilder was created via ``client.query.builder(table)``. - At least one of ``select()``, ``filter_*()``, ``where()``, or - ``top()`` must be called before ``to_dataframe()``; otherwise a - :class:`ValueError` is raised to prevent accidental full-table - scans. + At least one of ``select()``, ``where()``, or ``top()`` must be + called before ``to_dataframe()``; otherwise a :class:`ValueError` + is raised to prevent accidental full-table scans. :return: DataFrame containing all matching records. Returns an empty DataFrame when no records match. :rtype: ~pandas.DataFrame - :raises ValueError: If no ``select``, ``filter``, or ``top`` + :raises ValueError: If no ``select``, ``where``, or ``top`` constraint has been set. :raises RuntimeError: If the query was not created via ``client.query.builder()``. Example:: + from PowerPlatform.Dataverse.models.filters import col + df = (client.query.builder("account") .select("name", "telephone1") - .filter_eq("statecode", 0) + .where(col("statecode") == 0) .top(100) + .execute() .to_dataframe()) """ + warnings.warn( + "'QueryBuilder.to_dataframe()' is deprecated; use " "'QueryBuilder.execute().to_dataframe()' instead.", + DeprecationWarning, + stacklevel=2, + ) if self._query_ops is None: raise RuntimeError( "Cannot execute: query was not created via client.query.builder(). " - "Use build() and pass parameters to client.dataframe.get() instead." + "Use build() and pass parameters to client.records.list() instead." ) - self._validate_constraints() - params = self.build() - return self._query_ops._client.dataframe.get( - params["table"], - select=params.get("select"), - filter=params.get("filter"), - orderby=params.get("orderby"), - top=params.get("top"), - expand=params.get("expand"), - page_size=params.get("page_size"), - count=params.get("count", False), - include_annotations=params.get("include_annotations"), - ) + + result = self.execute() + if not result: + return pd.DataFrame(columns=self._select) if self._select else pd.DataFrame() + return result.to_dataframe() diff --git a/src/PowerPlatform/Dataverse/models/record.py b/src/PowerPlatform/Dataverse/models/record.py index 43641032..da65e7a5 100644 --- a/src/PowerPlatform/Dataverse/models/record.py +++ b/src/PowerPlatform/Dataverse/models/record.py @@ -6,9 +6,9 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any, Dict, Iterator, KeysView, Optional, ValuesView, ItemsView +from typing import Any, Dict, Iterator, KeysView, List, Optional, Union, ValuesView, ItemsView -__all__ = ["Record"] +__all__ = ["Record", "QueryResult"] _ODATA_PREFIX = "@odata." @@ -112,3 +112,55 @@ def from_api_response( def to_dict(self) -> Dict[str, Any]: """Return a plain dict copy of the record data (excludes metadata).""" return dict(self.data) + + +class QueryResult: + """Iterable wrapper around a list of :class:`Record` objects. + + Returned by :meth:`~PowerPlatform.Dataverse.models.query_builder.QueryBuilder.execute` + (flat mode) and :meth:`~PowerPlatform.Dataverse.operations.records.RecordOperations.list`. + + Backward-compatible: ``for r in result`` continues to work without change. + + :param records: Collected records from all pages. + :type records: list[:class:`Record`] + """ + + def __init__(self, records: List[Record]) -> None: + self.records: List[Record] = records + + def __iter__(self) -> Iterator[Record]: + return iter(self.records) + + def __len__(self) -> int: + return len(self.records) + + def __bool__(self) -> bool: + return bool(self.records) + + def __repr__(self) -> str: + return f"QueryResult({len(self.records)} records)" + + def __getitem__(self, index: Union[int, slice]) -> Union[Record, "QueryResult"]: + result = self.records[index] + return QueryResult(result) if isinstance(index, slice) else result + + def first(self) -> Optional[Record]: + """Return the first record, or ``None`` if the result is empty.""" + return self.records[0] if self.records else None + + def to_dataframe(self) -> Any: + """Return all records as a pandas DataFrame. + + :raises ImportError: If pandas is not installed. + :rtype: ~pandas.DataFrame + """ + try: + import pandas as pd + except ImportError as exc: + raise ImportError("pandas is required for to_dataframe(). " "Install it with: pip install pandas") from exc + + if not self.records: + return pd.DataFrame() + rows = [r.data if hasattr(r, "data") else dict(r) for r in self.records] + return pd.DataFrame.from_records(rows) diff --git a/src/PowerPlatform/Dataverse/operations/batch.py b/src/PowerPlatform/Dataverse/operations/batch.py index b2751d93..aa5d8391 100644 --- a/src/PowerPlatform/Dataverse/operations/batch.py +++ b/src/PowerPlatform/Dataverse/operations/batch.py @@ -5,6 +5,7 @@ from __future__ import annotations +import warnings from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union import pandas as pd @@ -18,6 +19,7 @@ _RecordUpdate, _RecordDelete, _RecordGet, + _RecordList, _RecordUpsert, _TableCreate, _TableDelete, @@ -43,6 +45,7 @@ if TYPE_CHECKING: from ..client import DataverseClient + from ..models.filters import FilterExpression __all__ = [ "BatchRecordOperations", @@ -166,11 +169,14 @@ class BatchRecordOperations: """ Record operations on a :class:`BatchRequest`. - Mirrors ``client.records`` exactly: same method names, same signatures. + Mirrors ``client.records``: same method names, same signatures. All methods return ``None``; results are available via :class:`~PowerPlatform.Dataverse.models.batch.BatchResult` after :meth:`BatchRequest.execute`. + GA methods: :meth:`retrieve` (single record) and :meth:`list` (multi-record, + single page). :meth:`get` is deprecated — use :meth:`retrieve` instead. + Do not instantiate directly; use ``batch.records``. """ @@ -253,16 +259,9 @@ def get( """ Add a single-record get operation to the batch. - Only the single-record overload (``record_id`` provided) is supported. - The paginated/multi-record overload of ``client.records.get()`` - (``filter``, ``orderby``, etc., without ``record_id``) is **not** - supported in batch — pagination requires following - ``@odata.nextLink`` across multiple round-trips, which is - incompatible with a single batch request. - - The response body will be available in - :attr:`~PowerPlatform.Dataverse.models.batch.BatchItemResponse.data` - after :meth:`BatchRequest.execute`. + .. deprecated:: + Use :meth:`retrieve` instead. ``batch.records.get()`` is deprecated + and will be removed in a future release. :param table: Table schema name. :type table: :class:`str` @@ -271,6 +270,11 @@ def get( :param select: Optional list of column names to include. :type select: list[str] or None """ + warnings.warn( + "'batch.records.get()' is deprecated; use 'batch.records.retrieve(table, record_id)' instead.", + DeprecationWarning, + stacklevel=2, + ) self._batch._items.append(_RecordGet(table=table, record_id=record_id, select=select)) def upsert( @@ -325,6 +329,136 @@ def upsert( raise TypeError("Each item must be an UpsertItem or a dict with 'alternate_key' and 'record' keys") self._batch._items.append(_RecordUpsert(table=table, items=normalized)) + def retrieve( + self, + table: str, + record_id: str, + *, + select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, + include_annotations: Optional[str] = None, + ) -> None: + """ + Add a single-record retrieve operation to the batch. + + GA replacement for the deprecated :meth:`get`. Enqueues a GET request + for one record by its GUID. The response body will be available in + :attr:`~PowerPlatform.Dataverse.models.batch.BatchItemResponse.data` + after :meth:`BatchRequest.execute`. + + :param table: Table schema name (e.g. ``"account"``). + :type table: :class:`str` + :param record_id: GUID of the record to retrieve. + :type record_id: :class:`str` + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :param expand: Optional list of navigation properties to expand. + Navigation property names are case-sensitive and must match the + entity's ``$metadata``. + :type expand: list[str] or None + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header (e.g. ``"*"`` or + ``"OData.Community.Display.V1.FormattedValue"``), or ``None``. + :type include_annotations: :class:`str` or None + + Example:: + + batch = client.batch.new() + batch.records.retrieve( + "account", account_id, + select=["name", "statuscode"], + expand=["primarycontactid"], + include_annotations="OData.Community.Display.V1.FormattedValue", + ) + result = batch.execute() + record = result.responses[0].data + contact = (record.get("primarycontactid") or {}) + print(contact.get("fullname")) + """ + self._batch._items.append( + _RecordGet( + table=table, + record_id=record_id, + select=select, + expand=expand, + include_annotations=include_annotations, + ) + ) + + def list( + self, + table: str, + *, + filter: "Optional[Union[str, FilterExpression]]" = None, + select: Optional[List[str]] = None, + orderby: Optional[List[str]] = None, + top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, + ) -> None: + """ + Add a multi-record list operation to the batch (single page, no pagination). + + Enqueues a GET request for multiple records. Because batch requests are + a single HTTP round-trip, pagination (``@odata.nextLink``) is **not** + supported — use ``top`` to bound the result size, or rely on the + server's default page limit. + + The response body (``{"value": [...]}`` JSON) will be available in + :attr:`~PowerPlatform.Dataverse.models.batch.BatchItemResponse.data` + after :meth:`BatchRequest.execute`. + + :param table: Table schema name (e.g. ``"account"``). + :type table: :class:`str` + :param filter: Optional OData ``$filter`` expression or :class:`FilterExpression`. + :type filter: str or FilterExpression or None + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :param orderby: Optional list of sort expressions (e.g. ``["name asc"]``). + :type orderby: list[str] or None + :param top: Maximum number of records to return. + :type top: int or None + :param expand: Optional list of navigation properties to expand. + :type expand: list[str] or None + :param page_size: Per-page size hint via ``Prefer: odata.maxpagesize``. + :type page_size: int or None + :param count: If ``True``, adds ``$count=true`` to the request. + :type count: bool + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header, or ``None``. + :type include_annotations: :class:`str` or None + + Example:: + + batch = client.batch.new() + batch.records.list( + "account", + filter="statecode eq 0", + select=["name", "statuscode"], + orderby=["name asc"], + top=50, + include_annotations="OData.Community.Display.V1.FormattedValue", + ) + result = batch.execute() + records = result.responses[0].data.get("value", []) + """ + filter_str: Optional[str] = str(filter) if filter is not None else None + self._batch._items.append( + _RecordList( + table=table, + select=select, + filter=filter_str, + orderby=orderby, + top=top, + expand=expand, + page_size=page_size, + count=count, + include_annotations=include_annotations, + ) + ) + class BatchTableOperations: """ diff --git a/src/PowerPlatform/Dataverse/operations/dataframe.py b/src/PowerPlatform/Dataverse/operations/dataframe.py index e6ec2033..28647c58 100644 --- a/src/PowerPlatform/Dataverse/operations/dataframe.py +++ b/src/PowerPlatform/Dataverse/operations/dataframe.py @@ -5,6 +5,7 @@ from __future__ import annotations +import warnings from typing import TYPE_CHECKING, Any, Dict, List, Optional import pandas as pd @@ -164,6 +165,12 @@ def get( df = client.dataframe.get("account", select=["name"], top=100) """ + warnings.warn( + "'dataframe.get()' is deprecated; use " + "client.query.builder(table).where(...).execute().to_dataframe() instead.", + DeprecationWarning, + stacklevel=2, + ) if record_id is not None: if not isinstance(record_id, str) or not record_id.strip(): raise ValueError("record_id must be a non-empty string") @@ -173,25 +180,30 @@ def get( "Cannot specify query parameters (filter, orderby, top, " "expand, page_size) when fetching a single record by ID" ) - result = self._client.records.get( - table, - record_id, - select=select, - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = self._client.records.get( + table, + record_id, + select=select, + ) return pd.DataFrame([result.data]) rows: List[dict] = [] - for batch in self._client.records.get( - table, - select=select, - filter=filter, - orderby=orderby, - top=top, - expand=expand, - page_size=page_size, - count=count, - include_annotations=include_annotations, - ): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + pages = self._client.records.get( + table, + select=select, + filter=filter, + orderby=orderby, + top=top, + expand=expand, + page_size=page_size, + count=count, + include_annotations=include_annotations, + ) + for batch in pages: rows.extend(row.data for row in batch) if not rows: diff --git a/src/PowerPlatform/Dataverse/operations/query.py b/src/PowerPlatform/Dataverse/operations/query.py index 83e82677..1f3c8ef2 100644 --- a/src/PowerPlatform/Dataverse/operations/query.py +++ b/src/PowerPlatform/Dataverse/operations/query.py @@ -5,9 +5,13 @@ from __future__ import annotations +import warnings +import xml.etree.ElementTree as _ET from typing import Any, Dict, List, Optional, TYPE_CHECKING +from urllib.parse import quote as _url_quote -from ..core.errors import MetadataError +from ..core.errors import MetadataError, ValidationError +from ..models.fetchxml_query import FetchXmlQuery, _MAX_URL_LENGTH from ..models.record import Record from ..models.query_builder import QueryBuilder @@ -29,12 +33,14 @@ class QueryOperations: Example:: + from PowerPlatform.Dataverse.models.filters import col + client = DataverseClient(base_url, credential) # Fluent query builder (recommended) for record in (client.query.builder("account") .select("name", "revenue") - .filter_eq("statecode", 0) + .where(col("statecode") == 0) .order_by("revenue", descending=True) .top(100) .execute()): @@ -66,10 +72,12 @@ def builder(self, table: str) -> QueryBuilder: Example: Build and execute a query fluently:: + from PowerPlatform.Dataverse.models.filters import col + for record in (client.query.builder("account") .select("name", "revenue") - .filter_eq("statecode", 0) - .filter_gt("revenue", 1000000) + .where(col("statecode") == 0) + .where(col("revenue") > 1_000_000) .order_by("revenue", descending=True) .top(100) .page_size(50) @@ -78,11 +86,11 @@ def builder(self, table: str) -> QueryBuilder: With composable expression tree:: - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col for record in (client.query.builder("account") - .where((eq("statecode", 0) | eq("statecode", 1)) - & gt("revenue", 100000)) + .where((col("statecode") == 0) | (col("statecode") == 1)) + .where(col("revenue") > 100_000) .execute()): print(record["name"]) """ @@ -146,6 +154,80 @@ def sql(self, sql: str) -> List[Record]: rows = od._query_sql(sql) return [Record.from_api_response("", row) for row in rows] + # --------------------------------------------------------------- fetchxml + + def fetchxml(self, xml: str) -> FetchXmlQuery: + """Return an inert :class:`~PowerPlatform.Dataverse.models.fetchxml_query.FetchXmlQuery` object. + + No HTTP request is made until + :meth:`~PowerPlatform.Dataverse.models.fetchxml_query.FetchXmlQuery.execute` + or + :meth:`~PowerPlatform.Dataverse.models.fetchxml_query.FetchXmlQuery.execute_pages` + is called on the returned object. + + Use for SQL-JOIN scenarios, aggregate queries, or other operations that + the OData builder endpoint cannot express. + + :param xml: Well-formed FetchXML query string. The root ```` + element determines the entity set endpoint. + :type xml: :class:`str` + :return: Inert query object with ``.execute()`` and ``.execute_pages()`` methods. + :rtype: :class:`~PowerPlatform.Dataverse.models.fetchxml_query.FetchXmlQuery` + :raises ValueError: If the FetchXML is missing a root ```` element + or the entity ``name`` attribute. + + Example:: + + query = client.query.fetchxml(\"\"\" + + + + + + + + + \"\"\") + + # Eager — collect all pages: + result = query.execute() + df = result.to_dataframe() + + # Lazy — process one page at a time: + for page in query.execute_pages(): + process(page.to_dataframe()) + """ + if not isinstance(xml, str): + raise ValidationError("xml must be a string") + xml = xml.strip() + if not xml: + raise ValidationError("xml must not be empty") + # Fast-fail before any HTTP is attempted; execute_pages() re-checks the full URL + # (base + encoded XML) on each page. + if len(_url_quote(xml, safe="")) > _MAX_URL_LENGTH: + raise ValidationError( + f"FetchXML exceeds the Dataverse URL length limit ({_MAX_URL_LENGTH:,} characters) when encoded. " + "Use a $batch POST request to send FetchXML in the request body where the limit is 64 KB." + ) + # Parse only to verify well-formedness and extract the entity name needed for the + # request URL. Structural and semantic validation is intentionally left to the server + # to avoid duplicating rules that may diverge from Dataverse's own enforcement. + # ElementTree does not resolve external entities or expand recursive internal entity + # references, so pathological inputs of that kind raise ParseError rather than + # consuming resources. + try: + root_el = _ET.fromstring(xml) + except _ET.ParseError as exc: + raise ValidationError(f"xml is not well-formed: {exc}") from exc + entity_el = root_el.find("entity") + if entity_el is None: + raise ValueError("FetchXML must contain an child element") + entity_name = entity_el.get("name", "") + if not entity_name: + raise ValueError("FetchXML element must have a 'name' attribute") + return FetchXmlQuery(xml, entity_name, self._client) + # --------------------------------------------------------------- sql_columns def sql_columns( @@ -230,178 +312,6 @@ def sql_columns( result.sort(key=lambda x: (not x["is_pk"], not x["is_name"], x["name"])) return result - # --------------------------------------------------------------- sql_select - - def sql_select( - self, - table: str, - *, - include_system: bool = False, - ) -> str: - """Return a comma-separated column list for use in SQL SELECT. - - Excludes virtual columns and optionally system columns. The result - can be embedded directly in a SQL query string. - - :param table: Schema name of the table (e.g. ``"account"``). - :type table: :class:`str` - :param include_system: Include system columns (default ``False``). - :type include_system: :class:`bool` - - :return: Comma-separated column names. - :rtype: :class:`str` - - Example:: - - cols = client.query.sql_select("account") - sql = f"SELECT TOP 10 {cols} FROM account" - df = client.dataframe.sql(sql) - """ - columns = self.sql_columns(table, include_system=include_system) - return ", ".join(c["name"] for c in columns) - - # --------------------------------------------------------------- sql_joins - - def sql_joins( - self, - table: str, - ) -> List[Dict[str, Any]]: - """Discover all possible SQL JOINs from a table. - - Returns one entry per outgoing lookup relationship, with the - exact column names needed for SQL ``JOIN ... ON`` clauses. - - For **polymorphic** lookups (e.g. ``customerid`` targeting both - ``account`` and ``contact``), multiple entries are returned with - the same ``column`` but different ``target`` values. - - :param table: Schema name of the table (e.g. ``"contact"``). - :type table: :class:`str` - - :return: List of JOIN metadata dicts, each containing: - - - ``column`` -- the lookup attribute on this table (use in ON clause) - - ``target`` -- the referenced entity name - - ``target_pk`` -- the referenced entity's primary key column - - ``relationship`` -- the schema name of the relationship - - ``join_clause`` -- a ready-to-use ``JOIN ... ON ...`` fragment - - :rtype: list[dict[str, typing.Any]] - - .. note:: - - The ``join_clause`` value references the source table by its - **full name** (e.g. ``ON contact.col = ...``), so the FROM - clause must also use the unaliased table name. For queries - that need aliases, use :meth:`sql_join` instead. - - Example:: - - joins = client.query.sql_joins("contact") - for j in joins: - print(f"{j['column']:30s} -> {j['target']}.{j['target_pk']}") - print(f" {j['join_clause']}") - - # Use in a query (no alias on the FROM table) - j = next(j for j in joins if j['target'] == 'account') - sql = f"SELECT TOP 10 contact.fullname, a.name FROM contact {j['join_clause']}" - """ - table_lower = table.lower() - rels = self._client.tables.list_table_relationships(table) - - used_aliases: set = set() - result: List[Dict[str, Any]] = [] - for r in rels: - ref_entity = (r.get("ReferencingEntity") or "").lower() - if ref_entity != table_lower: - continue - col = r.get("ReferencingAttribute", "") - target = r.get("ReferencedEntity", "") - target_pk = r.get("ReferencedAttribute", "") - schema = r.get("SchemaName", "") - if not all([col, target, target_pk]): - continue - - # Generate a unique alias — add a numeric suffix on collision so - # two lookups to tables starting with the same letter (e.g. - # "account" and "annotation") or two lookups to the same table - # (e.g. "ownerid" and "createdby" both to "systemuser") produce - # distinct aliases and valid SQL. - base = target[0] if target else "j" - alias = base - counter = 2 - while alias in used_aliases: - alias = f"{base}{counter}" - counter += 1 - used_aliases.add(alias) - join_clause = f"JOIN {target} {alias} ON {table_lower}.{col} = {alias}.{target_pk}" - - result.append( - { - "column": col, - "target": target, - "target_pk": target_pk, - "relationship": schema, - "join_clause": join_clause, - } - ) - - result.sort(key=lambda x: (x["target"], x["column"])) - return result - - # --------------------------------------------------------------- sql_join - - def sql_join( - self, - from_table: str, - to_table: str, - *, - from_alias: Optional[str] = None, - to_alias: Optional[str] = None, - ) -> str: - """Generate a SQL JOIN clause between two tables. - - Discovers the relationship automatically via metadata. If multiple - relationships exist (e.g. polymorphic lookups), picks the first - match. Use :meth:`sql_joins` to see all options. - - :param from_table: Schema name of the FROM table (e.g. ``"contact"``). - :type from_table: :class:`str` - :param to_table: Schema name of the target table (e.g. ``"account"``). - :type to_table: :class:`str` - :param from_alias: Optional alias for the FROM table in the JOIN - clause. If ``None``, uses the full table name. - :type from_alias: :class:`str` or None - :param to_alias: Optional alias for the target table. If ``None``, - uses the first letter of the target table name. - :type to_alias: :class:`str` or None - - :return: A ready-to-use ``JOIN ... ON ...`` clause. - :rtype: :class:`str` - - :raises ValueError: If no relationship is found between the tables. - - Example:: - - j = client.query.sql_join("contact", "account", from_alias="c", to_alias="a") - # Returns: "JOIN account a ON c.parentcustomerid = a.accountid" - sql = f"SELECT TOP 10 c.fullname, a.name FROM contact c {j}" - df = client.dataframe.sql(sql) - """ - to_lower = to_table.lower() - joins = self.sql_joins(from_table) - match = [j for j in joins if j["target"].lower() == to_lower] - if not match: - raise ValueError( - f"No relationship found from '{from_table}' to '{to_table}'. " - f"Use client.query.sql_joins('{from_table}') to see available targets." - ) - - j = match[0] - src = from_alias or from_table.lower() - tgt = to_alias or to_lower[0] - return f"JOIN {to_lower} {tgt} " f"ON {src}.{j['column']} = {tgt}.{j['target_pk']}" - # =========================================================== # OData helpers -- eliminate friction for records.get() users # =========================================================== @@ -433,6 +343,12 @@ def odata_select( for r in page: print(r) """ + warnings.warn( + "'odata_select' is deprecated; use the typed builder (1.x) " + "or client.query.sql_columns() to discover columns.", + DeprecationWarning, + stacklevel=2, + ) columns = self.sql_columns(table, include_system=include_system) return [c["name"] for c in columns] @@ -545,6 +461,12 @@ def odata_expand( acct = r.get(nav) or {} print(f"{r['fullname']} -> {acct.get('name', 'N/A')}") """ + warnings.warn( + "'odata_expand' is deprecated; use the typed builder (1.x) " + "with .expand() or client.query.odata_expands() to discover navigation properties.", + DeprecationWarning, + stacklevel=2, + ) to_lower = to_table.lower() expands = self.odata_expands(from_table) match = [e for e in expands if e["target_table"].lower() == to_lower] @@ -594,6 +516,11 @@ def odata_bind( **bind, }) """ + warnings.warn( + "'odata_bind' is deprecated; use the typed builder (1.x) " "or pass the @odata.bind dict manually.", + DeprecationWarning, + stacklevel=2, + ) to_lower = to_table.lower() expands = self.odata_expands(from_table) match = [e for e in expands if e["target_table"].lower() == to_lower and e["target_entity_set"]] diff --git a/src/PowerPlatform/Dataverse/operations/records.py b/src/PowerPlatform/Dataverse/operations/records.py index ef867f52..c9c66119 100644 --- a/src/PowerPlatform/Dataverse/operations/records.py +++ b/src/PowerPlatform/Dataverse/operations/records.py @@ -5,12 +5,15 @@ from __future__ import annotations -from typing import Any, Dict, Iterable, List, Optional, Union, overload, TYPE_CHECKING +import warnings +from typing import Any, Dict, Iterable, Iterator, List, Optional, Union, overload, TYPE_CHECKING -from ..models.record import Record +from ..core.errors import HttpError +from ..models.record import QueryResult, Record from ..models.upsert import UpsertItem if TYPE_CHECKING: + from ..models.filters import FilterExpression from ..client import DataverseClient @@ -151,6 +154,7 @@ def update( [id1, id2], [{"name": "Name A"}, {"name": "Name B"}], ) + """ with self._client._scoped_odata() as od: if isinstance(ids, str): @@ -395,10 +399,12 @@ def get( Only used for multi-record queries. :type include_annotations: :class:`str` or None - :return: A single record dict when ``record_id`` is provided, or a - generator yielding pages (lists of record dicts) when fetching - multiple records. - :rtype: dict or collections.abc.Iterable[list[dict]] + :return: A single :class:`~PowerPlatform.Dataverse.models.record.Record` + when ``record_id`` is provided, or a generator yielding pages + (lists of :class:`~PowerPlatform.Dataverse.models.record.Record`) + when fetching multiple records. + :rtype: ~PowerPlatform.Dataverse.models.record.Record or + collections.abc.Iterable[list[~PowerPlatform.Dataverse.models.record.Record]] :raises TypeError: If ``record_id`` is provided but not a string. :raises ValueError: If query parameters are provided alongside @@ -424,6 +430,12 @@ def get( print(record["name"]) """ if record_id is not None: + warnings.warn( + "'records.get()' with a record_id is deprecated; " + "use 'client.records.retrieve(table, record_id)' instead.", + DeprecationWarning, + stacklevel=2, + ) if not isinstance(record_id, str): raise TypeError("record_id must be str") if ( @@ -444,6 +456,12 @@ def get( raw = od._get(table, record_id, select=select) return Record.from_api_response(table, raw, record_id=record_id) + warnings.warn( + "'records.get()' is deprecated; " "use 'client.records.list(table, filter=...)' instead.", + DeprecationWarning, + stacklevel=2, + ) + def _paged() -> Iterable[List[Record]]: with self._client._scoped_odata() as od: for page in od._get_multiple( @@ -461,6 +479,202 @@ def _paged() -> Iterable[List[Record]]: return _paged() + # --------------------------------------------------------------- retrieve + + def retrieve( + self, + table: str, + record_id: str, + *, + select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, + include_annotations: Optional[str] = None, + ) -> Optional[Record]: + """Fetch a single record by its GUID, returning ``None`` if not found. + + GA replacement for ``records.get(table, record_id)``. Returns ``None`` + instead of raising when the record does not exist (HTTP 404). + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param record_id: GUID of the record to retrieve. + :type record_id: :class:`str` + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :param expand: Optional list of navigation properties to expand (e.g. + ``["primarycontactid"]``). Navigation property names are + case-sensitive and must match the entity's ``$metadata``. + :type expand: list[str] or None + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header (e.g. ``"*"`` or + ``"OData.Community.Display.V1.FormattedValue"``), or ``None``. + :type include_annotations: :class:`str` or None + :return: Typed record, or ``None`` if not found. + :rtype: :class:`~PowerPlatform.Dataverse.models.record.Record` or None + + Example:: + + record = client.records.retrieve( + "account", account_id, + select=["name", "statuscode"], + expand=["primarycontactid"], + include_annotations="OData.Community.Display.V1.FormattedValue", + ) + if record is not None: + contact = record.get("primarycontactid") or {} + print(contact.get("fullname")) + """ + with self._client._scoped_odata() as od: + try: + raw = od._get(table, record_id, select=select, expand=expand, include_annotations=include_annotations) + except HttpError as exc: + if exc.status_code == 404: + return None + raise + return Record.from_api_response(table, raw, record_id=record_id) + + # ------------------------------------------------------------------ list + + def list( + self, + table: str, + *, + filter: Optional[Union[str, "FilterExpression"]] = None, + select: Optional[List[str]] = None, + orderby: Optional[List[str]] = None, + top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, + ) -> QueryResult: + """Fetch multiple records and return them as a :class:`QueryResult`. + + GA replacement for ``records.get(table, filter=...)``. All pages are + collected eagerly and returned as a single :class:`QueryResult`. + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param filter: Optional OData filter string or :class:`FilterExpression`. + :type filter: str or FilterExpression or None + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :param orderby: Optional list of sort expressions (e.g. ``["name asc", "createdon desc"]``). + :type orderby: list[str] or None + :param top: Maximum total number of records to return. + :type top: int or None + :param expand: Optional list of navigation properties to expand. + :type expand: list[str] or None + :param page_size: Per-page size hint via ``Prefer: odata.maxpagesize``. + :type page_size: int or None + :param count: If ``True``, adds ``$count=true`` to include a total record count. + :type count: bool + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header, or ``None``. + :type include_annotations: :class:`str` or None + :return: All matching records collected into a :class:`QueryResult`. + :rtype: :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + + Example:: + + from PowerPlatform.Dataverse import col + + result = client.records.list( + "account", + filter=col("statecode") == 0, + select=["name", "statuscode"], + orderby=["name asc"], + top=100, + include_annotations="OData.Community.Display.V1.FormattedValue", + ) + for record in result: + print(record["name"], record.get("statuscode@OData.Community.Display.V1.FormattedValue")) + """ + filter_str: Optional[str] = str(filter) if filter is not None else None + all_records: List[Record] = [] + with self._client._scoped_odata() as od: + for page in od._get_multiple( + table, + select=select, + filter=filter_str, + orderby=orderby, + top=top, + expand=expand, + page_size=page_size, + count=count, + include_annotations=include_annotations, + ): + all_records.extend(Record.from_api_response(table, row) for row in page) + return QueryResult(all_records) + + # --------------------------------------------------------------- list_pages + + def list_pages( + self, + table: str, + *, + filter: Optional[Union[str, "FilterExpression"]] = None, + select: Optional[List[str]] = None, + orderby: Optional[List[str]] = None, + top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, + ) -> Iterator[QueryResult]: + """Lazily yield one :class:`QueryResult` per HTTP page. + + Streaming counterpart to :meth:`list`. Each iteration triggers one + network request via ``@odata.nextLink``. One-shot — do not iterate + more than once. + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param filter: Optional OData filter string or :class:`FilterExpression`. + :type filter: str or FilterExpression or None + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :param orderby: Optional list of sort expressions (e.g. ``["name asc", "createdon desc"]``). + :type orderby: list[str] or None + :param top: Maximum total number of records to return. + :type top: int or None + :param expand: Optional list of navigation properties to expand. + :type expand: list[str] or None + :param page_size: Per-page size hint via ``Prefer: odata.maxpagesize``. + :type page_size: int or None + :param count: If ``True``, adds ``$count=true`` to include a total record count. + :type count: bool + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header, or ``None``. + :type include_annotations: :class:`str` or None + :return: Iterator of per-page :class:`QueryResult` objects. + :rtype: Iterator[:class:`~PowerPlatform.Dataverse.models.record.QueryResult`] + + Example:: + + for page in client.records.list_pages( + "account", + filter="statecode eq 0", + orderby=["name asc"], + page_size=200, + ): + process(page.to_dataframe()) + """ + filter_str: Optional[str] = str(filter) if filter is not None else None + with self._client._scoped_odata() as od: + for page in od._get_multiple( + table, + select=select, + filter=filter_str, + orderby=orderby, + top=top, + expand=expand, + page_size=page_size, + count=count, + include_annotations=include_annotations, + ): + yield QueryResult([Record.from_api_response(table, row) for row in page]) + # ------------------------------------------------------------------ upsert def upsert(self, table: str, items: List[Union[UpsertItem, Dict[str, Any]]]) -> None: diff --git a/tests/unit/data/test_batch_serialization.py b/tests/unit/data/test_batch_serialization.py index 9cdfcb24..a1c293df 100644 --- a/tests/unit/data/test_batch_serialization.py +++ b/tests/unit/data/test_batch_serialization.py @@ -14,6 +14,7 @@ _RecordCreate, _RecordDelete, _RecordGet, + _RecordList, _RecordUpdate, _RecordUpsert, _TableCreate, @@ -252,7 +253,89 @@ def test_resolve_record_get(self): op = _RecordGet(table="account", record_id="guid-1", select=["name"]) result = client._resolve_record_get(op) - od._build_get.assert_called_once_with("account", "guid-1", select=["name"]) + od._build_get.assert_called_once_with( + "account", "guid-1", select=["name"], expand=None, include_annotations=None + ) + self.assertEqual(result, [mock_req]) + + def test_resolve_record_get_with_expand(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_get.return_value = mock_req + + op = _RecordGet(table="account", record_id="guid-1", select=["name"], expand=["primarycontactid"]) + result = client._resolve_record_get(op) + + od._build_get.assert_called_once_with( + "account", "guid-1", select=["name"], expand=["primarycontactid"], include_annotations=None + ) + self.assertEqual(result, [mock_req]) + + def test_resolve_record_get_with_annotations(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_get.return_value = mock_req + + annotation = "OData.Community.Display.V1.FormattedValue" + op = _RecordGet(table="account", record_id="guid-1", select=["name"], include_annotations=annotation) + result = client._resolve_record_get(op) + + od._build_get.assert_called_once_with( + "account", "guid-1", select=["name"], expand=None, include_annotations=annotation + ) + self.assertEqual(result, [mock_req]) + + def test_resolve_record_list_basic(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_list.return_value = mock_req + + op = _RecordList(table="account", select=["name"], filter="statecode eq 0", top=10) + result = client._resolve_record_list(op) + + od._build_list.assert_called_once_with( + "account", + select=["name"], + filter="statecode eq 0", + orderby=None, + top=10, + expand=None, + page_size=None, + count=False, + include_annotations=None, + ) + self.assertEqual(result, [mock_req]) + + def test_resolve_record_list_all_params(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_list.return_value = mock_req + + annotation = "OData.Community.Display.V1.FormattedValue" + op = _RecordList( + table="account", + select=["name"], + filter="statecode eq 0", + orderby=["name asc"], + top=50, + expand=["primarycontactid"], + page_size=100, + count=True, + include_annotations=annotation, + ) + result = client._resolve_record_list(op) + + od._build_list.assert_called_once_with( + "account", + select=["name"], + filter="statecode eq 0", + orderby=["name asc"], + top=50, + expand=["primarycontactid"], + page_size=100, + count=True, + include_annotations=annotation, + ) self.assertEqual(result, [mock_req]) def test_resolve_record_delete_single(self): @@ -599,6 +682,75 @@ def test_upsert_multiple_items_all_normalised(self): self.assertEqual(intent.items[1].alternate_key, {"accountnumber": "B"}) +class TestBatchRecordOperationsList(unittest.TestCase): + """Tests for BatchRecordOperations.list() surface (operations/batch.py).""" + + def _make_batch(self): + from PowerPlatform.Dataverse.operations.batch import BatchRecordOperations + + batch = MagicMock() + batch._items = [] + return BatchRecordOperations(batch), batch + + def test_list_basic_appends_record_list(self): + rec_ops, batch = self._make_batch() + rec_ops.list("account", filter="statecode eq 0", select=["name"], top=10) + + self.assertEqual(len(batch._items), 1) + intent = batch._items[0] + self.assertIsInstance(intent, _RecordList) + self.assertEqual(intent.table, "account") + self.assertEqual(intent.filter, "statecode eq 0") + self.assertEqual(intent.select, ["name"]) + self.assertEqual(intent.top, 10) + + def test_list_passes_orderby(self): + rec_ops, batch = self._make_batch() + rec_ops.list("account", orderby=["name asc"]) + self.assertEqual(batch._items[0].orderby, ["name asc"]) + + def test_list_passes_expand(self): + rec_ops, batch = self._make_batch() + rec_ops.list("account", expand=["primarycontactid"]) + self.assertEqual(batch._items[0].expand, ["primarycontactid"]) + + def test_list_passes_page_size(self): + rec_ops, batch = self._make_batch() + rec_ops.list("account", page_size=200) + self.assertEqual(batch._items[0].page_size, 200) + + def test_list_passes_count(self): + rec_ops, batch = self._make_batch() + rec_ops.list("account", count=True) + self.assertTrue(batch._items[0].count) + + def test_list_passes_include_annotations(self): + annotation = "OData.Community.Display.V1.FormattedValue" + rec_ops, batch = self._make_batch() + rec_ops.list("account", include_annotations=annotation) + self.assertEqual(batch._items[0].include_annotations, annotation) + + def test_list_filter_expression_converted_to_str(self): + from PowerPlatform.Dataverse.models.filters import col + + rec_ops, batch = self._make_batch() + rec_ops.list("account", filter=col("statecode") == 0) + self.assertEqual(batch._items[0].filter, "statecode eq 0") + + def test_list_defaults(self): + rec_ops, batch = self._make_batch() + rec_ops.list("account") + intent = batch._items[0] + self.assertIsNone(intent.filter) + self.assertIsNone(intent.select) + self.assertIsNone(intent.orderby) + self.assertIsNone(intent.top) + self.assertIsNone(intent.expand) + self.assertIsNone(intent.page_size) + self.assertFalse(intent.count) + self.assertIsNone(intent.include_annotations) + + class TestRaiseTopLevelBatchError(unittest.TestCase): """_raise_top_level_batch_error surfaces Dataverse error details as HttpError.""" diff --git a/tests/unit/models/test_query_builder.py b/tests/unit/models/test_query_builder.py index 9f094912..f40d1e5d 100644 --- a/tests/unit/models/test_query_builder.py +++ b/tests/unit/models/test_query_builder.py @@ -51,349 +51,120 @@ def test_select_returns_self(self): self.assertIs(qb.select("name"), qb) -class TestComparisonFilters(unittest.TestCase): - """Tests for comparison filter methods.""" +class TestRemovedFilterMethods(unittest.TestCase): + """Verify all 16 filter_* builder methods were removed in 1.0 GA.""" - def test_filter_eq_string(self): - qb = QueryBuilder("account").filter_eq("name", "Contoso") - self.assertEqual(qb.build()["filter"], "name eq 'Contoso'") - - def test_filter_eq_integer(self): - qb = QueryBuilder("account").filter_eq("statecode", 0) - self.assertEqual(qb.build()["filter"], "statecode eq 0") - - def test_filter_eq_boolean_true(self): - qb = QueryBuilder("account").filter_eq("active", True) - self.assertEqual(qb.build()["filter"], "active eq true") - - def test_filter_eq_boolean_false(self): - qb = QueryBuilder("account").filter_eq("active", False) - self.assertEqual(qb.build()["filter"], "active eq false") - - def test_filter_eq_none(self): - qb = QueryBuilder("account").filter_eq("telephone1", None) - self.assertEqual(qb.build()["filter"], "telephone1 eq null") - - def test_filter_eq_float(self): - qb = QueryBuilder("account").filter_eq("revenue", 1000000.5) - self.assertEqual(qb.build()["filter"], "revenue eq 1000000.5") - - def test_filter_eq_datetime(self): - from datetime import datetime, timezone - - dt = datetime(2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc) - qb = QueryBuilder("account").filter_eq("createdon", dt) - self.assertEqual(qb.build()["filter"], "createdon eq 2024-01-15T10:30:00Z") - - -class TestFilterIn(unittest.TestCase): - """Tests for the filter_in() method.""" - - def test_filter_in_integers(self): - qb = QueryBuilder("account").filter_in("statecode", [0, 1, 2]) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1","2"])', - ) - - def test_filter_in_strings(self): - qb = QueryBuilder("account").filter_in("name", ["Contoso", "Fabrikam"]) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.In(PropertyName=\'name\',PropertyValues=["Contoso","Fabrikam"])', - ) - - def test_filter_in_single_value(self): - qb = QueryBuilder("account").filter_in("statecode", [0]) - self.assertEqual( - qb.build()["filter"], - "Microsoft.Dynamics.CRM.In(PropertyName='statecode',PropertyValues=[\"0\"])", - ) - - def test_filter_in_column_lowercased(self): - qb = QueryBuilder("account").filter_in("StateCode", [0, 1]) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1"])', - ) - - def test_filter_in_empty_raises(self): - with self.assertRaises(ValueError): - QueryBuilder("account").filter_in("statecode", []) - - def test_filter_in_returns_self(self): - qb = QueryBuilder("account") - self.assertIs(qb.filter_in("statecode", [0, 1]), qb) - - def test_filter_in_with_set(self): - qb = QueryBuilder("account").filter_in("statecode", {0, 1}) - result = qb.build()["filter"] - self.assertIn("Microsoft.Dynamics.CRM.In", result) - self.assertIn("statecode", result) - - def test_filter_in_with_tuple(self): - qb = QueryBuilder("account").filter_in("statecode", (0, 1, 2)) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1","2"])', - ) - - def test_filter_in_int_enum(self): - from enum import IntEnum - - class Priority(IntEnum): - LOW = 1 - HIGH = 3 - - qb = QueryBuilder("account").filter_in("priority", [Priority.LOW, Priority.HIGH]) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.In(PropertyName=\'priority\',PropertyValues=["1","3"])', - ) - - def test_filter_in_combined_with_other_filters(self): - qb = QueryBuilder("account").filter_eq("statecode", 0).filter_in("priority", [1, 2, 3]) - self.assertEqual( - qb.build()["filter"], - 'statecode eq 0 and Microsoft.Dynamics.CRM.In(PropertyName=\'priority\',PropertyValues=["1","2","3"])', - ) - - def test_filter_ne(self): - qb = QueryBuilder("account").filter_ne("statecode", 1) - self.assertEqual(qb.build()["filter"], "statecode ne 1") - - def test_filter_gt(self): - qb = QueryBuilder("account").filter_gt("revenue", 1000000) - self.assertEqual(qb.build()["filter"], "revenue gt 1000000") - - def test_filter_ge(self): - qb = QueryBuilder("account").filter_ge("revenue", 1000000) - self.assertEqual(qb.build()["filter"], "revenue ge 1000000") - - def test_filter_lt(self): - qb = QueryBuilder("account").filter_lt("revenue", 500000) - self.assertEqual(qb.build()["filter"], "revenue lt 500000") - - def test_filter_le(self): - qb = QueryBuilder("account").filter_le("revenue", 500000) - self.assertEqual(qb.build()["filter"], "revenue le 500000") - - def test_column_names_lowercased(self): - qb = QueryBuilder("account").filter_eq("StateCode", 0).order_by("Revenue") - params = qb.build() - self.assertEqual(params["filter"], "statecode eq 0") - self.assertEqual(params["orderby"], ["revenue"]) - - def test_string_with_quotes_escaped(self): - qb = QueryBuilder("account").filter_eq("name", "O'Brien's Corp") - self.assertEqual(qb.build()["filter"], "name eq 'O''Brien''s Corp'") - - def test_multiple_filters_and_joined(self): - qb = QueryBuilder("account").filter_eq("statecode", 0).filter_gt("revenue", 1000000) - self.assertEqual(qb.build()["filter"], "statecode eq 0 and revenue gt 1000000") - - -class TestStringFunctionFilters(unittest.TestCase): - """Tests for string function filter methods.""" - - def test_filter_contains(self): - qb = QueryBuilder("account").filter_contains("name", "Corp") - self.assertEqual(qb.build()["filter"], "contains(name, 'Corp')") - - def test_filter_startswith(self): - qb = QueryBuilder("account").filter_startswith("name", "Con") - self.assertEqual(qb.build()["filter"], "startswith(name, 'Con')") - - def test_filter_endswith(self): - qb = QueryBuilder("account").filter_endswith("name", "Ltd") - self.assertEqual(qb.build()["filter"], "endswith(name, 'Ltd')") - - def test_filter_contains_single_quotes(self): - qb = QueryBuilder("account").filter_contains("name", "O'Brien") - self.assertEqual(qb.build()["filter"], "contains(name, 'O''Brien')") - - -class TestNullFilters(unittest.TestCase): - """Tests for null/not-null filter methods.""" - - def test_filter_null(self): - qb = QueryBuilder("account").filter_null("telephone1") - self.assertEqual(qb.build()["filter"], "telephone1 eq null") - - def test_filter_not_null(self): - qb = QueryBuilder("account").filter_not_null("telephone1") - self.assertEqual(qb.build()["filter"], "telephone1 ne null") - - -class TestFilterBetween(unittest.TestCase): - """Tests for the filter_between() method.""" - - def test_filter_between_parenthesized(self): - qb = QueryBuilder("account").filter_between("revenue", 100000, 500000) - self.assertEqual( - qb.build()["filter"], - "(revenue ge 100000 and revenue le 500000)", - ) + def setUp(self): + self.qb = QueryBuilder("account") - def test_filter_between_column_lowercased(self): - qb = QueryBuilder("account").filter_between("Revenue", 100, 500) - self.assertEqual( - qb.build()["filter"], - "(revenue ge 100 and revenue le 500)", - ) - - def test_filter_between_returns_self(self): - qb = QueryBuilder("account") - self.assertIs(qb.filter_between("revenue", 100, 500), qb) + def test_filter_eq_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_eq("name", "Contoso") - def test_filter_between_combined_with_other_filters(self): - qb = QueryBuilder("account").filter_eq("statecode", 0).filter_between("revenue", 100000, 500000) - self.assertEqual( - qb.build()["filter"], - "statecode eq 0 and (revenue ge 100000 and revenue le 500000)", - ) + def test_filter_ne_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_ne("statecode", 1) - def test_filter_between_datetimes(self): - from datetime import datetime, timezone + def test_filter_gt_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_gt("revenue", 0) - start = datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc) - end = datetime(2024, 12, 31, 23, 59, 59, tzinfo=timezone.utc) - qb = QueryBuilder("account").filter_between("createdon", start, end) - self.assertEqual( - qb.build()["filter"], - "(createdon ge 2024-01-01T00:00:00Z and createdon le 2024-12-31T23:59:59Z)", - ) + def test_filter_ge_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_ge("revenue", 0) + def test_filter_lt_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_lt("revenue", 0) -class TestFilterNotIn(unittest.TestCase): - """Tests for the filter_not_in() method.""" + def test_filter_le_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_le("revenue", 0) - def test_filter_not_in_ints(self): - qb = QueryBuilder("account").filter_not_in("statecode", [2, 3]) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["2","3"])', - ) + def test_filter_contains_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_contains("name", "Corp") - def test_filter_not_in_strings(self): - qb = QueryBuilder("account").filter_not_in("name", ["Contoso", "Fabrikam"]) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'name\',PropertyValues=["Contoso","Fabrikam"])', - ) + def test_filter_startswith_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_startswith("name", "Con") - def test_filter_not_in_empty_raises(self): - with self.assertRaises(ValueError): - QueryBuilder("account").filter_not_in("statecode", []) + def test_filter_endswith_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_endswith("name", "Ltd") - def test_filter_not_in_returns_self(self): - qb = QueryBuilder("account") - self.assertIs(qb.filter_not_in("statecode", [0, 1]), qb) + def test_filter_null_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_null("telephone1") - def test_filter_not_in_combined_with_other_filters(self): - qb = QueryBuilder("account").filter_eq("statecode", 0).filter_not_in("priority", [1, 2]) - self.assertEqual( - qb.build()["filter"], - 'statecode eq 0 and Microsoft.Dynamics.CRM.NotIn(PropertyName=\'priority\',PropertyValues=["1","2"])', - ) + def test_filter_not_null_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_not_null("telephone1") - def test_filter_not_in_with_set(self): - qb = QueryBuilder("account").filter_not_in("statecode", {2, 3}) - result = qb.build()["filter"] - self.assertIn("Microsoft.Dynamics.CRM.NotIn", result) - self.assertIn("statecode", result) + def test_filter_in_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_in("statecode", [0, 1]) - def test_filter_not_in_with_tuple(self): - qb = QueryBuilder("account").filter_not_in("statecode", (2, 3)) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["2","3"])', - ) + def test_filter_not_in_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_not_in("statecode", [0, 1]) + def test_filter_between_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_between("revenue", 100, 500) -class TestFilterNotBetween(unittest.TestCase): - """Tests for the filter_not_between() method.""" + def test_filter_not_between_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_not_between("revenue", 100, 500) - def test_filter_not_between_ints(self): - qb = QueryBuilder("account").filter_not_between("revenue", 100000, 500000) - self.assertEqual( - qb.build()["filter"], - "not ((revenue ge 100000 and revenue le 500000))", - ) - - def test_filter_not_between_returns_self(self): - qb = QueryBuilder("account") - self.assertIs(qb.filter_not_between("revenue", 100, 500), qb) - - def test_filter_not_between_combined_with_other_filters(self): - qb = QueryBuilder("account").filter_eq("statecode", 0).filter_not_between("revenue", 100000, 500000) - self.assertEqual( - qb.build()["filter"], - "statecode eq 0 and not ((revenue ge 100000 and revenue le 500000))", - ) - - -class TestFilterRaw(unittest.TestCase): - """Tests for the filter_raw() method.""" - - def test_filter_raw(self): - qb = QueryBuilder("account").filter_raw("(statecode eq 0 or statecode eq 1)") - self.assertEqual(qb.build()["filter"], "(statecode eq 0 or statecode eq 1)") - - def test_filter_raw_returns_self(self): - qb = QueryBuilder("account") - self.assertIs(qb.filter_raw("a eq 1"), qb) - - def test_build_with_plain_string_filter_part(self): - """build() handles plain string entries in _filter_parts (internal path).""" - qb = QueryBuilder("account") - qb._filter_parts.append("name eq 'Contoso'") - self.assertEqual(qb.build()["filter"], "name eq 'Contoso'") + def test_filter_raw_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_raw("statecode eq 0") class TestWhere(unittest.TestCase): """Tests for the where() method with composable expressions.""" def test_where_simple(self): - from PowerPlatform.Dataverse.models.filters import eq + from PowerPlatform.Dataverse.models.filters import col - qb = QueryBuilder("account").where(eq("statecode", 0)) + qb = QueryBuilder("account").where(col("statecode") == 0) self.assertEqual(qb.build()["filter"], "statecode eq 0") def test_where_complex(self): - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col - expr = (eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000) + expr = ((col("statecode") == 0) | (col("statecode") == 1)) & (col("revenue") > 100000) qb = QueryBuilder("account").where(expr) self.assertEqual( qb.build()["filter"], "((statecode eq 0 or statecode eq 1) and revenue gt 100000)", ) - def test_where_combined_with_filter_methods(self): - from PowerPlatform.Dataverse.models.filters import gt + def test_where_combined_with_raw(self): + from PowerPlatform.Dataverse.models.filters import col, raw - qb = QueryBuilder("account").filter_eq("statecode", 0).where(gt("revenue", 100000)) + qb = QueryBuilder("account").where(raw("statecode eq 0")).where(col("revenue") > 100000) self.assertEqual(qb.build()["filter"], "statecode eq 0 and revenue gt 100000") def test_where_multiple_calls(self): - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col - qb = QueryBuilder("account").where(eq("statecode", 0)).where(gt("revenue", 100000)) + qb = QueryBuilder("account").where(col("statecode") == 0).where(col("revenue") > 100000) self.assertEqual(qb.build()["filter"], "statecode eq 0 and revenue gt 100000") def test_where_preserves_call_order(self): - """Interleaved filter_*() and where() should preserve call order.""" - from PowerPlatform.Dataverse.models.filters import eq, gt + """Multiple where() calls preserve insertion order.""" + from PowerPlatform.Dataverse.models.filters import col - qb = QueryBuilder("account").where(eq("a", 1)).filter_eq("b", 2).where(gt("c", 3)) + qb = QueryBuilder("account").where(col("a") == 1).where(col("b") == 2).where(col("c") > 3) self.assertEqual(qb.build()["filter"], "a eq 1 and b eq 2 and c gt 3") def test_where_returns_self(self): - from PowerPlatform.Dataverse.models.filters import eq + from PowerPlatform.Dataverse.models.filters import col qb = QueryBuilder("account") - self.assertIs(qb.where(eq("statecode", 0)), qb) + self.assertIs(qb.where(col("statecode") == 0), qb) def test_where_non_expression_raises(self): qb = QueryBuilder("account") @@ -401,21 +172,54 @@ def test_where_non_expression_raises(self): qb.where("statecode eq 0") # type: ignore def test_where_with_not(self): - from PowerPlatform.Dataverse.models.filters import eq + from PowerPlatform.Dataverse.models.filters import col - qb = QueryBuilder("account").where(~eq("statecode", 1)) + qb = QueryBuilder("account").where(~(col("statecode") == 1)) self.assertEqual(qb.build()["filter"], "not (statecode eq 1)") def test_where_with_filter_in(self): - from PowerPlatform.Dataverse.models.filters import filter_in, gt + from PowerPlatform.Dataverse.models.filters import col - expr = filter_in("statecode", [0, 1]) & gt("revenue", 100000) + expr = col("statecode").in_([0, 1]) & (col("revenue") > 100000) qb = QueryBuilder("account").where(expr) self.assertEqual( qb.build()["filter"], '(Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1"]) and revenue gt 100000)', ) + def test_where_with_raw_preserves_string(self): + from PowerPlatform.Dataverse.models.filters import raw + + qb = QueryBuilder("account").where(raw("(statecode eq 0 or statecode eq 1)")) + self.assertEqual(qb.build()["filter"], "(statecode eq 0 or statecode eq 1)") + + def test_where_negation_of_in(self): + from PowerPlatform.Dataverse.models.filters import col + + qb = QueryBuilder("account").where(col("statecode").not_in([2, 3])) + self.assertEqual( + qb.build()["filter"], + 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["2","3"])', + ) + + def test_where_between(self): + from PowerPlatform.Dataverse.models.filters import col + + qb = QueryBuilder("account").where(col("revenue").between(100000, 500000)) + self.assertEqual( + qb.build()["filter"], + "(revenue ge 100000 and revenue le 500000)", + ) + + def test_where_not_between(self): + from PowerPlatform.Dataverse.models.filters import col + + qb = QueryBuilder("account").where(col("revenue").not_between(100000, 500000)) + self.assertEqual( + qb.build()["filter"], + "not ((revenue ge 100000 and revenue le 500000))", + ) + class TestOrderBy(unittest.TestCase): """Tests for the order_by() method.""" @@ -635,11 +439,13 @@ def test_empty_builder_only_has_table(self): self.assertNotIn("page_size", params) def test_full_query_build(self): + from PowerPlatform.Dataverse.models.filters import col, raw + qb = ( QueryBuilder("account") .select("name", "revenue", "telephone1") - .filter_eq("statecode", 0) - .filter_gt("revenue", 1000000) + .where(raw("statecode eq 0")) + .where(col("revenue") > 1000000) .order_by("revenue", descending=True) .order_by("name") .expand("primarycontactid") @@ -663,33 +469,34 @@ def test_build_returns_fresh_lists(self): self.assertEqual(params1["select"], params2["select"]) self.assertIsNot(params1["select"], params2["select"]) + def test_build_with_plain_string_filter_part(self): + """build() handles plain string entries in _filter_parts (internal path).""" + qb = QueryBuilder("account") + qb._filter_parts.append("name eq 'Contoso'") + self.assertEqual(qb.build()["filter"], "name eq 'Contoso'") + + def test_build_mixed_string_and_expression_filter_parts(self): + """build() AND-joins raw strings and FilterExpression objects in order.""" + from PowerPlatform.Dataverse.models.filters import col + + qb = QueryBuilder("account") + qb._filter_parts.append("statecode eq 0") + qb.where(col("revenue") > 100000) + self.assertEqual(qb.build()["filter"], "statecode eq 0 and revenue gt 100000") + class TestMethodChainingReturnsSelf(unittest.TestCase): - """Verify all methods return self for chaining.""" + """Verify all public methods return self for chaining.""" def test_all_methods_return_self(self): - from PowerPlatform.Dataverse.models.filters import eq + from PowerPlatform.Dataverse.models.filters import col qb = QueryBuilder("account") self.assertIs(qb.select("name"), qb) - self.assertIs(qb.filter_eq("a", 1), qb) - self.assertIs(qb.filter_ne("b", 2), qb) - self.assertIs(qb.filter_gt("c", 3), qb) - self.assertIs(qb.filter_ge("d", 4), qb) - self.assertIs(qb.filter_lt("e", 5), qb) - self.assertIs(qb.filter_le("f", 6), qb) - self.assertIs(qb.filter_contains("g", "x"), qb) - self.assertIs(qb.filter_startswith("h", "y"), qb) - self.assertIs(qb.filter_endswith("i", "z"), qb) - self.assertIs(qb.filter_null("j"), qb) - self.assertIs(qb.filter_not_null("k"), qb) - self.assertIs(qb.filter_raw("l eq 1"), qb) - self.assertIs(qb.filter_in("m", [1, 2]), qb) - self.assertIs(qb.filter_between("n", 1, 10), qb) - self.assertIs(qb.where(eq("o", 1)), qb) - self.assertIs(qb.order_by("p"), qb) - self.assertIs(qb.expand("q"), qb) + self.assertIs(qb.where(col("statecode") == 0), qb) + self.assertIs(qb.order_by("name"), qb) + self.assertIs(qb.expand("primarycontactid"), qb) self.assertIs(qb.top(10), qb) self.assertIs(qb.page_size(5), qb) self.assertIs(qb.count(), qb) @@ -701,26 +508,40 @@ class TestExecute(unittest.TestCase): """Tests for the execute() terminal method.""" def test_execute_without_query_ops_raises(self): - qb = QueryBuilder("account").filter_eq("statecode", 0) + from PowerPlatform.Dataverse.models.filters import raw + + qb = QueryBuilder("account").where(raw("statecode eq 0")) with self.assertRaises(RuntimeError) as ctx: qb.execute() self.assertIn("client.query.builder()", str(ctx.exception)) - def test_execute_calls_records_get(self): - """execute() should delegate to client.records.get() with built params.""" + def _make_od(self, pages=None): + """Return (mock_query_ops, mock_od) with _get_multiple pre-wired.""" mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([[{"name": "Test"}]]) + mock_od = MagicMock() + mock_od._get_multiple.side_effect = lambda *a, **kw: iter(pages or []) + mock_query_ops._client._scoped_odata.return_value.__enter__.return_value = mock_od + return mock_query_ops, mock_od + def test_execute_calls_get_multiple(self): + """execute() calls _get_multiple() via _scoped_odata with all built params.""" + from PowerPlatform.Dataverse.models.filters import raw + + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops - qb.select("name", "revenue").filter_eq("statecode", 0).order_by("revenue", descending=True).top(100).page_size( - 50 - ).expand("primarycontactid") + ( + qb.select("name", "revenue") + .where(raw("statecode eq 0")) + .order_by("revenue", descending=True) + .top(100) + .page_size(50) + .expand("primarycontactid") + ) - list(qb.execute()) + qb.execute() - mock_client.records.get.assert_called_once_with( + mock_od._get_multiple.assert_called_once_with( "account", select=["name", "revenue"], filter="statecode eq 0", @@ -733,10 +554,7 @@ def test_execute_calls_records_get(self): ) def test_execute_returns_flat_records_by_default(self): - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([[{"name": "A"}, {"name": "B"}], [{"name": "C"}]]) - + mock_query_ops, _ = self._make_od([[{"name": "A"}, {"name": "B"}], [{"name": "C"}]]) qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name") @@ -748,66 +566,47 @@ def test_execute_returns_flat_records_by_default(self): self.assertEqual(records[2]["name"], "C") def test_execute_by_page_returns_pages(self): - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - - page1 = [{"name": "A"}, {"name": "B"}] - page2 = [{"name": "C"}] - mock_client.records.get.return_value = iter([page1, page2]) + from PowerPlatform.Dataverse.models.record import QueryResult + mock_query_ops, _ = self._make_od([[{"name": "A"}, {"name": "B"}], [{"name": "C"}]]) qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name") pages = list(qb.execute(by_page=True)) self.assertEqual(len(pages), 2) - self.assertEqual(pages[0], page1) - self.assertEqual(pages[1], page2) - - def test_execute_unbounded_raises(self): - """execute() with no select/filter/top should raise ValueError.""" - mock_query_ops = MagicMock() - qb = QueryBuilder("account") - qb._query_ops = mock_query_ops - with self.assertRaises(ValueError) as ctx: - qb.execute() - self.assertIn("Unbounded query", str(ctx.exception)) + self.assertIsInstance(pages[0], QueryResult) + self.assertEqual(len(pages[0]), 2) + self.assertEqual(len(pages[1]), 1) def test_execute_with_only_select_succeeds(self): """execute() with select only should not raise.""" - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) - + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name") - list(qb.execute()) # should not raise - mock_client.records.get.assert_called_once() + qb.execute() # should not raise + mock_od._get_multiple.assert_called_once() def test_execute_with_only_filter_succeeds(self): """execute() with filter only should not raise.""" - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) + from PowerPlatform.Dataverse.models.filters import raw + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops - qb.filter_eq("statecode", 0) - list(qb.execute()) # should not raise - mock_client.records.get.assert_called_once() + qb.where(raw("statecode eq 0")) + qb.execute() # should not raise + mock_od._get_multiple.assert_called_once() def test_execute_with_only_top_succeeds(self): """execute() with top only should not raise.""" - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) - + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.top(10) - list(qb.execute()) # should not raise - mock_client.records.get.assert_called_once() + qb.execute() # should not raise + mock_od._get_multiple.assert_called_once() def test_execute_with_only_expand_raises(self): """expand alone is not a sufficient constraint.""" @@ -828,51 +627,42 @@ def test_execute_with_only_count_raises(self): qb.execute() def test_execute_with_where_expressions(self): - from PowerPlatform.Dataverse.models.filters import eq, gt - - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) + from PowerPlatform.Dataverse.models.filters import col + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops - qb.where((eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000)) - list(qb.execute()) + qb.where(((col("statecode") == 0) | (col("statecode") == 1)) & (col("revenue") > 100000)) + qb.execute() - call_args = mock_client.records.get.call_args self.assertEqual( - call_args.kwargs["filter"], + mock_od._get_multiple.call_args.kwargs["filter"], "((statecode eq 0 or statecode eq 1) and revenue gt 100000)", ) def test_execute_with_filter_in(self): - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) + from PowerPlatform.Dataverse.models.filters import col + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops - qb.filter_in("statecode", [0, 1, 2]) - list(qb.execute()) + qb.where(col("statecode").in_([0, 1, 2])) + qb.execute() - call_args = mock_client.records.get.call_args self.assertEqual( - call_args.kwargs["filter"], + mock_od._get_multiple.call_args.kwargs["filter"], 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1","2"])', ) def test_execute_passes_count_and_annotations(self): - """execute() should forward count and include_annotations when set.""" - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) - + """execute() should forward count and include_annotations to _get_multiple.""" + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name").count().include_formatted_values() - list(qb.execute()) + qb.execute() - mock_client.records.get.assert_called_once_with( + mock_od._get_multiple.assert_called_once_with( "account", select=["name"], filter=None, @@ -885,64 +675,141 @@ def test_execute_passes_count_and_annotations(self): ) -class TestToDataframe(unittest.TestCase): - """Tests for the to_dataframe() terminal method.""" +class TestExecutePages(unittest.TestCase): + """Tests for execute_pages() — lazy per-page QueryResult iterator.""" - def test_to_dataframe_without_query_ops_raises(self): - qb = QueryBuilder("account").filter_eq("statecode", 0) - with self.assertRaises(RuntimeError) as ctx: - qb.to_dataframe() - self.assertIn("client.query.builder()", str(ctx.exception)) + def _make_qb(self): + mock_query_ops = MagicMock() + mock_od = MagicMock() + mock_od._get_multiple.side_effect = lambda *a, **kw: iter([[{"name": "A"}], [{"name": "B"}]]) + mock_query_ops._client._scoped_odata.return_value.__enter__.return_value = mock_od + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name") + return qb, mock_query_ops - def test_to_dataframe_delegates_to_dataframe_get(self): - """to_dataframe() should delegate to client.dataframe.get() with built params.""" - import pandas as pd + def test_execute_pages_returns_iterator(self): + qb, _ = self._make_qb() + result = qb.execute_pages() + import types - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - expected_df = pd.DataFrame([{"name": "Contoso", "revenue": 1000}]) - mock_client.dataframe.get.return_value = expected_df + self.assertIsInstance(result, types.GeneratorType) + + def test_execute_pages_yields_query_result_per_page(self): + from PowerPlatform.Dataverse.models.record import QueryResult + + qb, _ = self._make_qb() + pages = list(qb.execute_pages()) + self.assertEqual(len(pages), 2) + for page in pages: + self.assertIsInstance(page, QueryResult) + def test_execute_pages_page_contents(self): + qb, _ = self._make_qb() + pages = list(qb.execute_pages()) + self.assertEqual(pages[0].first()["name"], "A") + self.assertEqual(pages[1].first()["name"], "B") + + def test_execute_pages_without_query_ops_raises(self): + from PowerPlatform.Dataverse.models.filters import raw + + qb = QueryBuilder("account").where(raw("statecode eq 0")) + with self.assertRaises(RuntimeError): + list(qb.execute_pages()) + + def test_execute_pages_without_constraints_raises(self): + mock_query_ops = MagicMock() qb = QueryBuilder("account") qb._query_ops = mock_query_ops - qb.select("name", "revenue").filter_eq("statecode", 0).order_by("revenue", descending=True).top(100).page_size( - 50 - ).expand("primarycontactid") + with self.assertRaises(ValueError): + list(qb.execute_pages()) - result = qb.to_dataframe() - mock_client.dataframe.get.assert_called_once_with( - "account", - select=["name", "revenue"], - filter="statecode eq 0", - orderby=["revenue desc"], - top=100, - expand=["primarycontactid"], - page_size=50, - count=False, - include_annotations=None, - ) - pd.testing.assert_frame_equal(result, expected_df) +class TestByPageWarning(unittest.TestCase): + """execute(by_page=...) fires UserWarning; plain execute() does not.""" - def test_to_dataframe_unbounded_raises(self): - """to_dataframe() with no select/filter/top should raise ValueError.""" + def _make_qb(self): mock_query_ops = MagicMock() + mock_od = MagicMock() + mock_od._get_multiple.side_effect = lambda *a, **kw: iter([[{"name": "A"}]]) + mock_query_ops._client._scoped_odata.return_value.__enter__.return_value = mock_od qb = QueryBuilder("account") qb._query_ops = mock_query_ops - with self.assertRaises(ValueError) as ctx: + qb.select("name") + return qb + + def test_execute_no_flag_no_warning(self): + """execute() with no by_page argument fires no warning.""" + import warnings + + qb = self._make_qb() + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + qb.execute() + user_warnings = [w for w in caught if issubclass(w.category, UserWarning)] + self.assertEqual(len(user_warnings), 0) + + def test_execute_by_page_true_fires_user_warning(self): + """execute(by_page=True) fires UserWarning pointing to execute_pages().""" + qb = self._make_qb() + with self.assertWarns(UserWarning) as ctx: + list(qb.execute(by_page=True)) + self.assertIn("execute_pages()", str(ctx.warning)) + + def test_execute_by_page_false_fires_user_warning(self): + """execute(by_page=False) fires UserWarning — redundant flag.""" + qb = self._make_qb() + with self.assertWarns(UserWarning) as ctx: + qb.execute(by_page=False) + self.assertIn("redundant", str(ctx.warning)) + + def test_execute_by_page_true_still_functional(self): + """execute(by_page=True) still returns the raw page iterator.""" + import warnings + + qb = self._make_qb() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + result = qb.execute(by_page=True) + pages = list(result) + self.assertEqual(len(pages), 1) + + def test_execute_by_page_false_still_returns_query_result(self): + """execute(by_page=False) still returns QueryResult.""" + import warnings + + from PowerPlatform.Dataverse.models.record import QueryResult + + qb = self._make_qb() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + result = qb.execute(by_page=False) + self.assertIsInstance(result, QueryResult) + + +class TestToDataframe(unittest.TestCase): + """Tests for the to_dataframe() terminal method.""" + + def _make_od(self, pages=None): + mock_query_ops = MagicMock() + mock_od = MagicMock() + mock_od._get_multiple.side_effect = lambda *a, **kw: iter(pages or []) + mock_query_ops._client._scoped_odata.return_value.__enter__.return_value = mock_od + return mock_query_ops, mock_od + + def test_to_dataframe_without_query_ops_raises(self): + from PowerPlatform.Dataverse.models.filters import raw + + qb = QueryBuilder("account").where(raw("statecode eq 0")) + with self.assertRaises(RuntimeError) as ctx: qb.to_dataframe() - self.assertIn("Unbounded query", str(ctx.exception)) + self.assertIn("client.query.builder()", str(ctx.exception)) def test_to_dataframe_returns_dataframe(self): - """to_dataframe() should return a pandas DataFrame.""" + """to_dataframe() collects execute() results into a DataFrame.""" import pandas as pd - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.dataframe.get.return_value = pd.DataFrame( - [{"name": "A", "revenue": 100}, {"name": "B", "revenue": 200}] - ) - + mock_query_ops, _ = self._make_od([[{"name": "A", "revenue": 100}, {"name": "B", "revenue": 200}]]) qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name", "revenue") @@ -953,20 +820,63 @@ def test_to_dataframe_returns_dataframe(self): self.assertEqual(len(result), 2) self.assertListEqual(list(result.columns), ["name", "revenue"]) - def test_to_dataframe_forwards_count_and_annotations(self): - """to_dataframe() should forward count and include_annotations when set.""" + def test_to_dataframe_empty_result_returns_empty_dataframe(self): + """to_dataframe() with no matching records returns empty DataFrame.""" import pandas as pd - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.dataframe.get.return_value = pd.DataFrame() + mock_query_ops, _ = self._make_od() + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name", "revenue") + + result = qb.to_dataframe() + + self.assertIsInstance(result, pd.DataFrame) + self.assertEqual(len(result), 0) + + def test_to_dataframe_calls_get_multiple_with_params(self): + """to_dataframe() passes all built query params to _get_multiple.""" + import pandas as pd + from PowerPlatform.Dataverse.models.filters import raw + + mock_query_ops, mock_od = self._make_od([[{"name": "Contoso", "revenue": 1000}]]) + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + ( + qb.select("name", "revenue") + .where(raw("statecode eq 0")) + .order_by("revenue", descending=True) + .top(100) + .page_size(50) + .expand("primarycontactid") + ) + result = qb.to_dataframe() + + mock_od._get_multiple.assert_called_once_with( + "account", + select=["name", "revenue"], + filter="statecode eq 0", + orderby=["revenue desc"], + top=100, + expand=["primarycontactid"], + page_size=50, + count=False, + include_annotations=None, + ) + self.assertIsInstance(result, pd.DataFrame) + self.assertEqual(len(result), 1) + self.assertEqual(result.iloc[0]["name"], "Contoso") + + def test_to_dataframe_forwards_count_and_annotations(self): + """to_dataframe() should forward count and include_annotations when set.""" + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name").count().include_formatted_values() qb.to_dataframe() - mock_client.dataframe.get.assert_called_once_with( + mock_od._get_multiple.assert_called_once_with( "account", select=["name"], filter=None, @@ -978,6 +888,43 @@ def test_to_dataframe_forwards_count_and_annotations(self): include_annotations="OData.Community.Display.V1.FormattedValue", ) + def test_to_dataframe_with_record_objects(self): + """to_dataframe() handles Record objects (with .data attribute).""" + import pandas as pd + from PowerPlatform.Dataverse.models.record import Record + + mock_query_ops, _ = self._make_od( + [ + [ + {"name": "Contoso", "revenue": 1000}, + {"name": "Fabrikam", "revenue": 2000}, + ] + ] + ) + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name", "revenue") + + result = qb.to_dataframe() + + self.assertIsInstance(result, pd.DataFrame) + self.assertEqual(len(result), 2) + self.assertEqual(result.iloc[0]["name"], "Contoso") + self.assertEqual(result.iloc[1]["revenue"], 2000) + + def test_to_dataframe_emits_deprecation_warning(self): + """QueryBuilder.to_dataframe() fires DeprecationWarning; use execute().to_dataframe() instead.""" + mock_query_ops, _ = self._make_od() + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name") + + with self.assertWarns(DeprecationWarning) as ctx: + qb.to_dataframe() + + self.assertIn("QueryBuilder.to_dataframe()", str(ctx.warning)) + self.assertIn("execute().to_dataframe()", str(ctx.warning)) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 52d5a716..2ed03ddd 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -9,135 +9,87 @@ from PowerPlatform.Dataverse.client import DataverseClient -class TestDataverseClient(unittest.TestCase): +class TestDataverseClientConstruction(unittest.TestCase): + """Tests for DataverseClient construction and lifecycle.""" + + def test_empty_base_url_raises(self): + """DataverseClient raises ValueError when base_url is empty.""" + mock_credential = MagicMock(spec=TokenCredential) + with self.assertRaises(ValueError): + DataverseClient("", mock_credential) + + def test_trailing_slash_stripped(self): + """DataverseClient strips trailing slash from base_url.""" + mock_credential = MagicMock(spec=TokenCredential) + client = DataverseClient("https://example.crm.dynamics.com/", mock_credential) + self.assertEqual(client._base_url, "https://example.crm.dynamics.com") + + def test_namespace_attributes_present(self): + """Client exposes records, query, tables, files, dataframe, batch namespaces.""" + mock_credential = MagicMock(spec=TokenCredential) + client = DataverseClient("https://example.crm.dynamics.com", mock_credential) + for attr in ("records", "query", "tables", "files", "dataframe", "batch"): + self.assertTrue(hasattr(client, attr), f"Missing namespace: {attr}") + + +class TestRemovedClientMethods(unittest.TestCase): + """Verify all 12 deprecated flat methods were removed from DataverseClient in 1.0 GA. + + These methods previously delegated to namespace equivalents (records.*, query.*, + tables.*, files.*). They were fully removed; callers must use the namespaces directly. + """ + def setUp(self): - """Set up test fixtures before each test method.""" - # Create mock credential self.mock_credential = MagicMock(spec=TokenCredential) - self.base_url = "https://example.crm.dynamics.com" + self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) - # Initialize the client under test - self.client = DataverseClient(self.base_url, self.mock_credential) + def test_create_removed(self): + with self.assertRaises(AttributeError): + self.client.create("account", {"name": "Test"}) - # Mock the internal _odata client - # This ensures we verify logic without making actual HTTP calls - self.client._odata = MagicMock() - - def test_create_single(self): - """Test create method with a single record.""" - # Setup mock return values - # _create must return a GUID string - self.client._odata._create.return_value = "00000000-0000-0000-0000-000000000000" - # _entity_set_from_schema_name should return the plural entity set name - self.client._odata._entity_set_from_schema_name.return_value = "accounts" - - # Execute test - self.client.create("account", {"name": "Contoso Ltd"}) - - # Verify - # Ensure _entity_set_from_schema_name was called and its result ("accounts") was passed to _create - self.client._odata._create.assert_called_once_with("accounts", "account", {"name": "Contoso Ltd"}) - - def test_create_multiple(self): - """Test create method with multiple records.""" - payloads = [{"name": "Company A"}, {"name": "Company B"}, {"name": "Company C"}] - - # Setup mock return values - # _create_multiple must return a list of GUID strings - self.client._odata._create_multiple.return_value = [ - "00000000-0000-0000-0000-000000000001", - "00000000-0000-0000-0000-000000000002", - "00000000-0000-0000-0000-000000000003", - ] - self.client._odata._entity_set_from_schema_name.return_value = "accounts" - - # Execute test - self.client.create("account", payloads) - - # Verify - self.client._odata._create_multiple.assert_called_once_with("accounts", "account", payloads) - - def test_update_single(self): - """Test update method with a single record.""" - self.client.update("account", "00000000-0000-0000-0000-000000000000", {"telephone1": "555-0199"}) - self.client._odata._update.assert_called_once_with( - "account", "00000000-0000-0000-0000-000000000000", {"telephone1": "555-0199"} - ) + def test_update_removed(self): + with self.assertRaises(AttributeError): + self.client.update("account", "guid-1", {"name": "Test"}) - def test_update_multiple(self): - """Test update method with multiple records (broadcast).""" - ids = [ - "00000000-0000-0000-0000-000000000001", - "00000000-0000-0000-0000-000000000002", - ] - changes = {"statecode": 1} - - self.client.update("account", ids, changes) - self.client._odata._update_by_ids.assert_called_once_with("account", ids, changes) - - def test_delete_single(self): - """Test delete method with a single record.""" - self.client.delete("account", "00000000-0000-0000-0000-000000000000") - self.client._odata._delete.assert_called_once_with("account", "00000000-0000-0000-0000-000000000000") - - def test_delete_multiple(self): - """Test delete method with multiple records.""" - ids = [ - "00000000-0000-0000-0000-000000000001", - "00000000-0000-0000-0000-000000000002", - ] - # Mock return value for bulk delete job ID - self.client._odata._delete_multiple.return_value = "job-guid-123" - - job_id = self.client.delete("account", ids) - - self.client._odata._delete_multiple.assert_called_once_with("account", ids) - self.assertEqual(job_id, "job-guid-123") - - def test_get_single(self): - """Test get method with a single record ID.""" - # Setup mock return value - expected_record = {"accountid": "00000000-0000-0000-0000-000000000000", "name": "Contoso"} - self.client._odata._get.return_value = expected_record - - result = self.client.get("account", "00000000-0000-0000-0000-000000000000") - - self.client._odata._get.assert_called_once_with("account", "00000000-0000-0000-0000-000000000000", select=None) - self.assertEqual(result["accountid"], "00000000-0000-0000-0000-000000000000") - self.assertEqual(result["name"], "Contoso") - - def test_get_multiple(self): - """Test get method for querying multiple records.""" - # Setup mock return value (iterator) - expected_batch = [{"accountid": "1", "name": "A"}, {"accountid": "2", "name": "B"}] - self.client._odata._get_multiple.return_value = iter([expected_batch]) - - # Execute query - result_iterator = self.client.get("account", filter="statecode eq 0", top=10) - - # Consume iterator to verify content - results = list(result_iterator) - - self.client._odata._get_multiple.assert_called_once_with( - "account", - select=None, - filter="statecode eq 0", - orderby=None, - top=10, - expand=None, - page_size=None, - count=False, - include_annotations=None, - ) - self.assertEqual(len(results), 1) - self.assertEqual(len(results[0]), 2) - self.assertEqual(results[0][0]["name"], "A") - self.assertEqual(results[0][1]["name"], "B") + def test_delete_removed(self): + with self.assertRaises(AttributeError): + self.client.delete("account", "guid-1") - def test_empty_base_url_raises(self): - """DataverseClient raises ValueError when base_url is empty.""" - with self.assertRaises(ValueError): - DataverseClient("", self.mock_credential) + def test_get_removed(self): + with self.assertRaises(AttributeError): + self.client.get("account", "guid-1") + + def test_query_sql_removed(self): + with self.assertRaises(AttributeError): + self.client.query_sql("SELECT name FROM account") + + def test_get_table_info_removed(self): + with self.assertRaises(AttributeError): + self.client.get_table_info("account") + + def test_create_table_removed(self): + with self.assertRaises(AttributeError): + self.client.create_table("new_Test", {}) + + def test_delete_table_removed(self): + with self.assertRaises(AttributeError): + self.client.delete_table("new_Test") + + def test_list_tables_removed(self): + with self.assertRaises(AttributeError): + self.client.list_tables() + + def test_create_columns_removed(self): + with self.assertRaises(AttributeError): + self.client.create_columns("account", {}) + + def test_delete_columns_removed(self): + with self.assertRaises(AttributeError): + self.client.delete_columns("account", []) + + def test_upload_file_removed(self): + with self.assertRaises(AttributeError): + self.client.upload_file("account", "guid-1", "file_col", "/path/file.pdf") class TestCreateLookupField(unittest.TestCase): diff --git a/tests/unit/test_client_deprecations.py b/tests/unit/test_client_deprecations.py index fbf1382d..b11486be 100644 --- a/tests/unit/test_client_deprecations.py +++ b/tests/unit/test_client_deprecations.py @@ -1,12 +1,11 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Tests for deprecated flat methods on DataverseClient. +"""Tests confirming all 12 deprecated flat methods were removed from DataverseClient. -Each deprecated method on the client should: -1. Emit a DeprecationWarning. -2. Delegate to the correct namespace method (records / query / tables / files). -3. Return the expected value, including any backward-compatibility shims. +These methods previously delegated to namespace equivalents with a DeprecationWarning. +In 1.0 GA they are fully removed; each call now raises AttributeError. +Callers must use the operation namespaces directly (records.*, query.*, tables.*, files.*). """ import unittest @@ -18,281 +17,95 @@ class TestClientDeprecations(unittest.TestCase): - """Verify every deprecated flat method warns and delegates correctly.""" + """All formerly-deprecated flat methods are now removed and raise AttributeError.""" def setUp(self): - """Set up test fixtures before each test method.""" self.mock_credential = MagicMock(spec=TokenCredential) self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) - # Mock the internal OData client so namespace methods resolve without - # making real HTTP calls. - self.client._odata = MagicMock() - # ---------------------------------------------------------------- create + # ---------------------------------------------------------------- records - def test_create_warns(self): - """client.create() emits a DeprecationWarning.""" - self.client._odata._entity_set_from_schema_name.return_value = "accounts" - self.client._odata._create.return_value = "guid-123" - - with self.assertWarns(DeprecationWarning): + def test_create_removed(self): + """client.create() → use client.records.create()""" + with self.assertRaises(AttributeError): self.client.create("account", {"name": "Test"}) def test_create_single_returns_list(self): - """client.create() wraps a single GUID in a list for backward compat. - - records.create() returns a bare string for a single dict, but the - deprecated client.create() always returned list[str]. - """ - self.client._odata._entity_set_from_schema_name.return_value = "accounts" - self.client._odata._create.return_value = "guid-123" - - with self.assertWarns(DeprecationWarning): - result = self.client.create("account", {"name": "A"}) - - self.assertIsInstance(result, list) - self.assertEqual(result, ["guid-123"]) + """client.create() single-dict shim is gone; client.records.create() returns str.""" + with self.assertRaises(AttributeError): + self.client.create("account", {"name": "A"}) def test_create_bulk_returns_list(self): - """client.create() with a list payload returns list[str] directly.""" - self.client._odata._entity_set_from_schema_name.return_value = "accounts" - self.client._odata._create_multiple.return_value = ["guid-1", "guid-2"] - - with self.assertWarns(DeprecationWarning): - result = self.client.create("account", [{"name": "A"}, {"name": "B"}]) - - self.assertIsInstance(result, list) - self.assertEqual(result, ["guid-1", "guid-2"]) - - # ---------------------------------------------------------------- update + """client.create() list-payload shim is gone; client.records.create() returns list[str].""" + with self.assertRaises(AttributeError): + self.client.create("account", [{"name": "A"}, {"name": "B"}]) def test_update_warns_and_delegates(self): - """client.update() emits a DeprecationWarning and delegates to records.update.""" - with self.assertWarns(DeprecationWarning): - self.client.update( - "account", - "00000000-0000-0000-0000-000000000001", - {"telephone1": "555-0199"}, - ) - - self.client._odata._update.assert_called_once_with( - "account", - "00000000-0000-0000-0000-000000000001", - {"telephone1": "555-0199"}, - ) - - # ---------------------------------------------------------------- delete + """client.update() → use client.records.update()""" + with self.assertRaises(AttributeError): + self.client.update("account", "guid-1", {"telephone1": "555-0199"}) def test_delete_warns_and_delegates(self): - """client.delete() emits a DeprecationWarning and delegates to records.delete.""" - with self.assertWarns(DeprecationWarning): - self.client.delete("account", "00000000-0000-0000-0000-000000000001") - - self.client._odata._delete.assert_called_once_with("account", "00000000-0000-0000-0000-000000000001") - - # ------------------------------------------------------------------- get + """client.delete() → use client.records.delete()""" + with self.assertRaises(AttributeError): + self.client.delete("account", "guid-1") def test_get_single_warns(self): - """client.get() with record_id emits a DeprecationWarning and delegates - to records.get. - """ - expected = {"accountid": "guid-1", "name": "Contoso"} - self.client._odata._get.return_value = expected - - with self.assertWarns(DeprecationWarning): - result = self.client.get("account", record_id="guid-1") - - self.client._odata._get.assert_called_once_with("account", "guid-1", select=None) - self.assertEqual(result["accountid"], "guid-1") - self.assertEqual(result["name"], "Contoso") + """client.get(record_id=...) → use client.records.get()""" + with self.assertRaises(AttributeError): + self.client.get("account", record_id="guid-1") def test_get_multiple_warns(self): - """client.get() without record_id emits a DeprecationWarning and delegates - to records.get. - """ - page = [{"accountid": "1", "name": "A"}, {"accountid": "2", "name": "B"}] - self.client._odata._get_multiple.return_value = iter([page]) - - with self.assertWarns(DeprecationWarning): - result = self.client.get("account", filter="statecode eq 0", top=10) + """client.get(filter=...) → use client.records.get()""" + with self.assertRaises(AttributeError): + self.client.get("account", filter="statecode eq 0", top=10) - # The result is a generator; consume it. - pages = list(result) - self.assertEqual(len(pages), 1) - self.assertEqual(pages[0][0]["name"], "A") - self.assertEqual(pages[0][1]["name"], "B") - - self.client._odata._get_multiple.assert_called_once_with( - "account", - select=None, - filter="statecode eq 0", - orderby=None, - top=10, - expand=None, - page_size=None, - count=False, - include_annotations=None, - ) - - # ------------------------------------------------------------- query_sql + # ----------------------------------------------------------------- query def test_query_sql_warns(self): - """client.query_sql() emits a DeprecationWarning and delegates to - query.sql. - """ - expected_rows = [{"name": "Contoso"}, {"name": "Fabrikam"}] - self.client._odata._query_sql.return_value = expected_rows - - with self.assertWarns(DeprecationWarning): - result = self.client.query_sql("SELECT name FROM account") - - self.client._odata._query_sql.assert_called_once_with("SELECT name FROM account") - self.assertEqual(len(result), 2) - self.assertEqual(result[0]["name"], "Contoso") - self.assertEqual(result[1]["name"], "Fabrikam") + """client.query_sql() → use client.query.sql()""" + with self.assertRaises(AttributeError): + self.client.query_sql("SELECT name FROM account") - # -------------------------------------------------------- get_table_info + # --------------------------------------------------------------- tables def test_get_table_info_warns(self): - """client.get_table_info() emits a DeprecationWarning and delegates to - tables.get. - """ - expected_info = { - "table_schema_name": "new_MyTable", - "table_logical_name": "new_mytable", - "entity_set_name": "new_mytables", - "metadata_id": "meta-guid", - } - self.client._odata._get_table_info.return_value = expected_info - - with self.assertWarns(DeprecationWarning): - result = self.client.get_table_info("new_MyTable") - - self.client._odata._get_table_info.assert_called_once_with("new_MyTable") - self.assertEqual(result["table_schema_name"], "new_MyTable") - self.assertEqual(result["entity_set_name"], "new_mytables") - - # --------------------------------------------------------- create_table + """client.get_table_info() → use client.tables.get()""" + with self.assertRaises(AttributeError): + self.client.get_table_info("new_MyTable") def test_create_table_warns(self): - """client.create_table() emits a DeprecationWarning and maps legacy - parameter names (solution_unique_name -> solution, - primary_column_schema_name -> primary_column) when delegating to - tables.create. - """ - expected = { - "table_schema_name": "new_Product", - "entity_set_name": "new_products", - "table_logical_name": "new_product", - "metadata_id": "meta-guid", - "columns_created": ["new_Price"], - } - self.client._odata._create_table.return_value = expected - - with self.assertWarns(DeprecationWarning): - result = self.client.create_table( - "new_Product", - {"new_Price": "decimal"}, - solution_unique_name="MySolution", - primary_column_schema_name="new_ProductName", - ) - - # Verify that the internal _create_table received the mapped params. - self.client._odata._create_table.assert_called_once_with( - "new_Product", - {"new_Price": "decimal"}, - "MySolution", - "new_ProductName", - None, - ) - self.assertEqual(result["table_schema_name"], "new_Product") - self.assertEqual(result["columns_created"], ["new_Price"]) - - # --------------------------------------------------------- delete_table + """client.create_table() → use client.tables.create()""" + with self.assertRaises(AttributeError): + self.client.create_table("new_Product", {"new_Price": "decimal"}) def test_delete_table_warns(self): - """client.delete_table() emits a DeprecationWarning and delegates to - tables.delete. - """ - with self.assertWarns(DeprecationWarning): + """client.delete_table() → use client.tables.delete()""" + with self.assertRaises(AttributeError): self.client.delete_table("new_MyTestTable") - self.client._odata._delete_table.assert_called_once_with("new_MyTestTable") - - # ---------------------------------------------------------- list_tables - def test_list_tables_warns(self): - """client.list_tables() emits a DeprecationWarning and delegates to - tables.list. - """ - expected = [{"LogicalName": "account"}, {"LogicalName": "contact"}] - self.client._odata._list_tables.return_value = expected - - with self.assertWarns(DeprecationWarning): - result = self.client.list_tables() - - self.client._odata._list_tables.assert_called_once() - self.assertEqual(result, expected) - - # ------------------------------------------------------- create_columns + """client.list_tables() → use client.tables.list()""" + with self.assertRaises(AttributeError): + self.client.list_tables() def test_create_columns_warns(self): - """client.create_columns() emits a DeprecationWarning and delegates to - tables.add_columns. - """ - self.client._odata._create_columns.return_value = ["new_Notes", "new_Active"] - - with self.assertWarns(DeprecationWarning): - result = self.client.create_columns( - "new_MyTestTable", - {"new_Notes": "string", "new_Active": "bool"}, - ) - - self.client._odata._create_columns.assert_called_once_with( - "new_MyTestTable", - {"new_Notes": "string", "new_Active": "bool"}, - ) - self.assertEqual(result, ["new_Notes", "new_Active"]) - - # ------------------------------------------------------- delete_columns + """client.create_columns() → use client.tables.add_columns()""" + with self.assertRaises(AttributeError): + self.client.create_columns("new_MyTestTable", {"new_Notes": "string"}) def test_delete_columns_warns(self): - """client.delete_columns() emits a DeprecationWarning and delegates to - tables.remove_columns. - """ - self.client._odata._delete_columns.return_value = ["new_Notes", "new_Active"] - - with self.assertWarns(DeprecationWarning): - result = self.client.delete_columns( - "new_MyTestTable", - ["new_Notes", "new_Active"], - ) + """client.delete_columns() → use client.tables.remove_columns()""" + with self.assertRaises(AttributeError): + self.client.delete_columns("new_MyTestTable", ["new_Notes"]) - self.client._odata._delete_columns.assert_called_once_with( - "new_MyTestTable", - ["new_Notes", "new_Active"], - ) - self.assertEqual(result, ["new_Notes", "new_Active"]) - - # ----------------------------------------------------------- upload_file + # ----------------------------------------------------------------- files def test_upload_file_warns(self): - """client.upload_file() emits a DeprecationWarning and delegates - to files.upload. - """ - with self.assertWarns(DeprecationWarning): + """client.upload_file() → use client.files.upload()""" + with self.assertRaises(AttributeError): self.client.upload_file("account", "guid-1", "new_Document", "/path/to/file.pdf") - self.client._odata._upload_file.assert_called_once_with( - "account", - "guid-1", - "new_Document", - "/path/to/file.pdf", - mode=None, - mime_type=None, - if_none_match=True, - ) - if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_migration_tool.py b/tests/unit/test_migration_tool.py new file mode 100644 index 00000000..8f7af8ff --- /dev/null +++ b/tests/unit/test_migration_tool.py @@ -0,0 +1,351 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Unit tests for PowerPlatform/Dataverse/migration/migrate_v0_to_v1.py. + +Covers: +- QueryBuilder.to_dataframe() -> .execute().to_dataframe() (auto-rewrite) +- QueryResult.to_dataframe() left untouched (receiver is .execute()) +- QueryBuilder chain via .select(), .where(), .filter_eq() all trigger the rewrite +- client.get(t, id) -> client.records.get(t, id) (top-level shortcut) +- batch.records.get(t, id) -> batch.records.retrieve(t, id) +- .filter_eq / .filter_ne / .filter_gt -> .where(col(...) OP v) +- .filter_null / .filter_not_null -> .where(col(...).is_null/is_not_null()) +- .filter_raw / .filter -> .where(raw(...)) +- .execute(by_page=True) -> .execute_pages() +- .execute(by_page=False) -> .execute() with flag stripped +- find_manual_patterns: flags client.records.get(), execute(by_page=variable), client.dataframe.get() +""" + +import textwrap +import unittest + +try: + import libcst # noqa: F401 + + _LIBCST_AVAILABLE = True +except ImportError: + _LIBCST_AVAILABLE = False + +_skip_no_libcst = unittest.skipUnless(_LIBCST_AVAILABLE, "libcst not installed") + + +def _migrate(source: str, *, client_var: str = "client") -> str: + from PowerPlatform.Dataverse.migration.migrate_v0_to_v1 import migrate_source + + return migrate_source(textwrap.dedent(source), client_var=client_var) + + +def _find_manual(source: str, *, client_var: str = "client") -> list: + from PowerPlatform.Dataverse.migration.migrate_v0_to_v1 import find_manual_patterns + + return find_manual_patterns(textwrap.dedent(source), client_var=client_var) + + +# --------------------------------------------------------------------------- +# QueryBuilder.to_dataframe() -> .execute().to_dataframe() +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestToDataframeRewrite(unittest.TestCase): + """QueryBuilder.to_dataframe() receives .execute() insertion.""" + + def test_builder_chain_gets_execute_inserted(self): + src = "df = client.query.builder('account').select('name').to_dataframe()\n" + out = _migrate(src) + self.assertIn(".execute().to_dataframe()", out) + self.assertNotIn(".to_dataframe().to_dataframe()", out) + + def test_where_chain_triggers_rewrite(self): + src = "df = q.where(col('statecode') == 0).to_dataframe()\n" + out = _migrate(src) + self.assertIn(".execute().to_dataframe()", out) + + def test_filter_eq_chain_triggers_rewrite(self): + src = "df = q.filter_eq('statecode', 0).to_dataframe()\n" + out = _migrate(src) + self.assertIn(".execute().to_dataframe()", out) + + def test_select_alone_triggers_rewrite(self): + src = "df = q.select('name', 'revenue').to_dataframe()\n" + out = _migrate(src) + self.assertIn(".execute().to_dataframe()", out) + + def test_already_executed_not_double_wrapped(self): + src = "df = q.select('name').execute().to_dataframe()\n" + out = _migrate(src) + self.assertNotIn(".execute().execute()", out) + self.assertIn(".execute().to_dataframe()", out) + + def test_unrelated_to_dataframe_not_rewritten(self): + src = "df = some_result.to_dataframe()\n" + out = _migrate(src) + self.assertNotIn(".execute()", out) + self.assertIn("some_result.to_dataframe()", out) + + def test_full_chain_structure_preserved(self): + src = "df = client.query.builder('account')\\\n" " .select('name')\\\n" " .to_dataframe()\n" + out = _migrate(src) + # .execute() is inserted before .to_dataframe(); a line-continuation may separate them + self.assertIn(".execute()", out) + self.assertIn(".to_dataframe()", out) + self.assertNotIn(".get(", out) + + def test_rewrite_inside_assignment(self): + src = "result = builder.select('name').to_dataframe()\n" + out = _migrate(src) + self.assertIn(".execute().to_dataframe()", out) + + +# --------------------------------------------------------------------------- +# Top-level shortcut rewrites +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestClientShortcutRewrites(unittest.TestCase): + def test_client_get_becomes_records_get(self): + src = "r = client.get('account', 'abc')\n" + out = _migrate(src) + self.assertIn("client.records.get(", out) + self.assertNotIn("client.get(", out) + + def test_client_create_becomes_records_create(self): + src = "client.create('account', {'name': 'X'})\n" + out = _migrate(src) + self.assertIn("client.records.create(", out) + + def test_client_delete_becomes_records_delete(self): + src = "client.delete('account', 'abc')\n" + out = _migrate(src) + self.assertIn("client.records.delete(", out) + + def test_client_update_becomes_records_update(self): + src = "client.update('account', 'abc', {'name': 'Y'})\n" + out = _migrate(src) + self.assertIn("client.records.update(", out) + + def test_client_query_sql_becomes_query_sql(self): + src = "rows = client.query_sql('SELECT * FROM account')\n" + out = _migrate(src) + self.assertIn("client.query.sql(", out) + + def test_client_get_table_info_becomes_tables_get(self): + src = "info = client.get_table_info('account')\n" + out = _migrate(src) + self.assertIn("client.tables.get(", out) + + def test_client_list_tables_becomes_tables_list(self): + src = "tables = client.list_tables()\n" + out = _migrate(src) + self.assertIn("client.tables.list(", out) + + def test_client_var_override(self): + src = "r = svc.get('account', 'abc')\n" + out = _migrate(src, client_var="svc") + self.assertIn("svc.records.get(", out) + + def test_client_get_not_matched_on_other_receiver(self): + src = "v = record.get('name')\n" + out = _migrate(src) + self.assertIn("record.get(", out) + self.assertNotIn("record.records.get(", out) + + +# --------------------------------------------------------------------------- +# batch.records.get() -> batch.records.retrieve() +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestBatchRecordsGetRewrite(unittest.TestCase): + def test_batch_records_get_becomes_retrieve(self): + src = "batch.records.get('account', 'abc')\n" + out = _migrate(src) + self.assertIn("batch.records.retrieve(", out) + self.assertNotIn("batch.records.get(", out) + + def test_client_records_get_not_rewritten(self): + src = "client.records.get('account', 'abc')\n" + out = _migrate(src) + self.assertIn("client.records.get(", out) + self.assertNotIn("client.records.retrieve(", out) + + +# --------------------------------------------------------------------------- +# .filter_*() -> .where(col(...) ...) rewrites +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestFilterMethodRewrites(unittest.TestCase): + def test_filter_eq(self): + src = "q.filter_eq('statecode', 0)\n" + out = _migrate(src) + self.assertIn(".where(", out) + self.assertIn("col(", out) + + def test_filter_ne(self): + src = "q.filter_ne('statecode', 0)\n" + out = _migrate(src) + self.assertIn(".where(", out) + + def test_filter_gt(self): + src = "q.filter_gt('revenue', 1000)\n" + out = _migrate(src) + self.assertIn(".where(", out) + + def test_filter_null(self): + src = "q.filter_null('email')\n" + out = _migrate(src) + self.assertIn(".is_null()", out) + + def test_filter_not_null(self): + src = "q.filter_not_null('email')\n" + out = _migrate(src) + self.assertIn(".is_not_null()", out) + + def test_filter_raw(self): + src = "q.filter_raw('statecode eq 0')\n" + out = _migrate(src) + self.assertIn("raw(", out) + + def test_filter_string_literal(self): + src = "q.filter('statecode eq 0')\n" + out = _migrate(src) + self.assertIn(".where(raw(", out) + + def test_filter_between(self): + src = "q.filter_between('revenue', 1000, 5000)\n" + out = _migrate(src) + self.assertIn(".between(", out) + + def test_filter_in(self): + src = "q.filter_in('statecode', [0, 1])\n" + out = _migrate(src) + self.assertIn(".in_(", out) + + +# --------------------------------------------------------------------------- +# .execute(by_page=...) -> .execute_pages() / .execute() +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestExecuteByPageRewrite(unittest.TestCase): + def test_execute_by_page_true_becomes_execute_pages(self): + src = "result = q.execute(by_page=True)\n" + out = _migrate(src) + self.assertIn(".execute_pages()", out) + self.assertNotIn("by_page", out) + + def test_execute_by_page_false_strips_flag(self): + src = "result = q.execute(by_page=False)\n" + out = _migrate(src) + self.assertIn(".execute()", out) + self.assertNotIn("by_page", out) + self.assertNotIn("execute_pages", out) + + def test_execute_no_args_unchanged(self): + src = "result = q.execute()\n" + out = _migrate(src) + self.assertIn(".execute()", out) + self.assertNotIn("execute_pages", out) + + +# --------------------------------------------------------------------------- +# find_manual_patterns +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestFindManualPatterns(unittest.TestCase): + def test_client_records_get_flagged(self): + src = "client.records.get('account', 'abc')\n" + findings = _find_manual(src) + self.assertTrue(any("records.get" in f for f in findings)) + + def test_execute_by_page_variable_flagged(self): + src = "q.execute(by_page=flag)\n" + findings = _find_manual(src) + self.assertTrue(any("by_page" in f for f in findings)) + + def test_execute_by_page_literal_not_flagged(self): + src = "q.execute(by_page=True)\n" + findings = _find_manual(src) + self.assertFalse(any("by_page" in f for f in findings)) + + def test_client_dataframe_get_flagged(self): + src = "client.dataframe.get('account')\n" + findings = _find_manual(src) + self.assertTrue(any("dataframe.get" in f for f in findings)) + + def test_query_sql_select_flagged(self): + src = "client.query.sql_select('account', ['name'])\n" + findings = _find_manual(src) + self.assertTrue(any("sql_select" in f for f in findings)) + + def test_clean_code_has_no_findings(self): + src = ( + "result = client.records.retrieve('account', 'abc')\n" "pages = client.records.list('account').execute()\n" + ) + findings = _find_manual(src) + self.assertEqual(findings, []) + + def test_batch_records_get_not_flagged(self): + src = "batch.records.get('account', 'abc')\n" + findings = _find_manual(src) + self.assertFalse(any("records.get" in f for f in findings)) + + +# --------------------------------------------------------------------------- +# CLI: --help / -h handling +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestMainHelp(unittest.TestCase): + """``main()`` returns 0 and prints usage when --help / -h is passed. + + Regression guard for the UX gap where ``--help`` was treated as a positional + path argument and produced ``[WARN] Not a file or directory: --help``. + """ + + def _run_main_capture(self, argv): + import io + import contextlib + from PowerPlatform.Dataverse.migration.migrate_v0_to_v1 import main + + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + rc = main(argv) + return rc, buf.getvalue() + + def test_long_help_flag_returns_zero(self): + rc, _ = self._run_main_capture(["--help"]) + self.assertEqual(rc, 0) + + def test_short_help_flag_returns_zero(self): + rc, _ = self._run_main_capture(["-h"]) + self.assertEqual(rc, 0) + + def test_help_prints_usage_line(self): + _, out = self._run_main_capture(["--help"]) + self.assertIn("Usage:", out) + self.assertIn("dataverse-migrate", out) + + def test_help_takes_precedence_over_other_flags(self): + """--help with other flags still exits 0 without processing paths.""" + rc, _ = self._run_main_capture(["--dry-run", "--help", "/nonexistent/path"]) + self.assertEqual(rc, 0) + + def test_no_args_returns_one(self): + """No arguments still prints usage but returns 1 (error).""" + rc, out = self._run_main_capture([]) + self.assertEqual(rc, 1) + self.assertIn("Usage:", out) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_phase1_ga.py b/tests/unit/test_phase1_ga.py new file mode 100644 index 00000000..ee832029 --- /dev/null +++ b/tests/unit/test_phase1_ga.py @@ -0,0 +1,861 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Comprehensive Phase 1 GA regression tests. + +Verifies all Phase 1 breaking changes and deprecations in one place: + +1. All 12 deprecated client flat methods raise AttributeError (removed). +2. All 15 deprecated filter factory functions emit DeprecationWarning on CALL + (not on import). +3. GA filter functions col() and raw() emit NO deprecation warning. +4. dataframe.get() emits DeprecationWarning on call. +5. All deprecated factories remain functional (correct OData output). +6. All 16 filter_* builder methods raise AttributeError on QueryBuilder (removed). +7. No DeprecationWarning is emitted at module import time. +""" + +import unittest +import warnings +from unittest.mock import MagicMock + +from azure.core.credentials import TokenCredential + +from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.models.query_builder import QueryBuilder + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_client(): + """Create a DataverseClient with a mock credential and mock _odata.""" + credential = MagicMock(spec=TokenCredential) + client = DataverseClient("https://example.crm.dynamics.com", credential) + client._odata = MagicMock() + return client + + +def _catch_warnings(*categories): + """Context manager: catch all warnings, return the list.""" + return warnings.catch_warnings(record=True) + + +# --------------------------------------------------------------------------- +# 1. Removed client flat methods raise AttributeError +# --------------------------------------------------------------------------- + + +class TestRemovedClientFlatMethods(unittest.TestCase): + """All 12 formerly-deprecated client flat methods are removed in GA.""" + + def setUp(self): + self.client = _make_client() + + def _assert_removed(self, method_name, *args, **kwargs): + with self.assertRaises(AttributeError, msg=f"client.{method_name} should not exist"): + getattr(self.client, method_name)(*args, **kwargs) + + def test_create_removed(self): + self._assert_removed("create", "account", {"name": "Test"}) + + def test_update_removed(self): + self._assert_removed("update", "account", "guid-1", {"name": "Test"}) + + def test_delete_removed(self): + self._assert_removed("delete", "account", "guid-1") + + def test_get_removed(self): + self._assert_removed("get", "account", "guid-1") + + def test_query_sql_removed(self): + self._assert_removed("query_sql", "SELECT name FROM account") + + def test_get_table_info_removed(self): + self._assert_removed("get_table_info", "account") + + def test_create_table_removed(self): + self._assert_removed("create_table", "new_Test", {}) + + def test_delete_table_removed(self): + self._assert_removed("delete_table", "new_Test") + + def test_list_tables_removed(self): + self._assert_removed("list_tables") + + def test_create_columns_removed(self): + self._assert_removed("create_columns", "account", {}) + + def test_delete_columns_removed(self): + self._assert_removed("delete_columns", "account", []) + + def test_upload_file_removed(self): + self._assert_removed("upload_file", "account", "guid-1", "col", "/path") + + +# --------------------------------------------------------------------------- +# 2. Deprecated filter factories emit DeprecationWarning on CALL (not import) +# --------------------------------------------------------------------------- + + +class TestDeprecatedFilterFactoriesWarnOnCall(unittest.TestCase): + """All 15 deprecated filter factories emit DeprecationWarning when called. + + The warning must fire on CALL, not on import — importing the module must + be warning-free. + """ + + def _assert_warns_on_call(self, func, *args, **kwargs): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = func(*args, **kwargs) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertGreater(len(dep_warnings), 0, f"{func.__name__}() did not emit DeprecationWarning") + return result, dep_warnings + + def _assert_single_warning(self, func, *args, **kwargs): + """Verify exactly one DeprecationWarning is emitted (no chained warnings).""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + func(*args, **kwargs) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual( + len(dep_warnings), 1, f"{func.__name__}() emitted {len(dep_warnings)} warnings (expected exactly 1)" + ) + + def test_no_warning_on_import(self): + """Importing the filters module emits no DeprecationWarning.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + import importlib + import PowerPlatform.Dataverse.models.filters as _f + + importlib.reload(_f) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0, "Import must not emit DeprecationWarning") + + def test_eq_warns(self): + from PowerPlatform.Dataverse.models.filters import eq + + self._assert_warns_on_call(eq, "name", "Contoso") + + def test_ne_warns(self): + from PowerPlatform.Dataverse.models.filters import ne + + self._assert_warns_on_call(ne, "statecode", 1) + + def test_gt_warns(self): + from PowerPlatform.Dataverse.models.filters import gt + + self._assert_warns_on_call(gt, "revenue", 1000000) + + def test_ge_warns(self): + from PowerPlatform.Dataverse.models.filters import ge + + self._assert_warns_on_call(ge, "revenue", 1000000) + + def test_lt_warns(self): + from PowerPlatform.Dataverse.models.filters import lt + + self._assert_warns_on_call(lt, "revenue", 500000) + + def test_le_warns(self): + from PowerPlatform.Dataverse.models.filters import le + + self._assert_warns_on_call(le, "revenue", 500000) + + def test_contains_warns(self): + from PowerPlatform.Dataverse.models.filters import contains + + self._assert_warns_on_call(contains, "name", "Corp") + + def test_startswith_warns(self): + from PowerPlatform.Dataverse.models.filters import startswith + + self._assert_warns_on_call(startswith, "name", "Con") + + def test_endswith_warns(self): + from PowerPlatform.Dataverse.models.filters import endswith + + self._assert_warns_on_call(endswith, "name", "Ltd") + + def test_between_warns(self): + from PowerPlatform.Dataverse.models.filters import between + + self._assert_warns_on_call(between, "revenue", 100000, 500000) + + def test_is_null_warns(self): + from PowerPlatform.Dataverse.models.filters import is_null + + self._assert_warns_on_call(is_null, "telephone1") + + def test_is_not_null_warns(self): + from PowerPlatform.Dataverse.models.filters import is_not_null + + self._assert_warns_on_call(is_not_null, "telephone1") + + def test_filter_in_warns(self): + from PowerPlatform.Dataverse.models.filters import filter_in + + self._assert_warns_on_call(filter_in, "statecode", [0, 1]) + + def test_not_in_warns(self): + from PowerPlatform.Dataverse.models.filters import not_in + + self._assert_warns_on_call(not_in, "statecode", [2, 3]) + + def test_not_between_warns(self): + from PowerPlatform.Dataverse.models.filters import not_between + + self._assert_warns_on_call(not_between, "revenue", 100000, 500000) + + def test_between_emits_exactly_one_warning(self): + """between() must not chain through deprecated ge/le (would emit 3 warnings).""" + from PowerPlatform.Dataverse.models.filters import between + + self._assert_single_warning(between, "revenue", 100000, 500000) + + def test_not_between_emits_exactly_one_warning(self): + """not_between() must not chain through deprecated ge/le.""" + from PowerPlatform.Dataverse.models.filters import not_between + + self._assert_single_warning(not_between, "revenue", 100000, 500000) + + def test_warning_is_deprecation_warning_class(self): + """Each factory's warning category must be DeprecationWarning, not its subclass.""" + from PowerPlatform.Dataverse.models.filters import eq + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + eq("name", "Test") + w = caught[0] + self.assertIs(w.category, DeprecationWarning) + + def test_warning_message_names_replacement(self): + """Each warning message should name the col()-based replacement.""" + from PowerPlatform.Dataverse.models.filters import eq + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + eq("name", "Test") + self.assertIn("col(", str(caught[0].message)) + + +# --------------------------------------------------------------------------- +# 3. GA functions col() and raw() emit NO deprecation warning +# --------------------------------------------------------------------------- + + +class TestGAFunctionsNoWarning(unittest.TestCase): + """col() and raw() are GA — must never emit DeprecationWarning.""" + + def _assert_no_dep_warning(self, func, *args, **kwargs): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + func(*args, **kwargs) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual( + len(dep_warnings), 0, f"{func.__name__}() emitted unexpected DeprecationWarning: {dep_warnings}" + ) + + def test_col_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + self._assert_no_dep_warning(col, "statecode") + + def test_raw_no_warning(self): + from PowerPlatform.Dataverse.models.filters import raw + + self._assert_no_dep_warning(raw, "statecode eq 0") + + def test_col_eq_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("statecode") == 0 + _ = expr.to_odata() + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_comparison_chain_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = (col("statecode") == 0) & (col("revenue") > 100000) + _ = expr.to_odata() + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_between_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("revenue").between(100000, 500000) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_not_between_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("revenue").not_between(100000, 500000) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_in_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("statecode").in_([0, 1]) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_not_in_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("statecode").not_in([2, 3]) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_like_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("name").like("Contoso%") + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_not_like_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("name").not_like("%Corp%") + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_where_with_col_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + QueryBuilder("account").where(col("statecode") == 0) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + +# --------------------------------------------------------------------------- +# 4. dataframe.get() emits DeprecationWarning on call +# --------------------------------------------------------------------------- + + +class TestDataframeGetDeprecation(unittest.TestCase): + """dataframe.get() must emit DeprecationWarning on call (not on import).""" + + def setUp(self): + self.client = _make_client() + # Set up _odata so records.get() can be called + self.client._odata._get_multiple.return_value = iter([]) + self.client._odata._get.return_value = MagicMock(data={}) + + def test_dataframe_get_warns_on_call(self): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + try: + self.client.dataframe.get("account", select=["name"], top=10) + except Exception: + pass + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertGreater(len(dep_warnings), 0, "dataframe.get() did not emit DeprecationWarning") + + def test_dataframe_get_warning_message(self): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + try: + self.client.dataframe.get("account", select=["name"], top=10) + except Exception: + pass + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + msg = str(dep_warnings[0].message) + self.assertIn("dataframe.get()", msg) + self.assertIn("builder", msg) + + def test_dataframe_other_methods_no_warning(self): + """dataframe.sql(), dataframe.create(), etc. must NOT warn.""" + import pandas as pd + from PowerPlatform.Dataverse.models.record import Record + + self.client._odata._query_sql.return_value = [] + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.dataframe.sql("SELECT name FROM account") + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0, "dataframe.sql() must not warn") + + +# --------------------------------------------------------------------------- +# 5. Deprecated factories remain functional (correct OData output) +# --------------------------------------------------------------------------- + + +class TestDeprecatedFactoriesStillFunctional(unittest.TestCase): + """Despite the warning, deprecated factories produce correct OData strings.""" + + def _odata(self, func, *args): + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + return func(*args).to_odata() + + def test_eq_functional(self): + from PowerPlatform.Dataverse.models.filters import eq + + self.assertEqual(self._odata(eq, "name", "Contoso"), "name eq 'Contoso'") + + def test_ne_functional(self): + from PowerPlatform.Dataverse.models.filters import ne + + self.assertEqual(self._odata(ne, "statecode", 1), "statecode ne 1") + + def test_gt_functional(self): + from PowerPlatform.Dataverse.models.filters import gt + + self.assertEqual(self._odata(gt, "revenue", 1000000), "revenue gt 1000000") + + def test_ge_functional(self): + from PowerPlatform.Dataverse.models.filters import ge + + self.assertEqual(self._odata(ge, "revenue", 1000000), "revenue ge 1000000") + + def test_lt_functional(self): + from PowerPlatform.Dataverse.models.filters import lt + + self.assertEqual(self._odata(lt, "revenue", 500000), "revenue lt 500000") + + def test_le_functional(self): + from PowerPlatform.Dataverse.models.filters import le + + self.assertEqual(self._odata(le, "revenue", 500000), "revenue le 500000") + + def test_contains_functional(self): + from PowerPlatform.Dataverse.models.filters import contains + + self.assertEqual(self._odata(contains, "name", "Corp"), "contains(name, 'Corp')") + + def test_startswith_functional(self): + from PowerPlatform.Dataverse.models.filters import startswith + + self.assertEqual(self._odata(startswith, "name", "Con"), "startswith(name, 'Con')") + + def test_endswith_functional(self): + from PowerPlatform.Dataverse.models.filters import endswith + + self.assertEqual(self._odata(endswith, "name", "Ltd"), "endswith(name, 'Ltd')") + + def test_between_functional(self): + from PowerPlatform.Dataverse.models.filters import between + + self.assertEqual( + self._odata(between, "revenue", 100000, 500000), + "(revenue ge 100000 and revenue le 500000)", + ) + + def test_is_null_functional(self): + from PowerPlatform.Dataverse.models.filters import is_null + + self.assertEqual(self._odata(is_null, "telephone1"), "telephone1 eq null") + + def test_is_not_null_functional(self): + from PowerPlatform.Dataverse.models.filters import is_not_null + + self.assertEqual(self._odata(is_not_null, "telephone1"), "telephone1 ne null") + + def test_filter_in_functional(self): + from PowerPlatform.Dataverse.models.filters import filter_in + + self.assertEqual( + self._odata(filter_in, "statecode", [0, 1]), + 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1"])', + ) + + def test_not_in_functional(self): + from PowerPlatform.Dataverse.models.filters import not_in + + self.assertEqual( + self._odata(not_in, "statecode", [2, 3]), + 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["2","3"])', + ) + + def test_not_between_functional(self): + from PowerPlatform.Dataverse.models.filters import not_between + + self.assertEqual( + self._odata(not_between, "revenue", 100000, 500000), + "not ((revenue ge 100000 and revenue le 500000))", + ) + + def test_deprecated_factory_usable_in_where(self): + """Deprecated factories still produce valid FilterExpression for where().""" + from PowerPlatform.Dataverse.models.filters import eq + + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + expr = eq("statecode", 0) + qb = QueryBuilder("account").where(expr) + self.assertEqual(qb.build()["filter"], "statecode eq 0") + + def test_deprecated_factories_composable(self): + """Deprecated factories still compose correctly with & and |.""" + from PowerPlatform.Dataverse.models.filters import eq, gt + + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + expr = (eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000) + self.assertEqual( + expr.to_odata(), + "((statecode eq 0 or statecode eq 1) and revenue gt 100000)", + ) + + +# --------------------------------------------------------------------------- +# 6. All 16 filter_* builder methods raise AttributeError on QueryBuilder +# --------------------------------------------------------------------------- + + +class TestRemovedBuilderFilterMethods(unittest.TestCase): + """All 16 filter_* methods were removed from QueryBuilder in GA.""" + + def setUp(self): + self.qb = QueryBuilder("account") + + def _assert_removed(self, method_name, *args): + with self.assertRaises(AttributeError, msg=f"QueryBuilder.{method_name} should not exist"): + getattr(self.qb, method_name)(*args) + + def test_filter_eq_removed(self): + self._assert_removed("filter_eq", "name", "Contoso") + + def test_filter_ne_removed(self): + self._assert_removed("filter_ne", "statecode", 1) + + def test_filter_gt_removed(self): + self._assert_removed("filter_gt", "revenue", 0) + + def test_filter_ge_removed(self): + self._assert_removed("filter_ge", "revenue", 0) + + def test_filter_lt_removed(self): + self._assert_removed("filter_lt", "revenue", 0) + + def test_filter_le_removed(self): + self._assert_removed("filter_le", "revenue", 0) + + def test_filter_contains_removed(self): + self._assert_removed("filter_contains", "name", "Corp") + + def test_filter_startswith_removed(self): + self._assert_removed("filter_startswith", "name", "Con") + + def test_filter_endswith_removed(self): + self._assert_removed("filter_endswith", "name", "Ltd") + + def test_filter_null_removed(self): + self._assert_removed("filter_null", "telephone1") + + def test_filter_not_null_removed(self): + self._assert_removed("filter_not_null", "telephone1") + + def test_filter_in_removed(self): + self._assert_removed("filter_in", "statecode", [0, 1]) + + def test_filter_not_in_removed(self): + self._assert_removed("filter_not_in", "statecode", [0, 1]) + + def test_filter_between_removed(self): + self._assert_removed("filter_between", "revenue", 100, 500) + + def test_filter_not_between_removed(self): + self._assert_removed("filter_not_between", "revenue", 100, 500) + + def test_filter_raw_removed(self): + self._assert_removed("filter_raw", "statecode eq 0") + + +# --------------------------------------------------------------------------- +# 7. ColumnProxy (col()) correctness — all operators and methods +# --------------------------------------------------------------------------- + + +class TestColumnProxyOperators(unittest.TestCase): + """col() proxy covers all operators and methods correctly.""" + + def test_eq(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("name") == "Contoso").to_odata(), "name eq 'Contoso'") + + def test_ne(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("statecode") != 1).to_odata(), "statecode ne 1") + + def test_gt(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("revenue") > 1000000).to_odata(), "revenue gt 1000000") + + def test_ge(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("revenue") >= 1000000).to_odata(), "revenue ge 1000000") + + def test_lt(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("revenue") < 500000).to_odata(), "revenue lt 500000") + + def test_le(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("revenue") <= 500000).to_odata(), "revenue le 500000") + + def test_is_null(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("telephone1").is_null().to_odata(), "telephone1 eq null") + + def test_is_not_null(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("telephone1").is_not_null().to_odata(), "telephone1 ne null") + + def test_in_(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("statecode").in_([0, 1]).to_odata(), + 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1"])', + ) + + def test_not_in(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("statecode").not_in([2, 3]).to_odata(), + 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["2","3"])', + ) + + def test_between(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("revenue").between(100000, 500000).to_odata(), + "(revenue ge 100000 and revenue le 500000)", + ) + + def test_not_between(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("revenue").not_between(100000, 500000).to_odata(), + "not ((revenue ge 100000 and revenue le 500000))", + ) + + def test_contains(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").contains("Corp").to_odata(), "contains(name, 'Corp')") + + def test_startswith(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").startswith("Con").to_odata(), "startswith(name, 'Con')") + + def test_endswith(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").endswith("Ltd").to_odata(), "endswith(name, 'Ltd')") + + def test_like_startswith_pattern(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").like("Contoso%").to_odata(), "startswith(name, 'Contoso')") + + def test_like_endswith_pattern(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").like("%Ltd").to_odata(), "endswith(name, 'Ltd')") + + def test_like_contains_pattern(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").like("%Corp%").to_odata(), "contains(name, 'Corp')") + + def test_like_no_wildcard_equality(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").like("Contoso").to_odata(), "name eq 'Contoso'") + + def test_like_interior_wildcard_raises(self): + from PowerPlatform.Dataverse.models.filters import col + + with self.assertRaises(ValueError): + col("name").like("Con%oso") + + def test_not_like_startswith_pattern(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("name").not_like("Contoso%").to_odata(), + "not (startswith(name, 'Contoso'))", + ) + + def test_not_like_endswith_pattern(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("name").not_like("%Ltd").to_odata(), + "not (endswith(name, 'Ltd'))", + ) + + def test_not_like_contains_pattern(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("name").not_like("%Corp%").to_odata(), + "not (contains(name, 'Corp'))", + ) + + def test_not_like_no_wildcard_negated_equality(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("name").not_like("Contoso").to_odata(), + "not (name eq 'Contoso')", + ) + + def test_not_like_interior_wildcard_raises(self): + from PowerPlatform.Dataverse.models.filters import col + + with self.assertRaises(ValueError): + col("name").not_like("Con%oso") + + def test_column_name_lowercased(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("StateCode") == 0).to_odata(), "statecode eq 0") + + def test_empty_column_name_raises(self): + from PowerPlatform.Dataverse.models.filters import col + + with self.assertRaises(ValueError): + col("") + + def test_whitespace_column_name_raises(self): + from PowerPlatform.Dataverse.models.filters import col + + with self.assertRaises(ValueError): + col(" ") + + def test_boolean_eq(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("active") == True).to_odata(), "active eq true") # noqa: E712 + self.assertEqual((col("active") == False).to_odata(), "active eq false") # noqa: E712 + + def test_none_eq(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("telephone1") == None).to_odata(), "telephone1 eq null") # noqa: E711 + + def test_and_composition(self): + from PowerPlatform.Dataverse.models.filters import col + + expr = (col("statecode") == 0) & (col("revenue") > 100000) + self.assertEqual(expr.to_odata(), "(statecode eq 0 and revenue gt 100000)") + + def test_or_composition(self): + from PowerPlatform.Dataverse.models.filters import col + + expr = (col("statecode") == 0) | (col("statecode") == 1) + self.assertEqual(expr.to_odata(), "(statecode eq 0 or statecode eq 1)") + + def test_not_composition(self): + from PowerPlatform.Dataverse.models.filters import col + + expr = ~(col("statecode") == 1) + self.assertEqual(expr.to_odata(), "not (statecode eq 1)") + + def test_in_empty_raises(self): + from PowerPlatform.Dataverse.models.filters import col + + with self.assertRaises(ValueError): + col("statecode").in_([]) + + def test_not_in_empty_raises(self): + from PowerPlatform.Dataverse.models.filters import col + + with self.assertRaises(ValueError): + col("statecode").not_in([]) + + +# --------------------------------------------------------------------------- +# 8. GA namespace-level fluent builder (where + col) end-to-end +# --------------------------------------------------------------------------- + + +class TestGABuilderEndToEnd(unittest.TestCase): + """End-to-end verification of the GA query builder without any deprecated APIs.""" + + def setUp(self): + self.client = _make_client() + self.client._odata._get_multiple.return_value = iter([]) + + def test_builder_with_col_exprs_no_warnings(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + qb = ( + self.client.query.builder("account") + .select("name", "revenue") + .where((col("statecode") == 0) | (col("statecode") == 1)) + .where(col("revenue") > 100000) + .order_by("revenue", descending=True) + .top(50) + ) + params = qb.build() + + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0, f"GA API emitted warnings: {dep_warnings}") + self.assertEqual( + params["filter"], + "(statecode eq 0 or statecode eq 1) and revenue gt 100000", + ) + self.assertEqual(params["select"], ["name", "revenue"]) + self.assertEqual(params["top"], 50) + + def test_builder_with_raw_no_warnings(self): + from PowerPlatform.Dataverse.models.filters import raw + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + qb = self.client.query.builder("account").select("name").where(raw("statecode eq 0")).top(10) + params = qb.build() + + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + self.assertEqual(params["filter"], "statecode eq 0") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_phase2_ga.py b/tests/unit/test_phase2_ga.py new file mode 100644 index 00000000..5809e1be --- /dev/null +++ b/tests/unit/test_phase2_ga.py @@ -0,0 +1,545 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Phase 2 GA regression tests. + +Covers: +- QueryResult class (models/record.py) +- execute() returns QueryResult in flat mode +- execute(by_page=True) still returns Iterable[list[Record]] +- col, raw, QueryResult re-exports from models/__init__ and package root +- pyproject.toml migration optional dep +""" + +import unittest +import warnings +from unittest.mock import MagicMock + +from azure.core.credentials import TokenCredential + +from PowerPlatform.Dataverse.models.record import QueryResult, Record +from PowerPlatform.Dataverse.models.query_builder import QueryBuilder + + +def _make_client(): + cred = MagicMock(spec=TokenCredential) + from PowerPlatform.Dataverse.client import DataverseClient + + client = DataverseClient("https://example.crm.dynamics.com", cred) + client._odata = MagicMock() + client._odata._get_multiple = MagicMock() + client._odata._get_single = MagicMock() + return client + + +class TestQueryResultClass(unittest.TestCase): + """Unit tests for QueryResult.""" + + def _records(self, n=3): + return [Record(id=f"id-{i}", table="account", data={"name": f"R{i}"}) for i in range(n)] + + # ----- construction / dunder + + def test_init_stores_records(self): + recs = self._records(2) + qr = QueryResult(recs) + self.assertIs(qr.records, recs) + + def test_empty_result(self): + qr = QueryResult([]) + self.assertEqual(len(qr), 0) + self.assertFalse(bool(qr)) + self.assertIsNone(qr.first()) + + def test_len(self): + self.assertEqual(len(QueryResult(self._records(5))), 5) + + def test_bool_true_when_nonempty(self): + self.assertTrue(bool(QueryResult(self._records(1)))) + + def test_bool_false_when_empty(self): + self.assertFalse(bool(QueryResult([]))) + + def test_iter_yields_records(self): + recs = self._records(3) + qr = QueryResult(recs) + self.assertEqual(list(qr), recs) + + def test_iter_multiple_times(self): + recs = self._records(2) + qr = QueryResult(recs) + self.assertEqual(list(qr), list(qr)) + + def test_repr_contains_count(self): + qr = QueryResult(self._records(7)) + self.assertIn("7", repr(qr)) + + # ----- first() + + def test_first_returns_first_record(self): + recs = self._records(3) + qr = QueryResult(recs) + self.assertIs(qr.first(), recs[0]) + + def test_first_returns_none_when_empty(self): + self.assertIsNone(QueryResult([]).first()) + + # ----- __getitem__ + + def test_getitem_int_returns_record(self): + recs = self._records(3) + qr = QueryResult(recs) + self.assertIs(qr[0], recs[0]) + self.assertIs(qr[2], recs[2]) + + def test_getitem_negative_index(self): + recs = self._records(3) + qr = QueryResult(recs) + self.assertIs(qr[-1], recs[-1]) + + def test_getitem_out_of_range_raises(self): + qr = QueryResult(self._records(2)) + with self.assertRaises(IndexError): + _ = qr[99] + + def test_getitem_slice_returns_query_result(self): + recs = self._records(5) + qr = QueryResult(recs) + sliced = qr[1:3] + self.assertIsInstance(sliced, QueryResult) + self.assertEqual(list(sliced), recs[1:3]) + + def test_getitem_slice_empty(self): + qr = QueryResult(self._records(3)) + sliced = qr[10:] + self.assertIsInstance(sliced, QueryResult) + self.assertEqual(len(sliced), 0) + + # ----- to_dataframe() + + def test_to_dataframe_nonempty(self): + import pandas as pd + + recs = [ + Record(id="id-1", table="account", data={"name": "Contoso", "revenue": 1000}), + Record(id="id-2", table="account", data={"name": "Fabrikam", "revenue": 2000}), + ] + qr = QueryResult(recs) + df = qr.to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 2) + self.assertIn("name", df.columns) + self.assertIn("revenue", df.columns) + self.assertEqual(df.iloc[0]["name"], "Contoso") + self.assertEqual(df.iloc[1]["revenue"], 2000) + + def test_to_dataframe_empty_returns_empty_df(self): + import pandas as pd + + df = QueryResult([]).to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 0) + + def test_to_dataframe_handles_plain_dicts(self): + """QueryResult.to_dataframe() handles plain dicts (no .data attribute).""" + import pandas as pd + + qr = QueryResult([{"name": "A"}, {"name": "B"}]) # type: ignore[arg-type] + df = qr.to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 2) + + # ----- for r in result (backward compat) + + def test_backward_compat_for_loop(self): + recs = self._records(3) + qr = QueryResult(recs) + collected = [] + for r in qr: + collected.append(r) + self.assertEqual(collected, recs) + + def test_list_conversion(self): + recs = self._records(4) + qr = QueryResult(recs) + self.assertEqual(list(qr), recs) + + +class TestQueryResultListLikeContract(unittest.TestCase): + """Contract tests: QueryResult must be substitutable for ``list[Record]`` + in the patterns shown in the class docstring, examples, and skill docs. + + These tests exist to catch the gap from PR #175 where ``__getitem__`` was + missing despite docs and examples treating ``QueryResult`` as list-like. + They drive the contract from the *caller's* perspective rather than + introspecting which dunders are implemented, so removing any single dunder + breaks at least one assertion here with a clear signal. + """ + + def _records(self, n=3): + return [Record(id=f"id-{i}", table="account", data={"name": f"R{i}"}) for i in range(n)] + + def test_contract_index_then_field_access(self): + """Pattern from examples/advanced/fetchxml.py: ``row = result[0]; row.get(...)``.""" + qr = QueryResult(self._records(3)) + row = qr[0] + self.assertEqual(row.get("name"), "R0") + + def test_contract_single_loop_field_access(self): + """Pattern from examples/basic/installation_example.py: ``for r in result: r["name"]``.""" + qr = QueryResult(self._records(3)) + names = [r["name"] for r in qr] + self.assertEqual(names, ["R0", "R1", "R2"]) + + def test_contract_first_with_none_guard(self): + """Pattern recommended by Copilot review: ``result.first()`` with None-check.""" + empty = QueryResult([]) + nonempty = QueryResult(self._records(2)) + self.assertIsNone(empty.first()) + first = nonempty.first() + self.assertIsNotNone(first) + self.assertEqual(first.get("name"), "R0") # type: ignore[union-attr] + + def test_contract_truthy_guard(self): + """Pattern: ``if result: ...`` to skip empty results before indexing.""" + if QueryResult(self._records(1)): + ok = True + else: + ok = False + self.assertTrue(ok) + self.assertFalse(bool(QueryResult([]))) + + def test_contract_len_for_size_check(self): + """Pattern: ``f"{len(result)} rows"`` in log/print statements.""" + self.assertEqual(len(QueryResult(self._records(7))), 7) + self.assertEqual(len(QueryResult([])), 0) + + def test_contract_slice_returns_list_like(self): + """Slicing must yield something that supports iteration and len().""" + qr = QueryResult(self._records(5)) + page = qr[1:4] + self.assertEqual(len(page), 3) + self.assertEqual([r.get("name") for r in page], ["R1", "R2", "R3"]) + + def test_contract_negative_index(self): + """``result[-1]`` for "last record" is a common Python idiom.""" + qr = QueryResult(self._records(3)) + self.assertEqual(qr[-1].get("name"), "R2") + + def test_contract_list_conversion_round_trip(self): + """``list(result)`` must yield the same records iteration yields.""" + recs = self._records(4) + qr = QueryResult(recs) + self.assertEqual(list(qr), recs) + self.assertEqual(list(qr), [r for r in qr]) + + def test_contract_iteration_does_not_consume(self): + """Multiple ``for`` loops over the same result must all see records. + + Guards against an accidental refactor to a single-shot iterator. + """ + qr = QueryResult(self._records(3)) + first_pass = list(qr) + second_pass = list(qr) + self.assertEqual(first_pass, second_pass) + self.assertEqual(len(first_pass), 3) + + def test_contract_nested_loop_iterates_records_not_fields(self): + """Regression guard for the bug in installation_example.py (PR #175 review #3). + + ``for page in result: for r in page: ...`` would iterate Record keys + if QueryResult were a flat iterable of Records (each Record is also + iterable over its keys). This test makes it explicit that the outer + loop yields Records, not pages — so callers know to use a single loop. + """ + qr = QueryResult(self._records(2)) + outer = list(qr) + self.assertTrue(all(isinstance(r, Record) for r in outer)) + + def test_contract_records_attribute_is_underlying_list(self): + """``result.records`` is the documented escape hatch for list-only APIs.""" + recs = self._records(3) + qr = QueryResult(recs) + self.assertIsInstance(qr.records, list) + self.assertIs(qr.records, recs) + + +class TestExecuteReturnsQueryResult(unittest.TestCase): + """execute() flat mode returns QueryResult.""" + + def setUp(self): + self.client = _make_client() + + def test_execute_flat_returns_query_result(self): + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}], + [{"name": "B", "accountid": "2"}], + ] + ) + result = self.client.query.builder("account").select("name").execute() + self.assertIsInstance(result, QueryResult) + + def test_execute_flat_collects_all_pages(self): + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}], + [{"name": "B", "accountid": "2"}, {"name": "C", "accountid": "3"}], + ] + ) + result = self.client.query.builder("account").select("name").execute() + self.assertEqual(len(result), 3) + + def test_execute_flat_records_accessible(self): + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "Contoso", "accountid": "abc"}], + ] + ) + result = self.client.query.builder("account").select("name").execute() + first = result.first() + self.assertIsNotNone(first) + self.assertEqual(first["name"], "Contoso") + + def test_execute_flat_for_loop_backward_compat(self): + """for r in execute() still works — backward-compatible iteration.""" + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}, {"name": "B", "accountid": "2"}], + ] + ) + records = [] + for r in self.client.query.builder("account").select("name").execute(): + records.append(r) + self.assertEqual(len(records), 2) + + def test_execute_flat_list_backward_compat(self): + """list(execute()) still works.""" + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "X", "accountid": "x"}], + ] + ) + records = list(self.client.query.builder("account").select("name").execute()) + self.assertEqual(len(records), 1) + self.assertEqual(records[0]["name"], "X") + + def test_execute_by_page_not_query_result(self): + """execute(by_page=True) still returns page iterator, not QueryResult.""" + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}], + [{"name": "B", "accountid": "2"}], + ] + ) + result = self.client.query.builder("account").select("name").execute(by_page=True) + self.assertNotIsInstance(result, QueryResult) + pages = list(result) + self.assertEqual(len(pages), 2) + + def test_execute_empty_returns_empty_query_result(self): + self.client._odata._get_multiple.return_value = iter([]) + result = self.client.query.builder("account").select("name").execute() + self.assertIsInstance(result, QueryResult) + self.assertEqual(len(result), 0) + self.assertFalse(bool(result)) + self.assertIsNone(result.first()) + + def test_execute_result_to_dataframe(self): + """execute().to_dataframe() works end-to-end.""" + import pandas as pd + + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "Contoso", "accountid": "1"}, {"name": "Fabrikam", "accountid": "2"}], + ] + ) + df = self.client.query.builder("account").select("name").execute().to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 2) + + def test_execute_no_deprecation_warnings(self): + """execute() flat mode emits no DeprecationWarnings.""" + from PowerPlatform.Dataverse.models.filters import col + + self.client._odata._get_multiple.return_value = iter([]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + (self.client.query.builder("account").select("name").where(col("statecode") == 0).execute()) + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"Unexpected warnings: {dep}") + + +class TestQueryBuilderToDataframe(unittest.TestCase): + """to_dataframe() delegates to QueryResult.to_dataframe() after execute().""" + + def setUp(self): + self.client = _make_client() + + def test_to_dataframe_returns_dataframe(self): + import pandas as pd + + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}], + ] + ) + df = self.client.query.builder("account").select("name").to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 1) + + def test_to_dataframe_empty_preserves_select_columns(self): + """to_dataframe() on empty result keeps column names from select().""" + import pandas as pd + + self.client._odata._get_multiple.return_value = iter([]) + df = self.client.query.builder("account").select("name", "revenue").to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 0) + self.assertListEqual(list(df.columns), ["name", "revenue"]) + + def test_to_dataframe_empty_no_select(self): + """to_dataframe() on empty result with no select() returns bare empty DataFrame.""" + import pandas as pd + + self.client._odata._get_multiple.return_value = iter([]) + df = self.client.query.builder("account").top(10).to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 0) + + +class TestExports(unittest.TestCase): + """col, raw, QueryResult are importable from models and package root.""" + + def test_col_importable_from_models(self): + from PowerPlatform.Dataverse.models import col + + self.assertIsNotNone(col) + + def test_raw_importable_from_models(self): + from PowerPlatform.Dataverse.models import raw + + self.assertIsNotNone(raw) + + def test_query_result_importable_from_models(self): + from PowerPlatform.Dataverse.models import QueryResult + + self.assertIsNotNone(QueryResult) + + def test_col_importable_from_package_root(self): + from PowerPlatform.Dataverse import col + + self.assertIsNotNone(col) + + def test_raw_importable_from_package_root(self): + from PowerPlatform.Dataverse import raw + + self.assertIsNotNone(raw) + + def test_query_result_importable_from_package_root(self): + from PowerPlatform.Dataverse import QueryResult + + self.assertIsNotNone(QueryResult) + + def test_col_from_root_produces_filter_expression(self): + from PowerPlatform.Dataverse import col as root_col + from PowerPlatform.Dataverse.models.filters import FilterExpression + + expr = root_col("statecode") == 0 + self.assertIsInstance(expr, FilterExpression) + + def test_raw_from_root_produces_filter_expression(self): + from PowerPlatform.Dataverse import raw as root_raw + from PowerPlatform.Dataverse.models.filters import FilterExpression + + expr = root_raw("statecode eq 0") + self.assertIsInstance(expr, FilterExpression) + + def test_query_result_from_root_is_correct_class(self): + from PowerPlatform.Dataverse import QueryResult as root_qr + + qr = root_qr([]) + self.assertIsInstance(qr, root_qr) + self.assertEqual(len(qr), 0) + + def test_col_from_package_root_no_warning(self): + """Importing col from package root fires no DeprecationWarning.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + from PowerPlatform.Dataverse import col # noqa: F401 + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"Unexpected warnings: {dep}") + + def test_col_call_no_warning(self): + """col() emits no DeprecationWarning.""" + from PowerPlatform.Dataverse import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = col("statecode") == 0 + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"Unexpected warnings: {dep}") + self.assertIsNotNone(result) + + def test_raw_call_no_warning(self): + """raw() emits no DeprecationWarning.""" + from PowerPlatform.Dataverse import raw + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = raw("statecode eq 0") + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"Unexpected warnings: {dep}") + self.assertIsNotNone(result) + + +class TestQueryResultAcceptanceCriteria(unittest.TestCase): + """Verify all QueryResult acceptance criteria from the spec.""" + + def setUp(self): + self.client = _make_client() + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "Contoso", "accountid": "id-1"}], + ] + ) + + def _execute(self): + return self.client.query.builder("account").select("name").execute() + + def test_result_is_query_result(self): + result = self._execute() + self.assertIsInstance(result, QueryResult) + + def test_for_loop_still_works(self): + result = self._execute() + records = [r for r in result] + self.assertEqual(len(records), 1) + + def test_first_returns_record_or_none(self): + result = self._execute() + r = result.first() + self.assertIsNotNone(r) + self.assertIsInstance(r, Record) + + def test_to_dataframe_returns_dataframe(self): + import pandas as pd + + result = self._execute() + df = result.to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + + def test_builder_to_dataframe_returns_dataframe(self): + import pandas as pd + + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "Contoso", "accountid": "id-1"}], + ] + ) + df = self.client.query.builder("account").select("name").to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_phase3_ga.py b/tests/unit/test_phase3_ga.py new file mode 100644 index 00000000..f9e53f9f --- /dev/null +++ b/tests/unit/test_phase3_ga.py @@ -0,0 +1,404 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Phase 3 GA regression tests. + +Covers: +- records.get() deprecation (DeprecationWarning, still functional) +- records.retrieve() — single record, None on 404 +- records.list() — QueryResult, accepts str/FilterExpression filter +- DataverseModel Protocol definition and isinstance() check +- DataverseModel exported from models and package root +- execute() emits zero DeprecationWarning (internal records.get() suppressed) + +Note: records.create(DataverseModel) and records.update(DataverseModel) dispatch +are deferred to post-GA and are not covered here. +""" + +import unittest +import warnings +from dataclasses import dataclass +from unittest.mock import MagicMock, patch + +from azure.core.credentials import TokenCredential + +from PowerPlatform.Dataverse.core.errors import HttpError +from PowerPlatform.Dataverse.models.record import QueryResult, Record +from PowerPlatform.Dataverse.models.protocol import DataverseModel + + +def _make_client(): + cred = MagicMock(spec=TokenCredential) + from PowerPlatform.Dataverse.client import DataverseClient + + client = DataverseClient("https://example.crm.dynamics.com", cred) + client._odata = MagicMock() + client._odata._get_multiple = MagicMock() + client._odata._get_single = MagicMock() + client._odata._get = MagicMock() + client._odata._create = MagicMock() + client._odata._create_multiple = MagicMock() + client._odata._update = MagicMock() + client._odata._update_by_ids = MagicMock() + client._odata._entity_set_from_schema_name = MagicMock(side_effect=lambda t: t + "s") + return client + + +# --------------------------------------------------------------------------- +# Sample DataverseModel implementation for tests + + +@dataclass +class _Account: + __entity_logical_name__ = "account" + __entity_set_name__ = "accounts" + name: str = "" + telephone1: str = "" + + def to_dict(self) -> dict: + return {"name": self.name, "telephone1": self.telephone1} + + @classmethod + def from_dict(cls, data: dict) -> "_Account": + return cls(name=data.get("name", ""), telephone1=data.get("telephone1", "")) + + +# --------------------------------------------------------------------------- + + +class TestDataverseModelProtocol(unittest.TestCase): + """DataverseModel Protocol structural checks.""" + + def test_account_satisfies_protocol(self): + self.assertIsInstance(_Account(), DataverseModel) + + def test_plain_dict_does_not_satisfy_protocol(self): + self.assertNotIsInstance({"name": "X"}, DataverseModel) + + def test_missing_entity_logical_name_fails(self): + class _Bad: + __entity_set_name__ = "bads" + + def to_dict(self): + return {} + + @classmethod + def from_dict(cls, d): + return cls() + + self.assertNotIsInstance(_Bad(), DataverseModel) + + def test_missing_to_dict_fails(self): + class _Bad: + __entity_logical_name__ = "bad" + __entity_set_name__ = "bads" + + @classmethod + def from_dict(cls, d): + return cls() + + self.assertNotIsInstance(_Bad(), DataverseModel) + + def test_importable_from_models(self): + from PowerPlatform.Dataverse.models import DataverseModel as dm + + self.assertIsNotNone(dm) + + def test_importable_from_package_root(self): + from PowerPlatform.Dataverse import DataverseModel as dm + + self.assertIsNotNone(dm) + + def test_import_no_deprecation_warning(self): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + from PowerPlatform.Dataverse import DataverseModel # noqa: F401 + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, []) + + +class TestRecordsGetDeprecation(unittest.TestCase): + """records.get() fires DeprecationWarning, still functional.""" + + def setUp(self): + self.client = _make_client() + + def test_get_single_warns(self): + self.client._odata._get.return_value = {"accountid": "1", "name": "Contoso"} + with self.assertWarns(DeprecationWarning) as ctx: + self.client.records.get("account", "guid-1") + self.assertIn("retrieve", str(ctx.warning)) + + def test_get_multiple_warns(self): + self.client._odata._get_multiple.return_value = iter([]) + with self.assertWarns(DeprecationWarning) as ctx: + list(self.client.records.get("account", filter="statecode eq 0")) + self.assertIn("list", str(ctx.warning)) + + def test_get_single_still_returns_record(self): + self.client._odata._get.return_value = {"accountid": "1", "name": "Contoso"} + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + record = self.client.records.get("account", "guid-1") + self.assertIsInstance(record, Record) + self.assertEqual(record["name"], "Contoso") + + def test_get_multiple_still_returns_pages(self): + self.client._odata._get_multiple.return_value = iter([[{"name": "A", "accountid": "1"}]]) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + pages = list(self.client.records.get("account", filter="statecode eq 0")) + self.assertEqual(len(pages), 1) + + def test_get_warning_message_single_id(self): + self.client._odata._get.return_value = {"accountid": "1"} + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.records.get("account", "guid-1") + msgs = [str(w.message) for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertTrue(any("retrieve" in m for m in msgs)) + + def test_get_warning_message_filter_form(self): + self.client._odata._get_multiple.return_value = iter([]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + list(self.client.records.get("account", filter="statecode eq 0")) + msgs = [str(w.message) for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertTrue(any("list" in m for m in msgs)) + + +class TestRecordsRetrieve(unittest.TestCase): + """records.retrieve() returns Record or None, no warning.""" + + def setUp(self): + self.client = _make_client() + + def test_retrieve_returns_record(self): + self.client._odata._get.return_value = {"accountid": "abc", "name": "Contoso"} + record = self.client.records.retrieve("account", "abc") + self.assertIsInstance(record, Record) + self.assertEqual(record["name"], "Contoso") + + def test_retrieve_passes_select(self): + self.client._odata._get.return_value = {"accountid": "abc", "name": "Contoso"} + self.client.records.retrieve("account", "abc", select=["name"]) + self.client._odata._get.assert_called_once_with( + "account", "abc", select=["name"], expand=None, include_annotations=None + ) + + def test_retrieve_passes_expand(self): + self.client._odata._get.return_value = { + "accountid": "abc", + "name": "Contoso", + "primarycontactid": {"contactid": "cid", "fullname": "John Doe"}, + } + record = self.client.records.retrieve("account", "abc", expand=["primarycontactid"]) + self.client._odata._get.assert_called_once_with( + "account", "abc", select=None, expand=["primarycontactid"], include_annotations=None + ) + self.assertEqual(record["primarycontactid"]["fullname"], "John Doe") + + def test_retrieve_passes_select_and_expand(self): + self.client._odata._get.return_value = { + "name": "Contoso", + "primarycontactid": {"fullname": "John Doe"}, + } + self.client.records.retrieve("account", "abc", select=["name"], expand=["primarycontactid"]) + self.client._odata._get.assert_called_once_with( + "account", "abc", select=["name"], expand=["primarycontactid"], include_annotations=None + ) + + def test_retrieve_passes_include_annotations(self): + annotation = "OData.Community.Display.V1.FormattedValue" + self.client._odata._get.return_value = { + "accountid": "abc", + "statuscode": 1, + f"statuscode@{annotation}": "Active", + } + record = self.client.records.retrieve("account", "abc", include_annotations=annotation) + self.client._odata._get.assert_called_once_with( + "account", "abc", select=None, expand=None, include_annotations=annotation + ) + self.assertEqual(record[f"statuscode@{annotation}"], "Active") + + def test_retrieve_no_deprecation_warning(self): + self.client._odata._get.return_value = {"accountid": "abc", "name": "Contoso"} + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.records.retrieve("account", "abc") + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"retrieve() must not emit DeprecationWarning: {dep}") + + def test_retrieve_returns_none_on_404(self): + self.client._odata._get.side_effect = HttpError("Not Found", status_code=404) + result = self.client.records.retrieve("account", "nonexistent-guid") + self.assertIsNone(result) + + def test_retrieve_reraises_non_404(self): + self.client._odata._get.side_effect = HttpError("Server Error", status_code=500) + with self.assertRaises(HttpError): + self.client.records.retrieve("account", "some-guid") + + def test_retrieve_reraises_non_http_errors(self): + self.client._odata._get.side_effect = ValueError("Bad input") + with self.assertRaises(ValueError): + self.client.records.retrieve("account", "some-guid") + + def test_retrieve_record_id_set(self): + self.client._odata._get.return_value = {"name": "Contoso"} + record = self.client.records.retrieve("account", "my-id") + self.assertEqual(record.id, "my-id") + + def test_retrieve_table_set(self): + self.client._odata._get.return_value = {"name": "Contoso"} + record = self.client.records.retrieve("account", "my-id") + self.assertEqual(record.table, "account") + + +class TestRecordsList(unittest.TestCase): + """records.list() returns QueryResult, no warning.""" + + def setUp(self): + self.client = _make_client() + + def test_list_returns_query_result(self): + self.client._odata._get_multiple.return_value = iter([]) + result = self.client.records.list("account") + self.assertIsInstance(result, QueryResult) + + def test_list_collects_all_pages(self): + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}], + [{"name": "B", "accountid": "2"}, {"name": "C", "accountid": "3"}], + ] + ) + result = self.client.records.list("account") + self.assertEqual(len(result), 3) + + def test_list_no_deprecation_warning(self): + self.client._odata._get_multiple.return_value = iter([]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.records.list("account", filter="statecode eq 0") + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"list() must not emit DeprecationWarning: {dep}") + + def test_list_passes_string_filter(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", filter="statecode eq 0") + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["filter"], "statecode eq 0") + + def test_list_passes_filter_expression(self): + from PowerPlatform.Dataverse.models.filters import col + + self.client._odata._get_multiple.return_value = iter([]) + expr = col("statecode") == 0 + self.client.records.list("account", filter=expr) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["filter"], "statecode eq 0") + + def test_list_passes_select(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", select=["name", "revenue"]) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["select"], ["name", "revenue"]) + + def test_list_passes_top(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", top=50) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["top"], 50) + + def test_list_none_filter_passes_none(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account") + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertIsNone(call_kwargs["filter"]) + + def test_list_result_iterable(self): + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "X", "accountid": "1"}], + ] + ) + result = self.client.records.list("account") + records = list(result) + self.assertEqual(len(records), 1) + self.assertEqual(records[0]["name"], "X") + + def test_list_result_to_dataframe(self): + import pandas as pd + + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}, {"name": "B", "accountid": "2"}], + ] + ) + df = self.client.records.list("account", select=["name"]).to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 2) + + def test_list_passes_orderby(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", orderby=["name asc"]) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["orderby"], ["name asc"]) + + def test_list_passes_expand(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", expand=["primarycontactid"]) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["expand"], ["primarycontactid"]) + + def test_list_passes_page_size(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", page_size=200) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["page_size"], 200) + + def test_list_passes_count(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", count=True) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertTrue(call_kwargs["count"]) + + def test_list_passes_include_annotations(self): + annotation = "OData.Community.Display.V1.FormattedValue" + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", include_annotations=annotation) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["include_annotations"], annotation) + + +class TestExecuteNoDeprecationFromRecordsGet(unittest.TestCase): + """execute() suppresses DeprecationWarning from the internal records.get() call.""" + + def setUp(self): + self.client = _make_client() + self.client._odata._get_multiple.return_value = iter([]) + + def test_execute_flat_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.query.builder("account").select("name").where(col("statecode") == 0).execute() + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"execute() leaked DeprecationWarning: {dep}") + + def test_to_dataframe_no_records_get_warning(self): + """to_dataframe() emits its own deprecation but must not leak records.get()'s warning.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.query.builder("account").select("name").to_dataframe() + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + # Exactly one DeprecationWarning: from QueryBuilder.to_dataframe() itself. + # The internal records.get() deprecation must remain suppressed. + self.assertEqual(len(dep), 1, f"Unexpected warnings: {dep}") + self.assertIn("QueryBuilder.to_dataframe()", str(dep[0].message)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_phase4_ga.py b/tests/unit/test_phase4_ga.py new file mode 100644 index 00000000..879a9f78 --- /dev/null +++ b/tests/unit/test_phase4_ga.py @@ -0,0 +1,629 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Phase 4 GA regression tests. + +Covers: +- fetchxml(): basic, pagination, missing-entity-element error +- sql_select / sql_join / sql_joins raise AttributeError (removed at GA) +- odata_select / odata_expand / odata_bind emit DeprecationWarning (deprecated at GA) +- sql_columns / odata_expands / sql() emit zero DeprecationWarning (still GA-clean) +""" + +import unittest +import warnings +import xml.etree.ElementTree as ET +from unittest.mock import MagicMock, patch + +from azure.core.credentials import TokenCredential + +from PowerPlatform.Dataverse.models.record import QueryResult, Record + + +def _make_client(): + cred = MagicMock(spec=TokenCredential) + from PowerPlatform.Dataverse.client import DataverseClient + + client = DataverseClient("https://example.crm.dynamics.com", cred) + client._odata = MagicMock() + client._odata._entity_set_from_schema_name = MagicMock(side_effect=lambda t: t + "s") + client._odata.api = "https://example.crm.dynamics.com/api/data/v9.2" + return client + + +# --------------------------------------------------------------------------- +# fetchxml() +# --------------------------------------------------------------------------- + + +class TestFetchXml(unittest.TestCase): + + def setUp(self): + self.client = _make_client() + + def _fetch_xml(self, entity="account"): + return f"""""" + + def _mock_response(self, records, more=False, cookie=""): + resp = MagicMock() + payload = {"value": records} + if more: + payload["@Microsoft.Dynamics.CRM.morerecords"] = True + payload["@Microsoft.Dynamics.CRM.fetchxmlpagingcookie"] = cookie + else: + payload["@Microsoft.Dynamics.CRM.morerecords"] = False + resp.json.return_value = payload + return resp + + def test_fetchxml_inert_no_http_request(self): + """fetchxml() alone must not fire any HTTP request.""" + from PowerPlatform.Dataverse.models.fetchxml_query import FetchXmlQuery + + query = self.client.query.fetchxml(self._fetch_xml()) + self.assertIsInstance(query, FetchXmlQuery) + self.client._odata._request.assert_not_called() + + def test_basic_returns_query_result(self): + self.client._odata._request.return_value = self._mock_response([{"name": "Contoso", "accountid": "1"}]) + result = self.client.query.fetchxml(self._fetch_xml()).execute() + self.assertIsInstance(result, QueryResult) + + def test_basic_record_count(self): + self.client._odata._request.return_value = self._mock_response([{"name": "A"}, {"name": "B"}]) + result = self.client.query.fetchxml(self._fetch_xml()).execute() + self.assertEqual(len(result), 2) + + def test_record_values_accessible(self): + self.client._odata._request.return_value = self._mock_response([{"name": "Contoso", "accountid": "abc-123"}]) + result = self.client.query.fetchxml(self._fetch_xml()).execute() + self.assertEqual(result.first()["name"], "Contoso") + + def test_empty_result_returns_empty_query_result(self): + self.client._odata._request.return_value = self._mock_response([]) + result = self.client.query.fetchxml(self._fetch_xml()).execute() + self.assertIsInstance(result, QueryResult) + self.assertEqual(len(result), 0) + self.assertFalse(result) + + def test_pagination_fetches_all_pages(self): + """execute_pages() drives the HTTP loop; each page yields one QueryResult.""" + # Annotation is outer XML; pagingcookie attribute is double URL-encoded inner cookie. + cookie_raw = '' + page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + pages = list(self.client.query.fetchxml(self._fetch_xml()).execute_pages()) + self.assertEqual(len(pages), 2) + self.assertEqual(self.client._odata._request.call_count, 2) + + def test_pagination_second_request_includes_page_and_cookie(self): + """execute_pages() injects the decoded paging cookie into the second request.""" + # pagingcookie="%253Cc%252F%253E": double URL-decode gives "" (the inner cookie XML). + cookie_raw = '' + page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + list(self.client.query.fetchxml(self._fetch_xml()).execute_pages()) + + second_call_kwargs = self.client._odata._request.call_args_list[1] + params = ( + second_call_kwargs.kwargs.get("params") or second_call_kwargs.args[2] + if len(second_call_kwargs.args) > 2 + else {} + ) + if not params: + params = second_call_kwargs[1].get("params", {}) + xml_sent = params.get("fetchXml", "") + fetch_el = ET.fromstring(xml_sent) + self.assertEqual(fetch_el.get("page"), "2") + self.assertIsNotNone(fetch_el.get("paging-cookie")) + + def test_missing_entity_element_raises_value_error(self): + with self.assertRaises(ValueError) as ctx: + self.client.query.fetchxml("") + self.assertIn("entity", str(ctx.exception).lower()) + + def test_entity_missing_name_attr_raises_value_error(self): + with self.assertRaises(ValueError) as ctx: + self.client.query.fetchxml("") + self.assertIn("name", str(ctx.exception).lower()) + + def test_entity_set_resolved_from_entity_name(self): + self.client._odata._request.return_value = self._mock_response([]) + self.client.query.fetchxml(self._fetch_xml("account")).execute() + self.client._odata._entity_set_from_schema_name.assert_called_with("account") + + def test_request_uses_prefer_header(self): + self.client._odata._request.return_value = self._mock_response([]) + self.client.query.fetchxml(self._fetch_xml()).execute() + call_kwargs = self.client._odata._request.call_args + headers = call_kwargs.kwargs.get("headers", {}) + self.assertIn("Prefer", headers) + self.assertIn("fetchxmlpagingcookie", headers["Prefer"]) + + def test_result_iterable(self): + self.client._odata._request.return_value = self._mock_response([{"name": "A"}, {"name": "B"}]) + result = self.client.query.fetchxml(self._fetch_xml()).execute() + names = [r["name"] for r in result] + self.assertEqual(names, ["A", "B"]) + + def test_result_to_dataframe(self): + try: + import pandas as pd + except ImportError: + self.skipTest("pandas not installed") + self.client._odata._request.return_value = self._mock_response([{"name": "Contoso"}, {"name": "Fabrikam"}]) + result = self.client.query.fetchxml(self._fetch_xml()).execute() + df = result.to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 2) + + def test_no_deprecation_warning_emitted(self): + self.client._odata._request.return_value = self._mock_response([]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.query.fetchxml(self._fetch_xml()).execute() + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(deprecations), 0, "fetchxml().execute() should not emit DeprecationWarning") + + def test_execute_pages_returns_iterator_of_query_result(self): + """execute_pages() yields QueryResult objects, one per HTTP page.""" + cookie_raw = '' + page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + pages = list(self.client.query.fetchxml(self._fetch_xml()).execute_pages()) + self.assertEqual(len(pages), 2) + for page in pages: + self.assertIsInstance(page, QueryResult) + + def test_execute_pages_one_http_call_per_page(self): + """Each execute_pages() iteration fires exactly one HTTP request.""" + cookie_raw = '' + page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + count = 0 + for _page in self.client.query.fetchxml(self._fetch_xml()).execute_pages(): + count += 1 + self.assertEqual(self.client._odata._request.call_count, 2) + self.assertEqual(count, 2) + + def test_execute_pages_per_page_records(self): + """Each page yielded by execute_pages() contains only its own records.""" + cookie_raw = '' + page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) + page2 = self._mock_response([{"name": "B"}, {"name": "C"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + pages = list(self.client.query.fetchxml(self._fetch_xml()).execute_pages()) + self.assertEqual(len(pages[0]), 1) + self.assertEqual(len(pages[1]), 2) + self.assertEqual(pages[0].first()["name"], "A") + self.assertEqual(pages[1].first()["name"], "B") + + # ------------------------------------------------------------------ + # Input validation + # ------------------------------------------------------------------ + + def test_non_string_input_raises_validation_error(self): + from PowerPlatform.Dataverse.core.errors import ValidationError + + with self.assertRaises(ValidationError): + self.client.query.fetchxml(123) + + def test_empty_string_raises_validation_error(self): + from PowerPlatform.Dataverse.core.errors import ValidationError + + with self.assertRaises(ValidationError): + self.client.query.fetchxml("") + + def test_whitespace_only_raises_validation_error(self): + from PowerPlatform.Dataverse.core.errors import ValidationError + + with self.assertRaises(ValidationError): + self.client.query.fetchxml(" ") + + def test_malformed_xml_raises_validation_error(self): + from PowerPlatform.Dataverse.core.errors import ValidationError + + with self.assertRaises(ValidationError): + self.client.query.fetchxml("") + + def test_url_too_long_raises_validation_error(self): + """XML whose URL-encoded form exceeds 32,768 chars is rejected before any HTTP.""" + from PowerPlatform.Dataverse.core.errors import ValidationError + + # Alphanumeric chars are URL-safe and don't expand; a 32,769-char name attribute + # value pushes the encoded XML over the limit. + long_name = "a" * 32_769 + big_xml = f'' + with self.assertRaises(ValidationError): + self.client.query.fetchxml(big_xml) + + # ------------------------------------------------------------------ + # Paging behaviour + # ------------------------------------------------------------------ + + def test_morerecords_string_true_continues_paging(self): + """morerecords annotation as string "true" (not bool) is handled correctly.""" + cookie_raw = '' + page1_payload = { + "value": [{"name": "A"}], + "@Microsoft.Dynamics.CRM.morerecords": "true", + "@Microsoft.Dynamics.CRM.fetchxmlpagingcookie": cookie_raw, + } + page2_payload = { + "value": [{"name": "B"}], + "@Microsoft.Dynamics.CRM.morerecords": False, + } + r1, r2 = MagicMock(), MagicMock() + r1.json.return_value = page1_payload + r2.json.return_value = page2_payload + self.client._odata._request.side_effect = [r1, r2] + + result = self.client.query.fetchxml(self._fetch_xml()).execute() + self.assertEqual(len(result), 2) + self.assertEqual(self.client._odata._request.call_count, 2) + + def test_simple_paging_fallback_emits_user_warning(self): + """No cookie returned with morerecords=True triggers a UserWarning.""" + page1 = self._mock_response([{"name": "A"}], more=True, cookie="") + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + list(self.client.query.fetchxml(self._fetch_xml()).execute_pages()) + + user_warnings = [w for w in caught if issubclass(w.category, UserWarning)] + self.assertEqual(len(user_warnings), 1) + self.assertIn("simple paging", str(user_warnings[0].message).lower()) + + def test_simple_paging_fallback_fetches_all_pages(self): + """Simple paging fallback continues iterating; caller still gets all records.""" + page1 = self._mock_response([{"name": "A"}], more=True, cookie="") + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + result = self.client.query.fetchxml(self._fetch_xml()).execute() + + self.assertEqual(len(result), 2) + self.assertEqual(self.client._odata._request.call_count, 2) + + def test_malformed_cookie_xml_warns_distinctly(self): + """A cookie that is not valid XML emits a 'could not be parsed' warning, not the no-cookie warning.""" + page1 = self._mock_response([{"name": "A"}], more=True, cookie="not-valid-xml") + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = self.client.query.fetchxml(self._fetch_xml()).execute() + + self.assertEqual(len(result), 2) + user_warnings = [w for w in caught if issubclass(w.category, UserWarning)] + self.assertEqual(len(user_warnings), 1) + self.assertIn("could not be parsed", str(user_warnings[0].message).lower()) + + def test_corrupt_pagenumber_warns_distinctly(self): + """Valid XML cookie with non-integer pagenumber emits a 'could not be parsed' warning.""" + bad_cookie = '' + page1 = self._mock_response([{"name": "A"}], more=True, cookie=bad_cookie) + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = self.client.query.fetchxml(self._fetch_xml()).execute() + + self.assertEqual(len(result), 2) + user_warnings = [w for w in caught if issubclass(w.category, UserWarning)] + self.assertEqual(len(user_warnings), 1) + self.assertIn("could not be parsed", str(user_warnings[0].message).lower()) + + def test_max_pages_exceeded_raises(self): + """Paging loop raises ValidationError after exceeding _MAX_PAGES.""" + from PowerPlatform.Dataverse.core.errors import ValidationError + + cookie_raw = '' + always_more = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) + self.client._odata._request.return_value = always_more + + with patch("PowerPlatform.Dataverse.models.fetchxml_query._MAX_PAGES", 3): + with self.assertRaises(ValidationError): + list(self.client.query.fetchxml(self._fetch_xml()).execute_pages()) + + +# --------------------------------------------------------------------------- +# Removed SQL helpers — raise AttributeError +# --------------------------------------------------------------------------- + + +class TestRemovedSqlHelpers(unittest.TestCase): + + def setUp(self): + self.client = _make_client() + + def test_sql_select_raises_attribute_error(self): + with self.assertRaises(AttributeError): + self.client.query.sql_select("account") + + def test_sql_joins_raises_attribute_error(self): + with self.assertRaises(AttributeError): + self.client.query.sql_joins("contact") + + def test_sql_join_raises_attribute_error(self): + with self.assertRaises(AttributeError): + self.client.query.sql_join("contact", "account") + + +# --------------------------------------------------------------------------- +# Deprecated OData helpers — emit DeprecationWarning, still functional +# --------------------------------------------------------------------------- + + +class TestDeprecatedOdataHelpers(unittest.TestCase): + + def setUp(self): + self.client = _make_client() + + # --- odata_select --- + + def test_odata_select_emits_deprecation_warning(self): + self.client._odata._list_columns.return_value = [] + with self.assertWarns(DeprecationWarning): + self.client.query.odata_select("account") + + def test_odata_select_still_returns_list(self): + self.client._odata._list_columns.return_value = [ + { + "LogicalName": "name", + "AttributeType": "String", + "IsPrimaryId": False, + "IsPrimaryName": True, + "DisplayName": {}, + } + ] + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + cols = self.client.query.odata_select("account") + self.assertIsInstance(cols, list) + self.assertIn("name", cols) + + # --- odata_expand --- + + def _contact_to_account_rel(self): + return [ + { + "ReferencingEntity": "contact", + "ReferencingAttribute": "parentcustomerid", + "ReferencedEntity": "account", + "ReferencedAttribute": "accountid", + "ReferencingEntityNavigationPropertyName": "parentcustomerid_account", + "SchemaName": "contact_customer_accounts", + } + ] + + def test_odata_expand_emits_deprecation_warning(self): + self.client._odata._list_table_relationships.return_value = self._contact_to_account_rel() + with self.assertWarns(DeprecationWarning): + self.client.query.odata_expand("contact", "account") + + def test_odata_expand_still_returns_nav_property(self): + self.client._odata._list_table_relationships.return_value = self._contact_to_account_rel() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + nav = self.client.query.odata_expand("contact", "account") + self.assertEqual(nav, "parentcustomerid_account") + + def test_odata_expand_no_match_raises_value_error(self): + self.client._odata._list_table_relationships.return_value = [] + with self.assertRaises(ValueError): + self.client.query.odata_expand("contact", "nonexistent") + + # --- odata_bind --- + + def test_odata_bind_emits_deprecation_warning(self): + self.client._odata._list_table_relationships.return_value = self._contact_to_account_rel() + with self.assertWarns(DeprecationWarning): + self.client.query.odata_bind("contact", "account", "some-guid") + + def test_odata_bind_still_returns_bind_dict(self): + self.client._odata._list_table_relationships.return_value = self._contact_to_account_rel() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = self.client.query.odata_bind("contact", "account", "guid-123") + self.assertIsInstance(result, dict) + key = list(result.keys())[0] + self.assertEqual(key, "parentcustomerid_account@odata.bind") + self.assertIn("guid-123", result[key]) + + def test_odata_bind_no_match_raises_value_error(self): + self.client._odata._list_table_relationships.return_value = [] + with self.assertRaises(ValueError): + self.client.query.odata_bind("contact", "nonexistent", "guid") + + +# --------------------------------------------------------------------------- +# GA-clean methods: no DeprecationWarning +# --------------------------------------------------------------------------- + + +class TestGaCleanMethods(unittest.TestCase): + + def setUp(self): + self.client = _make_client() + + def test_sql_columns_no_warning(self): + self.client._odata._list_columns.return_value = [] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.query.sql_columns("account") + deps = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(deps), 0) + + def test_odata_expands_no_warning(self): + self.client._odata._list_table_relationships.return_value = [] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.query.odata_expands("contact") + deps = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(deps), 0) + + def test_sql_no_warning(self): + self.client._odata._query_sql.return_value = [] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.query.sql("SELECT name FROM account") + deps = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(deps), 0) + + def test_builder_no_warning(self): + self.client._odata._get_multiple.return_value = iter([[]]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + list(self.client.query.builder("account").select("name").execute()) + deps = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(deps), 0) + + +# --------------------------------------------------------------------------- +# Codemod — execute(by_page=...) transforms +# --------------------------------------------------------------------------- + + +class TestCodemodByPage(unittest.TestCase): + """migrate_source() rewrites literal by_page arguments.""" + + @classmethod + def setUpClass(cls): + try: + from PowerPlatform.Dataverse.migration.migrate_v0_to_v1 import migrate_source + + cls.migrate = staticmethod(migrate_source) + except ImportError: + cls.migrate = None + + def setUp(self): + if self.migrate is None: + self.skipTest("libcst not installed") + + def test_execute_by_page_true_becomes_execute_pages(self): + src = "result = builder.execute(by_page=True)\n" + out = self.migrate(src) + self.assertIn("execute_pages()", out) + self.assertNotIn("by_page", out) + self.assertNotIn("execute(", out) + + def test_execute_by_page_false_removes_flag(self): + src = "result = builder.execute(by_page=False)\n" + out = self.migrate(src) + self.assertIn("execute()", out) + self.assertNotIn("by_page", out) + + def test_execute_by_page_variable_not_rewritten(self): + """Variable by_page argument must not be rewritten — requires manual review.""" + src = "result = builder.execute(by_page=flag)\n" + out = self.migrate(src) + self.assertIn("by_page=flag", out) + + def test_idempotent_execute_pages(self): + """Codemod is idempotent — running again changes nothing.""" + src = "result = builder.execute(by_page=True)\n" + once = self.migrate(src) + twice = self.migrate(once) + self.assertEqual(once, twice) + + def test_idempotent_execute_no_flag(self): + src = "result = builder.execute(by_page=False)\n" + once = self.migrate(src) + twice = self.migrate(once) + self.assertEqual(once, twice) + + def test_client_var_default_rewrites_client(self): + """Default client_var='client' rewrites client.create(...).""" + src = "client.create('account', data)\n" + out = self.migrate(src) + self.assertIn("client.records.create", out) + + def test_client_var_custom_rewrites_matching_name(self): + """custom client_var rewrites that variable name, not 'client'.""" + src = "svc.create('account', data)\n" + out = self.migrate(src, client_var="svc") + self.assertIn("svc.records.create", out) + + def test_client_var_custom_does_not_rewrite_default_name(self): + """When client_var='svc', the literal name 'client' is left untouched.""" + src = "client.create('account', data)\n" + out = self.migrate(src, client_var="svc") + self.assertNotIn("client.records.create", out) + self.assertIn("client.create", out) + + +class TestManualReviewFinder(unittest.TestCase): + """find_manual_patterns() detects patterns that require manual migration.""" + + @classmethod + def setUpClass(cls): + try: + from PowerPlatform.Dataverse.migration.migrate_v0_to_v1 import find_manual_patterns + + cls.find = staticmethod(find_manual_patterns) + except ImportError: + cls.find = None + + def setUp(self): + if self.find is None: + self.skipTest("libcst not installed") + + def test_records_get_flagged(self): + src = "result = client.records.get('account', record_id)\n" + findings = self.find(src) + self.assertTrue(any("records.get" in f for f in findings)) + + def test_dataframe_get_flagged(self): + src = "df = client.dataframe.get('account', select=['name'])\n" + findings = self.find(src) + self.assertTrue(any("dataframe.get" in f for f in findings)) + + def test_execute_by_page_variable_flagged(self): + src = "result = builder.execute(by_page=flag)\n" + findings = self.find(src) + self.assertTrue(any("by_page" in f for f in findings)) + + def test_execute_by_page_literal_not_flagged(self): + """Literal True/False is handled by the transformer — not a manual item.""" + src = "result = builder.execute(by_page=True)\n" + findings = self.find(src) + self.assertFalse(any("by_page" in f for f in findings)) + + def test_sql_select_flagged(self): + src = "cols = client.query.sql_select('account')\n" + findings = self.find(src) + self.assertTrue(any("sql_select" in f for f in findings)) + + def test_sql_join_flagged(self): + src = "j = client.query.sql_join('account', 'contact')\n" + findings = self.find(src) + self.assertTrue(any("sql_join" in f for f in findings)) + + def test_clean_code_no_findings(self): + src = "result = client.records.list('account', filter='statecode eq 0')\n" + self.assertEqual(self.find(src), []) + + def test_custom_client_var(self): + src = "svc.records.get('account', guid)\n" + self.assertEqual(self.find(src, client_var="client"), []) + findings = self.find(src, client_var="svc") + self.assertTrue(any("records.get" in f for f in findings)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_query_operations.py b/tests/unit/test_query_operations.py index 1add6288..c6520e74 100644 --- a/tests/unit/test_query_operations.py +++ b/tests/unit/test_query_operations.py @@ -179,9 +179,13 @@ def test_builder_returns_query_builder(self): def test_builder_execute_flat_default(self): """builder().execute() should return flat records by default.""" + from PowerPlatform.Dataverse.models.filters import col + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1", "name": "Test"}]]) - records = list(self.client.query.builder("account").select("name").filter_eq("statecode", 0).top(10).execute()) + records = list( + self.client.query.builder("account").select("name").where(col("statecode") == 0).top(10).execute() + ) self.client._odata._get_multiple.assert_called_once_with( "account", @@ -220,13 +224,15 @@ def test_builder_execute_by_page(self): def test_builder_execute_all_params(self): """builder().execute() should forward all parameters.""" + from PowerPlatform.Dataverse.models.filters import col + self.client._odata._get_multiple.return_value = iter([[{"name": "Test"}]]) list( self.client.query.builder("account") .select("name", "revenue") - .filter_eq("statecode", 0) - .filter_gt("revenue", 1000000) + .where(col("statecode") == 0) + .where(col("revenue") > 1000000) .order_by("revenue", descending=True) .expand("primarycontactid") .top(50) @@ -248,13 +254,13 @@ def test_builder_execute_all_params(self): def test_builder_execute_with_where(self): """builder().where().execute() should compile expression to filter.""" - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col self.client._odata._get_multiple.return_value = iter([[{"name": "Test"}]]) list( self.client.query.builder("account") - .where((eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000)) + .where(((col("statecode") == 0) | (col("statecode") == 1)) & (col("revenue") > 100000)) .execute() ) @@ -265,10 +271,12 @@ def test_builder_execute_with_where(self): ) def test_builder_execute_with_filter_in(self): - """builder().filter_in().execute() should forward CRM.In filter to _get_multiple.""" + """builder().where(col().in_()).execute() should forward CRM.In filter to _get_multiple.""" + from PowerPlatform.Dataverse.models.filters import col + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) - list(self.client.query.builder("account").select("name").filter_in("statecode", [0, 1, 2]).execute()) + list(self.client.query.builder("account").select("name").where(col("statecode").in_([0, 1, 2])).execute()) call_kwargs = self.client._odata._get_multiple.call_args self.assertEqual( @@ -277,13 +285,15 @@ def test_builder_execute_with_filter_in(self): ) def test_builder_execute_with_where_filter_in(self): - """builder().where(filter_in(...) & ...).execute() should compile composed expression.""" - from PowerPlatform.Dataverse.models.filters import filter_in, gt + """builder().where(col().in_() & ...).execute() should compile composed expression.""" + from PowerPlatform.Dataverse.models.filters import col self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) list( - self.client.query.builder("account").where(filter_in("statecode", [0, 1]) & gt("revenue", 100000)).execute() + self.client.query.builder("account") + .where(col("statecode").in_([0, 1]) & (col("revenue") > 100000)) + .execute() ) call_kwargs = self.client._odata._get_multiple.call_args @@ -293,14 +303,15 @@ def test_builder_execute_with_where_filter_in(self): ) def test_builder_execute_with_filter_between_datetimes(self): - """builder().filter_between() with datetimes should forward correct OData.""" + """builder().where(col().between()).execute() should forward correct OData.""" from datetime import datetime, timezone + from PowerPlatform.Dataverse.models.filters import col self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) start = datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc) end = datetime(2024, 12, 31, 23, 59, 59, tzinfo=timezone.utc) - list(self.client.query.builder("account").filter_between("createdon", start, end).execute()) + list(self.client.query.builder("account").where(col("createdon").between(start, end)).execute()) call_kwargs = self.client._odata._get_multiple.call_args self.assertEqual( @@ -309,10 +320,12 @@ def test_builder_execute_with_filter_between_datetimes(self): ) def test_builder_execute_with_filter_not_in(self): - """builder().filter_not_in().execute() should forward CRM.NotIn filter.""" + """builder().where(col().not_in()).execute() should forward CRM.NotIn filter.""" + from PowerPlatform.Dataverse.models.filters import col + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) - list(self.client.query.builder("account").select("name").filter_not_in("statecode", [2, 3]).execute()) + list(self.client.query.builder("account").select("name").where(col("statecode").not_in([2, 3])).execute()) call_kwargs = self.client._odata._get_multiple.call_args self.assertEqual( @@ -321,10 +334,12 @@ def test_builder_execute_with_filter_not_in(self): ) def test_builder_execute_with_filter_not_between(self): - """builder().filter_not_between().execute() should forward negated between filter.""" + """builder().where(col().not_between()).execute() should forward negated between filter.""" + from PowerPlatform.Dataverse.models.filters import col + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) - list(self.client.query.builder("account").filter_not_between("revenue", 100000, 500000).execute()) + list(self.client.query.builder("account").where(col("revenue").not_between(100000, 500000)).execute()) call_kwargs = self.client._odata._get_multiple.call_args self.assertEqual( @@ -334,6 +349,8 @@ def test_builder_execute_with_filter_not_between(self): def test_builder_full_fluent_workflow(self): """End-to-end test of the fluent query workflow.""" + from PowerPlatform.Dataverse.models.filters import col + expected_records = [ {"accountid": "1", "name": "Big Corp", "revenue": 5000000}, {"accountid": "2", "name": "Mega Inc", "revenue": 4000000}, @@ -343,8 +360,8 @@ def test_builder_full_fluent_workflow(self): records = list( self.client.query.builder("account") .select("name", "revenue") - .filter_eq("statecode", 0) - .filter_gt("revenue", 1000000) + .where(col("statecode") == 0) + .where(col("revenue") > 1000000) .order_by("revenue", descending=True) .expand("primarycontactid") .top(10) @@ -357,23 +374,24 @@ def test_builder_full_fluent_workflow(self): self.assertEqual(records[1]["name"], "Mega Inc") def test_builder_to_dataframe(self): - """builder().to_dataframe() should delegate to client.dataframe.get().""" + """builder().to_dataframe() should collect records into a DataFrame.""" import pandas as pd + from PowerPlatform.Dataverse.models.filters import raw - expected_df = pd.DataFrame([{"name": "Contoso", "revenue": 1000}]) - self.client.dataframe = MagicMock() - self.client.dataframe.get.return_value = expected_df + expected_records = [{"name": "Contoso", "revenue": 1000}] + self.client._odata._get_multiple.return_value = iter([expected_records]) result = ( self.client.query.builder("account") .select("name", "revenue") - .filter_eq("statecode", 0) + .where(raw("statecode eq 0")) .order_by("name") .top(50) + .execute() .to_dataframe() ) - self.client.dataframe.get.assert_called_once_with( + self.client._odata._get_multiple.assert_called_once_with( "account", select=["name", "revenue"], filter="statecode eq 0", @@ -384,7 +402,9 @@ def test_builder_to_dataframe(self): count=False, include_annotations=None, ) - pd.testing.assert_frame_equal(result, expected_df) + self.assertIsInstance(result, pd.DataFrame) + self.assertEqual(len(result), 1) + self.assertEqual(result.iloc[0]["name"], "Contoso") # =================================================================== @@ -526,239 +546,25 @@ def test_excludes_attribute_of_columns(self): self.assertNotIn("createdbyname", names) -class TestSqlSelect(unittest.TestCase): - """Tests for client.query.sql_select().""" - - def setUp(self): - self.mock_credential = MagicMock(spec=TokenCredential) - self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) - self.client._odata = MagicMock() - - def test_returns_comma_separated(self): - self.client._odata._list_columns.return_value = [ - { - "LogicalName": "accountid", - "AttributeType": "Uniqueidentifier", - "IsPrimaryId": True, - "IsPrimaryName": False, - "DisplayName": {}, - }, - { - "LogicalName": "name", - "AttributeType": "String", - "IsPrimaryId": False, - "IsPrimaryName": True, - "DisplayName": {}, - }, - { - "LogicalName": "revenue", - "AttributeType": "Money", - "IsPrimaryId": False, - "IsPrimaryName": False, - "DisplayName": {}, - }, - ] - result = self.client.query.sql_select("account") - self.assertIn("accountid", result) - self.assertIn("name", result) - self.assertIn("revenue", result) - self.assertEqual(result.count(","), 2) # 3 cols = 2 commas - - -class TestSqlJoins(unittest.TestCase): - """Tests for client.query.sql_joins().""" - - def setUp(self): - self.mock_credential = MagicMock(spec=TokenCredential) - self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) - self.client._odata = MagicMock() - - def _mock_rels(self, rels): - self.client._odata._list_table_relationships.return_value = rels - - def test_outgoing_lookups(self): - self._mock_rels( - [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "parentcustomerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "SchemaName": "contact_customer_accounts", - }, - ] - ) - joins = self.client.query.sql_joins("contact") - self.assertEqual(len(joins), 1) - j = joins[0] - self.assertEqual(j["column"], "parentcustomerid") - self.assertEqual(j["target"], "account") - self.assertEqual(j["target_pk"], "accountid") - self.assertIn("JOIN account", j["join_clause"]) - self.assertIn("parentcustomerid", j["join_clause"]) - - def test_ignores_incoming_rels(self): - self._mock_rels( - [ - # This is an incoming relationship (account is referenced, not referencing) - { - "ReferencingEntity": "opportunity", - "ReferencingAttribute": "customerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "SchemaName": "opp_customer_accounts", - }, - ] - ) - joins = self.client.query.sql_joins("account") - self.assertEqual(len(joins), 0) - - def test_polymorphic_returns_multiple(self): - self._mock_rels( - [ - { - "ReferencingEntity": "opportunity", - "ReferencingAttribute": "customerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "SchemaName": "opp_customer_accounts", - }, - { - "ReferencingEntity": "opportunity", - "ReferencingAttribute": "customerid", - "ReferencedEntity": "contact", - "ReferencedAttribute": "contactid", - "SchemaName": "opp_customer_contacts", - }, - ] - ) - joins = self.client.query.sql_joins("opportunity") - self.assertEqual(len(joins), 2) - targets = {j["target"] for j in joins} - self.assertEqual(targets, {"account", "contact"}) - # Both use the same source column - self.assertTrue(all(j["column"] == "customerid" for j in joins)) - - def test_empty_relationships(self): - self._mock_rels([]) - joins = self.client.query.sql_joins("account") - self.assertEqual(joins, []) - - def test_alias_collision_same_first_letter(self): - """Two targets starting with the same letter get distinct aliases.""" - self._mock_rels( - [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "parentcustomerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "SchemaName": "contact_customer_accounts", - }, - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "regardingobjectid", - "ReferencedEntity": "annotation", - "ReferencedAttribute": "annotationid", - "SchemaName": "contact_annotation", - }, - ] - ) - joins = self.client.query.sql_joins("contact") - self.assertEqual(len(joins), 2) - aliases = [j["join_clause"].split()[2] for j in joins] - self.assertEqual(len(set(aliases)), 2, "aliases must be unique") - self.assertNotEqual(aliases[0], aliases[1]) - - def test_alias_collision_same_target_table(self): - """Two lookups to the same table (e.g. ownerid + createdby -> systemuser) get distinct aliases.""" - self._mock_rels( - [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "ownerid", - "ReferencedEntity": "systemuser", - "ReferencedAttribute": "systemuserid", - "SchemaName": "contact_ownerid_systemuser", - }, - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "createdby", - "ReferencedEntity": "systemuser", - "ReferencedAttribute": "systemuserid", - "SchemaName": "contact_createdby_systemuser", - }, - ] - ) - joins = self.client.query.sql_joins("contact") - self.assertEqual(len(joins), 2) - aliases = [j["join_clause"].split()[2] for j in joins] - self.assertEqual(len(set(aliases)), 2, "aliases must be unique") - self.assertNotEqual(aliases[0], aliases[1]) - - -class TestSqlJoin(unittest.TestCase): - """Tests for client.query.sql_join().""" +class TestRemovedSqlHelpers(unittest.TestCase): + """sql_select(), sql_join(), sql_joins() are removed at GA — raise AttributeError.""" def setUp(self): self.mock_credential = MagicMock(spec=TokenCredential) self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) self.client._odata = MagicMock() - def _mock_rels(self, rels): - self.client._odata._list_table_relationships.return_value = rels - - def test_generates_join_clause(self): - self._mock_rels( - [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "parentcustomerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "SchemaName": "contact_customer_accounts", - }, - ] - ) - result = self.client.query.sql_join("contact", "account", from_alias="c", to_alias="a") - self.assertEqual(result, "JOIN account a ON c.parentcustomerid = a.accountid") - - def test_default_aliases(self): - self._mock_rels( - [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "parentcustomerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "SchemaName": "contact_customer_accounts", - }, - ] - ) - result = self.client.query.sql_join("contact", "account") - self.assertIn("JOIN account a ON contact.parentcustomerid = a.accountid", result) + def test_sql_select_removed(self): + with self.assertRaises(AttributeError): + self.client.query.sql_select("account") - def test_no_relationship_raises(self): - self._mock_rels([]) - with self.assertRaises(ValueError) as ctx: - self.client.query.sql_join("contact", "nonexistent") - self.assertIn("No relationship found", str(ctx.exception)) + def test_sql_joins_removed(self): + with self.assertRaises(AttributeError): + self.client.query.sql_joins("contact") - def test_case_insensitive_target(self): - self._mock_rels( - [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "ownerid", - "ReferencedEntity": "systemuser", - "ReferencedAttribute": "systemuserid", - "SchemaName": "contact_owner", - }, - ] - ) - result = self.client.query.sql_join("contact", "SystemUser", from_alias="c", to_alias="su") - self.assertIn("JOIN systemuser su", result) - self.assertIn("c.ownerid = su.systemuserid", result) + def test_sql_join_removed(self): + with self.assertRaises(AttributeError): + self.client.query.sql_join("contact", "account") # =================================================================== @@ -767,13 +573,18 @@ def test_case_insensitive_target(self): class TestOdataSelect(unittest.TestCase): - """Tests for client.query.odata_select().""" + """Tests for client.query.odata_select() — deprecated at GA, still functional.""" def setUp(self): self.mock_credential = MagicMock(spec=TokenCredential) self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) self.client._odata = MagicMock() + def test_emits_deprecation_warning(self): + self.client._odata._list_columns.return_value = [] + with self.assertWarns(DeprecationWarning): + self.client.query.odata_select("account") + def test_returns_list_of_strings(self): self.client._odata._list_columns.return_value = [ { @@ -791,7 +602,11 @@ def test_returns_list_of_strings(self): "DisplayName": {}, }, ] - result = self.client.query.odata_select("account") + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = self.client.query.odata_select("account") self.assertIsInstance(result, list) self.assertIn("accountid", result) self.assertIn("name", result) @@ -807,7 +622,11 @@ def test_result_usable_in_records_get(self): "DisplayName": {}, }, ] - cols = self.client.query.odata_select("account") + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + cols = self.client.query.odata_select("account") self.assertEqual(cols, ["name"]) @@ -914,13 +733,28 @@ def test_metadata_error_on_entity_set_resolution_is_swallowed(self): class TestOdataExpand(unittest.TestCase): - """Tests for client.query.odata_expand().""" + """Tests for client.query.odata_expand() — deprecated at GA, still functional.""" def setUp(self): self.mock_credential = MagicMock(spec=TokenCredential) self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) self.client._odata = MagicMock() + def test_emits_deprecation_warning(self): + self.client._odata._list_table_relationships.return_value = [ + { + "ReferencingEntity": "contact", + "ReferencingAttribute": "parentcustomerid", + "ReferencedEntity": "account", + "ReferencedAttribute": "accountid", + "ReferencingEntityNavigationPropertyName": "parentcustomerid_account", + "SchemaName": "contact_customer_accounts", + }, + ] + self.client._odata._entity_set_from_schema_name.return_value = "accounts" + with self.assertWarns(DeprecationWarning): + self.client.query.odata_expand("contact", "account") + def test_returns_nav_property(self): self.client._odata._list_table_relationships.return_value = [ { @@ -933,7 +767,11 @@ def test_returns_nav_property(self): }, ] self.client._odata._entity_set_from_schema_name.return_value = "accounts" - result = self.client.query.odata_expand("contact", "account") + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = self.client.query.odata_expand("contact", "account") self.assertEqual(result, "parentcustomerid_account") def test_no_relationship_raises(self): @@ -954,20 +792,24 @@ def test_case_insensitive_target(self): }, ] self.client._odata._entity_set_from_schema_name.return_value = "systemusers" - result = self.client.query.odata_expand("contact", "SystemUser") + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = self.client.query.odata_expand("contact", "SystemUser") self.assertEqual(result, "ownerid_systemuser") class TestOdataBind(unittest.TestCase): - """Tests for client.query.odata_bind().""" + """Tests for client.query.odata_bind() — deprecated at GA, still functional.""" def setUp(self): self.mock_credential = MagicMock(spec=TokenCredential) self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) self.client._odata = MagicMock() - def test_returns_bind_dict(self): - self.client._odata._list_table_relationships.return_value = [ + def _rel(self): + return [ { "ReferencingEntity": "contact", "ReferencingAttribute": "parentcustomerid", @@ -977,10 +819,23 @@ def test_returns_bind_dict(self): "SchemaName": "contact_customer_accounts", }, ] + + def test_emits_deprecation_warning(self): + self.client._odata._list_table_relationships.return_value = self._rel() + self.client._odata._entity_set_from_schema_name.return_value = "accounts" + with self.assertWarns(DeprecationWarning): + self.client.query.odata_bind("contact", "account", "some-guid") + + def test_returns_bind_dict(self): + self.client._odata._list_table_relationships.return_value = self._rel() self.client._odata._entity_set_from_schema_name.return_value = "accounts" guid = "12345678-1234-1234-1234-123456789abc" - result = self.client.query.odata_bind("contact", "account", guid) + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = self.client.query.odata_bind("contact", "account", guid) self.assertIsInstance(result, dict) self.assertEqual(len(result), 1) key = list(result.keys())[0] @@ -994,19 +849,14 @@ def test_no_relationship_raises(self): def test_usable_in_create_payload(self): """Result can be merged into a create payload via **spread.""" - self.client._odata._list_table_relationships.return_value = [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "parentcustomerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "ReferencingEntityNavigationPropertyName": "parentcustomerid_account", - "SchemaName": "contact_customer_accounts", - }, - ] + self.client._odata._list_table_relationships.return_value = self._rel() self.client._odata._entity_set_from_schema_name.return_value = "accounts" - bind = self.client.query.odata_bind("contact", "account", "some-guid") + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + bind = self.client.query.odata_bind("contact", "account", "some-guid") payload = {"firstname": "Jane", "lastname": "Doe", **bind} self.assertIn("parentcustomerid_account@odata.bind", payload) self.assertEqual(payload["firstname"], "Jane") diff --git a/tests/unit/test_records_operations.py b/tests/unit/test_records_operations.py index 09df6869..b9ae98d8 100644 --- a/tests/unit/test_records_operations.py +++ b/tests/unit/test_records_operations.py @@ -389,5 +389,95 @@ def test_upsert_composite_alternate_key(self): self.client._odata._upsert_multiple.assert_not_called() +class TestListPages(unittest.TestCase): + """Unit tests for records.list_pages().""" + + def setUp(self): + self.mock_credential = MagicMock(spec=TokenCredential) + self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) + self.client._odata = MagicMock() + + def test_list_pages_returns_iterator(self): + self.client._odata._get_multiple.return_value = iter([[{"name": "A"}], [{"name": "B"}]]) + result = self.client.records.list_pages("account") + import types + + self.assertIsInstance(result, types.GeneratorType) + + def test_list_pages_yields_query_result_per_page(self): + from PowerPlatform.Dataverse.models.record import QueryResult + + self.client._odata._get_multiple.return_value = iter([[{"name": "A"}], [{"name": "B"}]]) + pages = list(self.client.records.list_pages("account")) + self.assertEqual(len(pages), 2) + for page in pages: + self.assertIsInstance(page, QueryResult) + + def test_list_pages_page_contents(self): + self.client._odata._get_multiple.return_value = iter([[{"name": "A"}], [{"name": "B"}, {"name": "C"}]]) + pages = list(self.client.records.list_pages("account")) + self.assertEqual(len(pages[0]), 1) + self.assertEqual(len(pages[1]), 2) + + def test_list_pages_passes_filter(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", filter="statecode eq 0")) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("filter") or call_kwargs[1].get("filter"), "statecode eq 0") + + def test_list_pages_passes_select(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", select=["name"])) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("select") or call_kwargs[1].get("select"), ["name"]) + + def test_list_pages_passes_top(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", top=50)) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("top") or call_kwargs[1].get("top"), 50) + + def test_list_pages_no_deprecation_warning(self): + import warnings + + self.client._odata._get_multiple.return_value = iter([]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + list(self.client.records.list_pages("account", filter="statecode eq 0")) + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(deprecations), 0) + + def test_list_pages_passes_orderby(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", orderby=["name asc"])) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("orderby"), ["name asc"]) + + def test_list_pages_passes_expand(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", expand=["primarycontactid"])) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("expand"), ["primarycontactid"]) + + def test_list_pages_passes_page_size(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", page_size=200)) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("page_size"), 200) + + def test_list_pages_passes_count(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", count=True)) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertTrue(call_kwargs.kwargs.get("count")) + + def test_list_pages_passes_include_annotations(self): + annotation = "OData.Community.Display.V1.FormattedValue" + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", include_annotations=annotation)) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("include_annotations"), annotation) + + if __name__ == "__main__": unittest.main()